1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * BPF Jit compiler for s390. 4 * 5 * Minimum build requirements: 6 * 7 * - HAVE_MARCH_Z196_FEATURES: laal, laalg 8 * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj 9 * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf 10 * - 64BIT 11 * 12 * Copyright IBM Corp. 2012,2015 13 * 14 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 15 * Michael Holzheu <holzheu@linux.vnet.ibm.com> 16 */ 17 18 #define pr_fmt(fmt) "bpf_jit: " fmt 19 20 #include <linux/netdevice.h> 21 #include <linux/filter.h> 22 #include <linux/init.h> 23 #include <linux/bpf.h> 24 #include <linux/cfi.h> 25 #include <linux/mm.h> 26 #include <linux/kernel.h> 27 #include <asm/cacheflush.h> 28 #include <asm/extable.h> 29 #include <asm/dis.h> 30 #include <asm/facility.h> 31 #include <asm/lowcore.h> 32 #include <asm/nospec-branch.h> 33 #include <asm/set_memory.h> 34 #include <asm/text-patching.h> 35 #include <asm/unwind.h> 36 37 struct bpf_jit { 38 u32 seen; /* Flags to remember seen eBPF instructions */ 39 u16 seen_regs; /* Mask to remember which registers are used */ 40 u32 *addrs; /* Array with relative instruction addresses */ 41 u8 *prg_buf; /* Start of program */ 42 int size; /* Size of program and literal pool */ 43 int size_prg; /* Size of program */ 44 int prg; /* Current position in program */ 45 int lit32_start; /* Start of 32-bit literal pool */ 46 int lit32; /* Current position in 32-bit literal pool */ 47 int lit64_start; /* Start of 64-bit literal pool */ 48 int lit64; /* Current position in 64-bit literal pool */ 49 int base_ip; /* Base address for literal pool */ 50 int exit_ip; /* Address of exit */ 51 int tail_call_start; /* Tail call start offset */ 52 int excnt; /* Number of exception table entries */ 53 int prologue_plt_ret; /* Return address for prologue hotpatch PLT */ 54 int prologue_plt; /* Start of prologue hotpatch PLT */ 55 int kern_arena; /* Pool offset of kernel arena address */ 56 u64 user_arena; /* User arena address */ 57 u32 frame_off; /* Offset of struct bpf_prog from %r15 */ 58 }; 59 60 #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */ 61 #define SEEN_LITERAL BIT(1) /* code uses literals */ 62 #define SEEN_FUNC BIT(2) /* calls C functions */ 63 #define SEEN_STACK (SEEN_FUNC | SEEN_MEM) 64 65 #define NVREGS 0xffc0 /* %r6-%r15 */ 66 67 /* 68 * s390 registers 69 */ 70 #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ 71 #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ 72 #define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */ 73 #define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */ 74 #define REG_0 REG_W0 /* Register 0 */ 75 #define REG_1 REG_W1 /* Register 1 */ 76 #define REG_2 BPF_REG_1 /* Register 2 */ 77 #define REG_3 BPF_REG_2 /* Register 3 */ 78 #define REG_4 BPF_REG_3 /* Register 4 */ 79 #define REG_7 BPF_REG_6 /* Register 7 */ 80 #define REG_8 BPF_REG_7 /* Register 8 */ 81 #define REG_14 BPF_REG_0 /* Register 14 */ 82 83 /* 84 * Mapping of BPF registers to s390 registers 85 */ 86 static const int reg2hex[] = { 87 /* Return code */ 88 [BPF_REG_0] = 14, 89 /* Function parameters */ 90 [BPF_REG_1] = 2, 91 [BPF_REG_2] = 3, 92 [BPF_REG_3] = 4, 93 [BPF_REG_4] = 5, 94 [BPF_REG_5] = 6, 95 /* Call saved registers */ 96 [BPF_REG_6] = 7, 97 [BPF_REG_7] = 8, 98 [BPF_REG_8] = 9, 99 [BPF_REG_9] = 10, 100 /* BPF stack pointer */ 101 [BPF_REG_FP] = 13, 102 /* Register for blinding */ 103 [BPF_REG_AX] = 12, 104 /* Work registers for s390x backend */ 105 [REG_W0] = 0, 106 [REG_W1] = 1, 107 [REG_L] = 11, 108 [REG_15] = 15, 109 }; 110 111 static inline u32 reg(u32 dst_reg, u32 src_reg) 112 { 113 return reg2hex[dst_reg] << 4 | reg2hex[src_reg]; 114 } 115 116 static inline u32 reg_high(u32 reg) 117 { 118 return reg2hex[reg] << 4; 119 } 120 121 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 122 { 123 u32 r1 = reg2hex[b1]; 124 125 if (r1 >= 6 && r1 <= 15) 126 jit->seen_regs |= (1 << r1); 127 } 128 129 static s32 off_to_pcrel(struct bpf_jit *jit, u32 off) 130 { 131 return off - jit->prg; 132 } 133 134 static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr) 135 { 136 if (jit->prg_buf) 137 return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg); 138 return 0; 139 } 140 141 #define REG_SET_SEEN(b1) \ 142 ({ \ 143 reg_set_seen(jit, b1); \ 144 }) 145 146 /* 147 * EMIT macros for code generation 148 */ 149 150 #define _EMIT2(op) \ 151 ({ \ 152 if (jit->prg_buf) \ 153 *(u16 *) (jit->prg_buf + jit->prg) = (op); \ 154 jit->prg += 2; \ 155 }) 156 157 #define EMIT2(op, b1, b2) \ 158 ({ \ 159 _EMIT2((op) | reg(b1, b2)); \ 160 REG_SET_SEEN(b1); \ 161 REG_SET_SEEN(b2); \ 162 }) 163 164 #define _EMIT4(op) \ 165 ({ \ 166 if (jit->prg_buf) \ 167 *(u32 *) (jit->prg_buf + jit->prg) = (op); \ 168 jit->prg += 4; \ 169 }) 170 171 #define EMIT4(op, b1, b2) \ 172 ({ \ 173 _EMIT4((op) | reg(b1, b2)); \ 174 REG_SET_SEEN(b1); \ 175 REG_SET_SEEN(b2); \ 176 }) 177 178 #define EMIT4_RRF(op, b1, b2, b3) \ 179 ({ \ 180 _EMIT4((op) | reg_high(b3) << 8 | reg(b1, b2)); \ 181 REG_SET_SEEN(b1); \ 182 REG_SET_SEEN(b2); \ 183 REG_SET_SEEN(b3); \ 184 }) 185 186 #define _EMIT4_DISP(op, disp) \ 187 ({ \ 188 unsigned int __disp = (disp) & 0xfff; \ 189 _EMIT4((op) | __disp); \ 190 }) 191 192 #define EMIT4_DISP(op, b1, b2, disp) \ 193 ({ \ 194 _EMIT4_DISP((op) | reg_high(b1) << 16 | \ 195 reg_high(b2) << 8, (disp)); \ 196 REG_SET_SEEN(b1); \ 197 REG_SET_SEEN(b2); \ 198 }) 199 200 #define EMIT4_IMM(op, b1, imm) \ 201 ({ \ 202 unsigned int __imm = (imm) & 0xffff; \ 203 _EMIT4((op) | reg_high(b1) << 16 | __imm); \ 204 REG_SET_SEEN(b1); \ 205 }) 206 207 #define EMIT4_PCREL(op, pcrel) \ 208 ({ \ 209 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 210 _EMIT4((op) | __pcrel); \ 211 }) 212 213 #define EMIT4_PCREL_RIC(op, mask, target) \ 214 ({ \ 215 int __rel = off_to_pcrel(jit, target) / 2; \ 216 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \ 217 }) 218 219 #define _EMIT6(op1, op2) \ 220 ({ \ 221 if (jit->prg_buf) { \ 222 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \ 223 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \ 224 } \ 225 jit->prg += 6; \ 226 }) 227 228 #define _EMIT6_DISP(op1, op2, disp) \ 229 ({ \ 230 unsigned int __disp = (disp) & 0xfff; \ 231 _EMIT6((op1) | __disp, op2); \ 232 }) 233 234 #define _EMIT6_DISP_LH(op1, op2, disp) \ 235 ({ \ 236 u32 _disp = (u32) (disp); \ 237 unsigned int __disp_h = _disp & 0xff000; \ 238 unsigned int __disp_l = _disp & 0x00fff; \ 239 _EMIT6((op1) | __disp_l, (op2) | __disp_h >> 4); \ 240 }) 241 242 #define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \ 243 ({ \ 244 _EMIT6_DISP_LH((op1) | reg(b1, b2) << 16 | \ 245 reg_high(b3) << 8, op2, disp); \ 246 REG_SET_SEEN(b1); \ 247 REG_SET_SEEN(b2); \ 248 REG_SET_SEEN(b3); \ 249 }) 250 251 #define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \ 252 ({ \ 253 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 254 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \ 255 (op2) | (mask) << 12); \ 256 REG_SET_SEEN(b1); \ 257 REG_SET_SEEN(b2); \ 258 }) 259 260 #define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \ 261 ({ \ 262 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 263 _EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \ 264 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \ 265 REG_SET_SEEN(b1); \ 266 BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \ 267 }) 268 269 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \ 270 ({ \ 271 int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\ 272 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\ 273 REG_SET_SEEN(b1); \ 274 REG_SET_SEEN(b2); \ 275 }) 276 277 static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel) 278 { 279 u32 pc32dbl = (s32)(pcrel / 2); 280 281 _EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff); 282 } 283 284 static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel) 285 { 286 emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel); 287 REG_SET_SEEN(b); 288 } 289 290 #define EMIT6_PCREL_RILB(op, b, target) \ 291 emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target)) 292 293 #define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \ 294 emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr)) 295 296 static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel) 297 { 298 emit6_pcrel_ril(jit, op | mask << 20, pcrel); 299 } 300 301 #define EMIT6_PCREL_RILC(op, mask, target) \ 302 emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target)) 303 304 #define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \ 305 emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr)) 306 307 #define _EMIT6_IMM(op, imm) \ 308 ({ \ 309 unsigned int __imm = (imm); \ 310 _EMIT6((op) | (__imm >> 16), __imm & 0xffff); \ 311 }) 312 313 #define EMIT6_IMM(op, b1, imm) \ 314 ({ \ 315 _EMIT6_IMM((op) | reg_high(b1) << 16, imm); \ 316 REG_SET_SEEN(b1); \ 317 }) 318 319 #define _EMIT_CONST_U32(val) \ 320 ({ \ 321 unsigned int ret; \ 322 ret = jit->lit32; \ 323 if (jit->prg_buf) \ 324 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\ 325 jit->lit32 += 4; \ 326 ret; \ 327 }) 328 329 #define EMIT_CONST_U32(val) \ 330 ({ \ 331 jit->seen |= SEEN_LITERAL; \ 332 _EMIT_CONST_U32(val) - jit->base_ip; \ 333 }) 334 335 #define _EMIT_CONST_U64(val) \ 336 ({ \ 337 unsigned int ret; \ 338 ret = jit->lit64; \ 339 if (jit->prg_buf) \ 340 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\ 341 jit->lit64 += 8; \ 342 ret; \ 343 }) 344 345 #define EMIT_CONST_U64(val) \ 346 ({ \ 347 jit->seen |= SEEN_LITERAL; \ 348 _EMIT_CONST_U64(val) - jit->base_ip; \ 349 }) 350 351 #define EMIT_ZERO(b1) \ 352 ({ \ 353 if (!fp->aux->verifier_zext) { \ 354 /* llgfr %dst,%dst (zero extend to 64 bit) */ \ 355 EMIT4(0xb9160000, b1, b1); \ 356 REG_SET_SEEN(b1); \ 357 } \ 358 }) 359 360 static inline void emit_u32_data(const u32 data, struct bpf_jit *jit) 361 { 362 if (jit->prg_buf) 363 *(u32 *)(jit->prg_buf + jit->prg) = data; 364 jit->prg += 4; 365 } 366 367 static inline void emit_kcfi(u32 hash, struct bpf_jit *jit) 368 { 369 if (IS_ENABLED(CONFIG_CFI)) 370 emit_u32_data(hash, jit); 371 } 372 373 /* 374 * Return whether this is the first pass. The first pass is special, since we 375 * don't know any sizes yet, and thus must be conservative. 376 */ 377 static bool is_first_pass(struct bpf_jit *jit) 378 { 379 return jit->size == 0; 380 } 381 382 /* 383 * Return whether this is the code generation pass. The code generation pass is 384 * special, since we should change as little as possible. 385 */ 386 static bool is_codegen_pass(struct bpf_jit *jit) 387 { 388 return jit->prg_buf; 389 } 390 391 /* 392 * Return whether "rel" can be encoded as a short PC-relative offset 393 */ 394 static bool is_valid_rel(int rel) 395 { 396 return rel >= -65536 && rel <= 65534; 397 } 398 399 /* 400 * Return whether "off" can be reached using a short PC-relative offset 401 */ 402 static bool can_use_rel(struct bpf_jit *jit, int off) 403 { 404 return is_valid_rel(off - jit->prg); 405 } 406 407 /* 408 * Return whether given displacement can be encoded using 409 * Long-Displacement Facility 410 */ 411 static bool is_valid_ldisp(int disp) 412 { 413 return disp >= -524288 && disp <= 524287; 414 } 415 416 /* 417 * Return whether the next 32-bit literal pool entry can be referenced using 418 * Long-Displacement Facility 419 */ 420 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit) 421 { 422 return is_valid_ldisp(jit->lit32 - jit->base_ip); 423 } 424 425 /* 426 * Return whether the next 64-bit literal pool entry can be referenced using 427 * Long-Displacement Facility 428 */ 429 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit) 430 { 431 return is_valid_ldisp(jit->lit64 - jit->base_ip); 432 } 433 434 /* 435 * Fill whole space with illegal instructions 436 */ 437 static void jit_fill_hole(void *area, unsigned int size) 438 { 439 memset(area, 0, size); 440 } 441 442 /* 443 * Caller-allocated part of the frame. 444 * Thanks to packed stack, its otherwise unused initial part can be used for 445 * the BPF stack and for the next frame. 446 */ 447 struct prog_frame { 448 u64 unused[8]; 449 /* BPF stack starts here and grows towards 0 */ 450 u32 tail_call_cnt; 451 u32 pad; 452 u64 r6[10]; /* r6 - r15 */ 453 u64 backchain; 454 } __packed; 455 456 /* 457 * Save registers from "rs" (register start) to "re" (register end) on stack 458 */ 459 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) 460 { 461 u32 off = offsetof(struct prog_frame, r6) + (rs - 6) * 8; 462 463 if (rs == re) 464 /* stg %rs,off(%r15) */ 465 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); 466 else 467 /* stmg %rs,%re,off(%r15) */ 468 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); 469 } 470 471 /* 472 * Restore registers from "rs" (register start) to "re" (register end) on stack 473 */ 474 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) 475 { 476 u32 off = jit->frame_off + offsetof(struct prog_frame, r6) + (rs - 6) * 8; 477 478 if (rs == re) 479 /* lg %rs,off(%r15) */ 480 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); 481 else 482 /* lmg %rs,%re,off(%r15) */ 483 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); 484 } 485 486 /* 487 * Return first seen register (from start) 488 */ 489 static int get_start(u16 seen_regs, int start) 490 { 491 int i; 492 493 for (i = start; i <= 15; i++) { 494 if (seen_regs & (1 << i)) 495 return i; 496 } 497 return 0; 498 } 499 500 /* 501 * Return last seen register (from start) (gap >= 2) 502 */ 503 static int get_end(u16 seen_regs, int start) 504 { 505 int i; 506 507 for (i = start; i < 15; i++) { 508 if (!(seen_regs & (3 << i))) 509 return i - 1; 510 } 511 return (seen_regs & (1 << 15)) ? 15 : 14; 512 } 513 514 #define REGS_SAVE 1 515 #define REGS_RESTORE 0 516 /* 517 * Save and restore clobbered registers (6-15) on stack. 518 * We save/restore registers in chunks with gap >= 2 registers. 519 */ 520 static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs) 521 { 522 u16 seen_regs = jit->seen_regs | extra_regs; 523 const int last = 15, save_restore_size = 6; 524 int re = 6, rs; 525 526 if (is_first_pass(jit)) { 527 /* 528 * We don't know yet which registers are used. Reserve space 529 * conservatively. 530 */ 531 jit->prg += (last - re + 1) * save_restore_size; 532 return; 533 } 534 535 do { 536 rs = get_start(seen_regs, re); 537 if (!rs) 538 break; 539 re = get_end(seen_regs, rs + 1); 540 if (op == REGS_SAVE) 541 save_regs(jit, rs, re); 542 else 543 restore_regs(jit, rs, re); 544 re++; 545 } while (re <= last); 546 } 547 548 static void bpf_skip(struct bpf_jit *jit, int size) 549 { 550 if (size >= 6 && !is_valid_rel(size)) { 551 /* brcl 0xf,size */ 552 EMIT6_PCREL_RILC(0xc0040000, 0xf, size); 553 size -= 6; 554 } else if (size >= 4 && is_valid_rel(size)) { 555 /* brc 0xf,size */ 556 EMIT4_PCREL(0xa7f40000, size); 557 size -= 4; 558 } 559 while (size >= 2) { 560 /* bcr 0,%0 */ 561 _EMIT2(0x0700); 562 size -= 2; 563 } 564 } 565 566 /* 567 * PLT for hotpatchable calls. The calling convention is the same as for the 568 * ftrace hotpatch trampolines: %r0 is return address, %r1 is clobbered. 569 */ 570 struct bpf_plt { 571 char code[16]; 572 void *ret; 573 void *target; 574 } __packed; 575 extern const struct bpf_plt bpf_plt; 576 asm( 577 ".pushsection .rodata\n" 578 " .balign 8\n" 579 "bpf_plt:\n" 580 " lgrl %r0,bpf_plt_ret\n" 581 " lgrl %r1,bpf_plt_target\n" 582 " br %r1\n" 583 " .balign 8\n" 584 "bpf_plt_ret: .quad 0\n" 585 "bpf_plt_target: .quad 0\n" 586 " .popsection\n" 587 ); 588 589 static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target) 590 { 591 memcpy(plt, &bpf_plt, sizeof(*plt)); 592 plt->ret = ret; 593 /* 594 * (target == NULL) implies that the branch to this PLT entry was 595 * patched and became a no-op. However, some CPU could have jumped 596 * to this PLT entry before patching and may be still executing it. 597 * 598 * Since the intention in this case is to make the PLT entry a no-op, 599 * make the target point to the return label instead of NULL. 600 */ 601 plt->target = target ?: ret; 602 } 603 604 /* 605 * Emit function prologue 606 * 607 * Save registers and create stack frame if necessary. 608 * Stack frame layout is described by struct prog_frame. 609 */ 610 static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) 611 { 612 BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); 613 614 emit_kcfi(bpf_is_subprog(fp) ? cfi_bpf_subprog_hash : cfi_bpf_hash, jit); 615 616 /* No-op for hotpatching */ 617 /* brcl 0,prologue_plt */ 618 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); 619 jit->prologue_plt_ret = jit->prg; 620 621 if (!bpf_is_subprog(fp)) { 622 /* Initialize the tail call counter in the main program. */ 623 /* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */ 624 _EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt), 625 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 626 } else { 627 /* 628 * Skip the tail call counter initialization in subprograms. 629 * Insert nops in order to have tail_call_start at a 630 * predictable offset. 631 */ 632 bpf_skip(jit, 6); 633 } 634 /* Tail calls have to skip above initialization */ 635 jit->tail_call_start = jit->prg - cfi_get_offset(); 636 if (fp->aux->exception_cb) { 637 /* 638 * Switch stack, the new address is in the 2nd parameter. 639 * 640 * Arrange the restoration of %r6-%r15 in the epilogue. 641 * Do not restore them now, the prog does not need them. 642 */ 643 /* lgr %r15,%r3 */ 644 EMIT4(0xb9040000, REG_15, REG_3); 645 jit->seen_regs |= NVREGS; 646 } else { 647 /* Save registers */ 648 save_restore_regs(jit, REGS_SAVE, 649 fp->aux->exception_boundary ? NVREGS : 0); 650 } 651 /* Setup literal pool */ 652 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) { 653 if (!is_first_pass(jit) && 654 is_valid_ldisp(jit->size - (jit->prg + 2))) { 655 /* basr %l,0 */ 656 EMIT2(0x0d00, REG_L, REG_0); 657 jit->base_ip = jit->prg; 658 } else { 659 /* larl %l,lit32_start */ 660 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); 661 jit->base_ip = jit->lit32_start; 662 } 663 } 664 /* Setup stack and backchain */ 665 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) { 666 /* lgr %w1,%r15 (backchain) */ 667 EMIT4(0xb9040000, REG_W1, REG_15); 668 /* la %bfp,unused_end(%r15) (BPF frame pointer) */ 669 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, 670 offsetofend(struct prog_frame, unused)); 671 /* aghi %r15,-frame_off */ 672 EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off); 673 /* stg %w1,backchain(%r15) */ 674 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, 675 REG_15, 676 offsetof(struct prog_frame, backchain)); 677 } 678 } 679 680 /* 681 * Jump using a register either directly or via an expoline thunk 682 */ 683 #define EMIT_JUMP_REG(reg) do { \ 684 if (nospec_uses_trampoline()) \ 685 /* brcl 0xf,__s390_indirect_jump_rN */ \ 686 EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \ 687 __s390_indirect_jump_r ## reg); \ 688 else \ 689 /* br %rN */ \ 690 _EMIT2(0x07f0 | reg); \ 691 } while (0) 692 693 /* 694 * Function epilogue 695 */ 696 static void bpf_jit_epilogue(struct bpf_jit *jit) 697 { 698 jit->exit_ip = jit->prg; 699 /* Load exit code: lgr %r2,%b0 */ 700 EMIT4(0xb9040000, REG_2, BPF_REG_0); 701 /* Restore registers */ 702 save_restore_regs(jit, REGS_RESTORE, 0); 703 EMIT_JUMP_REG(14); 704 705 jit->prg = ALIGN(jit->prg, 8); 706 jit->prologue_plt = jit->prg; 707 if (jit->prg_buf) 708 bpf_jit_plt((struct bpf_plt *)(jit->prg_buf + jit->prg), 709 jit->prg_buf + jit->prologue_plt_ret, NULL); 710 jit->prg += sizeof(struct bpf_plt); 711 } 712 713 bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) 714 { 715 regs->psw.addr = extable_fixup(x); 716 if (x->data != -1) 717 regs->gprs[x->data] = 0; 718 return true; 719 } 720 721 /* 722 * A single BPF probe instruction 723 */ 724 struct bpf_jit_probe { 725 int prg; /* JITed instruction offset */ 726 int reg; /* Register to clear on exception */ 727 int arena_reg; /* Register to use for arena addressing */ 728 }; 729 730 static void bpf_jit_probe_init(struct bpf_jit_probe *probe) 731 { 732 probe->prg = -1; 733 probe->reg = -1; 734 probe->arena_reg = REG_0; 735 } 736 737 static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, 738 struct bpf_jit_probe *probe) 739 { 740 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 741 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 742 BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 743 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 744 return; 745 746 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 747 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { 748 /* lgrl %r1,kern_arena */ 749 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 750 probe->arena_reg = REG_W1; 751 } 752 probe->prg = jit->prg; 753 probe->reg = reg2hex[insn->dst_reg]; 754 } 755 756 static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, 757 struct bpf_jit_probe *probe) 758 { 759 if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 760 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 761 return; 762 763 /* lgrl %r1,kern_arena */ 764 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 765 probe->arena_reg = REG_W1; 766 probe->prg = jit->prg; 767 } 768 769 static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, 770 struct bpf_insn *insn, 771 struct bpf_jit_probe *probe) 772 { 773 int load_reg; 774 775 if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 776 return; 777 778 /* lgrl %r1,kern_arena */ 779 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 780 /* agr %r1,%dst */ 781 EMIT4(0xb9080000, REG_W1, insn->dst_reg); 782 probe->arena_reg = REG_W1; 783 probe->prg = jit->prg; 784 /* 785 * A read-modify-write carrying BPF_FETCH reads the old value into 786 * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on 787 * fault, the remaining atomics only write memory. 788 */ 789 load_reg = bpf_atomic_load_reg(insn); 790 if (load_reg >= 0) 791 probe->reg = reg2hex[load_reg]; 792 } 793 794 static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, 795 struct bpf_jit_probe *probe) 796 { 797 struct exception_table_entry *ex; 798 s64 delta; 799 800 if (probe->prg == -1) 801 /* The probe is not armed. */ 802 return 0; 803 if (!fp->aux->extable) 804 /* Do nothing during early JIT passes. */ 805 return 0; 806 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) 807 /* Verifier bug - not enough entries. */ 808 return -1; 809 ex = &fp->aux->extable[jit->excnt]; 810 delta = jit->prg_buf + probe->prg - (u8 *)&ex->insn; 811 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 812 /* JIT bug - code and extable must be close. */ 813 return -1; 814 ex->insn = delta; 815 /* 816 * Land on the current instruction. Note that the extable 817 * infrastructure ignores the fixup field; it is handled by 818 * ex_handler_bpf(). 819 */ 820 delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; 821 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 822 /* JIT bug - landing pad and extable must be close. */ 823 return -1; 824 ex->fixup = delta; 825 ex->type = EX_TYPE_BPF; 826 ex->data = probe->reg; 827 jit->excnt++; 828 return 0; 829 } 830 831 static void emit_shift_count32(struct bpf_jit *jit, u32 src_reg) 832 { 833 /* risbg %w1,%src,59,128+63,0 */ 834 _EMIT6(0xec000000 | reg(REG_W1, src_reg) << 16 | 59 << 8 | (0x80 | 63), 0x0055); 835 REG_SET_SEEN(REG_W1); 836 REG_SET_SEEN(src_reg); 837 } 838 839 static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) 840 { 841 struct bpf_jit_probe probe; 842 843 bpf_jit_probe_init(&probe); 844 bpf_jit_probe_load_pre(jit, insn, &probe); 845 846 switch (BPF_SIZE(insn->code)) { 847 case BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 848 /* llgc %dst,off(%src,%arena) */ 849 EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg, 850 probe.arena_reg, insn->off); 851 break; 852 case BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 853 /* llgh %dst,off(%src,%arena) */ 854 EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg, 855 probe.arena_reg, insn->off); 856 break; 857 case BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 858 /* llgf %dst,off(%src,%arena) */ 859 EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg, 860 probe.arena_reg, insn->off); 861 break; 862 case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 863 /* lg %dst,off(%src,%arena) */ 864 EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg, 865 probe.arena_reg, insn->off); 866 break; 867 } 868 869 return bpf_jit_probe_post(jit, fp, &probe); 870 } 871 872 static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) 873 { 874 struct bpf_jit_probe probe; 875 876 bpf_jit_probe_init(&probe); 877 bpf_jit_probe_store_pre(jit, insn, &probe); 878 879 switch (BPF_SIZE(insn->code)) { 880 case BPF_B: /* *(u8 *)(dst + off) = src_reg */ 881 /* stcy %src,off(%dst,%arena) */ 882 EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg, 883 probe.arena_reg, insn->off); 884 break; 885 case BPF_H: /* (u16 *)(dst + off) = src */ 886 /* sthy %src,off(%dst,%arena) */ 887 EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg, 888 probe.arena_reg, insn->off); 889 break; 890 case BPF_W: /* *(u32 *)(dst + off) = src */ 891 /* sty %src,off(%dst,%arena) */ 892 EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg, 893 probe.arena_reg, insn->off); 894 break; 895 case BPF_DW: /* (u64 *)(dst + off) = src */ 896 /* stg %src,off(%dst,%arena) */ 897 EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg, 898 probe.arena_reg, insn->off); 899 break; 900 } 901 902 return bpf_jit_probe_post(jit, fp, &probe); 903 } 904 905 /* 906 * Sign- or zero-extend the register if necessary 907 */ 908 static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) 909 { 910 switch (size) { 911 case 1: 912 if (flags & BTF_FMODEL_SIGNED_ARG) 913 /* lgbr %r,%r */ 914 EMIT4(0xb9060000, r, r); 915 else 916 /* llgcr %r,%r */ 917 EMIT4(0xb9840000, r, r); 918 return 0; 919 case 2: 920 if (flags & BTF_FMODEL_SIGNED_ARG) 921 /* lghr %r,%r */ 922 EMIT4(0xb9070000, r, r); 923 else 924 /* llghr %r,%r */ 925 EMIT4(0xb9850000, r, r); 926 return 0; 927 case 4: 928 if (flags & BTF_FMODEL_SIGNED_ARG) 929 /* lgfr %r,%r */ 930 EMIT4(0xb9140000, r, r); 931 else 932 /* llgfr %r,%r */ 933 EMIT4(0xb9160000, r, r); 934 return 0; 935 case 8: 936 return 0; 937 default: 938 return -1; 939 } 940 } 941 942 /* 943 * Compile one eBPF instruction into s390x code 944 * 945 * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of 946 * stack space for the large switch statement. 947 */ 948 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 949 int i, bool extra_pass) 950 { 951 struct bpf_insn *insn = &fp->insnsi[i]; 952 s32 branch_oc_off = insn->off; 953 u32 dst_reg = insn->dst_reg; 954 u32 src_reg = insn->src_reg; 955 struct bpf_jit_probe probe; 956 int last, insn_count = 1; 957 u32 *addrs = jit->addrs; 958 s32 imm = insn->imm; 959 s16 off = insn->off; 960 unsigned int mask; 961 int err; 962 963 bpf_jit_probe_init(&probe); 964 965 switch (insn->code) { 966 /* 967 * BPF_MOV 968 */ 969 case BPF_ALU | BPF_MOV | BPF_X: 970 switch (insn->off) { 971 case 0: /* DST = (u32) SRC */ 972 /* llgfr %dst,%src */ 973 EMIT4(0xb9160000, dst_reg, src_reg); 974 if (insn_is_zext(&insn[1])) 975 insn_count = 2; 976 break; 977 case 8: /* DST = (u32)(s8) SRC */ 978 /* lbr %dst,%src */ 979 EMIT4(0xb9260000, dst_reg, src_reg); 980 /* llgfr %dst,%dst */ 981 EMIT4(0xb9160000, dst_reg, dst_reg); 982 break; 983 case 16: /* DST = (u32)(s16) SRC */ 984 /* lhr %dst,%src */ 985 EMIT4(0xb9270000, dst_reg, src_reg); 986 /* llgfr %dst,%dst */ 987 EMIT4(0xb9160000, dst_reg, dst_reg); 988 break; 989 } 990 break; 991 case BPF_ALU64 | BPF_MOV | BPF_X: 992 if (insn_is_cast_user(insn)) { 993 int patch_brc; 994 995 /* ltgr %dst,%src */ 996 EMIT4(0xb9020000, dst_reg, src_reg); 997 /* brc 8,0f */ 998 patch_brc = jit->prg; 999 EMIT4_PCREL_RIC(0xa7040000, 8, 0); 1000 /* iihf %dst,user_arena>>32 */ 1001 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); 1002 /* 0: */ 1003 if (jit->prg_buf) 1004 *(u16 *)(jit->prg_buf + patch_brc + 2) = 1005 (jit->prg - patch_brc) >> 1; 1006 break; 1007 } 1008 if (insn_is_mov_percpu_addr(insn)) { 1009 const u64 *percpu_offset = &get_lowcore()->percpu_offset; 1010 1011 if (dst_reg != src_reg) 1012 /* lgr %dst,%src */ 1013 EMIT4(0xb9040000, dst_reg, src_reg); 1014 /* ag %dst,percpu_offset */ 1015 EMIT6_DISP_LH(0xe3000000, 0x0008, dst_reg, REG_0, REG_0, 1016 (unsigned long)percpu_offset); 1017 break; 1018 } 1019 switch (insn->off) { 1020 case 0: /* DST = SRC */ 1021 /* lgr %dst,%src */ 1022 EMIT4(0xb9040000, dst_reg, src_reg); 1023 break; 1024 case 8: /* DST = (s8) SRC */ 1025 /* lgbr %dst,%src */ 1026 EMIT4(0xb9060000, dst_reg, src_reg); 1027 break; 1028 case 16: /* DST = (s16) SRC */ 1029 /* lghr %dst,%src */ 1030 EMIT4(0xb9070000, dst_reg, src_reg); 1031 break; 1032 case 32: /* DST = (s32) SRC */ 1033 /* lgfr %dst,%src */ 1034 EMIT4(0xb9140000, dst_reg, src_reg); 1035 break; 1036 } 1037 break; 1038 case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ 1039 /* llilf %dst,imm */ 1040 EMIT6_IMM(0xc00f0000, dst_reg, imm); 1041 if (insn_is_zext(&insn[1])) 1042 insn_count = 2; 1043 break; 1044 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ 1045 /* lgfi %dst,imm */ 1046 EMIT6_IMM(0xc0010000, dst_reg, imm); 1047 break; 1048 /* 1049 * BPF_LD 64 1050 */ 1051 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 1052 { 1053 /* 16 byte instruction that uses two 'struct bpf_insn' */ 1054 u64 imm64; 1055 1056 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; 1057 /* lgrl %dst,imm */ 1058 EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64)); 1059 insn_count = 2; 1060 break; 1061 } 1062 /* 1063 * BPF_ADD 1064 */ 1065 case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ 1066 /* ar %dst,%src */ 1067 EMIT2(0x1a00, dst_reg, src_reg); 1068 EMIT_ZERO(dst_reg); 1069 break; 1070 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ 1071 /* agr %dst,%src */ 1072 EMIT4(0xb9080000, dst_reg, src_reg); 1073 break; 1074 case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ 1075 if (imm != 0) { 1076 /* alfi %dst,imm */ 1077 EMIT6_IMM(0xc20b0000, dst_reg, imm); 1078 } 1079 EMIT_ZERO(dst_reg); 1080 break; 1081 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ 1082 if (!imm) 1083 break; 1084 /* agfi %dst,imm */ 1085 EMIT6_IMM(0xc2080000, dst_reg, imm); 1086 break; 1087 /* 1088 * BPF_SUB 1089 */ 1090 case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ 1091 /* sr %dst,%src */ 1092 EMIT2(0x1b00, dst_reg, src_reg); 1093 EMIT_ZERO(dst_reg); 1094 break; 1095 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ 1096 /* sgr %dst,%src */ 1097 EMIT4(0xb9090000, dst_reg, src_reg); 1098 break; 1099 case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ 1100 if (imm != 0) { 1101 /* alfi %dst,-imm */ 1102 EMIT6_IMM(0xc20b0000, dst_reg, -imm); 1103 } 1104 EMIT_ZERO(dst_reg); 1105 break; 1106 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ 1107 if (!imm) 1108 break; 1109 if (imm == -0x80000000) { 1110 /* algfi %dst,0x80000000 */ 1111 EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000); 1112 } else { 1113 /* agfi %dst,-imm */ 1114 EMIT6_IMM(0xc2080000, dst_reg, -imm); 1115 } 1116 break; 1117 /* 1118 * BPF_MUL 1119 */ 1120 case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ 1121 /* msr %dst,%src */ 1122 EMIT4(0xb2520000, dst_reg, src_reg); 1123 EMIT_ZERO(dst_reg); 1124 break; 1125 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ 1126 /* msgr %dst,%src */ 1127 EMIT4(0xb90c0000, dst_reg, src_reg); 1128 break; 1129 case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ 1130 if (imm != 1) { 1131 /* msfi %r5,imm */ 1132 EMIT6_IMM(0xc2010000, dst_reg, imm); 1133 } 1134 EMIT_ZERO(dst_reg); 1135 break; 1136 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */ 1137 if (imm == 1) 1138 break; 1139 /* msgfi %dst,imm */ 1140 EMIT6_IMM(0xc2000000, dst_reg, imm); 1141 break; 1142 /* 1143 * BPF_DIV / BPF_MOD 1144 */ 1145 case BPF_ALU | BPF_DIV | BPF_X: 1146 case BPF_ALU | BPF_MOD | BPF_X: 1147 { 1148 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1149 1150 switch (off) { 1151 case 0: /* dst = (u32) dst {/,%} (u32) src */ 1152 /* xr %w0,%w0 */ 1153 EMIT2(0x1700, REG_W0, REG_W0); 1154 /* lr %w1,%dst */ 1155 EMIT2(0x1800, REG_W1, dst_reg); 1156 /* dlr %w0,%src */ 1157 EMIT4(0xb9970000, REG_W0, src_reg); 1158 break; 1159 case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */ 1160 /* lgfr %r1,%dst */ 1161 EMIT4(0xb9140000, REG_W1, dst_reg); 1162 /* dsgfr %r0,%src */ 1163 EMIT4(0xb91d0000, REG_W0, src_reg); 1164 break; 1165 } 1166 /* llgfr %dst,%rc */ 1167 EMIT4(0xb9160000, dst_reg, rc_reg); 1168 if (insn_is_zext(&insn[1])) 1169 insn_count = 2; 1170 break; 1171 } 1172 case BPF_ALU64 | BPF_DIV | BPF_X: 1173 case BPF_ALU64 | BPF_MOD | BPF_X: 1174 { 1175 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1176 1177 switch (off) { 1178 case 0: /* dst = dst {/,%} src */ 1179 /* lghi %w0,0 */ 1180 EMIT4_IMM(0xa7090000, REG_W0, 0); 1181 /* lgr %w1,%dst */ 1182 EMIT4(0xb9040000, REG_W1, dst_reg); 1183 /* dlgr %w0,%src */ 1184 EMIT4(0xb9870000, REG_W0, src_reg); 1185 break; 1186 case 1: /* dst = (s64) dst {/,%} (s64) src */ 1187 /* lgr %w1,%dst */ 1188 EMIT4(0xb9040000, REG_W1, dst_reg); 1189 /* dsgr %w0,%src */ 1190 EMIT4(0xb90d0000, REG_W0, src_reg); 1191 break; 1192 } 1193 /* lgr %dst,%rc */ 1194 EMIT4(0xb9040000, dst_reg, rc_reg); 1195 break; 1196 } 1197 case BPF_ALU | BPF_DIV | BPF_K: 1198 case BPF_ALU | BPF_MOD | BPF_K: 1199 { 1200 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1201 1202 if (imm == 1) { 1203 if (BPF_OP(insn->code) == BPF_MOD) 1204 /* lghi %dst,0 */ 1205 EMIT4_IMM(0xa7090000, dst_reg, 0); 1206 else 1207 EMIT_ZERO(dst_reg); 1208 break; 1209 } 1210 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) { 1211 switch (off) { 1212 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1213 /* xr %w0,%w0 */ 1214 EMIT2(0x1700, REG_W0, REG_W0); 1215 /* lr %w1,%dst */ 1216 EMIT2(0x1800, REG_W1, dst_reg); 1217 /* dl %w0,<d(imm)>(%l) */ 1218 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, 1219 REG_L, EMIT_CONST_U32(imm)); 1220 break; 1221 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1222 /* lgfr %r1,%dst */ 1223 EMIT4(0xb9140000, REG_W1, dst_reg); 1224 /* dsgf %r0,<d(imm)>(%l) */ 1225 EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0, 1226 REG_L, EMIT_CONST_U32(imm)); 1227 break; 1228 } 1229 } else { 1230 switch (off) { 1231 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1232 /* xr %w0,%w0 */ 1233 EMIT2(0x1700, REG_W0, REG_W0); 1234 /* lr %w1,%dst */ 1235 EMIT2(0x1800, REG_W1, dst_reg); 1236 /* lrl %dst,imm */ 1237 EMIT6_PCREL_RILB(0xc40d0000, dst_reg, 1238 _EMIT_CONST_U32(imm)); 1239 jit->seen |= SEEN_LITERAL; 1240 /* dlr %w0,%dst */ 1241 EMIT4(0xb9970000, REG_W0, dst_reg); 1242 break; 1243 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1244 /* lgfr %w1,%dst */ 1245 EMIT4(0xb9140000, REG_W1, dst_reg); 1246 /* lgfrl %dst,imm */ 1247 EMIT6_PCREL_RILB(0xc40c0000, dst_reg, 1248 _EMIT_CONST_U32(imm)); 1249 jit->seen |= SEEN_LITERAL; 1250 /* dsgr %w0,%dst */ 1251 EMIT4(0xb90d0000, REG_W0, dst_reg); 1252 break; 1253 } 1254 } 1255 /* llgfr %dst,%rc */ 1256 EMIT4(0xb9160000, dst_reg, rc_reg); 1257 if (insn_is_zext(&insn[1])) 1258 insn_count = 2; 1259 break; 1260 } 1261 case BPF_ALU64 | BPF_DIV | BPF_K: 1262 case BPF_ALU64 | BPF_MOD | BPF_K: 1263 { 1264 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1265 1266 if (imm == 1) { 1267 if (BPF_OP(insn->code) == BPF_MOD) 1268 /* lhgi %dst,0 */ 1269 EMIT4_IMM(0xa7090000, dst_reg, 0); 1270 break; 1271 } 1272 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1273 switch (off) { 1274 case 0: /* dst = dst {/,%} imm */ 1275 /* lghi %w0,0 */ 1276 EMIT4_IMM(0xa7090000, REG_W0, 0); 1277 /* lgr %w1,%dst */ 1278 EMIT4(0xb9040000, REG_W1, dst_reg); 1279 /* dlg %w0,<d(imm)>(%l) */ 1280 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, 1281 REG_L, EMIT_CONST_U64(imm)); 1282 break; 1283 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1284 /* lgr %w1,%dst */ 1285 EMIT4(0xb9040000, REG_W1, dst_reg); 1286 /* dsg %w0,<d(imm)>(%l) */ 1287 EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0, 1288 REG_L, EMIT_CONST_U64(imm)); 1289 break; 1290 } 1291 } else { 1292 switch (off) { 1293 case 0: /* dst = dst {/,%} imm */ 1294 /* lghi %w0,0 */ 1295 EMIT4_IMM(0xa7090000, REG_W0, 0); 1296 /* lgr %w1,%dst */ 1297 EMIT4(0xb9040000, REG_W1, dst_reg); 1298 /* lgrl %dst,imm */ 1299 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1300 _EMIT_CONST_U64(imm)); 1301 jit->seen |= SEEN_LITERAL; 1302 /* dlgr %w0,%dst */ 1303 EMIT4(0xb9870000, REG_W0, dst_reg); 1304 break; 1305 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1306 /* lgr %w1,%dst */ 1307 EMIT4(0xb9040000, REG_W1, dst_reg); 1308 /* lgrl %dst,imm */ 1309 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1310 _EMIT_CONST_U64(imm)); 1311 jit->seen |= SEEN_LITERAL; 1312 /* dsgr %w0,%dst */ 1313 EMIT4(0xb90d0000, REG_W0, dst_reg); 1314 break; 1315 } 1316 } 1317 /* lgr %dst,%rc */ 1318 EMIT4(0xb9040000, dst_reg, rc_reg); 1319 break; 1320 } 1321 /* 1322 * BPF_AND 1323 */ 1324 case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */ 1325 /* nr %dst,%src */ 1326 EMIT2(0x1400, dst_reg, src_reg); 1327 EMIT_ZERO(dst_reg); 1328 break; 1329 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 1330 /* ngr %dst,%src */ 1331 EMIT4(0xb9800000, dst_reg, src_reg); 1332 break; 1333 case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */ 1334 /* nilf %dst,imm */ 1335 EMIT6_IMM(0xc00b0000, dst_reg, imm); 1336 EMIT_ZERO(dst_reg); 1337 break; 1338 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 1339 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1340 /* ng %dst,<d(imm)>(%l) */ 1341 EMIT6_DISP_LH(0xe3000000, 0x0080, 1342 dst_reg, REG_0, REG_L, 1343 EMIT_CONST_U64(imm)); 1344 } else { 1345 /* lgrl %w0,imm */ 1346 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1347 _EMIT_CONST_U64(imm)); 1348 jit->seen |= SEEN_LITERAL; 1349 /* ngr %dst,%w0 */ 1350 EMIT4(0xb9800000, dst_reg, REG_W0); 1351 } 1352 break; 1353 /* 1354 * BPF_OR 1355 */ 1356 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 1357 /* or %dst,%src */ 1358 EMIT2(0x1600, dst_reg, src_reg); 1359 EMIT_ZERO(dst_reg); 1360 break; 1361 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 1362 /* ogr %dst,%src */ 1363 EMIT4(0xb9810000, dst_reg, src_reg); 1364 break; 1365 case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */ 1366 /* oilf %dst,imm */ 1367 EMIT6_IMM(0xc00d0000, dst_reg, imm); 1368 EMIT_ZERO(dst_reg); 1369 break; 1370 case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */ 1371 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1372 /* og %dst,<d(imm)>(%l) */ 1373 EMIT6_DISP_LH(0xe3000000, 0x0081, 1374 dst_reg, REG_0, REG_L, 1375 EMIT_CONST_U64(imm)); 1376 } else { 1377 /* lgrl %w0,imm */ 1378 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1379 _EMIT_CONST_U64(imm)); 1380 jit->seen |= SEEN_LITERAL; 1381 /* ogr %dst,%w0 */ 1382 EMIT4(0xb9810000, dst_reg, REG_W0); 1383 } 1384 break; 1385 /* 1386 * BPF_XOR 1387 */ 1388 case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */ 1389 /* xr %dst,%src */ 1390 EMIT2(0x1700, dst_reg, src_reg); 1391 EMIT_ZERO(dst_reg); 1392 break; 1393 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */ 1394 /* xgr %dst,%src */ 1395 EMIT4(0xb9820000, dst_reg, src_reg); 1396 break; 1397 case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */ 1398 if (imm != 0) { 1399 /* xilf %dst,imm */ 1400 EMIT6_IMM(0xc0070000, dst_reg, imm); 1401 } 1402 EMIT_ZERO(dst_reg); 1403 break; 1404 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */ 1405 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1406 /* xg %dst,<d(imm)>(%l) */ 1407 EMIT6_DISP_LH(0xe3000000, 0x0082, 1408 dst_reg, REG_0, REG_L, 1409 EMIT_CONST_U64(imm)); 1410 } else { 1411 /* lgrl %w0,imm */ 1412 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1413 _EMIT_CONST_U64(imm)); 1414 jit->seen |= SEEN_LITERAL; 1415 /* xgr %dst,%w0 */ 1416 EMIT4(0xb9820000, dst_reg, REG_W0); 1417 } 1418 break; 1419 /* 1420 * BPF_LSH 1421 */ 1422 case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << ((u32) src & 31) */ 1423 emit_shift_count32(jit, src_reg); 1424 /* sll %dst,0(%w1) */ 1425 EMIT4_DISP(0x89000000, dst_reg, REG_W1, 0); 1426 EMIT_ZERO(dst_reg); 1427 break; 1428 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ 1429 /* sllg %dst,%dst,0(%src) */ 1430 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); 1431 break; 1432 case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */ 1433 if (imm != 0) { 1434 /* sll %dst,imm(%r0) */ 1435 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); 1436 } 1437 EMIT_ZERO(dst_reg); 1438 break; 1439 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */ 1440 if (imm == 0) 1441 break; 1442 /* sllg %dst,%dst,imm(%r0) */ 1443 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); 1444 break; 1445 /* 1446 * BPF_RSH 1447 */ 1448 case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> ((u32) src & 31) */ 1449 emit_shift_count32(jit, src_reg); 1450 /* srl %dst,0(%w1) */ 1451 EMIT4_DISP(0x88000000, dst_reg, REG_W1, 0); 1452 EMIT_ZERO(dst_reg); 1453 break; 1454 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ 1455 /* srlg %dst,%dst,0(%src) */ 1456 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); 1457 break; 1458 case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */ 1459 if (imm != 0) { 1460 /* srl %dst,imm(%r0) */ 1461 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); 1462 } 1463 EMIT_ZERO(dst_reg); 1464 break; 1465 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */ 1466 if (imm == 0) 1467 break; 1468 /* srlg %dst,%dst,imm(%r0) */ 1469 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); 1470 break; 1471 /* 1472 * BPF_ARSH 1473 */ 1474 case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= (u32) src & 31 */ 1475 emit_shift_count32(jit, src_reg); 1476 /* sra %dst,0(%w1) */ 1477 EMIT4_DISP(0x8a000000, dst_reg, REG_W1, 0); 1478 EMIT_ZERO(dst_reg); 1479 break; 1480 case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ 1481 /* srag %dst,%dst,0(%src) */ 1482 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); 1483 break; 1484 case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */ 1485 if (imm != 0) { 1486 /* sra %dst,imm(%r0) */ 1487 EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); 1488 } 1489 EMIT_ZERO(dst_reg); 1490 break; 1491 case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ 1492 if (imm == 0) 1493 break; 1494 /* srag %dst,%dst,imm(%r0) */ 1495 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); 1496 break; 1497 /* 1498 * BPF_NEG 1499 */ 1500 case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */ 1501 /* lcr %dst,%dst */ 1502 EMIT2(0x1300, dst_reg, dst_reg); 1503 EMIT_ZERO(dst_reg); 1504 break; 1505 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 1506 /* lcgr %dst,%dst */ 1507 EMIT4(0xb9030000, dst_reg, dst_reg); 1508 break; 1509 /* 1510 * BPF_FROM_BE/LE 1511 */ 1512 case BPF_ALU | BPF_END | BPF_FROM_BE: 1513 /* s390 is big endian, therefore only clear high order bytes */ 1514 switch (imm) { 1515 case 16: /* dst = (u16) cpu_to_be16(dst) */ 1516 /* llghr %dst,%dst */ 1517 EMIT4(0xb9850000, dst_reg, dst_reg); 1518 if (insn_is_zext(&insn[1])) 1519 insn_count = 2; 1520 break; 1521 case 32: /* dst = (u32) cpu_to_be32(dst) */ 1522 if (!fp->aux->verifier_zext) 1523 /* llgfr %dst,%dst */ 1524 EMIT4(0xb9160000, dst_reg, dst_reg); 1525 break; 1526 case 64: /* dst = (u64) cpu_to_be64(dst) */ 1527 break; 1528 } 1529 break; 1530 case BPF_ALU | BPF_END | BPF_FROM_LE: 1531 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1532 switch (imm) { 1533 case 16: /* dst = (u16) cpu_to_le16(dst) */ 1534 /* lrvr %dst,%dst */ 1535 EMIT4(0xb91f0000, dst_reg, dst_reg); 1536 /* srl %dst,16(%r0) */ 1537 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); 1538 /* llghr %dst,%dst */ 1539 EMIT4(0xb9850000, dst_reg, dst_reg); 1540 if (insn_is_zext(&insn[1])) 1541 insn_count = 2; 1542 break; 1543 case 32: /* dst = (u32) cpu_to_le32(dst) */ 1544 /* lrvr %dst,%dst */ 1545 EMIT4(0xb91f0000, dst_reg, dst_reg); 1546 if (!fp->aux->verifier_zext) 1547 /* llgfr %dst,%dst */ 1548 EMIT4(0xb9160000, dst_reg, dst_reg); 1549 break; 1550 case 64: /* dst = (u64) cpu_to_le64(dst) */ 1551 /* lrvgr %dst,%dst */ 1552 EMIT4(0xb90f0000, dst_reg, dst_reg); 1553 break; 1554 } 1555 break; 1556 /* 1557 * BPF_NOSPEC (speculation barrier) 1558 */ 1559 case BPF_ST | BPF_NOSPEC: 1560 break; 1561 /* 1562 * BPF_ST(X) 1563 */ 1564 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ 1565 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1566 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 1567 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1568 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 1569 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1570 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 1571 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1572 err = emit_stx(jit, fp, insn); 1573 if (err < 0) 1574 return err; 1575 jit->seen |= SEEN_MEM; 1576 break; 1577 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 1578 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1579 /* lhi %w0,imm */ 1580 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); 1581 bpf_jit_probe_store_pre(jit, insn, &probe); 1582 /* stcy %w0,off(%dst,%arena) */ 1583 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, 1584 probe.arena_reg, off); 1585 err = bpf_jit_probe_post(jit, fp, &probe); 1586 if (err < 0) 1587 return err; 1588 jit->seen |= SEEN_MEM; 1589 break; 1590 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 1591 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1592 /* lhi %w0,imm */ 1593 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); 1594 bpf_jit_probe_store_pre(jit, insn, &probe); 1595 /* sthy %w0,off(%dst,%arena) */ 1596 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, 1597 probe.arena_reg, off); 1598 err = bpf_jit_probe_post(jit, fp, &probe); 1599 if (err < 0) 1600 return err; 1601 jit->seen |= SEEN_MEM; 1602 break; 1603 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 1604 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1605 /* llilf %w0,imm */ 1606 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); 1607 bpf_jit_probe_store_pre(jit, insn, &probe); 1608 /* sty %w0,off(%dst,%arena) */ 1609 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, 1610 probe.arena_reg, off); 1611 err = bpf_jit_probe_post(jit, fp, &probe); 1612 if (err < 0) 1613 return err; 1614 jit->seen |= SEEN_MEM; 1615 break; 1616 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 1617 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1618 /* lgfi %w0,imm */ 1619 EMIT6_IMM(0xc0010000, REG_W0, imm); 1620 bpf_jit_probe_store_pre(jit, insn, &probe); 1621 /* stg %w0,off(%dst,%arena) */ 1622 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, 1623 probe.arena_reg, off); 1624 err = bpf_jit_probe_post(jit, fp, &probe); 1625 if (err < 0) 1626 return err; 1627 jit->seen |= SEEN_MEM; 1628 break; 1629 /* 1630 * BPF_ATOMIC 1631 */ 1632 case BPF_STX | BPF_ATOMIC | BPF_B: 1633 case BPF_STX | BPF_ATOMIC | BPF_H: 1634 case BPF_STX | BPF_ATOMIC | BPF_DW: 1635 case BPF_STX | BPF_ATOMIC | BPF_W: 1636 case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: 1637 case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: 1638 case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1639 case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1640 { 1641 bool is32 = BPF_SIZE(insn->code) == BPF_W; 1642 1643 /* 1644 * Unlike loads and stores, s390 atomics have only a base 1645 * register, but no index register. For the non-arena case, 1646 * simply use %dst as a base. For the arena case, use the 1647 * work register %r1: first, load the arena base into it, 1648 * and then add %dst to it. 1649 */ 1650 probe.arena_reg = dst_reg; 1651 1652 switch (insn->imm) { 1653 #define EMIT_ATOMIC(op32, op64) do { \ 1654 bpf_jit_probe_atomic_pre(jit, insn, &probe); \ 1655 /* {op32|op64} {%w0|%src},%src,off(%arena) */ \ 1656 EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ 1657 (insn->imm & BPF_FETCH) ? src_reg : REG_W0, \ 1658 src_reg, probe.arena_reg, off); \ 1659 err = bpf_jit_probe_post(jit, fp, &probe); \ 1660 if (err < 0) \ 1661 return err; \ 1662 if (insn->imm & BPF_FETCH) { \ 1663 /* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ 1664 _EMIT2(0x07e0); \ 1665 if (is32) \ 1666 EMIT_ZERO(src_reg); \ 1667 } \ 1668 } while (0) 1669 case BPF_ADD: 1670 case BPF_ADD | BPF_FETCH: 1671 /* {laal|laalg} */ 1672 EMIT_ATOMIC(0x00fa, 0x00ea); 1673 break; 1674 case BPF_AND: 1675 case BPF_AND | BPF_FETCH: 1676 /* {lan|lang} */ 1677 EMIT_ATOMIC(0x00f4, 0x00e4); 1678 break; 1679 case BPF_OR: 1680 case BPF_OR | BPF_FETCH: 1681 /* {lao|laog} */ 1682 EMIT_ATOMIC(0x00f6, 0x00e6); 1683 break; 1684 case BPF_XOR: 1685 case BPF_XOR | BPF_FETCH: 1686 /* {lax|laxg} */ 1687 EMIT_ATOMIC(0x00f7, 0x00e7); 1688 break; 1689 #undef EMIT_ATOMIC 1690 case BPF_XCHG: { 1691 struct bpf_jit_probe load_probe = probe; 1692 int loop_start; 1693 1694 bpf_jit_probe_atomic_pre(jit, insn, &load_probe); 1695 /* {ly|lg} %w0,off(%arena) */ 1696 EMIT6_DISP_LH(0xe3000000, 1697 is32 ? 0x0058 : 0x0004, REG_W0, REG_0, 1698 load_probe.arena_reg, off); 1699 /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ 1700 if (load_probe.prg != -1) { 1701 probe.prg = jit->prg; 1702 probe.arena_reg = load_probe.arena_reg; 1703 probe.reg = load_probe.reg; 1704 } 1705 loop_start = jit->prg; 1706 /* 0: {csy|csg} %w0,%src,off(%arena) */ 1707 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1708 REG_W0, src_reg, probe.arena_reg, off); 1709 /* brc 4,0b */ 1710 EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); 1711 /* {llgfr|lgr} %src,%w0 */ 1712 EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); 1713 /* Both probes should land here on exception. */ 1714 err = bpf_jit_probe_post(jit, fp, &load_probe); 1715 if (err < 0) 1716 return err; 1717 err = bpf_jit_probe_post(jit, fp, &probe); 1718 if (err < 0) 1719 return err; 1720 if (is32 && insn_is_zext(&insn[1])) 1721 insn_count = 2; 1722 break; 1723 } 1724 case BPF_CMPXCHG: 1725 bpf_jit_probe_atomic_pre(jit, insn, &probe); 1726 /* 0: {csy|csg} %b0,%src,off(%arena) */ 1727 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1728 BPF_REG_0, src_reg, 1729 probe.arena_reg, off); 1730 err = bpf_jit_probe_post(jit, fp, &probe); 1731 if (err < 0) 1732 return err; 1733 break; 1734 case BPF_LOAD_ACQ: 1735 /* s390 has strong ordering, just use load */ 1736 err = emit_ldx(jit, fp, insn); 1737 if (err < 0) 1738 return err; 1739 break; 1740 case BPF_STORE_REL: 1741 /* s390 has strong ordering, just use store */ 1742 err = emit_stx(jit, fp, insn); 1743 if (err < 0) 1744 return err; 1745 break; 1746 default: 1747 pr_err("Unknown atomic operation %02x\n", insn->imm); 1748 return -1; 1749 } 1750 1751 jit->seen |= SEEN_MEM; 1752 break; 1753 } 1754 /* 1755 * BPF_LDX 1756 */ 1757 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 1758 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1759 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1760 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 1761 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1762 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1763 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 1764 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1765 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1766 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 1767 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1768 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1769 err = emit_ldx(jit, fp, insn); 1770 if (err < 0) 1771 return err; 1772 jit->seen |= SEEN_MEM; 1773 if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1])) 1774 insn_count = 2; 1775 break; 1776 case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ 1777 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1778 bpf_jit_probe_load_pre(jit, insn, &probe); 1779 /* lgb %dst,off(%src) */ 1780 EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off); 1781 err = bpf_jit_probe_post(jit, fp, &probe); 1782 if (err < 0) 1783 return err; 1784 jit->seen |= SEEN_MEM; 1785 break; 1786 case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ 1787 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1788 bpf_jit_probe_load_pre(jit, insn, &probe); 1789 /* lgh %dst,off(%src) */ 1790 EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off); 1791 err = bpf_jit_probe_post(jit, fp, &probe); 1792 if (err < 0) 1793 return err; 1794 jit->seen |= SEEN_MEM; 1795 break; 1796 case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ 1797 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1798 bpf_jit_probe_load_pre(jit, insn, &probe); 1799 /* lgf %dst,off(%src) */ 1800 jit->seen |= SEEN_MEM; 1801 EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off); 1802 err = bpf_jit_probe_post(jit, fp, &probe); 1803 if (err < 0) 1804 return err; 1805 break; 1806 /* 1807 * BPF_JMP / CALL 1808 */ 1809 case BPF_JMP | BPF_CALL: 1810 { 1811 const struct btf_func_model *m; 1812 bool func_addr_fixed; 1813 int j, ret; 1814 u64 func; 1815 1816 /* Implement helper call to bpf_get_smp_processor_id() inline */ 1817 if (insn->src_reg == 0 && 1818 insn->imm == BPF_FUNC_get_smp_processor_id) { 1819 const u32 *cpu_nr = &get_lowcore()->cpu_nr; 1820 1821 /* llgf %b0, cpu_nr */ 1822 EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, 1823 (unsigned long)cpu_nr); 1824 break; 1825 } 1826 1827 /* Implement helper call to bpf_get_current_task/_btf() inline */ 1828 if (insn->src_reg == 0 && 1829 (insn->imm == BPF_FUNC_get_current_task || 1830 insn->imm == BPF_FUNC_get_current_task_btf)) { 1831 const u64 *current_task = 1832 &get_lowcore()->current_task; 1833 1834 /* lg %b0, current_task */ 1835 EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0, 1836 (unsigned long)current_task); 1837 break; 1838 } 1839 1840 ret = bpf_jit_get_func_addr(fp, insn, extra_pass, 1841 &func, &func_addr_fixed); 1842 if (ret < 0) 1843 return -1; 1844 1845 REG_SET_SEEN(BPF_REG_5); 1846 jit->seen |= SEEN_FUNC; 1847 1848 /* 1849 * Copy the tail call counter to where the callee expects it. 1850 */ 1851 1852 if (insn->src_reg == BPF_PSEUDO_CALL) 1853 /* 1854 * mvc tail_call_cnt(4,%r15), 1855 * frame_off+tail_call_cnt(%r15) 1856 */ 1857 _EMIT6(0xd203f000 | offsetof(struct prog_frame, 1858 tail_call_cnt), 1859 0xf000 | (jit->frame_off + 1860 offsetof(struct prog_frame, 1861 tail_call_cnt))); 1862 1863 /* Sign-extend the kfunc arguments. */ 1864 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1865 m = bpf_jit_find_kfunc_model(fp, insn); 1866 if (!m) 1867 return -1; 1868 1869 for (j = 0; j < m->nr_args; j++) { 1870 if (sign_zero_extend(jit, BPF_REG_1 + j, 1871 m->arg_size[j], 1872 m->arg_flags[j])) 1873 return -1; 1874 } 1875 } 1876 1877 if ((void *)func == arch_bpf_timed_may_goto) { 1878 /* 1879 * arch_bpf_timed_may_goto() has a special ABI: the 1880 * parameter is in BPF_REG_AX; the return value is in 1881 * BPF_REG_AX; and all GPRs except REG_W0, REG_W1, and 1882 * BPF_REG_AX are callee-saved. 1883 */ 1884 1885 /* brasl %r0,func */ 1886 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_0, (void *)func); 1887 } else { 1888 /* brasl %r14,func */ 1889 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, (void *)func); 1890 /* lgr %b0,%r2: load return value into %b0 */ 1891 EMIT4(0xb9040000, BPF_REG_0, REG_2); 1892 } 1893 1894 /* 1895 * Copy the potentially updated tail call counter back. 1896 */ 1897 1898 if (insn->src_reg == BPF_PSEUDO_CALL) 1899 /* 1900 * mvc frame_off+tail_call_cnt(%r15), 1901 * tail_call_cnt(4,%r15) 1902 */ 1903 _EMIT6(0xd203f000 | (jit->frame_off + 1904 offsetof(struct prog_frame, 1905 tail_call_cnt)), 1906 0xf000 | offsetof(struct prog_frame, 1907 tail_call_cnt)); 1908 1909 break; 1910 } 1911 case BPF_JMP | BPF_TAIL_CALL: { 1912 int patch_1_clrj, patch_2_clij, patch_3_brc; 1913 1914 /* 1915 * Implicit input: 1916 * B1: pointer to ctx 1917 * B2: pointer to bpf_array 1918 * B3: index in bpf_array 1919 * 1920 * if (index >= array->map.max_entries) 1921 * goto out; 1922 */ 1923 1924 /* llgf %w1,map.max_entries(%b2) */ 1925 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2, 1926 offsetof(struct bpf_array, map.max_entries)); 1927 /* if ((u32)%b3 >= (u32)%w1) goto out; */ 1928 /* clrj %b3,%w1,0xa,out */ 1929 patch_1_clrj = jit->prg; 1930 EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa, 1931 jit->prg); 1932 1933 /* 1934 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) 1935 * goto out; 1936 * 1937 * tail_call_cnt is read into %w0, which needs to be preserved 1938 * until it's incremented and flushed. 1939 */ 1940 1941 off = jit->frame_off + 1942 offsetof(struct prog_frame, tail_call_cnt); 1943 /* ly %w0,off(%r15) */ 1944 EMIT6_DISP_LH(0xe3000000, 0x0058, REG_W0, REG_0, REG_15, off); 1945 /* clij %w0,MAX_TAIL_CALL_CNT,0xa,out */ 1946 patch_2_clij = jit->prg; 1947 EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W0, MAX_TAIL_CALL_CNT, 1948 0xa, jit->prg); 1949 1950 /* 1951 * prog = array->ptrs[index]; 1952 * if (prog == NULL) 1953 * goto out; 1954 */ 1955 1956 /* llgfr %r1,%b3: %r1 = (u32) index */ 1957 EMIT4(0xb9160000, REG_1, BPF_REG_3); 1958 /* sllg %r1,%r1,3: %r1 *= 8 */ 1959 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3); 1960 /* ltg %r1,prog(%b2,%r1) */ 1961 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2, 1962 REG_1, offsetof(struct bpf_array, ptrs)); 1963 /* brc 0x8,out */ 1964 patch_3_brc = jit->prg; 1965 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); 1966 1967 /* tail_call_cnt++; */ 1968 /* ahi %w0,1 */ 1969 EMIT4_IMM(0xa70a0000, REG_W0, 1); 1970 /* sty %w0,off(%r15) */ 1971 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, REG_0, REG_15, off); 1972 1973 /* 1974 * Restore registers before calling function 1975 */ 1976 save_restore_regs(jit, REGS_RESTORE, 0); 1977 1978 /* 1979 * goto *(prog->bpf_func + tail_call_start); 1980 */ 1981 1982 /* lg %r1,bpf_func(%r1) */ 1983 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0, 1984 offsetof(struct bpf_prog, bpf_func)); 1985 if (nospec_uses_trampoline()) { 1986 jit->seen |= SEEN_FUNC; 1987 /* aghi %r1,tail_call_start */ 1988 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); 1989 /* brcl 0xf,__s390_indirect_jump_r1 */ 1990 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 1991 __s390_indirect_jump_r1); 1992 } else { 1993 /* bc 0xf,tail_call_start(%r1) */ 1994 _EMIT4(0x47f01000 + jit->tail_call_start); 1995 } 1996 /* out: */ 1997 if (jit->prg_buf) { 1998 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) = 1999 (jit->prg - patch_1_clrj) >> 1; 2000 *(u16 *)(jit->prg_buf + patch_2_clij + 2) = 2001 (jit->prg - patch_2_clij) >> 1; 2002 *(u16 *)(jit->prg_buf + patch_3_brc + 2) = 2003 (jit->prg - patch_3_brc) >> 1; 2004 } 2005 break; 2006 } 2007 case BPF_JMP | BPF_EXIT: /* return b0 */ 2008 last = (i == fp->len - 1) ? 1 : 0; 2009 if (last) 2010 break; 2011 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip)) 2012 /* brc 0xf, <exit> */ 2013 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); 2014 else 2015 /* brcl 0xf, <exit> */ 2016 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); 2017 break; 2018 /* 2019 * Branch relative (number of skipped instructions) to offset on 2020 * condition. 2021 * 2022 * Condition code to mask mapping: 2023 * 2024 * CC | Description | Mask 2025 * ------------------------------ 2026 * 0 | Operands equal | 8 2027 * 1 | First operand low | 4 2028 * 2 | First operand high | 2 2029 * 3 | Unused | 1 2030 * 2031 * For s390x relative branches: ip = ip + off_bytes 2032 * For BPF relative branches: insn = insn + off_insns + 1 2033 * 2034 * For example for s390x with offset 0 we jump to the branch 2035 * instruction itself (loop) and for BPF with offset 0 we 2036 * branch to the instruction behind the branch. 2037 */ 2038 case BPF_JMP32 | BPF_JA: /* if (true) */ 2039 branch_oc_off = imm; 2040 fallthrough; 2041 case BPF_JMP | BPF_JA: /* if (true) */ 2042 mask = 0xf000; /* j */ 2043 goto branch_oc; 2044 case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */ 2045 case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */ 2046 mask = 0x2000; /* jh */ 2047 goto branch_ks; 2048 case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */ 2049 case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */ 2050 mask = 0x4000; /* jl */ 2051 goto branch_ks; 2052 case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */ 2053 case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */ 2054 mask = 0xa000; /* jhe */ 2055 goto branch_ks; 2056 case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */ 2057 case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */ 2058 mask = 0xc000; /* jle */ 2059 goto branch_ks; 2060 case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */ 2061 case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */ 2062 mask = 0x2000; /* jh */ 2063 goto branch_ku; 2064 case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */ 2065 case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */ 2066 mask = 0x4000; /* jl */ 2067 goto branch_ku; 2068 case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */ 2069 case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */ 2070 mask = 0xa000; /* jhe */ 2071 goto branch_ku; 2072 case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */ 2073 case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */ 2074 mask = 0xc000; /* jle */ 2075 goto branch_ku; 2076 case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */ 2077 case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */ 2078 mask = 0x7000; /* jne */ 2079 goto branch_ku; 2080 case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */ 2081 case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */ 2082 mask = 0x8000; /* je */ 2083 goto branch_ku; 2084 case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */ 2085 case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */ 2086 mask = 0x7000; /* jnz */ 2087 if (BPF_CLASS(insn->code) == BPF_JMP32) { 2088 /* llilf %w1,imm (load zero extend imm) */ 2089 EMIT6_IMM(0xc00f0000, REG_W1, imm); 2090 /* nr %w1,%dst */ 2091 EMIT2(0x1400, REG_W1, dst_reg); 2092 } else { 2093 /* lgfi %w1,imm (load sign extend imm) */ 2094 EMIT6_IMM(0xc0010000, REG_W1, imm); 2095 /* ngr %w1,%dst */ 2096 EMIT4(0xb9800000, REG_W1, dst_reg); 2097 } 2098 goto branch_oc; 2099 2100 case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */ 2101 case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */ 2102 mask = 0x2000; /* jh */ 2103 goto branch_xs; 2104 case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */ 2105 case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */ 2106 mask = 0x4000; /* jl */ 2107 goto branch_xs; 2108 case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */ 2109 case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */ 2110 mask = 0xa000; /* jhe */ 2111 goto branch_xs; 2112 case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */ 2113 case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */ 2114 mask = 0xc000; /* jle */ 2115 goto branch_xs; 2116 case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */ 2117 case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */ 2118 mask = 0x2000; /* jh */ 2119 goto branch_xu; 2120 case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */ 2121 case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */ 2122 mask = 0x4000; /* jl */ 2123 goto branch_xu; 2124 case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */ 2125 case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */ 2126 mask = 0xa000; /* jhe */ 2127 goto branch_xu; 2128 case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */ 2129 case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */ 2130 mask = 0xc000; /* jle */ 2131 goto branch_xu; 2132 case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */ 2133 case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */ 2134 mask = 0x7000; /* jne */ 2135 goto branch_xu; 2136 case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */ 2137 case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */ 2138 mask = 0x8000; /* je */ 2139 goto branch_xu; 2140 case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */ 2141 case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */ 2142 { 2143 bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2144 2145 mask = 0x7000; /* jnz */ 2146 /* nrk or ngrk %w1,%dst,%src */ 2147 EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000), 2148 REG_W1, dst_reg, src_reg); 2149 goto branch_oc; 2150 branch_ks: 2151 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2152 /* cfi or cgfi %dst,imm */ 2153 EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000, 2154 dst_reg, imm); 2155 if (!is_first_pass(jit) && 2156 can_use_rel(jit, addrs[i + off + 1])) { 2157 /* brc mask,off */ 2158 EMIT4_PCREL_RIC(0xa7040000, 2159 mask >> 12, addrs[i + off + 1]); 2160 } else { 2161 /* brcl mask,off */ 2162 EMIT6_PCREL_RILC(0xc0040000, 2163 mask >> 12, addrs[i + off + 1]); 2164 } 2165 break; 2166 branch_ku: 2167 /* lgfi %w1,imm (load sign extend imm) */ 2168 src_reg = REG_1; 2169 EMIT6_IMM(0xc0010000, src_reg, imm); 2170 goto branch_xu; 2171 branch_xs: 2172 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2173 if (!is_first_pass(jit) && 2174 can_use_rel(jit, addrs[i + off + 1])) { 2175 /* crj or cgrj %dst,%src,mask,off */ 2176 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064), 2177 dst_reg, src_reg, i, off, mask); 2178 } else { 2179 /* cr or cgr %dst,%src */ 2180 if (is_jmp32) 2181 EMIT2(0x1900, dst_reg, src_reg); 2182 else 2183 EMIT4(0xb9200000, dst_reg, src_reg); 2184 /* brcl mask,off */ 2185 EMIT6_PCREL_RILC(0xc0040000, 2186 mask >> 12, addrs[i + off + 1]); 2187 } 2188 break; 2189 branch_xu: 2190 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2191 if (!is_first_pass(jit) && 2192 can_use_rel(jit, addrs[i + off + 1])) { 2193 /* clrj or clgrj %dst,%src,mask,off */ 2194 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065), 2195 dst_reg, src_reg, i, off, mask); 2196 } else { 2197 /* clr or clgr %dst,%src */ 2198 if (is_jmp32) 2199 EMIT2(0x1500, dst_reg, src_reg); 2200 else 2201 EMIT4(0xb9210000, dst_reg, src_reg); 2202 /* brcl mask,off */ 2203 EMIT6_PCREL_RILC(0xc0040000, 2204 mask >> 12, addrs[i + off + 1]); 2205 } 2206 break; 2207 branch_oc: 2208 if (!is_first_pass(jit) && 2209 can_use_rel(jit, addrs[i + branch_oc_off + 1])) { 2210 /* brc mask,off */ 2211 EMIT4_PCREL_RIC(0xa7040000, 2212 mask >> 12, 2213 addrs[i + branch_oc_off + 1]); 2214 } else { 2215 /* brcl mask,off */ 2216 EMIT6_PCREL_RILC(0xc0040000, 2217 mask >> 12, 2218 addrs[i + branch_oc_off + 1]); 2219 } 2220 break; 2221 } 2222 default: /* too complex, give up */ 2223 pr_err("Unknown opcode %02x\n", insn->code); 2224 return -1; 2225 } 2226 2227 return insn_count; 2228 } 2229 2230 /* 2231 * Return whether new i-th instruction address does not violate any invariant 2232 */ 2233 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i) 2234 { 2235 /* On the first pass anything goes */ 2236 if (is_first_pass(jit)) 2237 return true; 2238 2239 /* The codegen pass must not change anything */ 2240 if (is_codegen_pass(jit)) 2241 return jit->addrs[i] == jit->prg; 2242 2243 /* Passes in between must not increase code size */ 2244 return jit->addrs[i] >= jit->prg; 2245 } 2246 2247 /* 2248 * Update the address of i-th instruction 2249 */ 2250 static int bpf_set_addr(struct bpf_jit *jit, int i) 2251 { 2252 int delta; 2253 2254 if (is_codegen_pass(jit)) { 2255 delta = jit->prg - jit->addrs[i]; 2256 if (delta < 0) 2257 bpf_skip(jit, -delta); 2258 } 2259 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i))) 2260 return -1; 2261 jit->addrs[i] = jit->prg; 2262 return 0; 2263 } 2264 2265 /* 2266 * Compile eBPF program into s390x code 2267 */ 2268 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, 2269 bool extra_pass) 2270 { 2271 int i, insn_count, lit32_size, lit64_size; 2272 u64 kern_arena; 2273 2274 jit->lit32 = jit->lit32_start; 2275 jit->lit64 = jit->lit64_start; 2276 jit->prg = 0; 2277 jit->excnt = 0; 2278 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) 2279 jit->frame_off = sizeof(struct prog_frame) - 2280 offsetofend(struct prog_frame, unused) + 2281 round_up(fp->aux->stack_depth, 8); 2282 else 2283 jit->frame_off = 0; 2284 2285 if (bpf_prog_uses_arena_insns(fp)) { 2286 kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); 2287 jit->kern_arena = _EMIT_CONST_U64(kern_arena); 2288 jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); 2289 } 2290 2291 bpf_jit_prologue(jit, fp); 2292 if (bpf_set_addr(jit, 0) < 0) 2293 return -1; 2294 for (i = 0; i < fp->len; i += insn_count) { 2295 insn_count = bpf_jit_insn(jit, fp, i, extra_pass); 2296 if (insn_count < 0) 2297 return -1; 2298 /* Next instruction address */ 2299 if (bpf_set_addr(jit, i + insn_count) < 0) 2300 return -1; 2301 } 2302 bpf_jit_epilogue(jit); 2303 2304 lit32_size = jit->lit32 - jit->lit32_start; 2305 lit64_size = jit->lit64 - jit->lit64_start; 2306 jit->lit32_start = jit->prg; 2307 if (lit32_size) 2308 jit->lit32_start = ALIGN(jit->lit32_start, 4); 2309 jit->lit64_start = jit->lit32_start + lit32_size; 2310 if (lit64_size) 2311 jit->lit64_start = ALIGN(jit->lit64_start, 8); 2312 jit->size = jit->lit64_start + lit64_size; 2313 jit->size_prg = jit->prg; 2314 2315 if (WARN_ON_ONCE(fp->aux->extable && 2316 jit->excnt != fp->aux->num_exentries)) 2317 /* Verifier bug - too many entries. */ 2318 return -1; 2319 2320 return 0; 2321 } 2322 2323 bool bpf_jit_needs_zext(void) 2324 { 2325 return true; 2326 } 2327 2328 struct s390_jit_data { 2329 struct bpf_binary_header *header; 2330 struct bpf_jit ctx; 2331 int pass; 2332 }; 2333 2334 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, 2335 struct bpf_prog *fp) 2336 { 2337 struct bpf_binary_header *header; 2338 struct bpf_insn *insn; 2339 u32 extable_size; 2340 u32 code_size; 2341 int i; 2342 2343 for (i = 0; i < fp->len; i++) { 2344 insn = &fp->insnsi[i]; 2345 2346 if (BPF_CLASS(insn->code) == BPF_STX && 2347 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && 2348 (BPF_SIZE(insn->code) == BPF_DW || 2349 BPF_SIZE(insn->code) == BPF_W) && 2350 insn->imm == BPF_XCHG) 2351 /* 2352 * bpf_jit_insn() emits a load and a compare-and-swap, 2353 * both of which need to be probed. 2354 */ 2355 fp->aux->num_exentries += 1; 2356 } 2357 code_size = roundup(jit->size, 2358 __alignof__(struct exception_table_entry)); 2359 extable_size = fp->aux->num_exentries * 2360 sizeof(struct exception_table_entry); 2361 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf, 2362 8, jit_fill_hole); 2363 if (!header) 2364 return NULL; 2365 fp->aux->extable = (struct exception_table_entry *) 2366 (jit->prg_buf + code_size); 2367 return header; 2368 } 2369 2370 /* 2371 * Compile eBPF program "fp" 2372 */ 2373 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 2374 { 2375 struct bpf_binary_header *header; 2376 struct s390_jit_data *jit_data; 2377 bool extra_pass = false; 2378 struct bpf_jit jit; 2379 int pass; 2380 2381 if (!fp->jit_requested) 2382 return fp; 2383 2384 jit_data = fp->aux->jit_data; 2385 if (!jit_data) { 2386 jit_data = kzalloc_obj(*jit_data); 2387 if (!jit_data) 2388 return fp; 2389 fp->aux->jit_data = jit_data; 2390 } 2391 if (jit_data->ctx.addrs) { 2392 jit = jit_data->ctx; 2393 header = jit_data->header; 2394 extra_pass = true; 2395 pass = jit_data->pass + 1; 2396 goto skip_init_ctx; 2397 } 2398 2399 memset(&jit, 0, sizeof(jit)); 2400 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); 2401 if (jit.addrs == NULL) 2402 goto out_err; 2403 /* 2404 * Three initial passes: 2405 * - 1/2: Determine clobbered registers 2406 * - 3: Calculate program size and addrs array 2407 */ 2408 for (pass = 1; pass <= 3; pass++) { 2409 if (bpf_jit_prog(&jit, fp, extra_pass)) 2410 goto out_err; 2411 } 2412 /* 2413 * Final pass: Allocate and generate program 2414 */ 2415 header = bpf_jit_alloc(&jit, fp); 2416 if (!header) 2417 goto out_err; 2418 skip_init_ctx: 2419 if (bpf_jit_prog(&jit, fp, extra_pass)) { 2420 bpf_jit_binary_free(header); 2421 goto out_err; 2422 } 2423 if (bpf_jit_enable > 1) { 2424 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); 2425 print_fn_code(jit.prg_buf, jit.size_prg); 2426 } 2427 if (!fp->is_func || extra_pass) { 2428 if (bpf_jit_binary_lock_ro(header)) { 2429 bpf_jit_binary_free(header); 2430 goto out_err; 2431 } 2432 } else { 2433 jit_data->header = header; 2434 jit_data->ctx = jit; 2435 jit_data->pass = pass; 2436 } 2437 fp->bpf_func = (void *)jit.prg_buf + cfi_get_offset(); 2438 fp->jited = 1; 2439 fp->jited_len = jit.size - cfi_get_offset(); 2440 2441 if (!fp->is_func || extra_pass) { 2442 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); 2443 free_addrs: 2444 kvfree(jit.addrs); 2445 kfree(jit_data); 2446 fp->aux->jit_data = NULL; 2447 } 2448 2449 return fp; 2450 2451 out_err: 2452 if (extra_pass) { 2453 fp->bpf_func = NULL; 2454 fp->jited = 0; 2455 fp->jited_len = 0; 2456 } 2457 goto free_addrs; 2458 } 2459 2460 bool bpf_jit_supports_kfunc_call(void) 2461 { 2462 return true; 2463 } 2464 2465 bool bpf_jit_supports_far_kfunc_call(void) 2466 { 2467 return true; 2468 } 2469 2470 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 2471 enum bpf_text_poke_type new_t, void *old_addr, 2472 void *new_addr) 2473 { 2474 struct bpf_plt expected_plt, current_plt, new_plt, *plt; 2475 struct { 2476 u16 opc; 2477 s32 disp; 2478 } __packed insn; 2479 char *ret; 2480 int err; 2481 2482 /* Verify the branch to be patched. */ 2483 err = copy_from_kernel_nofault(&insn, ip, sizeof(insn)); 2484 if (err < 0) 2485 return err; 2486 if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0))) 2487 return -EINVAL; 2488 2489 if ((new_t == BPF_MOD_JUMP || old_t == BPF_MOD_JUMP) && 2490 insn.disp == ((char *)new_addr - (char *)ip) >> 1) { 2491 /* 2492 * The branch already points to the destination, 2493 * there is no PLT. 2494 */ 2495 } else { 2496 /* Verify the PLT. */ 2497 plt = ip + (insn.disp << 1); 2498 err = copy_from_kernel_nofault(¤t_plt, plt, 2499 sizeof(current_plt)); 2500 if (err < 0) 2501 return err; 2502 ret = (char *)ip + 6; 2503 bpf_jit_plt(&expected_plt, ret, old_addr); 2504 if (memcmp(¤t_plt, &expected_plt, sizeof(current_plt))) 2505 return -EINVAL; 2506 /* Adjust the call address. */ 2507 bpf_jit_plt(&new_plt, ret, new_addr); 2508 s390_kernel_write(&plt->target, &new_plt.target, 2509 sizeof(void *)); 2510 } 2511 2512 /* Adjust the mask of the branch. */ 2513 insn.opc = 0xc004 | (new_addr ? 0xf0 : 0); 2514 s390_kernel_write((char *)ip + 1, (char *)&insn.opc + 1, 1); 2515 2516 /* Make the new code visible to the other CPUs. */ 2517 text_poke_sync_lock(); 2518 2519 return 0; 2520 } 2521 2522 struct bpf_tramp_jit { 2523 struct bpf_jit common; 2524 int orig_stack_args_off;/* Offset of arguments placed on stack by the 2525 * func_addr's original caller 2526 */ 2527 int stack_size; /* Trampoline stack size */ 2528 int backchain_off; /* Offset of backchain */ 2529 int stack_args_off; /* Offset of stack arguments for calling 2530 * func_addr, has to be at the top 2531 */ 2532 int reg_args_off; /* Offset of register arguments for calling 2533 * func_addr 2534 */ 2535 int ip_off; /* For bpf_get_func_ip(), has to be at 2536 * (ctx - 16) 2537 */ 2538 int func_meta_off; /* For bpf_get_func_arg_cnt()/fsession, has 2539 * to be at (ctx - 8) 2540 */ 2541 int bpf_args_off; /* Offset of BPF_PROG context, which consists 2542 * of BPF arguments followed by return value 2543 */ 2544 int retval_off; /* Offset of return value (see above) */ 2545 int r7_r8_off; /* Offset of saved %r7 and %r8, which are used 2546 * for __bpf_prog_enter() return value and 2547 * func_addr respectively 2548 */ 2549 int run_ctx_off; /* Offset of struct bpf_tramp_run_ctx */ 2550 int tccnt_off; /* Offset of saved tailcall counter */ 2551 int r14_off; /* Offset of saved %r14, has to be at the 2552 * bottom */ 2553 int do_fexit; /* do_fexit: label */ 2554 int skip[BPF_MAX_TRAMP_LINKS]; /* skip: labels after each prog */ 2555 int nr_progs; 2556 }; 2557 2558 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) 2559 { 2560 /* llihf %dst_reg,val_hi */ 2561 EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32)); 2562 /* oilf %rdst_reg,val_lo */ 2563 EMIT6_IMM(0xc00d0000, dst_reg, val); 2564 } 2565 2566 static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm) 2567 { 2568 load_imm64(jit, tmp_reg, imm); 2569 /* stg %tmp_reg,stack_off(%r15) */ 2570 EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off); 2571 } 2572 2573 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, 2574 struct bpf_tramp_image *im, 2575 const struct btf_func_model *m, 2576 struct bpf_tramp_node *node, bool save_ret) 2577 { 2578 struct bpf_jit *jit = &tjit->common; 2579 int cookie_off = tjit->run_ctx_off + 2580 offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 2581 struct bpf_prog *p = node->link->prog; 2582 void *skip = jit->prg_buf + jit->prg; 2583 int idx = tjit->nr_progs++; 2584 int patch; 2585 2586 if (idx >= ARRAY_SIZE(tjit->skip)) 2587 return -E2BIG; 2588 2589 /* 2590 * nop, patched to skip this prog when it is detached 2591 */ 2592 2593 /* brcl 0,skip */ 2594 EMIT6_PCREL_RILC(0xc0040000, 0, tjit->skip[idx]); 2595 2596 /* 2597 * run_ctx.cookie = node->cookie; 2598 */ 2599 2600 emit_store_stack_imm64(jit, REG_W0, cookie_off, node->cookie); 2601 2602 /* 2603 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) 2604 * goto skip; 2605 */ 2606 2607 /* %r2 = p */ 2608 load_imm64(jit, REG_2, (u64)p); 2609 /* la %r3,run_ctx_off(%r15) */ 2610 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); 2611 /* brasl %r14,__bpf_prog_enter */ 2612 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_enter(p)); 2613 /* ltgr %r7,%r2 */ 2614 EMIT4(0xb9020000, REG_7, REG_2); 2615 /* brcl 8,skip */ 2616 patch = jit->prg; 2617 EMIT6_PCREL_RILC(0xc0040000, 8, 0); 2618 2619 /* 2620 * retval = bpf_func(args, p->insnsi); 2621 */ 2622 2623 /* la %r2,bpf_args_off(%r15) */ 2624 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); 2625 /* %r3 = p->insnsi */ 2626 if (!p->jited) 2627 load_imm64(jit, REG_3, (u64)p->insnsi); 2628 /* brasl %r14,p->bpf_func */ 2629 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func); 2630 /* stg %r2,retval_off(%r15) */ 2631 if (save_ret) { 2632 if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags)) 2633 return -1; 2634 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2635 tjit->retval_off); 2636 } 2637 2638 /* skip: */ 2639 if (jit->prg_buf) 2640 *(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1; 2641 2642 /* 2643 * __bpf_prog_exit(p, start, &run_ctx); 2644 */ 2645 2646 /* %r2 = p */ 2647 load_imm64(jit, REG_2, (u64)p); 2648 /* lgr %r3,%r7 */ 2649 EMIT4(0xb9040000, REG_3, REG_7); 2650 /* la %r4,run_ctx_off(%r15) */ 2651 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); 2652 /* brasl %r14,__bpf_prog_exit */ 2653 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); 2654 2655 /* skip: */ 2656 tjit->skip[idx] = jit->prg; 2657 bpf_tramp_image_add_skip(im, p, skip, jit->prg_buf + jit->prg); 2658 2659 return 0; 2660 } 2661 2662 static int invoke_bpf(struct bpf_tramp_jit *tjit, 2663 struct bpf_tramp_image *im, 2664 const struct btf_func_model *m, 2665 struct bpf_tramp_nodes *tn, bool save_ret, 2666 u64 func_meta, int cookie_off) 2667 { 2668 int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64); 2669 struct bpf_jit *jit = &tjit->common; 2670 2671 for (i = 0; i < tn->nr_nodes; i++) { 2672 if (bpf_prog_calls_session_cookie(tn->nodes[i])) { 2673 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); 2674 2675 emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); 2676 cur_cookie--; 2677 } 2678 if (invoke_bpf_prog(tjit, im, m, tn->nodes[i], save_ret)) 2679 return -EINVAL; 2680 } 2681 2682 return 0; 2683 } 2684 2685 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) 2686 { 2687 int stack_offset = tjit->stack_size; 2688 2689 tjit->stack_size += size; 2690 return stack_offset; 2691 } 2692 2693 /* ABI uses %r2 - %r6 for parameter passing. */ 2694 #define MAX_NR_REG_ARGS 5 2695 2696 /* The "L" field of the "mvc" instruction is 8 bits. */ 2697 #define MAX_MVC_SIZE 256 2698 #define MAX_NR_STACK_ARGS (MAX_MVC_SIZE / sizeof(u64)) 2699 2700 /* -mfentry generates a 6-byte nop on s390x. */ 2701 #define S390X_PATCH_SIZE 6 2702 2703 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 2704 struct bpf_tramp_jit *tjit, 2705 const struct btf_func_model *m, 2706 u32 flags, 2707 struct bpf_tramp_nodes *tnodes, 2708 void *func_addr) 2709 { 2710 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 2711 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 2712 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 2713 int nr_bpf_args, nr_reg_args, nr_stack_args; 2714 int cookie_cnt, cookie_off, fsession_cnt; 2715 struct bpf_jit *jit = &tjit->common; 2716 int arg, bpf_arg_off; 2717 u64 func_meta; 2718 int i, j; 2719 2720 /* The skip labels are taken from the previous pass. */ 2721 tjit->nr_progs = 0; 2722 if (im) 2723 im->nr_skips = 0; 2724 2725 /* Support as many stack arguments as "mvc" instruction can handle. */ 2726 nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); 2727 nr_stack_args = m->nr_args - nr_reg_args; 2728 if (nr_stack_args > MAX_NR_STACK_ARGS) 2729 return -ENOTSUPP; 2730 2731 /* Return to %r14 in the struct_ops case. */ 2732 if (flags & BPF_TRAMP_F_INDIRECT) { 2733 flags |= BPF_TRAMP_F_SKIP_FRAME; 2734 emit_kcfi(cfi_get_func_hash(func_addr), jit); 2735 } 2736 2737 /* 2738 * Compute how many arguments we need to pass to BPF programs. 2739 * BPF ABI mirrors that of x86_64: arguments that are 16 bytes or 2740 * smaller are packed into 1 or 2 registers; larger arguments are 2741 * passed via pointers. 2742 * In s390x ABI, arguments that are 8 bytes or smaller are packed into 2743 * a register; larger arguments are passed via pointers. 2744 * We need to deal with this difference. 2745 */ 2746 nr_bpf_args = 0; 2747 for (i = 0; i < m->nr_args; i++) { 2748 if (m->arg_size[i] <= 8) 2749 nr_bpf_args += 1; 2750 else if (m->arg_size[i] <= 16) 2751 nr_bpf_args += 2; 2752 else 2753 return -ENOTSUPP; 2754 } 2755 2756 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 2757 fsession_cnt = bpf_fsession_cnt(tnodes); 2758 2759 /* 2760 * Calculate the stack layout. 2761 */ 2762 2763 /* 2764 * Allocate STACK_FRAME_OVERHEAD bytes for the callees. As the s390x 2765 * ABI requires, put our backchain at the end of the allocated memory. 2766 */ 2767 tjit->stack_size = STACK_FRAME_OVERHEAD; 2768 tjit->backchain_off = tjit->stack_size - sizeof(u64); 2769 tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); 2770 tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); 2771 cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64)); 2772 tjit->ip_off = alloc_stack(tjit, sizeof(u64)); 2773 tjit->func_meta_off = alloc_stack(tjit, sizeof(u64)); 2774 tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); 2775 tjit->retval_off = alloc_stack(tjit, sizeof(u64)); 2776 tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); 2777 tjit->run_ctx_off = alloc_stack(tjit, 2778 sizeof(struct bpf_tramp_run_ctx)); 2779 tjit->tccnt_off = alloc_stack(tjit, sizeof(u64)); 2780 tjit->r14_off = alloc_stack(tjit, sizeof(u64) * 2); 2781 /* 2782 * In accordance with the s390x ABI, the caller has allocated 2783 * STACK_FRAME_OVERHEAD bytes for us. 8 of them contain the caller's 2784 * backchain, and the rest we can use. 2785 */ 2786 tjit->stack_size -= STACK_FRAME_OVERHEAD - sizeof(u64); 2787 tjit->orig_stack_args_off = tjit->stack_size + STACK_FRAME_OVERHEAD; 2788 2789 /* lgr %r1,%r15 */ 2790 EMIT4(0xb9040000, REG_1, REG_15); 2791 /* aghi %r15,-stack_size */ 2792 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); 2793 /* stg %r1,backchain_off(%r15) */ 2794 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, 2795 tjit->backchain_off); 2796 /* mvc tccnt_off(4,%r15),stack_size+tail_call_cnt(%r15) */ 2797 _EMIT6(0xd203f000 | tjit->tccnt_off, 2798 0xf000 | (tjit->stack_size + 2799 offsetof(struct prog_frame, tail_call_cnt))); 2800 /* stmg %r2,%rN,fwd_reg_args_off(%r15) */ 2801 if (nr_reg_args) 2802 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, 2803 REG_2 + (nr_reg_args - 1), REG_15, 2804 tjit->reg_args_off); 2805 for (i = 0, j = 0; i < m->nr_args; i++) { 2806 if (i < MAX_NR_REG_ARGS) 2807 arg = REG_2 + i; 2808 else 2809 arg = tjit->orig_stack_args_off + 2810 (i - MAX_NR_REG_ARGS) * sizeof(u64); 2811 bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64); 2812 if (m->arg_size[i] <= 8) { 2813 if (i < MAX_NR_REG_ARGS) 2814 /* stg %arg,bpf_arg_off(%r15) */ 2815 EMIT6_DISP_LH(0xe3000000, 0x0024, arg, 2816 REG_0, REG_15, bpf_arg_off); 2817 else 2818 /* mvc bpf_arg_off(8,%r15),arg(%r15) */ 2819 _EMIT6(0xd207f000 | bpf_arg_off, 2820 0xf000 | arg); 2821 j += 1; 2822 } else { 2823 if (i < MAX_NR_REG_ARGS) { 2824 /* mvc bpf_arg_off(16,%r15),0(%arg) */ 2825 _EMIT6(0xd20ff000 | bpf_arg_off, 2826 reg2hex[arg] << 12); 2827 } else { 2828 /* lg %r1,arg(%r15) */ 2829 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0, 2830 REG_15, arg); 2831 /* mvc bpf_arg_off(16,%r15),0(%r1) */ 2832 _EMIT6(0xd20ff000 | bpf_arg_off, 0x1000); 2833 } 2834 j += 2; 2835 } 2836 } 2837 /* stmg %r7,%r8,r7_r8_off(%r15) */ 2838 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15, 2839 tjit->r7_r8_off); 2840 /* stg %r14,r14_off(%r15) */ 2841 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); 2842 2843 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2844 /* 2845 * The ftrace trampoline puts the return address (which is the 2846 * address of the original function + S390X_PATCH_SIZE) into 2847 * %r0; see ftrace_shared_hotpatch_trampoline_br and 2848 * ftrace_init_nop() for details. 2849 */ 2850 2851 /* lgr %r8,%r0 */ 2852 EMIT4(0xb9040000, REG_8, REG_0); 2853 } 2854 2855 /* 2856 * ip = func_addr; 2857 * arg_cnt = m->nr_args; 2858 */ 2859 2860 if (flags & BPF_TRAMP_F_IP_ARG) 2861 emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr); 2862 func_meta = nr_bpf_args; 2863 /* lghi %r0,func_meta */ 2864 EMIT4_IMM(0xa7090000, REG_0, func_meta); 2865 /* stg %r0,func_meta_off(%r15) */ 2866 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2867 tjit->func_meta_off); 2868 2869 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2870 /* 2871 * __bpf_tramp_enter(im); 2872 */ 2873 2874 /* %r2 = im */ 2875 load_imm64(jit, REG_2, (u64)im); 2876 /* brasl %r14,__bpf_tramp_enter */ 2877 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter); 2878 } 2879 2880 if (fsession_cnt) { 2881 /* Clear all the session cookies' value. */ 2882 for (i = 0; i < cookie_cnt; i++) 2883 emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0); 2884 /* Clear the return value to make sure fentry always gets 0. */ 2885 emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); 2886 } 2887 2888 if (invoke_bpf(tjit, im, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, 2889 func_meta, cookie_off)) 2890 return -EINVAL; 2891 2892 if (fmod_ret->nr_nodes) { 2893 /* 2894 * retval = 0; 2895 */ 2896 2897 /* xc retval_off(8,%r15),retval_off(%r15) */ 2898 _EMIT6(0xd707f000 | tjit->retval_off, 2899 0xf000 | tjit->retval_off); 2900 2901 for (i = 0; i < fmod_ret->nr_nodes; i++) { 2902 if (invoke_bpf_prog(tjit, im, m, fmod_ret->nodes[i], true)) 2903 return -EINVAL; 2904 2905 /* 2906 * if (retval) 2907 * goto do_fexit; 2908 */ 2909 2910 /* ltg %r0,retval_off(%r15) */ 2911 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15, 2912 tjit->retval_off); 2913 /* brcl 7,do_fexit */ 2914 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); 2915 } 2916 } 2917 2918 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2919 /* 2920 * retval = func_addr(args); 2921 */ 2922 2923 /* lmg %r2,%rN,reg_args_off(%r15) */ 2924 if (nr_reg_args) 2925 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2926 REG_2 + (nr_reg_args - 1), REG_15, 2927 tjit->reg_args_off); 2928 /* mvc stack_args_off(N,%r15),orig_stack_args_off(%r15) */ 2929 if (nr_stack_args) 2930 _EMIT6(0xd200f000 | 2931 (nr_stack_args * sizeof(u64) - 1) << 16 | 2932 tjit->stack_args_off, 2933 0xf000 | tjit->orig_stack_args_off); 2934 /* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2935 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), 2936 0xf000 | tjit->tccnt_off); 2937 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2938 if (nospec_uses_trampoline()) 2939 /* brasl %r14,__s390_indirect_jump_r8 */ 2940 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2941 __s390_indirect_jump_r8); 2942 else 2943 /* basr %r14,%r8 */ 2944 EMIT2(0x0d00, REG_14, REG_8); 2945 } else { 2946 /* brasl %r14,func_addr+S390X_PATCH_SIZE */ 2947 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2948 func_addr + S390X_PATCH_SIZE); 2949 } 2950 /* stg %r2,retval_off(%r15) */ 2951 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2952 tjit->retval_off); 2953 /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ 2954 _EMIT6(0xd203f000 | tjit->tccnt_off, 2955 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 2956 } 2957 2958 /* Set the "is_return" flag for fsession. */ 2959 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 2960 if (fsession_cnt) 2961 emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off, 2962 func_meta); 2963 2964 /* do_fexit: */ 2965 tjit->do_fexit = jit->prg; 2966 if (invoke_bpf(tjit, im, m, fexit, false, func_meta, cookie_off)) 2967 return -EINVAL; 2968 2969 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2970 /* 2971 * __bpf_tramp_exit(im); 2972 */ 2973 2974 /* %r2 = im */ 2975 load_imm64(jit, REG_2, (u64)im); 2976 /* brasl %r14,__bpf_tramp_exit */ 2977 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_exit); 2978 } 2979 2980 /* lmg %r2,%rN,reg_args_off(%r15) */ 2981 if ((flags & BPF_TRAMP_F_RESTORE_REGS) && nr_reg_args) 2982 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2983 REG_2 + (nr_reg_args - 1), REG_15, 2984 tjit->reg_args_off); 2985 /* lgr %r1,%r8 */ 2986 if (!(flags & BPF_TRAMP_F_SKIP_FRAME) && 2987 (flags & BPF_TRAMP_F_ORIG_STACK)) 2988 EMIT4(0xb9040000, REG_1, REG_8); 2989 /* lmg %r7,%r8,r7_r8_off(%r15) */ 2990 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, 2991 tjit->r7_r8_off); 2992 /* lg %r14,r14_off(%r15) */ 2993 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); 2994 /* lg %r2,retval_off(%r15) */ 2995 if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET)) 2996 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, 2997 tjit->retval_off); 2998 /* mvc stack_size+tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2999 _EMIT6(0xd203f000 | (tjit->stack_size + 3000 offsetof(struct prog_frame, tail_call_cnt)), 3001 0xf000 | tjit->tccnt_off); 3002 /* aghi %r15,stack_size */ 3003 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); 3004 if (flags & BPF_TRAMP_F_SKIP_FRAME) 3005 EMIT_JUMP_REG(14); 3006 else if (flags & BPF_TRAMP_F_ORIG_STACK) 3007 EMIT_JUMP_REG(1); 3008 else 3009 /* brcl 0xf,func_addr+S390X_PATCH_SIZE */ 3010 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 3011 func_addr + S390X_PATCH_SIZE); 3012 return 0; 3013 } 3014 3015 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 3016 struct bpf_tramp_nodes *tnodes, void *orig_call) 3017 { 3018 struct bpf_tramp_image im = {}; 3019 struct bpf_tramp_jit tjit; 3020 int ret; 3021 3022 memset(&tjit, 0, sizeof(tjit)); 3023 3024 ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, 3025 tnodes, orig_call); 3026 3027 return ret < 0 ? ret : tjit.common.prg; 3028 } 3029 3030 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 3031 void *image_end, const struct btf_func_model *m, 3032 u32 flags, struct bpf_tramp_nodes *tnodes, 3033 void *func_addr) 3034 { 3035 struct bpf_tramp_jit tjit; 3036 int ret; 3037 3038 /* Compute offsets, check whether the code fits. */ 3039 memset(&tjit, 0, sizeof(tjit)); 3040 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 3041 tnodes, func_addr); 3042 3043 if (ret < 0) 3044 return ret; 3045 if (tjit.common.prg > (char *)image_end - (char *)image) 3046 /* 3047 * Use the same error code as for exceeding 3048 * BPF_MAX_TRAMP_LINKS. 3049 */ 3050 return -E2BIG; 3051 3052 tjit.common.prg = 0; 3053 tjit.common.prg_buf = image; 3054 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 3055 tnodes, func_addr); 3056 3057 return ret < 0 ? ret : tjit.common.prg; 3058 } 3059 3060 bool bpf_jit_supports_subprog_tailcalls(void) 3061 { 3062 return true; 3063 } 3064 3065 bool bpf_jit_supports_arena(void) 3066 { 3067 return true; 3068 } 3069 3070 bool bpf_jit_supports_fsession(void) 3071 { 3072 return true; 3073 } 3074 3075 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 3076 { 3077 if (!in_arena) 3078 return true; 3079 switch (insn->code) { 3080 case BPF_LDX | BPF_MEMSX | BPF_B: 3081 case BPF_LDX | BPF_MEMSX | BPF_H: 3082 case BPF_LDX | BPF_MEMSX | BPF_W: 3083 return false; 3084 } 3085 return true; 3086 } 3087 3088 bool bpf_jit_supports_exceptions(void) 3089 { 3090 /* 3091 * Exceptions require unwinding support, which is always available, 3092 * because the kernel is always built with backchain. 3093 */ 3094 return true; 3095 } 3096 3097 void arch_bpf_stack_walk(bool (*consume_fn)(void *, u64, u64, u64), 3098 void *cookie) 3099 { 3100 unsigned long addr, prev_addr = 0; 3101 struct unwind_state state; 3102 3103 unwind_for_each_frame(&state, NULL, NULL, 0) { 3104 addr = unwind_get_return_address(&state); 3105 if (!addr) 3106 break; 3107 /* 3108 * addr is a return address and state.sp is the value of %r15 3109 * at this address. exception_cb needs %r15 at entry to the 3110 * function containing addr, so take the next state.sp. 3111 * 3112 * There is no bp, and the exception_cb prog does not need one 3113 * to perform a quasi-longjmp. The common code requires a 3114 * non-zero bp, so pass sp there as well. 3115 */ 3116 if (prev_addr && !consume_fn(cookie, prev_addr, state.sp, 3117 state.sp)) 3118 break; 3119 prev_addr = addr; 3120 } 3121 } 3122 3123 bool bpf_jit_supports_timed_may_goto(void) 3124 { 3125 return true; 3126 } 3127 3128 bool bpf_jit_supports_percpu_insn(void) 3129 { 3130 return true; 3131 } 3132 3133 bool bpf_jit_inlines_helper_call(s32 imm) 3134 { 3135 switch (imm) { 3136 case BPF_FUNC_get_smp_processor_id: 3137 case BPF_FUNC_get_current_task: 3138 case BPF_FUNC_get_current_task_btf: 3139 return true; 3140 default: 3141 return false; 3142 } 3143 } 3144