Lines Matching +full:0 +full:xc0010000
59 #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */
64 #define NVREGS 0xffc0 /* %r6-%r15 */
69 #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */
73 #define REG_0 REG_W0 /* Register 0 */
104 [REG_W0] = 0,
137 return 0; in ptr_to_pcrel()
187 unsigned int __disp = (disp) & 0xfff; \
201 unsigned int __imm = (imm) & 0xffff; \
208 long __pcrel = ((pcrel) >> 1) & 0xffff; \
215 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \
229 unsigned int __disp = (disp) & 0xfff; \
236 unsigned int __disp_h = _disp & 0xff000; \
237 unsigned int __disp_l = _disp & 0x00fff; \
253 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \
263 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \
265 BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \
271 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\
280 _EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff); in emit6_pcrel_ril()
309 _EMIT6((op) | (__imm >> 16), __imm & 0xffff); \
354 EMIT4(0xb9160000, b1, b1); \
365 return jit->size == 0; in is_first_pass()
425 memset(area, 0, size); in jit_fill_hole()
435 /* BPF stack starts here and grows towards 0 */
451 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); in save_regs()
454 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); in save_regs()
466 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); in restore_regs()
469 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); in restore_regs()
483 return 0; in get_start()
501 #define REGS_RESTORE 0
537 /* brcl 0xf,size */ in bpf_skip()
538 EMIT6_PCREL_RILC(0xc0040000, 0xf, size); in bpf_skip()
541 /* brc 0xf,size */ in bpf_skip()
542 EMIT4_PCREL(0xa7f40000, size); in bpf_skip()
546 /* bcr 0,%0 */ in bpf_skip()
547 _EMIT2(0x0700); in bpf_skip()
570 "bpf_plt_ret: .quad 0\n"
571 "bpf_plt_target: .quad 0\n"
601 /* brcl 0,prologue_plt */ in bpf_jit_prologue()
602 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); in bpf_jit_prologue()
608 _EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt), in bpf_jit_prologue()
609 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); in bpf_jit_prologue()
628 EMIT4(0xb9040000, REG_15, REG_3); in bpf_jit_prologue()
633 fp->aux->exception_boundary ? NVREGS : 0); in bpf_jit_prologue()
639 /* basr %l,0 */ in bpf_jit_prologue()
640 EMIT2(0x0d00, REG_L, REG_0); in bpf_jit_prologue()
644 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); in bpf_jit_prologue()
651 EMIT4(0xb9040000, REG_W1, REG_15); in bpf_jit_prologue()
653 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, in bpf_jit_prologue()
656 EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off); in bpf_jit_prologue()
658 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, in bpf_jit_prologue()
669 /* brcl 0xf,__s390_indirect_jump_rN */ \
670 EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \
674 _EMIT2(0x07f0 | reg); \
675 } while (0)
684 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, in call_r1()
688 EMIT2(0x0d00, REG_14, REG_1); in call_r1()
698 EMIT4(0xb9040000, REG_2, BPF_REG_0); in bpf_jit_epilogue()
700 save_restore_regs(jit, REGS_RESTORE, 0); in bpf_jit_epilogue()
715 regs->gprs[x->data] = 0; in ex_handler_bpf()
751 /* bcr 0,%0 */ in bpf_jit_probe_emit_nop()
752 _EMIT2(0x0700); in bpf_jit_probe_emit_nop()
765 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_load_pre()
779 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_store_pre()
792 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_atomic_pre()
794 EMIT4(0xb9080000, REG_W1, insn->dst_reg); in bpf_jit_probe_atomic_pre()
809 return 0; in bpf_jit_probe_post()
813 return 0; in bpf_jit_probe_post()
818 for (i = 0; i < 2; i++) { in bpf_jit_probe_post()
824 prg = i == 0 ? probe->prg : probe->nop_prg; in bpf_jit_probe_post()
844 return 0; in bpf_jit_probe_post()
853 return 0; in sign_extend()
858 EMIT4(0xb9060000, r, r); in sign_extend()
859 return 0; in sign_extend()
862 EMIT4(0xb9070000, r, r); in sign_extend()
863 return 0; in sign_extend()
866 EMIT4(0xb9140000, r, r); in sign_extend()
867 return 0; in sign_extend()
869 return 0; in sign_extend()
904 case 0: /* DST = (u32) SRC */ in bpf_jit_insn()
906 EMIT4(0xb9160000, dst_reg, src_reg); in bpf_jit_insn()
912 EMIT4(0xb9260000, dst_reg, src_reg); in bpf_jit_insn()
914 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
918 EMIT4(0xb9270000, dst_reg, src_reg); in bpf_jit_insn()
920 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
929 EMIT4(0xb9020000, dst_reg, src_reg); in bpf_jit_insn()
930 /* brc 8,0f */ in bpf_jit_insn()
932 EMIT4_PCREL_RIC(0xa7040000, 8, 0); in bpf_jit_insn()
934 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); in bpf_jit_insn()
935 /* 0: */ in bpf_jit_insn()
942 case 0: /* DST = SRC */ in bpf_jit_insn()
944 EMIT4(0xb9040000, dst_reg, src_reg); in bpf_jit_insn()
948 EMIT4(0xb9060000, dst_reg, src_reg); in bpf_jit_insn()
952 EMIT4(0xb9070000, dst_reg, src_reg); in bpf_jit_insn()
956 EMIT4(0xb9140000, dst_reg, src_reg); in bpf_jit_insn()
962 EMIT6_IMM(0xc00f0000, dst_reg, imm); in bpf_jit_insn()
968 EMIT6_IMM(0xc0010000, dst_reg, imm); in bpf_jit_insn()
978 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; in bpf_jit_insn()
980 EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64)); in bpf_jit_insn()
989 EMIT2(0x1a00, dst_reg, src_reg); in bpf_jit_insn()
994 EMIT4(0xb9080000, dst_reg, src_reg); in bpf_jit_insn()
997 if (imm != 0) { in bpf_jit_insn()
999 EMIT6_IMM(0xc20b0000, dst_reg, imm); in bpf_jit_insn()
1007 EMIT6_IMM(0xc2080000, dst_reg, imm); in bpf_jit_insn()
1014 EMIT2(0x1b00, dst_reg, src_reg); in bpf_jit_insn()
1019 EMIT4(0xb9090000, dst_reg, src_reg); in bpf_jit_insn()
1022 if (imm != 0) { in bpf_jit_insn()
1024 EMIT6_IMM(0xc20b0000, dst_reg, -imm); in bpf_jit_insn()
1031 if (imm == -0x80000000) { in bpf_jit_insn()
1032 /* algfi %dst,0x80000000 */ in bpf_jit_insn()
1033 EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000); in bpf_jit_insn()
1036 EMIT6_IMM(0xc2080000, dst_reg, -imm); in bpf_jit_insn()
1044 EMIT4(0xb2520000, dst_reg, src_reg); in bpf_jit_insn()
1049 EMIT4(0xb90c0000, dst_reg, src_reg); in bpf_jit_insn()
1054 EMIT6_IMM(0xc2010000, dst_reg, imm); in bpf_jit_insn()
1062 EMIT6_IMM(0xc2000000, dst_reg, imm); in bpf_jit_insn()
1073 case 0: /* dst = (u32) dst {/,%} (u32) src */ in bpf_jit_insn()
1075 EMIT2(0x1700, REG_W0, REG_W0); in bpf_jit_insn()
1077 EMIT2(0x1800, REG_W1, dst_reg); in bpf_jit_insn()
1079 EMIT4(0xb9970000, REG_W0, src_reg); in bpf_jit_insn()
1083 EMIT4(0xb9140000, REG_W1, dst_reg); in bpf_jit_insn()
1085 EMIT4(0xb91d0000, REG_W0, src_reg); in bpf_jit_insn()
1089 EMIT4(0xb9160000, dst_reg, rc_reg); in bpf_jit_insn()
1100 case 0: /* dst = dst {/,%} src */ in bpf_jit_insn()
1101 /* lghi %w0,0 */ in bpf_jit_insn()
1102 EMIT4_IMM(0xa7090000, REG_W0, 0); in bpf_jit_insn()
1104 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1106 EMIT4(0xb9870000, REG_W0, src_reg); in bpf_jit_insn()
1110 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1112 EMIT4(0xb90d0000, REG_W0, src_reg); in bpf_jit_insn()
1116 EMIT4(0xb9040000, dst_reg, rc_reg); in bpf_jit_insn()
1126 /* lghi %dst,0 */ in bpf_jit_insn()
1127 EMIT4_IMM(0xa7090000, dst_reg, 0); in bpf_jit_insn()
1134 case 0: /* dst = (u32) dst {/,%} (u32) imm */ in bpf_jit_insn()
1136 EMIT2(0x1700, REG_W0, REG_W0); in bpf_jit_insn()
1138 EMIT2(0x1800, REG_W1, dst_reg); in bpf_jit_insn()
1140 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, in bpf_jit_insn()
1145 EMIT4(0xb9140000, REG_W1, dst_reg); in bpf_jit_insn()
1147 EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0, in bpf_jit_insn()
1153 case 0: /* dst = (u32) dst {/,%} (u32) imm */ in bpf_jit_insn()
1155 EMIT2(0x1700, REG_W0, REG_W0); in bpf_jit_insn()
1157 EMIT2(0x1800, REG_W1, dst_reg); in bpf_jit_insn()
1159 EMIT6_PCREL_RILB(0xc40d0000, dst_reg, in bpf_jit_insn()
1163 EMIT4(0xb9970000, REG_W0, dst_reg); in bpf_jit_insn()
1167 EMIT4(0xb9140000, REG_W1, dst_reg); in bpf_jit_insn()
1169 EMIT6_PCREL_RILB(0xc40c0000, dst_reg, in bpf_jit_insn()
1173 EMIT4(0xb90d0000, REG_W0, dst_reg); in bpf_jit_insn()
1178 EMIT4(0xb9160000, dst_reg, rc_reg); in bpf_jit_insn()
1190 /* lhgi %dst,0 */ in bpf_jit_insn()
1191 EMIT4_IMM(0xa7090000, dst_reg, 0); in bpf_jit_insn()
1196 case 0: /* dst = dst {/,%} imm */ in bpf_jit_insn()
1197 /* lghi %w0,0 */ in bpf_jit_insn()
1198 EMIT4_IMM(0xa7090000, REG_W0, 0); in bpf_jit_insn()
1200 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1202 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, in bpf_jit_insn()
1207 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1209 EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0, in bpf_jit_insn()
1215 case 0: /* dst = dst {/,%} imm */ in bpf_jit_insn()
1216 /* lghi %w0,0 */ in bpf_jit_insn()
1217 EMIT4_IMM(0xa7090000, REG_W0, 0); in bpf_jit_insn()
1219 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1221 EMIT6_PCREL_RILB(0xc4080000, dst_reg, in bpf_jit_insn()
1225 EMIT4(0xb9870000, REG_W0, dst_reg); in bpf_jit_insn()
1229 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1231 EMIT6_PCREL_RILB(0xc4080000, dst_reg, in bpf_jit_insn()
1235 EMIT4(0xb90d0000, REG_W0, dst_reg); in bpf_jit_insn()
1240 EMIT4(0xb9040000, dst_reg, rc_reg); in bpf_jit_insn()
1248 EMIT2(0x1400, dst_reg, src_reg); in bpf_jit_insn()
1253 EMIT4(0xb9800000, dst_reg, src_reg); in bpf_jit_insn()
1257 EMIT6_IMM(0xc00b0000, dst_reg, imm); in bpf_jit_insn()
1263 EMIT6_DISP_LH(0xe3000000, 0x0080, in bpf_jit_insn()
1268 EMIT6_PCREL_RILB(0xc4080000, REG_W0, in bpf_jit_insn()
1272 EMIT4(0xb9800000, dst_reg, REG_W0); in bpf_jit_insn()
1280 EMIT2(0x1600, dst_reg, src_reg); in bpf_jit_insn()
1285 EMIT4(0xb9810000, dst_reg, src_reg); in bpf_jit_insn()
1289 EMIT6_IMM(0xc00d0000, dst_reg, imm); in bpf_jit_insn()
1295 EMIT6_DISP_LH(0xe3000000, 0x0081, in bpf_jit_insn()
1300 EMIT6_PCREL_RILB(0xc4080000, REG_W0, in bpf_jit_insn()
1304 EMIT4(0xb9810000, dst_reg, REG_W0); in bpf_jit_insn()
1312 EMIT2(0x1700, dst_reg, src_reg); in bpf_jit_insn()
1317 EMIT4(0xb9820000, dst_reg, src_reg); in bpf_jit_insn()
1320 if (imm != 0) { in bpf_jit_insn()
1322 EMIT6_IMM(0xc0070000, dst_reg, imm); in bpf_jit_insn()
1329 EMIT6_DISP_LH(0xe3000000, 0x0082, in bpf_jit_insn()
1334 EMIT6_PCREL_RILB(0xc4080000, REG_W0, in bpf_jit_insn()
1338 EMIT4(0xb9820000, dst_reg, REG_W0); in bpf_jit_insn()
1345 /* sll %dst,0(%src) */ in bpf_jit_insn()
1346 EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); in bpf_jit_insn()
1350 /* sllg %dst,%dst,0(%src) */ in bpf_jit_insn()
1351 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); in bpf_jit_insn()
1354 if (imm != 0) { in bpf_jit_insn()
1356 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); in bpf_jit_insn()
1361 if (imm == 0) in bpf_jit_insn()
1364 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); in bpf_jit_insn()
1370 /* srl %dst,0(%src) */ in bpf_jit_insn()
1371 EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); in bpf_jit_insn()
1375 /* srlg %dst,%dst,0(%src) */ in bpf_jit_insn()
1376 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); in bpf_jit_insn()
1379 if (imm != 0) { in bpf_jit_insn()
1381 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); in bpf_jit_insn()
1386 if (imm == 0) in bpf_jit_insn()
1389 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); in bpf_jit_insn()
1395 /* sra %dst,%dst,0(%src) */ in bpf_jit_insn()
1396 EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); in bpf_jit_insn()
1400 /* srag %dst,%dst,0(%src) */ in bpf_jit_insn()
1401 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); in bpf_jit_insn()
1404 if (imm != 0) { in bpf_jit_insn()
1406 EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); in bpf_jit_insn()
1411 if (imm == 0) in bpf_jit_insn()
1414 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); in bpf_jit_insn()
1421 EMIT2(0x1300, dst_reg, dst_reg); in bpf_jit_insn()
1426 EMIT4(0xb9030000, dst_reg, dst_reg); in bpf_jit_insn()
1436 EMIT4(0xb9850000, dst_reg, dst_reg); in bpf_jit_insn()
1443 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
1454 EMIT4(0xb91f0000, dst_reg, dst_reg); in bpf_jit_insn()
1456 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); in bpf_jit_insn()
1458 EMIT4(0xb9850000, dst_reg, dst_reg); in bpf_jit_insn()
1464 EMIT4(0xb91f0000, dst_reg, dst_reg); in bpf_jit_insn()
1467 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
1471 EMIT4(0xb90f0000, dst_reg, dst_reg); in bpf_jit_insn()
1487 EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, in bpf_jit_insn()
1490 if (err < 0) in bpf_jit_insn()
1498 EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, in bpf_jit_insn()
1501 if (err < 0) in bpf_jit_insn()
1509 EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, in bpf_jit_insn()
1512 if (err < 0) in bpf_jit_insn()
1520 EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, in bpf_jit_insn()
1523 if (err < 0) in bpf_jit_insn()
1530 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); in bpf_jit_insn()
1533 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, in bpf_jit_insn()
1536 if (err < 0) in bpf_jit_insn()
1543 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); in bpf_jit_insn()
1546 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, in bpf_jit_insn()
1549 if (err < 0) in bpf_jit_insn()
1556 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); in bpf_jit_insn()
1559 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, in bpf_jit_insn()
1562 if (err < 0) in bpf_jit_insn()
1569 EMIT6_IMM(0xc0010000, REG_W0, imm); in bpf_jit_insn()
1572 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, in bpf_jit_insn()
1575 if (err < 0) in bpf_jit_insn()
1602 EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ in bpf_jit_insn()
1606 if (err < 0) \ in bpf_jit_insn()
1609 /* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ in bpf_jit_insn()
1610 _EMIT2(0x07e0); \ in bpf_jit_insn()
1614 } while (0) in bpf_jit_insn()
1618 EMIT_ATOMIC(0x00fa, 0x00ea); in bpf_jit_insn()
1623 EMIT_ATOMIC(0x00f4, 0x00e4); in bpf_jit_insn()
1628 EMIT_ATOMIC(0x00f6, 0x00e6); in bpf_jit_insn()
1633 EMIT_ATOMIC(0x00f7, 0x00e7); in bpf_jit_insn()
1642 EMIT6_DISP_LH(0xe3000000, in bpf_jit_insn()
1643 is32 ? 0x0058 : 0x0004, REG_W0, REG_0, in bpf_jit_insn()
1652 /* 0: {csy|csg} %w0,%src,off(%arena) */ in bpf_jit_insn()
1653 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, in bpf_jit_insn()
1656 /* brc 4,0b */ in bpf_jit_insn()
1657 EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); in bpf_jit_insn()
1659 EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); in bpf_jit_insn()
1662 if (err < 0) in bpf_jit_insn()
1665 if (err < 0) in bpf_jit_insn()
1673 /* 0: {csy|csg} %b0,%src,off(%arena) */ in bpf_jit_insn()
1674 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, in bpf_jit_insn()
1678 if (err < 0) in bpf_jit_insn()
1697 EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, in bpf_jit_insn()
1700 if (err < 0) in bpf_jit_insn()
1710 EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off); in bpf_jit_insn()
1712 if (err < 0) in bpf_jit_insn()
1721 EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, in bpf_jit_insn()
1724 if (err < 0) in bpf_jit_insn()
1734 EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off); in bpf_jit_insn()
1736 if (err < 0) in bpf_jit_insn()
1746 EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, in bpf_jit_insn()
1749 if (err < 0) in bpf_jit_insn()
1759 EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off); in bpf_jit_insn()
1761 if (err < 0) in bpf_jit_insn()
1770 EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, in bpf_jit_insn()
1773 if (err < 0) in bpf_jit_insn()
1788 if (ret < 0) in bpf_jit_insn()
1804 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), in bpf_jit_insn()
1805 0xf000 | (jit->frame_off + in bpf_jit_insn()
1814 for (j = 0; j < m->nr_args; j++) { in bpf_jit_insn()
1823 EMIT6_PCREL_RILB(0xc4080000, REG_W1, _EMIT_CONST_U64(func)); in bpf_jit_insn()
1827 EMIT4(0xb9040000, BPF_REG_0, REG_2); in bpf_jit_insn()
1844 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2, in bpf_jit_insn()
1847 /* clrj %b3,%w1,0xa,out */ in bpf_jit_insn()
1849 EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa, in bpf_jit_insn()
1860 EMIT4_IMM(0xa7080000, REG_W0, 1); in bpf_jit_insn()
1862 EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off); in bpf_jit_insn()
1863 /* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */ in bpf_jit_insn()
1865 EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1, in bpf_jit_insn()
1875 EMIT4(0xb9160000, REG_1, BPF_REG_3); in bpf_jit_insn()
1877 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3); in bpf_jit_insn()
1879 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2, in bpf_jit_insn()
1881 /* brc 0x8,out */ in bpf_jit_insn()
1883 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); in bpf_jit_insn()
1888 save_restore_regs(jit, REGS_RESTORE, 0); in bpf_jit_insn()
1895 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0, in bpf_jit_insn()
1900 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); in bpf_jit_insn()
1901 /* brcl 0xf,__s390_indirect_jump_r1 */ in bpf_jit_insn()
1902 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, in bpf_jit_insn()
1905 /* bc 0xf,tail_call_start(%r1) */ in bpf_jit_insn()
1906 _EMIT4(0x47f01000 + jit->tail_call_start); in bpf_jit_insn()
1920 last = (i == fp->len - 1) ? 1 : 0; in bpf_jit_insn()
1924 /* brc 0xf, <exit> */ in bpf_jit_insn()
1925 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); in bpf_jit_insn()
1927 /* brcl 0xf, <exit> */ in bpf_jit_insn()
1928 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); in bpf_jit_insn()
1938 * 0 | Operands equal | 8 in bpf_jit_insn()
1946 * For example for s390x with offset 0 we jump to the branch in bpf_jit_insn()
1947 * instruction itself (loop) and for BPF with offset 0 we in bpf_jit_insn()
1954 mask = 0xf000; /* j */ in bpf_jit_insn()
1958 mask = 0x2000; /* jh */ in bpf_jit_insn()
1962 mask = 0x4000; /* jl */ in bpf_jit_insn()
1966 mask = 0xa000; /* jhe */ in bpf_jit_insn()
1970 mask = 0xc000; /* jle */ in bpf_jit_insn()
1974 mask = 0x2000; /* jh */ in bpf_jit_insn()
1978 mask = 0x4000; /* jl */ in bpf_jit_insn()
1982 mask = 0xa000; /* jhe */ in bpf_jit_insn()
1986 mask = 0xc000; /* jle */ in bpf_jit_insn()
1990 mask = 0x7000; /* jne */ in bpf_jit_insn()
1994 mask = 0x8000; /* je */ in bpf_jit_insn()
1998 mask = 0x7000; /* jnz */ in bpf_jit_insn()
2001 EMIT6_IMM(0xc00f0000, REG_W1, imm); in bpf_jit_insn()
2003 EMIT2(0x1400, REG_W1, dst_reg); in bpf_jit_insn()
2006 EMIT6_IMM(0xc0010000, REG_W1, imm); in bpf_jit_insn()
2008 EMIT4(0xb9800000, REG_W1, dst_reg); in bpf_jit_insn()
2014 mask = 0x2000; /* jh */ in bpf_jit_insn()
2018 mask = 0x4000; /* jl */ in bpf_jit_insn()
2022 mask = 0xa000; /* jhe */ in bpf_jit_insn()
2026 mask = 0xc000; /* jle */ in bpf_jit_insn()
2030 mask = 0x2000; /* jh */ in bpf_jit_insn()
2034 mask = 0x4000; /* jl */ in bpf_jit_insn()
2038 mask = 0xa000; /* jhe */ in bpf_jit_insn()
2042 mask = 0xc000; /* jle */ in bpf_jit_insn()
2046 mask = 0x7000; /* jne */ in bpf_jit_insn()
2050 mask = 0x8000; /* je */ in bpf_jit_insn()
2057 mask = 0x7000; /* jnz */ in bpf_jit_insn()
2059 EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000), in bpf_jit_insn()
2065 EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000, in bpf_jit_insn()
2070 EMIT4_PCREL_RIC(0xa7040000, in bpf_jit_insn()
2074 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2081 EMIT6_IMM(0xc0010000, src_reg, imm); in bpf_jit_insn()
2088 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064), in bpf_jit_insn()
2093 EMIT2(0x1900, dst_reg, src_reg); in bpf_jit_insn()
2095 EMIT4(0xb9200000, dst_reg, src_reg); in bpf_jit_insn()
2097 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2106 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065), in bpf_jit_insn()
2111 EMIT2(0x1500, dst_reg, src_reg); in bpf_jit_insn()
2113 EMIT4(0xb9210000, dst_reg, src_reg); in bpf_jit_insn()
2115 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2123 EMIT4_PCREL_RIC(0xa7040000, in bpf_jit_insn()
2128 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2168 if (delta < 0) in bpf_set_addr()
2174 return 0; in bpf_set_addr()
2188 jit->prg = 0; in bpf_jit_prog()
2189 jit->excnt = 0; in bpf_jit_prog()
2195 jit->frame_off = 0; in bpf_jit_prog()
2203 if (bpf_set_addr(jit, 0) < 0) in bpf_jit_prog()
2205 for (i = 0; i < fp->len; i += insn_count) { in bpf_jit_prog()
2207 if (insn_count < 0) in bpf_jit_prog()
2210 if (bpf_set_addr(jit, i + insn_count) < 0) in bpf_jit_prog()
2231 return 0; in bpf_jit_prog()
2254 for (i = 0; i < fp->len; i++) { in bpf_jit_alloc()
2329 memset(&jit, 0, sizeof(jit)); in bpf_int_jit_compile()
2416 if (err < 0) in bpf_arch_text_poke()
2418 if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0))) in bpf_arch_text_poke()
2432 if (err < 0) in bpf_arch_text_poke()
2445 insn.opc = 0xc004 | (new_addr ? 0xf0 : 0); in bpf_arch_text_poke()
2451 return 0; in bpf_arch_text_poke()
2491 EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32)); in load_imm64()
2493 EMIT6_IMM(0xc00d0000, dst_reg, val); in load_imm64()
2513 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off); in invoke_bpf_prog()
2516 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) in invoke_bpf_prog()
2525 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); in invoke_bpf_prog()
2529 EMIT4(0xb9020000, REG_7, REG_2); in invoke_bpf_prog()
2532 EMIT6_PCREL_RILC(0xc0040000, 8, 0); in invoke_bpf_prog()
2541 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); in invoke_bpf_prog()
2551 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, in invoke_bpf_prog()
2568 EMIT4(0xb9040000, REG_3, REG_7); in invoke_bpf_prog()
2570 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); in invoke_bpf_prog()
2574 return 0; in invoke_bpf_prog()
2629 nr_bpf_args = 0; in __arch_prepare_bpf_trampoline()
2630 for (i = 0; i < m->nr_args; i++) { in __arch_prepare_bpf_trampoline()
2669 EMIT4(0xb9040000, REG_1, REG_15); in __arch_prepare_bpf_trampoline()
2671 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); in __arch_prepare_bpf_trampoline()
2673 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2676 _EMIT6(0xd203f000 | tjit->tccnt_off, in __arch_prepare_bpf_trampoline()
2677 0xf000 | (tjit->stack_size + in __arch_prepare_bpf_trampoline()
2681 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, in __arch_prepare_bpf_trampoline()
2684 for (i = 0, j = 0; i < m->nr_args; i++) { in __arch_prepare_bpf_trampoline()
2694 EMIT6_DISP_LH(0xe3000000, 0x0024, arg, in __arch_prepare_bpf_trampoline()
2698 _EMIT6(0xd207f000 | bpf_arg_off, in __arch_prepare_bpf_trampoline()
2699 0xf000 | arg); in __arch_prepare_bpf_trampoline()
2703 /* mvc bpf_arg_off(16,%r15),0(%arg) */ in __arch_prepare_bpf_trampoline()
2704 _EMIT6(0xd20ff000 | bpf_arg_off, in __arch_prepare_bpf_trampoline()
2708 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0, in __arch_prepare_bpf_trampoline()
2710 /* mvc bpf_arg_off(16,%r15),0(%r1) */ in __arch_prepare_bpf_trampoline()
2711 _EMIT6(0xd20ff000 | bpf_arg_off, 0x1000); in __arch_prepare_bpf_trampoline()
2717 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15, in __arch_prepare_bpf_trampoline()
2720 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); in __arch_prepare_bpf_trampoline()
2731 EMIT4(0xb9040000, REG_8, REG_0); in __arch_prepare_bpf_trampoline()
2746 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2750 EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args); in __arch_prepare_bpf_trampoline()
2752 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2768 for (i = 0; i < fentry->nr_links; i++) in __arch_prepare_bpf_trampoline()
2775 * retval = 0; in __arch_prepare_bpf_trampoline()
2779 _EMIT6(0xd707f000 | tjit->retval_off, in __arch_prepare_bpf_trampoline()
2780 0xf000 | tjit->retval_off); in __arch_prepare_bpf_trampoline()
2782 for (i = 0; i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
2792 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2795 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); in __arch_prepare_bpf_trampoline()
2806 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, in __arch_prepare_bpf_trampoline()
2811 _EMIT6(0xd200f000 | in __arch_prepare_bpf_trampoline()
2814 0xf000 | tjit->orig_stack_args_off); in __arch_prepare_bpf_trampoline()
2816 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), in __arch_prepare_bpf_trampoline()
2817 0xf000 | tjit->tccnt_off); in __arch_prepare_bpf_trampoline()
2819 EMIT4(0xb9040000, REG_1, REG_8); in __arch_prepare_bpf_trampoline()
2823 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2832 /* brcl 0,im->ip_epilogue */ in __arch_prepare_bpf_trampoline()
2833 EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); in __arch_prepare_bpf_trampoline()
2838 for (i = 0; i < fexit->nr_links; i++) in __arch_prepare_bpf_trampoline()
2859 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, in __arch_prepare_bpf_trampoline()
2864 EMIT4(0xb9040000, REG_1, REG_8); in __arch_prepare_bpf_trampoline()
2866 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, in __arch_prepare_bpf_trampoline()
2869 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); in __arch_prepare_bpf_trampoline()
2872 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2875 _EMIT6(0xd203f000 | (tjit->stack_size + in __arch_prepare_bpf_trampoline()
2877 0xf000 | tjit->tccnt_off); in __arch_prepare_bpf_trampoline()
2879 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); in __arch_prepare_bpf_trampoline()
2885 return 0; in __arch_prepare_bpf_trampoline()
2895 memset(&tjit, 0, sizeof(tjit)); in arch_bpf_trampoline_size()
2900 return ret < 0 ? ret : tjit.common.prg; in arch_bpf_trampoline_size()
2912 memset(&tjit, 0, sizeof(tjit)); in arch_prepare_bpf_trampoline()
2916 if (ret < 0) in arch_prepare_bpf_trampoline()
2925 tjit.common.prg = 0; in arch_prepare_bpf_trampoline()
2930 return ret < 0 ? ret : tjit.common.prg; in arch_prepare_bpf_trampoline()
2970 unsigned long addr, prev_addr = 0; in arch_bpf_stack_walk()
2973 unwind_for_each_frame(&state, NULL, NULL, 0) { in arch_bpf_stack_walk()