112 static inline u32 reg(u32 dst_reg, u32 src_reg)  in reg()  argument
114 	return reg2hex[dst_reg] << 4 | reg2hex[src_reg];  in reg()
746 	probe->reg = reg2hex[insn->dst_reg];  in bpf_jit_probe_load_pre()
771 	EMIT4(0xb9080000, REG_W1, insn->dst_reg);  in bpf_jit_probe_atomic_pre()
863 	u32 dst_reg = insn->dst_reg;  in bpf_jit_insn()  local
883 			EMIT4(0xb9160000, dst_reg, src_reg);  in bpf_jit_insn()
889 			EMIT4(0xb9260000, dst_reg, src_reg);  in bpf_jit_insn()
891 			EMIT4(0xb9160000, dst_reg, dst_reg);  in bpf_jit_insn()
895 			EMIT4(0xb9270000, dst_reg, src_reg);  in bpf_jit_insn()
897 			EMIT4(0xb9160000, dst_reg, dst_reg);  in bpf_jit_insn()
906 			EMIT4(0xb9020000, dst_reg, src_reg);  in bpf_jit_insn()
911 			EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32);  in bpf_jit_insn()
921 			EMIT4(0xb9040000, dst_reg, src_reg);  in bpf_jit_insn()
925 			EMIT4(0xb9060000, dst_reg, src_reg);  in bpf_jit_insn()
929 			EMIT4(0xb9070000, dst_reg, src_reg);  in bpf_jit_insn()
933 			EMIT4(0xb9140000, dst_reg, src_reg);  in bpf_jit_insn()
939 		EMIT6_IMM(0xc00f0000, dst_reg, imm);  in bpf_jit_insn()
945 		EMIT6_IMM(0xc0010000, dst_reg, imm);  in bpf_jit_insn()
957 		EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64));  in bpf_jit_insn()
966 		EMIT2(0x1a00, dst_reg, src_reg);  in bpf_jit_insn()
967 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
971 		EMIT4(0xb9080000, dst_reg, src_reg);  in bpf_jit_insn()
976 			EMIT6_IMM(0xc20b0000, dst_reg, imm);  in bpf_jit_insn()
978 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
984 		EMIT6_IMM(0xc2080000, dst_reg, imm);  in bpf_jit_insn()
991 		EMIT2(0x1b00, dst_reg, src_reg);  in bpf_jit_insn()
992 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
996 		EMIT4(0xb9090000, dst_reg, src_reg);  in bpf_jit_insn()
1001 			EMIT6_IMM(0xc20b0000, dst_reg, -imm);  in bpf_jit_insn()
1003 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1010 			EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000);  in bpf_jit_insn()
1013 			EMIT6_IMM(0xc2080000, dst_reg, -imm);  in bpf_jit_insn()
1021 		EMIT4(0xb2520000, dst_reg, src_reg);  in bpf_jit_insn()
1022 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1026 		EMIT4(0xb90c0000, dst_reg, src_reg);  in bpf_jit_insn()
1031 			EMIT6_IMM(0xc2010000, dst_reg, imm);  in bpf_jit_insn()
1033 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1039 		EMIT6_IMM(0xc2000000, dst_reg, imm);  in bpf_jit_insn()
1054 			EMIT2(0x1800, REG_W1, dst_reg);  in bpf_jit_insn()
1060 			EMIT4(0xb9140000, REG_W1, dst_reg);  in bpf_jit_insn()
1066 		EMIT4(0xb9160000, dst_reg, rc_reg);  in bpf_jit_insn()
1081 			EMIT4(0xb9040000, REG_W1, dst_reg);  in bpf_jit_insn()
1087 			EMIT4(0xb9040000, REG_W1, dst_reg);  in bpf_jit_insn()
1093 		EMIT4(0xb9040000, dst_reg, rc_reg);  in bpf_jit_insn()
1104 				EMIT4_IMM(0xa7090000, dst_reg, 0);  in bpf_jit_insn()
1106 				EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1115 				EMIT2(0x1800, REG_W1, dst_reg);  in bpf_jit_insn()
1122 				EMIT4(0xb9140000, REG_W1, dst_reg);  in bpf_jit_insn()
1134 				EMIT2(0x1800, REG_W1, dst_reg);  in bpf_jit_insn()
1136 				EMIT6_PCREL_RILB(0xc40d0000, dst_reg,  in bpf_jit_insn()
1140 				EMIT4(0xb9970000, REG_W0, dst_reg);  in bpf_jit_insn()
1144 				EMIT4(0xb9140000, REG_W1, dst_reg);  in bpf_jit_insn()
1146 				EMIT6_PCREL_RILB(0xc40c0000, dst_reg,  in bpf_jit_insn()
1150 				EMIT4(0xb90d0000, REG_W0, dst_reg);  in bpf_jit_insn()
1155 		EMIT4(0xb9160000, dst_reg, rc_reg);  in bpf_jit_insn()
1168 				EMIT4_IMM(0xa7090000, dst_reg, 0);  in bpf_jit_insn()
1177 				EMIT4(0xb9040000, REG_W1, dst_reg);  in bpf_jit_insn()
1184 				EMIT4(0xb9040000, REG_W1, dst_reg);  in bpf_jit_insn()
1196 				EMIT4(0xb9040000, REG_W1, dst_reg);  in bpf_jit_insn()
1198 				EMIT6_PCREL_RILB(0xc4080000, dst_reg,  in bpf_jit_insn()
1202 				EMIT4(0xb9870000, REG_W0, dst_reg);  in bpf_jit_insn()
1206 				EMIT4(0xb9040000, REG_W1, dst_reg);  in bpf_jit_insn()
1208 				EMIT6_PCREL_RILB(0xc4080000, dst_reg,  in bpf_jit_insn()
1212 				EMIT4(0xb90d0000, REG_W0, dst_reg);  in bpf_jit_insn()
1217 		EMIT4(0xb9040000, dst_reg, rc_reg);  in bpf_jit_insn()
1225 		EMIT2(0x1400, dst_reg, src_reg);  in bpf_jit_insn()
1226 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1230 		EMIT4(0xb9800000, dst_reg, src_reg);  in bpf_jit_insn()
1234 		EMIT6_IMM(0xc00b0000, dst_reg, imm);  in bpf_jit_insn()
1235 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1241 				      dst_reg, REG_0, REG_L,  in bpf_jit_insn()
1249 			EMIT4(0xb9800000, dst_reg, REG_W0);  in bpf_jit_insn()
1257 		EMIT2(0x1600, dst_reg, src_reg);  in bpf_jit_insn()
1258 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1262 		EMIT4(0xb9810000, dst_reg, src_reg);  in bpf_jit_insn()
1266 		EMIT6_IMM(0xc00d0000, dst_reg, imm);  in bpf_jit_insn()
1267 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1273 				      dst_reg, REG_0, REG_L,  in bpf_jit_insn()
1281 			EMIT4(0xb9810000, dst_reg, REG_W0);  in bpf_jit_insn()
1289 		EMIT2(0x1700, dst_reg, src_reg);  in bpf_jit_insn()
1290 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1294 		EMIT4(0xb9820000, dst_reg, src_reg);  in bpf_jit_insn()
1299 			EMIT6_IMM(0xc0070000, dst_reg, imm);  in bpf_jit_insn()
1301 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1307 				      dst_reg, REG_0, REG_L,  in bpf_jit_insn()
1315 			EMIT4(0xb9820000, dst_reg, REG_W0);  in bpf_jit_insn()
1323 		EMIT4_DISP(0x89000000, dst_reg, src_reg, 0);  in bpf_jit_insn()
1324 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1328 		EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0);  in bpf_jit_insn()
1333 			EMIT4_DISP(0x89000000, dst_reg, REG_0, imm);  in bpf_jit_insn()
1335 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1341 		EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm);  in bpf_jit_insn()
1348 		EMIT4_DISP(0x88000000, dst_reg, src_reg, 0);  in bpf_jit_insn()
1349 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1353 		EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0);  in bpf_jit_insn()
1358 			EMIT4_DISP(0x88000000, dst_reg, REG_0, imm);  in bpf_jit_insn()
1360 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1366 		EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm);  in bpf_jit_insn()
1373 		EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0);  in bpf_jit_insn()
1374 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1378 		EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);  in bpf_jit_insn()
1383 			EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm);  in bpf_jit_insn()
1385 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1391 		EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm);  in bpf_jit_insn()
1398 		EMIT2(0x1300, dst_reg, dst_reg);  in bpf_jit_insn()
1399 		EMIT_ZERO(dst_reg);  in bpf_jit_insn()
1403 		EMIT4(0xb9030000, dst_reg, dst_reg);  in bpf_jit_insn()
1413 			EMIT4(0xb9850000, dst_reg, dst_reg);  in bpf_jit_insn()
1420 				EMIT4(0xb9160000, dst_reg, dst_reg);  in bpf_jit_insn()
1431 			EMIT4(0xb91f0000, dst_reg, dst_reg);  in bpf_jit_insn()
1433 			EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);  in bpf_jit_insn()
1435 			EMIT4(0xb9850000, dst_reg, dst_reg);  in bpf_jit_insn()
1441 			EMIT4(0xb91f0000, dst_reg, dst_reg);  in bpf_jit_insn()
1444 				EMIT4(0xb9160000, dst_reg, dst_reg);  in bpf_jit_insn()
1448 			EMIT4(0xb90f0000, dst_reg, dst_reg);  in bpf_jit_insn()
1464 		EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg,  in bpf_jit_insn()
1475 		EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg,  in bpf_jit_insn()
1486 		EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg,  in bpf_jit_insn()
1497 		EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg,  in bpf_jit_insn()
1510 		EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg,  in bpf_jit_insn()
1523 		EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg,  in bpf_jit_insn()
1536 		EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg,  in bpf_jit_insn()
1549 		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg,  in bpf_jit_insn()
1573 		probe.arena_reg = dst_reg;  in bpf_jit_insn()
1674 		EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg,  in bpf_jit_insn()
1687 		EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off);  in bpf_jit_insn()
1698 		EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg,  in bpf_jit_insn()
1711 		EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off);  in bpf_jit_insn()
1723 		EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg,  in bpf_jit_insn()
1736 		EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off);  in bpf_jit_insn()
1747 		EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg,  in bpf_jit_insn()
1980 			EMIT2(0x1400, REG_W1, dst_reg);  in bpf_jit_insn()
1985 			EMIT4(0xb9800000, REG_W1, dst_reg);  in bpf_jit_insn()
2037 			  REG_W1, dst_reg, src_reg);  in bpf_jit_insn()
2043 			  dst_reg, imm);  in bpf_jit_insn()
2066 				    dst_reg, src_reg, i, off, mask);  in bpf_jit_insn()
2070 				EMIT2(0x1900, dst_reg, src_reg);  in bpf_jit_insn()
2072 				EMIT4(0xb9200000, dst_reg, src_reg);  in bpf_jit_insn()
2084 				    dst_reg, src_reg, i, off, mask);  in bpf_jit_insn()
2088 				EMIT2(0x1500, dst_reg, src_reg);  in bpf_jit_insn()
2090 				EMIT4(0xb9210000, dst_reg, src_reg);  in bpf_jit_insn()
2460 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)  in load_imm64()  argument
2463 	EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32));  in load_imm64()
2465 	EMIT6_IMM(0xc00d0000, dst_reg, val);  in load_imm64()