248 				     const struct bpf_insn *patch, u32 len)  in bpf_patch_insn_data()  argument
264 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);  in bpf_patch_insn_data()
585 	struct bpf_insn *patch;  in bpf_opt_subreg_zext_lo32_rnd_hi32()  local
639 			patch = rnd_hi32_patch;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
668 		patch = zext_patch;  in bpf_opt_subreg_zext_lo32_rnd_hi32()
671 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);  in bpf_opt_subreg_zext_lo32_rnd_hi32()
766 			struct bpf_insn *patch = insn_buf;  in bpf_convert_ctx_accesses()  local
768 			*patch++ = BPF_ST_NOSPEC();  in bpf_convert_ctx_accesses()
769 			*patch++ = *insn;  in bpf_convert_ctx_accesses()
770 			cnt = patch - insn_buf;  in bpf_convert_ctx_accesses()
839 			struct bpf_insn *patch = insn_buf;  in bpf_convert_ctx_accesses()  local
841 			*patch++ = *insn;  in bpf_convert_ctx_accesses()
842 			*patch++ = BPF_ST_NOSPEC();  in bpf_convert_ctx_accesses()
843 			cnt = patch - insn_buf;  in bpf_convert_ctx_accesses()
1433 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)  in add_hidden_subprog()  argument
1448 	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);  in add_hidden_subprog()
1481 		struct bpf_insn *patch = insn_buf;  in bpf_do_misc_fixups()  local
1483 		*patch++ = env->prog->insnsi[insn_cnt - 1];  in bpf_do_misc_fixups()
1484 		*patch++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);  in bpf_do_misc_fixups()
1485 		*patch++ = BPF_EXIT_INSN();  in bpf_do_misc_fixups()
1486 		ret = add_hidden_subprog(env, insn_buf, patch - insn_buf);  in bpf_do_misc_fixups()
1522 			struct bpf_insn *patch = insn_buf;  in bpf_do_misc_fixups()  local
1525 				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |  in bpf_do_misc_fixups()
1529 				*patch++ = BPF_MOV32_IMM(insn->dst_reg, 0);  in bpf_do_misc_fixups()
1531 			cnt = patch - insn_buf;  in bpf_do_misc_fixups()
1552 			struct bpf_insn *patch = insn_buf;  in bpf_do_misc_fixups()  local
1559 				*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);  in bpf_do_misc_fixups()
1560 				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |  in bpf_do_misc_fixups()
1563 				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |  in bpf_do_misc_fixups()
1566 				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |  in bpf_do_misc_fixups()
1569 				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |  in bpf_do_misc_fixups()
1573 				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |  in bpf_do_misc_fixups()
1576 				*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);  in bpf_do_misc_fixups()
1577 				*patch++ = *insn;  in bpf_do_misc_fixups()
1578 				cnt = patch - insn_buf;  in bpf_do_misc_fixups()
1582 				*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);  in bpf_do_misc_fixups()
1583 				*patch++ = BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |  in bpf_do_misc_fixups()
1586 				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |  in bpf_do_misc_fixups()
1589 				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |  in bpf_do_misc_fixups()
1592 				*patch++ = BPF_MOV32_IMM(insn->dst_reg, 0);  in bpf_do_misc_fixups()
1593 				*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);  in bpf_do_misc_fixups()
1594 				*patch++ = *insn;  in bpf_do_misc_fixups()
1597 					*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);  in bpf_do_misc_fixups()
1598 					*patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg);  in bpf_do_misc_fixups()
1600 				cnt = patch - insn_buf;  in bpf_do_misc_fixups()
1603 				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |  in bpf_do_misc_fixups()
1606 				*patch++ = BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg);  in bpf_do_misc_fixups()
1607 				*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);  in bpf_do_misc_fixups()
1608 				*patch++ = *insn;  in bpf_do_misc_fixups()
1609 				cnt = patch - insn_buf;  in bpf_do_misc_fixups()
1612 				*patch++ = BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |  in bpf_do_misc_fixups()
1615 				*patch++ = *insn;  in bpf_do_misc_fixups()
1618 					*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);  in bpf_do_misc_fixups()
1619 					*patch++ = BPF_MOV32_REG(insn->dst_reg, insn->dst_reg);  in bpf_do_misc_fixups()
1621 				cnt = patch - insn_buf;  in bpf_do_misc_fixups()
1638 			struct bpf_insn *patch = insn_buf;  in bpf_do_misc_fixups()  local
1644 			*patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);  in bpf_do_misc_fixups()
1646 				*patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off);  in bpf_do_misc_fixups()
1647 			*patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32);  in bpf_do_misc_fixups()
1648 			*patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2);  in bpf_do_misc_fixups()
1649 			*patch++ = *insn;  in bpf_do_misc_fixups()
1650 			*patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);  in bpf_do_misc_fixups()
1651 			*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);  in bpf_do_misc_fixups()
1653 			cnt = patch - insn_buf;  in bpf_do_misc_fixups()
1689 			struct bpf_insn *patch = insn_buf;  in bpf_do_misc_fixups()  local
1705 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);  in bpf_do_misc_fixups()
1708 					*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);  in bpf_do_misc_fixups()
1709 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);  in bpf_do_misc_fixups()
1710 				*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);  in bpf_do_misc_fixups()
1711 				*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);  in bpf_do_misc_fixups()
1712 				*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);  in bpf_do_misc_fixups()
1713 				*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);  in bpf_do_misc_fixups()
1714 				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);  in bpf_do_misc_fixups()
1717 				*patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);  in bpf_do_misc_fixups()
1722 			*patch++ = *insn;  in bpf_do_misc_fixups()
1724 				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);  in bpf_do_misc_fixups()
1725 			cnt = patch - insn_buf;  in bpf_do_misc_fixups()