1 // SPDX-License-Identifier: GPL-2.0-only
10 #include <asm/text-patching.h>
12 #include <asm/insn-eval.h>
21 #define MAX_PATCH_LEN (255-1)
42 __setup("debug-alternative", debug_alt);
51 __setup("noreplace-smp", setup_noreplace_smp);
68 		for (j = 0; j < (len) - 1; j++)				\
128 	void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *),  in __its_alloc()
133 	pages->pages = tmp;  in __its_alloc()
134 	pages->pages[pages->num++] = page;  in __its_alloc()
164 		reg -= 8;  in its_init_thunk()
175 	for (int i = 0; i < pages->num; i++) {  in its_pages_protect()
176 		void *page = pages->pages[i];  in its_pages_protect()
211 		its_pages_protect(&mod->arch.its_pages);  in its_fini_mod()
219 	for (int i = 0; i < mod->arch.its_pages.num; i++) {  in its_free_mod()
220 		void *page = mod->arch.its_pages.pages[i];  in its_free_mod()
223 	kfree(mod->arch.its_pages.pages);  in its_free_mod()
234 		pages = &its_mod->arch.its_pages;  in its_alloc()
263 	if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {  in its_allocate_thunk()
277 	if ((its_offset + size - 1) % 64 < 32)  in its_allocate_thunk()
278 		its_offset = ((its_offset - 1) | 0x3F) + 33;  in its_allocate_thunk()
293 		return thunk - 2;  in its_static_thunk()
310  * eventually text-poked into the kernel image.
320  * for every single-byte NOP, try to generate the maximally available NOP of
322  * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
350  * Find the offset of the first non-NOP instruction starting at @offset
391 			add_nop(buf + nop, next - nop);  in optimize_nops()
406  *   src_imm = target - src_next_ip                  (1)
410  *   dst_imm = target - dst_next_ip                  (2)
418  *   dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
429  *   dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
430  *           = src_imm + src - dst + ip_offset - ip_offset
431  *           = src_imm + src - dst                   (5)
440 		BUG_ON((v >> 31) != (v >> (n_-1)));		\
491 					    repl - instr);  in __apply_relocation()
499 				imm += repl - instr;  in __apply_relocation()
500 				imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;  in __apply_relocation()
505 					memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2);  in __apply_relocation()
515 					    repl - instr);  in __apply_relocation()
527 /* Low-level backend functions usable from alternative code replacements. */
549 	if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) {  in alt_replace_call()
550 		pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n");  in alt_replace_call()
554 	if (a->instrlen != 6 ||  in alt_replace_call()
566 	target = *(void **)(instr + a->instrlen + disp);  in alt_replace_call()
575 	/* (BUG_func - .) + (target - BUG_func) := target - . */  in alt_replace_call()
576 	*(s32 *)(insn_buff + 1) += target - bug;  in alt_replace_call()
586 	return (u8 *)&i->instr_offset + i->instr_offset;  in instr_va()
602 	ps->instr = instr_va(start);  in analyze_patch_site()
611 	for (; alt < end && instr_va(alt) == ps->instr; alt++) {  in analyze_patch_site()
612 		ps->len = max(ps->len, alt->instrlen);  in analyze_patch_site()
614 		BUG_ON(alt->cpuid >= (NCAPINTS + NBUGINTS) * 32);  in analyze_patch_site()
617 		 * - feature is present  in analyze_patch_site()
618 		 * - feature not present but ALT_FLAG_NOT is set to mean,  in analyze_patch_site()
621 		if (!boot_cpu_has(alt->cpuid) != !(alt->flags & ALT_FLAG_NOT))  in analyze_patch_site()
622 			ps->alt = alt;  in analyze_patch_site()
625 	BUG_ON(ps->len > sizeof(ps->buff));  in analyze_patch_site()
632 	struct alt_instr *alt = ps->alt;  in prep_patch_site()
638 		memcpy(ps->buff, ps->instr, ps->len);  in prep_patch_site()
642 	repl = (u8 *)&alt->repl_offset + alt->repl_offset;  in prep_patch_site()
644 		alt->cpuid >> 5, alt->cpuid & 0x1f,  in prep_patch_site()
645 		ps->instr, ps->instr, ps->len,  in prep_patch_site()
646 		repl, alt->replacementlen, alt->flags);  in prep_patch_site()
648 	memcpy(ps->buff, repl, alt->replacementlen);  in prep_patch_site()
649 	buff_sz = alt->replacementlen;  in prep_patch_site()
651 	if (alt->flags & ALT_FLAG_DIRECT_CALL)  in prep_patch_site()
652 		buff_sz = alt_replace_call(ps->instr, ps->buff, alt);  in prep_patch_site()
654 	for (; buff_sz < ps->len; buff_sz++)  in prep_patch_site()
655 		ps->buff[buff_sz] = 0x90;  in prep_patch_site()
657 	__apply_relocation(ps->buff, ps->instr, ps->len, repl, alt->replacementlen);  in prep_patch_site()
659 	DUMP_BYTES(ALT, ps->instr, ps->len, "%px:   old_insn: ", ps->instr);  in prep_patch_site()
660 	DUMP_BYTES(ALT, repl, alt->replacementlen, "%px:   rpl_insn: ", repl);  in prep_patch_site()
661 	DUMP_BYTES(ALT, ps->buff, ps->len, "%px: final_insn: ", ps->instr);  in prep_patch_site()
666 	optimize_nops(ps->instr, ps->buff, ps->len);  in patch_site()
667 	text_poke_early(ps->instr, ps->buff, ps->len);  in patch_site()
685 	DPRINTK(ALT, "alt table %px, -> %px", start, end);  in apply_alternatives()
691 	 * conversion and triggers a false-positive out-of-bound report.  in apply_alternatives()
723 	/* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */  in is_jcc32()
724 	return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;  in is_jcc32()
741 	len -= 2 + (reg >= 8);  in emit_indirect()
761 		return -1;  in emit_indirect()
764 	while (cs--)  in emit_indirect()
765 		bytes[i++] = 0x2e; /* CS-prefix */  in emit_indirect()
769 		reg -= 8;  in emit_indirect()
778 	while (bp--)  in emit_indirect()
787 	u8 op = insn->opcode.bytes[0];  in __emit_trampoline()
792 	 * tail-calls. Deal with them.  in __emit_trampoline()
796 		op = insn->opcode.bytes[1];  in __emit_trampoline()
800 	if (insn->length == 6)  in __emit_trampoline()
801 		bytes[i++] = 0x2e; /* CS-prefix */  in __emit_trampoline()
821 		return -1;  in __emit_trampoline()
824 	WARN_ON_ONCE(i != insn->length);  in __emit_trampoline()
848 /* Check if an indirect branch is at ITS-unsafe address */
857 	/* Lower-half of the cacheline? */  in cpu_wants_indirect_its_thunk_at()
893 	target = addr + insn->length + insn->immediate.value;  in patch_retpoline()
894 	reg = target - __x86_indirect_thunk_array;  in patch_retpoline()
897 		return -1;  in patch_retpoline()
907 		return -1;  in patch_retpoline()
910 	op = insn->opcode.bytes[0];  in patch_retpoline()
926 		cc = insn->opcode.bytes[1] & 0xf;  in patch_retpoline()
930 		bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */  in patch_retpoline()
947 	 * Check if the address of last byte of emitted-indirect is in  in patch_retpoline()
948 	 * lower-half of the cacheline. Such branches need ITS mitigation.  in patch_retpoline()
954 	ret = emit_indirect(op, reg, bytes + i, insn->length - i);  in patch_retpoline()
959 	for (; i < insn->length;)  in patch_retpoline()
966  * Generated by 'objtool --retpoline'.
997 			if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) {  in apply_retpolines()
1044  * Rewrite the compiler generated return thunk tail-calls.
1067 	for (; i < insn->length;)  in patch_return()
1096 			      "missing return thunk: %pS-%pS: %*ph",  in apply_returns()
1171  * Generated by: objtool --ibt
1185 			poison_cfi(addr - 16);  in apply_seal_endbr()
1215 	func -= cfi_get_offset();  in cfi_get_func_hash()
1241 	if (get_kernel_nofault(disp, func - 4))  in cfi_get_func_arity()
1245 	return target - __bhi_args;  in cfi_get_func_arity()
1255  * Re-hash the CFI hash with a boot-time seed while making sure the result is
1261 	while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) {  in cfi_rehash()
1273 		return -EINVAL;  in cfi_parse_cmdline()
1296 			pr_alert("CFI: mismatch non-fatal!\n");  in cfi_parse_cmdline()
1342  *	endbr64					     nopl -42(%rax)
1346  *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%eax	// 5
1347  *	addl	$-15(%r11),%r10d	 // 4	     lea    -0x10(%r11),%r11	// 4
1354  * non-taken. This is based on Agner Fog's optimization manual, which states:
1357  *   for not-taken branches is better than for taken branches on most
1366  * 10:   0f 1f 40 d6             nopl   -0x2a(%rax)
1378 	"	nopl	-42(%rax)			\n"
1387 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
1388 #define fineibt_preamble_bhi  (fineibt_preamble_bhi - fineibt_preamble_start)
1395  *  5:   4d 8d 5b f0             lea    -0x10(%r11), %r11
1401 	"	lea	-0x10(%r11), %r11	\n"
1410 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
1413 #define fineibt_caller_jmp (fineibt_caller_size - 2)
1429  *  5:   41 3b 43 f5             cmp    -0x11(%r11), %eax
1430  *  9:   2e 4d 8d 5b <f0>        cs lea -0x10(%r11), %r11
1436  * avoiding a dependency. Again, using a non-taken (backwards) branch
1443 	"	cmpl	-11(%r11), %eax			\n"
1444 	"	cs lea	-0x10(%r11), %r11		\n"
1457 #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start)
1458 #define fineibt_paranoid_ind  (fineibt_paranoid_ind - fineibt_paranoid_start)
1468 			*reg = p[0] - 0xb8;  in decode_preamble_hash()
1479 	/* 41 ba 88 a9 cb ed       mov    $(-0x12345678),%r10d */  in decode_caller_hash()
1481 		return -*(u32 *)(addr + 2);  in decode_caller_hash()
1485 		return -*(u32 *)(addr + 2);  in decode_caller_hash()
1505 		addr -= fineibt_caller_size;  in cfi_disable_callers()
1519 	 * Re-enable kCFI, undo what cfi_disable_callers() did.  in cfi_enable_callers()
1528 		addr -= fineibt_caller_size;  in cfi_enable_callers()
1551 			return -EINVAL;  in cfi_rand_preamble()
1561  * Inline the bhi-arity 1 case:
1567  *  d: 2e 75 03                jne,pn    foo+0x3
1569  * foo:
1570  * 10: 0f 1f 40 <d6>           nopl -42(%rax)
1581 	"	nopl -42(%rax)				\n"
1589 #define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start)
1607 	 * return address will be foo+0.  in cfi_fineibt_bhi_preamble()
1643 			return -EINVAL;  in cfi_rewrite_preamble()
1683 		addr -= fineibt_caller_size;  in cfi_rand_callers()
1686 			hash = -cfi_rehash(hash);  in cfi_rand_callers()
1696 	u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2;  in emit_paranoid_trampoline()
1719 		addr -= fineibt_caller_size;  in cfi_rewrite_callers()
1750 			int len = fineibt_paranoid_size - fineibt_paranoid_ind;  in cfi_rewrite_callers()
1829 		pr_cfi_debug("CFI: re-enabling all indirect call checking\n");  in __apply_fineibt()
1841 		/* place the FineIBT preamble at func()-16 */  in __apply_fineibt()
1846 		/* rewrite the callers to target func()-16 */  in __apply_fineibt()
1896 		 *	nopl	-42(%rax)  in poison_cfi()
1900 		 *	nopl	-42(%rax)  in poison_cfi()
1926 #define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
1929  * When regs->ip points to a 0xD6 byte in the FineIBT preamble,
1937 	unsigned long addr = regs->ip - fineibt_preamble_ud;  in decode_fineibt_preamble()
1946 	*type = (u32)regs->ax + hash;  in decode_fineibt_preamble()
1949 	 * Since regs->ip points to the middle of an instruction; it cannot  in decode_fineibt_preamble()
1952 	regs->ip = *target;  in decode_fineibt_preamble()
1961  * regs->ip points to one of the UD2 in __bhi_args[].
1971 	if (regs->ip < (unsigned long)__bhi_args ||  in decode_fineibt_bhi()
1972 	    regs->ip >= (unsigned long)__bhi_args_end)  in decode_fineibt_bhi()
1981 	__get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);  in decode_fineibt_bhi()
1984 	addr -= fineibt_prefix_size;  in decode_fineibt_bhi()
1989 	*type = (u32)regs->ax + hash;  in decode_fineibt_bhi()
1993 	 * as such the non-fatal case can use the regular fixup.  in decode_fineibt_bhi()
2013  * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
2018 	unsigned long addr = regs->ip - fineibt_paranoid_ud;  in decode_fineibt_paranoid()
2023 	if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {  in decode_fineibt_paranoid()
2024 		*target = regs->r11 + fineibt_prefix_size;  in decode_fineibt_paranoid()
2025 		*type = regs->ax;  in decode_fineibt_paranoid()
2038 	 *  5:   41 3b 43 f7             cmp    -11(%r11), %eax  in decode_fineibt_paranoid()
2039 	 *  a:   2e 3d 8d 5b f0          cs lea -0x10(%r11), %r11  in decode_fineibt_paranoid()
2052 	if (is_paranoid_thunk(regs->ip)) {  in decode_fineibt_paranoid()
2053 		*target = regs->r11 + fineibt_prefix_size;  in decode_fineibt_paranoid()
2054 		*type = regs->ax;  in decode_fineibt_paranoid()
2056 		regs->ip = *target;  in decode_fineibt_paranoid()
2168 	smp->mod	= mod;  in alternatives_smp_module_add()
2169 	smp->name	= name;  in alternatives_smp_module_add()
2170 	smp->locks	= locks;  in alternatives_smp_module_add()
2171 	smp->locks_end	= locks_end;  in alternatives_smp_module_add()
2172 	smp->text	= text;  in alternatives_smp_module_add()
2173 	smp->text_end	= text_end;  in alternatives_smp_module_add()
2174 	DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n",  in alternatives_smp_module_add()
2175 		smp->locks, smp->locks_end,  in alternatives_smp_module_add()
2176 		smp->text, smp->text_end, smp->name);  in alternatives_smp_module_add()
2178 	list_add_tail(&smp->next, &smp_alt_modules);  in alternatives_smp_module_add()
2191 		if (mod != item->mod)  in alternatives_smp_module_del()
2193 		list_del(&item->next);  in alternatives_smp_module_del()
2215 			alternatives_smp_lock(mod->locks, mod->locks_end,  in alternatives_enable_smp()
2216 					      mod->text, mod->text_end);  in alternatives_enable_smp()
2223  * Return 1 if the address range is reserved for SMP-alternatives.
2236 		if (mod->text > text_end || mod->text_end < text_start)  in alternatives_text_reserved()
2238 		for (poff = mod->locks; poff < mod->locks_end; poff++) {  in alternatives_text_reserved()
2251  * Self-test for the INT3 based CALL emulation code.
2275 "	.size		int3_selftest_asm, . - int3_selftest_asm\n"
2288 "	.size		int3_selftest_callee, . - int3_selftest_callee\n"
2299 	struct pt_regs *regs = args->regs;  in int3_exception_notify()
2309 	if (regs->ip - INT3_INSN_SIZE != selftest)  in int3_exception_notify()
2321 		.priority	= INT_MAX-1, /* last */  in int3_selftest()
2328 	 * Basically: int3_selftest_callee(&val); but really complicated :-)  in int3_selftest()
2351 	 * instruction and additionally on x86_64 we get a RIP-relative LEA:  in alt_reloc_selftest()
2354 	 *   call   +0              # 5d5: R_X86_64_PLT32   __alt_reloc_selftest-0x4  in alt_reloc_selftest()
2397 	/* Keep CET-IBT disabled until caller/callee are patched */  in alternative_instructions()
2414 	 * Adjust all CALL instructions to point to func()-10, including  in alternative_instructions()
2451  * text_poke_early - Update instructions on a live kernel at boot time
2470 		 * Modules text is marked initially as non-executable, so the  in text_poke_early()
2471 		 * code cannot be running and speculative code-fetches are  in text_poke_early()
2553 	 * flush_tlb_mm_range(), which is intended for non-global PTEs.  in __text_poke()
2558 	 * The lock is not really needed, but this allows to avoid open-coding.  in __text_poke()
2598 	 * Loading the previous page-table hierarchy requires a serializing  in __text_poke()
2600 	 * Xen-PV is assumed to serialize execution in a similar manner.  in __text_poke()
2626  * text_poke - Update instructions on a live kernel
2649  * text_poke_kgdb - Update instructions on a live kernel by kgdb
2680 		s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);  in text_poke_copy_locked()
2689  * text_poke_copy - Copy instructions into (an unused part of) RX memory
2709  * text_poke_set - memset into (an unused part of) RX memory
2730 		s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);  in text_poke_set()
2799 	return _stext + tpl->rel_addr;  in text_poke_addr()
2805 		return -1;  in patch_cmp()
2822 	 * text_poke_array with non-zero refcount:  in smp_text_poke_int3_handler()
2836 	ip = (void *) regs->ip - INT3_INSN_SIZE;  in smp_text_poke_int3_handler()
2853 	ip += tpl->len;  in smp_text_poke_int3_handler()
2855 	switch (tpl->opcode) {  in smp_text_poke_int3_handler()
2868 		int3_emulate_call(regs, (long)ip + tpl->disp);  in smp_text_poke_int3_handler()
2873 		int3_emulate_jmp(regs, (long)ip + tpl->disp);  in smp_text_poke_int3_handler()
2877 		int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp);  in smp_text_poke_int3_handler()
2892  * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP
2898  * Modify multi-byte instructions by using INT3 breakpoints on SMP.
2900  * synchronization using INT3 breakpoints and SMP cross-calls.
2903  *	- For each entry in the vector:
2904  *		- add an INT3 trap to the address that will be patched
2905  *	- SMP sync all CPUs
2906  *	- For each entry in the vector:
2907  *		- update all but the first byte of the patched range
2908  *	- SMP sync all CPUs
2909  *	- For each entry in the vector:
2910  *		- replace the first byte (INT3) by the first byte of the
2912  *	- SMP sync all CPUs
2927 	 * ensure reading a non-zero refcount provides up to date text_poke_array data.  in smp_text_poke_batch_finish()
2967 		if (len - INT3_INSN_SIZE > 0) {  in smp_text_poke_batch_finish()
2970 			       len - INT3_INSN_SIZE);  in smp_text_poke_batch_finish()
2980 				  len - INT3_INSN_SIZE);  in smp_text_poke_batch_finish()
2989 		 *   - write INT3 byte  in smp_text_poke_batch_finish()
2990 		 *   - IPI-SYNC  in smp_text_poke_batch_finish()
2991 		 *   - write instruction tail  in smp_text_poke_batch_finish()
2996 		 *   - emit RECORD_TEXT_POKE with the new instruction  in smp_text_poke_batch_finish()
2997 		 *   - IPI-SYNC  in smp_text_poke_batch_finish()
2998 		 *   - write first byte  in smp_text_poke_batch_finish()
2999 		 *   - IPI-SYNC  in smp_text_poke_batch_finish()
3044 	 * Notably, if after step-3 above the INT3 got removed, then the  in smp_text_poke_batch_finish()
3046 	 * handlers and the below spin-wait will not happen.  in smp_text_poke_batch_finish()
3072 	memcpy((void *)tpl->text, opcode+i, len-i);  in __smp_text_poke_batch_add()
3079 	tpl->rel_addr = addr - (void *)_stext;  in __smp_text_poke_batch_add()
3080 	tpl->len = len;  in __smp_text_poke_batch_add()
3081 	tpl->opcode = insn.opcode.bytes[0];  in __smp_text_poke_batch_add()
3087 		tpl->opcode = insn.opcode.bytes[1] - 0x10;  in __smp_text_poke_batch_add()
3090 	switch (tpl->opcode) {  in __smp_text_poke_batch_add()
3099 			BUG_ON(tpl->text[i] != INT3_INSN_OPCODE);  in __smp_text_poke_batch_add()
3106 	switch (tpl->opcode) {  in __smp_text_poke_batch_add()
3115 		tpl->disp = insn.immediate.value;  in __smp_text_poke_batch_add()
3120 		case 2: /* NOP2 -- emulate as JMP8+0 */  in __smp_text_poke_batch_add()
3122 			tpl->opcode = JMP8_INSN_OPCODE;  in __smp_text_poke_batch_add()
3123 			tpl->disp = 0;  in __smp_text_poke_batch_add()
3126 		case 5: /* NOP5 -- emulate as JMP32+0 */  in __smp_text_poke_batch_add()
3128 			tpl->opcode = JMP32_INSN_OPCODE;  in __smp_text_poke_batch_add()
3129 			tpl->disp = 0;  in __smp_text_poke_batch_add()
3156 	if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr)  in text_poke_addr_ordered()
3163  * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched
3169  * Add a new instruction to the current queue of to-be-patched instructions
3183  * smp_text_poke_single() -- update instruction on live kernel on SMP immediately