1 // SPDX-License-Identifier: GPL-2.0-only
10 #include <asm/text-patching.h>
20 #define MAX_PATCH_LEN (255-1)
41 __setup("debug-alternative", debug_alt);
50 __setup("noreplace-smp", setup_noreplace_smp);
67 		for (j = 0; j < (len) - 1; j++)				\
127 	void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *),  in __its_alloc()
132 	pages->pages = tmp;  in __its_alloc()
133 	pages->pages[pages->num++] = page;  in __its_alloc()
162 		bytes[i++] = 0x41; /* REX.B prefix */  in its_init_thunk()
163 		reg -= 8;  in its_init_thunk()
174 	for (int i = 0; i < pages->num; i++) {  in its_pages_protect()
175 		void *page = pages->pages[i];  in its_pages_protect()
188 void its_init_mod(struct module *mod)  in its_init_mod()  argument
194 	its_mod = mod;  in its_init_mod()
198 void its_fini_mod(struct module *mod)  in its_fini_mod()  argument
203 	WARN_ON_ONCE(its_mod != mod);  in its_fini_mod()
210 		its_pages_protect(&mod->arch.its_pages);  in its_fini_mod()
213 void its_free_mod(struct module *mod)  in its_free_mod()  argument
218 	for (int i = 0; i < mod->arch.its_pages.num; i++) {  in its_free_mod()
219 		void *page = mod->arch.its_pages.pages[i];  in its_free_mod()
222 	kfree(mod->arch.its_pages.pages);  in its_free_mod()
233 		pages = &its_mod->arch.its_pages;  in its_alloc()
263 	if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {  in its_allocate_thunk()
277 	if ((its_offset + size - 1) % 64 < 32)  in its_allocate_thunk()
278 		its_offset = ((its_offset - 1) | 0x3F) + 33;  in its_allocate_thunk()
293 		return thunk - 2;  in its_static_thunk()
310  * eventually text-poked into the kernel image.
320  * for every single-byte NOP, try to generate the maximally available NOP of
322  * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
355 	if (insn->opcode.bytes[0] == 0x90 &&  in insn_is_nop()
356 	    (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3))  in insn_is_nop()
360 	if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F)  in insn_is_nop()
369  * Find the offset of the first non-NOP instruction starting at @offset
410 			add_nop(buf + nop, next - nop);  in optimize_nops()
425  *   src_imm = target - src_next_ip                  (1)
429  *   dst_imm = target - dst_next_ip                  (2)
437  *   dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
448  *   dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
449  *           = src_imm + src - dst + ip_offset - ip_offset
450  *           = src_imm + src - dst                   (5)
459 		BUG_ON((v >> 31) != (v >> (n_-1)));		\
510 					    repl - instr);  in __apply_relocation()
518 				imm += repl - instr;  in __apply_relocation()
519 				imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;  in __apply_relocation()
524 					memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2);  in __apply_relocation()
534 					    repl - instr);  in __apply_relocation()
546 /* Low-level backend functions usable from alternative code replacements. */
568 	if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) {  in alt_replace_call()
569 		pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n");  in alt_replace_call()
573 	if (a->instrlen != 6 ||  in alt_replace_call()
585 	target = *(void **)(instr + a->instrlen + disp);  in alt_replace_call()
594 	/* (BUG_func - .) + (target - BUG_func) := target - . */  in alt_replace_call()
595 	*(s32 *)(insn_buff + 1) += target - bug;  in alt_replace_call()
605 	return (u8 *)&i->instr_offset + i->instr_offset;  in instr_va()
623 	struct alt_instr *a, *b;  in apply_alternatives()  local
625 	DPRINTK(ALT, "alt table %px, -> %px", start, end);  in apply_alternatives()
631 	 * conversion and triggers a false-positive out-of-bound report.  in apply_alternatives()
655 		for (b = a+1; b < end && instr_va(b) == instr_va(a); b++) {  in apply_alternatives()
656 			u8 len = max(a->instrlen, b->instrlen);  in apply_alternatives()
657 			a->instrlen = b->instrlen = len;  in apply_alternatives()
661 		replacement = (u8 *)&a->repl_offset + a->repl_offset;  in apply_alternatives()
662 		BUG_ON(a->instrlen > sizeof(insn_buff));  in apply_alternatives()
663 		BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);  in apply_alternatives()
667 		 * - feature is present  in apply_alternatives()
668 		 * - feature not present but ALT_FLAG_NOT is set to mean,  in apply_alternatives()
671 		if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) {  in apply_alternatives()
672 			memcpy(insn_buff, instr, a->instrlen);  in apply_alternatives()
673 			optimize_nops(instr, insn_buff, a->instrlen);  in apply_alternatives()
674 			text_poke_early(instr, insn_buff, a->instrlen);  in apply_alternatives()
679 			a->cpuid >> 5,  in apply_alternatives()
680 			a->cpuid & 0x1f,  in apply_alternatives()
681 			instr, instr, a->instrlen,  in apply_alternatives()
682 			replacement, a->replacementlen, a->flags);  in apply_alternatives()
684 		memcpy(insn_buff, replacement, a->replacementlen);  in apply_alternatives()
685 		insn_buff_sz = a->replacementlen;  in apply_alternatives()
687 		if (a->flags & ALT_FLAG_DIRECT_CALL) {  in apply_alternatives()
693 		for (; insn_buff_sz < a->instrlen; insn_buff_sz++)  in apply_alternatives()
696 		text_poke_apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen);  in apply_alternatives()
698 		DUMP_BYTES(ALT, instr, a->instrlen, "%px:   old_insn: ", instr);  in apply_alternatives()
699 		DUMP_BYTES(ALT, replacement, a->replacementlen, "%px:   rpl_insn: ", replacement);  in apply_alternatives()
710 	/* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */  in is_jcc32()
711 	return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;  in is_jcc32()
735 		return -1;  in emit_indirect()
739 		bytes[i++] = 0x41; /* REX.B prefix */  in emit_indirect()
740 		reg -= 8;  in emit_indirect()
743 	modrm |= 0xc0; /* Mod = 3 */  in emit_indirect()
755 	u8 op = insn->opcode.bytes[0];  in __emit_trampoline()
760 	 * tail-calls. Deal with them.  in __emit_trampoline()
764 		op = insn->opcode.bytes[1];  in __emit_trampoline()
768 	if (insn->length == 6)  in __emit_trampoline()
769 		bytes[i++] = 0x2e; /* CS-prefix */  in __emit_trampoline()
789 		return -1;  in __emit_trampoline()
792 	WARN_ON_ONCE(i != insn->length);  in __emit_trampoline()
816 /* Check if an indirect branch is at ITS-unsafe address */
825 	/* Lower-half of the cacheline? */  in cpu_wants_indirect_its_thunk_at()
861 	target = addr + insn->length + insn->immediate.value;  in patch_retpoline()
862 	reg = target - __x86_indirect_thunk_array;  in patch_retpoline()
865 		return -1;  in patch_retpoline()
875 		return -1;  in patch_retpoline()
878 	op = insn->opcode.bytes[0];  in patch_retpoline()
894 		cc = insn->opcode.bytes[1] & 0xf;  in patch_retpoline()
898 		bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */  in patch_retpoline()
915 	 * Check if the address of last byte of emitted-indirect is in  in patch_retpoline()
916 	 * lower-half of the cacheline. Such branches need ITS mitigation.  in patch_retpoline()
933 	if (op == JMP32_INSN_OPCODE && i < insn->length)  in patch_retpoline()
936 	for (; i < insn->length;)  in patch_retpoline()
943  * Generated by 'objtool --retpoline'.
974 			if (dest[-1] == 0xea && (dest[0] & 0xf0) == 0x70) {  in apply_retpolines()
1021  * Rewrite the compiler generated return thunk tail-calls.
1044 	for (; i < insn->length;)  in patch_return()
1073 			      "missing return thunk: %pS-%pS: %*ph",  in apply_returns()
1148  * Generated by: objtool --ibt
1162 			poison_cfi(addr - 16);  in apply_seal_endbr()
1228 	func -= cfi_get_offset();  in cfi_get_func_hash()
1254 	if (get_kernel_nofault(disp, func - 4))  in cfi_get_func_arity()
1258 	return target - __bhi_args;  in cfi_get_func_arity()
1268  * Re-hash the CFI hash with a boot-time seed while making sure the result is
1274 	while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) {  in cfi_rehash()
1286 		return -EINVAL;  in cfi_parse_cmdline()
1307 			pr_alert("CFI mismatch non-fatal!\n");  in cfi_parse_cmdline()
1355  *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%r10d	// 6
1356  *	addl	$-15(%r11),%r10d	 // 4	     lea    -0x10(%r11),%r11	// 4
1366  *  4:   41 81 <ea> 78 56 34 12  sub    $0x12345678, %r10d
1367  *  b:   75 f9                   jne    6 <fineibt_preamble_start+0x6>
1388 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
1389 #define fineibt_preamble_bhi  (fineibt_preamble_bhi - fineibt_preamble_start)
1395  *  0:   41 ba 78 56 34 12       mov    $0x12345678, %r10d
1396  *  6:   4d 8d 5b f0             lea    -0x10(%r11), %r11
1402 	"	lea	-0x10(%r11), %r11	\n"
1411 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
1414 #define fineibt_caller_jmp (fineibt_caller_size - 2)
1429  *  0:   41 ba 78 56 34 12       mov    $0x12345678, %r10d
1430  *  6:   45 3b 53 f7             cmp    -0x9(%r11), %r10d
1431  *  a:   4d 8d 5b <f0>           lea    -0x10(%r11), %r11
1437  * avoiding a dependency. Again, using a non-taken (backwards) branch
1444 	"	cmpl	-9(%r11), %r10d			\n"
1445 	"	lea	-0x10(%r11), %r11		\n"
1458 #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start)
1459 #define fineibt_paranoid_ind  (fineibt_paranoid_ind - fineibt_paranoid_start)
1466 	/* b8+reg 78 56 34 12          movl    $0x12345678,\reg */  in decode_preamble_hash()
1469 			*reg = p[0] - 0xb8;  in decode_preamble_hash()
1480 	/* 41 ba 88 a9 cb ed       mov    $(-0x12345678),%r10d */  in decode_caller_hash()
1482 		return -*(u32 *)(addr + 2);  in decode_caller_hash()
1484 	/* e8 0c 88 a9 cb ed	   jmp.d8  +12 */  in decode_caller_hash()
1486 		return -*(u32 *)(addr + 2);  in decode_caller_hash()
1506 		addr -= fineibt_caller_size;  in cfi_disable_callers()
1520 	 * Re-enable kCFI, undo what cfi_disable_callers() did.  in cfi_enable_callers()
1529 		addr -= fineibt_caller_size;  in cfi_enable_callers()
1552 			return -EINVAL;  in cfi_rand_preamble()
1568 		 * Crazy scheme to allow arity-1 inline:  in cfi_fineibt_bhi_preamble()
1572 		 *  4: 41 81 <ea> 78 56 34 12  sub     0x12345678, %r10d  in cfi_fineibt_bhi_preamble()
1573 		 *  b: 49 0f 45 fa             cmovne  %r10, %rdi  in cfi_fineibt_bhi_preamble()
1628 			return -EINVAL;  in cfi_rewrite_preamble()
1668 		addr -= fineibt_caller_size;  in cfi_rand_callers()
1671 			hash = -cfi_rehash(hash);  in cfi_rand_callers()
1681 	u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2;  in emit_paranoid_trampoline()
1706 		addr -= fineibt_caller_size;  in cfi_rewrite_callers()
1811 		/* place the FineIBT preamble at func()-16 */  in __apply_fineibt()
1816 		/* rewrite the callers to target func()-16 */  in __apply_fineibt()
1895  * When regs->ip points to a 0xEA byte in the FineIBT preamble,
1903 	unsigned long addr = regs->ip - fineibt_preamble_ud;  in decode_fineibt_preamble()
1912 	*type = (u32)regs->r10 + hash;  in decode_fineibt_preamble()
1915 	 * Since regs->ip points to the middle of an instruction; it cannot  in decode_fineibt_preamble()
1918 	regs->ip = *target;  in decode_fineibt_preamble()
1927  * regs->ip points to one of the UD2 in __bhi_args[].
1937 	if (regs->ip < (unsigned long)__bhi_args ||  in decode_fineibt_bhi()
1938 	    regs->ip >= (unsigned long)__bhi_args_end)  in decode_fineibt_bhi()
1947 	__get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);  in decode_fineibt_bhi()
1950 	addr -= fineibt_preamble_size;  in decode_fineibt_bhi()
1955 	*type = (u32)regs->r10 + hash;  in decode_fineibt_bhi()
1959 	 * as such the non-fatal case can use the regular fixup.  in decode_fineibt_bhi()
1979  * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
1985 	unsigned long addr = regs->ip - fineibt_paranoid_ud;  in decode_fineibt_paranoid()
1990 	if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {  in decode_fineibt_paranoid()
1991 		*target = regs->r11 + fineibt_preamble_size;  in decode_fineibt_paranoid()
1992 		*type = regs->r10;  in decode_fineibt_paranoid()
2004 	 *  0:   41 ba 78 56 34 12       mov    $0x12345678, %r10d  in decode_fineibt_paranoid()
2005 	 *  6:   45 3b 53 f7             cmp    -0x9(%r11), %r10d  in decode_fineibt_paranoid()
2006 	 *  a:   4d 8d 5b f0             lea    -0x10(%r11), %r11  in decode_fineibt_paranoid()
2019 	if (is_paranoid_thunk(regs->ip)) {  in decode_fineibt_paranoid()
2020 		*target = regs->r11 + fineibt_preamble_size;  in decode_fineibt_paranoid()
2021 		*type = regs->r10;  in decode_fineibt_paranoid()
2023 		regs->ip = *target;  in decode_fineibt_paranoid()
2097 	struct module	*mod;  member
2113 void __init_or_module alternatives_smp_module_add(struct module *mod,  in alternatives_smp_module_add()  argument
2133 	smp->mod	= mod;  in alternatives_smp_module_add()
2134 	smp->name	= name;  in alternatives_smp_module_add()
2135 	smp->locks	= locks;  in alternatives_smp_module_add()
2136 	smp->locks_end	= locks_end;  in alternatives_smp_module_add()
2137 	smp->text	= text;  in alternatives_smp_module_add()
2138 	smp->text_end	= text_end;  in alternatives_smp_module_add()
2139 	DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n",  in alternatives_smp_module_add()
2140 		smp->locks, smp->locks_end,  in alternatives_smp_module_add()
2141 		smp->text, smp->text_end, smp->name);  in alternatives_smp_module_add()
2143 	list_add_tail(&smp->next, &smp_alt_modules);  in alternatives_smp_module_add()
2150 void __init_or_module alternatives_smp_module_del(struct module *mod)  in alternatives_smp_module_del()  argument
2156 		if (mod != item->mod)  in alternatives_smp_module_del()
2158 		list_del(&item->next);  in alternatives_smp_module_del()
2167 	struct smp_alt_module *mod;  in alternatives_enable_smp()  local
2179 		list_for_each_entry(mod, &smp_alt_modules, next)  in alternatives_enable_smp()
2180 			alternatives_smp_lock(mod->locks, mod->locks_end,  in alternatives_enable_smp()
2181 					      mod->text, mod->text_end);  in alternatives_enable_smp()
2188  * Return 1 if the address range is reserved for SMP-alternatives.
2193 	struct smp_alt_module *mod;  in alternatives_text_reserved()  local
2200 	list_for_each_entry(mod, &smp_alt_modules, next) {  in alternatives_text_reserved()
2201 		if (mod->text > text_end || mod->text_end < text_start)  in alternatives_text_reserved()
2203 		for (poff = mod->locks; poff < mod->locks_end; poff++) {  in alternatives_text_reserved()
2216  * Self-test for the INT3 based CALL emulation code.
2240 "	.size		int3_magic, .-int3_magic\n"
2251 	struct pt_regs *regs = args->regs;  in int3_exception_notify()
2261 	if (regs->ip - INT3_INSN_SIZE != selftest)  in int3_exception_notify()
2273 		.priority	= INT_MAX-1, /* last */  in int3_selftest()
2280 	 * Basically: int3_magic(&val); but really complicated :-)  in int3_selftest()
2311 	 * instruction and additionally on x86_64 we get a RIP-relative LEA:  in alt_reloc_selftest()
2314 	 *   call   +0              # 5d5: R_X86_64_PLT32   __alt_reloc_selftest-0x4  in alt_reloc_selftest()
2357 	/* Keep CET-IBT disabled until caller/callee are patched */  in alternative_instructions()
2373 	 * Adjust all CALL instructions to point to func()-10, including  in alternative_instructions()
2410  * text_poke_early - Update instructions on a live kernel at boot time
2429 		 * Modules text is marked initially as non-executable, so the  in text_poke_early()
2430 		 * code cannot be running and speculative code-fetches are  in text_poke_early()
2498 	 * flush_tlb_mm_range(), which is intended for non-global PTEs.  in __text_poke()
2503 	 * The lock is not really needed, but this allows to avoid open-coding.  in __text_poke()
2543 	 * Loading the previous page-table hierarchy requires a serializing  in __text_poke()
2545 	 * Xen-PV is assumed to serialize execution in a similar manner.  in __text_poke()
2571  * text_poke - Update instructions on a live kernel
2594  * text_poke_kgdb - Update instructions on a live kernel by kgdb
2625 		s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);  in text_poke_copy_locked()
2634  * text_poke_copy - Copy instructions into (an unused part of) RX memory
2654  * text_poke_set - memset into (an unused part of) RX memory
2675 		s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);  in text_poke_set()
2744 	return _stext + tpl->rel_addr;  in text_poke_addr()
2750 		return -1;  in patch_cmp()
2767 	 * text_poke_array with non-zero refcount:  in smp_text_poke_int3_handler()
2781 	ip = (void *) regs->ip - INT3_INSN_SIZE;  in smp_text_poke_int3_handler()
2798 	ip += tpl->len;  in smp_text_poke_int3_handler()
2800 	switch (tpl->opcode) {  in smp_text_poke_int3_handler()
2813 		int3_emulate_call(regs, (long)ip + tpl->disp);  in smp_text_poke_int3_handler()
2818 		int3_emulate_jmp(regs, (long)ip + tpl->disp);  in smp_text_poke_int3_handler()
2822 		int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp);  in smp_text_poke_int3_handler()
2837  * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP
2843  * Modify multi-byte instructions by using INT3 breakpoints on SMP.
2845  * synchronization using INT3 breakpoints and SMP cross-calls.
2848  *	- For each entry in the vector:
2849  *		- add an INT3 trap to the address that will be patched
2850  *	- SMP sync all CPUs
2851  *	- For each entry in the vector:
2852  *		- update all but the first byte of the patched range
2853  *	- SMP sync all CPUs
2854  *	- For each entry in the vector:
2855  *		- replace the first byte (INT3) by the first byte of the
2857  *	- SMP sync all CPUs
2872 	 * ensure reading a non-zero refcount provides up to date text_poke_array data.  in smp_text_poke_batch_finish()
2912 		if (len - INT3_INSN_SIZE > 0) {  in smp_text_poke_batch_finish()
2915 			       len - INT3_INSN_SIZE);  in smp_text_poke_batch_finish()
2925 				  len - INT3_INSN_SIZE);  in smp_text_poke_batch_finish()
2934 		 *   - write INT3 byte  in smp_text_poke_batch_finish()
2935 		 *   - IPI-SYNC  in smp_text_poke_batch_finish()
2936 		 *   - write instruction tail  in smp_text_poke_batch_finish()
2941 		 *   - emit RECORD_TEXT_POKE with the new instruction  in smp_text_poke_batch_finish()
2942 		 *   - IPI-SYNC  in smp_text_poke_batch_finish()
2943 		 *   - write first byte  in smp_text_poke_batch_finish()
2944 		 *   - IPI-SYNC  in smp_text_poke_batch_finish()
2989 	 * Notably, if after step-3 above the INT3 got removed, then the  in smp_text_poke_batch_finish()
2991 	 * handlers and the below spin-wait will not happen.  in smp_text_poke_batch_finish()
3017 	memcpy((void *)tpl->text, opcode+i, len-i);  in __smp_text_poke_batch_add()
3024 	tpl->rel_addr = addr - (void *)_stext;  in __smp_text_poke_batch_add()
3025 	tpl->len = len;  in __smp_text_poke_batch_add()
3026 	tpl->opcode = insn.opcode.bytes[0];  in __smp_text_poke_batch_add()
3032 		tpl->opcode = insn.opcode.bytes[1] - 0x10;  in __smp_text_poke_batch_add()
3035 	switch (tpl->opcode) {  in __smp_text_poke_batch_add()
3044 			BUG_ON(tpl->text[i] != INT3_INSN_OPCODE);  in __smp_text_poke_batch_add()
3051 	switch (tpl->opcode) {  in __smp_text_poke_batch_add()
3060 		tpl->disp = insn.immediate.value;  in __smp_text_poke_batch_add()
3065 		case 2: /* NOP2 -- emulate as JMP8+0 */  in __smp_text_poke_batch_add()
3067 			tpl->opcode = JMP8_INSN_OPCODE;  in __smp_text_poke_batch_add()
3068 			tpl->disp = 0;  in __smp_text_poke_batch_add()
3071 		case 5: /* NOP5 -- emulate as JMP32+0 */  in __smp_text_poke_batch_add()
3073 			tpl->opcode = JMP32_INSN_OPCODE;  in __smp_text_poke_batch_add()
3074 			tpl->disp = 0;  in __smp_text_poke_batch_add()
3101 	if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr)  in text_poke_addr_ordered()
3108  * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched
3114  * Add a new instruction to the current queue of to-be-patched instructions
3128  * smp_text_poke_single() -- update instruction on live kernel on SMP immediately