1 // SPDX-License-Identifier: GPL-2.0-only 2 #define pr_fmt(fmt) "SMP alternatives: " fmt 3 4 #include <linux/module.h> 5 #include <linux/sched.h> 6 #include <linux/perf_event.h> 7 #include <linux/mutex.h> 8 #include <linux/list.h> 9 #include <linux/stringify.h> 10 #include <linux/highmem.h> 11 #include <linux/mm.h> 12 #include <linux/vmalloc.h> 13 #include <linux/memory.h> 14 #include <linux/stop_machine.h> 15 #include <linux/slab.h> 16 #include <linux/kdebug.h> 17 #include <linux/kprobes.h> 18 #include <linux/mmu_context.h> 19 #include <linux/bsearch.h> 20 #include <linux/sync_core.h> 21 #include <asm/text-patching.h> 22 #include <asm/alternative.h> 23 #include <asm/sections.h> 24 #include <asm/mce.h> 25 #include <asm/nmi.h> 26 #include <asm/cacheflush.h> 27 #include <asm/tlbflush.h> 28 #include <asm/insn.h> 29 #include <asm/io.h> 30 #include <asm/fixmap.h> 31 #include <asm/paravirt.h> 32 #include <asm/asm-prototypes.h> 33 #include <asm/cfi.h> 34 35 int __read_mostly alternatives_patched; 36 37 EXPORT_SYMBOL_GPL(alternatives_patched); 38 39 #define MAX_PATCH_LEN (255-1) 40 41 #define DA_ALL (~0) 42 #define DA_ALT 0x01 43 #define DA_RET 0x02 44 #define DA_RETPOLINE 0x04 45 #define DA_ENDBR 0x08 46 #define DA_SMP 0x10 47 48 static unsigned int debug_alternative; 49 50 static int __init debug_alt(char *str) 51 { 52 if (str && *str == '=') 53 str++; 54 55 if (!str || kstrtouint(str, 0, &debug_alternative)) 56 debug_alternative = DA_ALL; 57 58 return 1; 59 } 60 __setup("debug-alternative", debug_alt); 61 62 static int noreplace_smp; 63 64 static int __init setup_noreplace_smp(char *str) 65 { 66 noreplace_smp = 1; 67 return 1; 68 } 69 __setup("noreplace-smp", setup_noreplace_smp); 70 71 #define DPRINTK(type, fmt, args...) \ 72 do { \ 73 if (debug_alternative & DA_##type) \ 74 printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \ 75 } while (0) 76 77 #define DUMP_BYTES(type, buf, len, fmt, args...) \ 78 do { \ 79 if (unlikely(debug_alternative & DA_##type)) { \ 80 int j; \ 81 \ 82 if (!(len)) \ 83 break; \ 84 \ 85 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 86 for (j = 0; j < (len) - 1; j++) \ 87 printk(KERN_CONT "%02hhx ", buf[j]); \ 88 printk(KERN_CONT "%02hhx\n", buf[j]); \ 89 } \ 90 } while (0) 91 92 static const unsigned char x86nops[] = 93 { 94 BYTES_NOP1, 95 BYTES_NOP2, 96 BYTES_NOP3, 97 BYTES_NOP4, 98 BYTES_NOP5, 99 BYTES_NOP6, 100 BYTES_NOP7, 101 BYTES_NOP8, 102 #ifdef CONFIG_64BIT 103 BYTES_NOP9, 104 BYTES_NOP10, 105 BYTES_NOP11, 106 #endif 107 }; 108 109 const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 110 { 111 NULL, 112 x86nops, 113 x86nops + 1, 114 x86nops + 1 + 2, 115 x86nops + 1 + 2 + 3, 116 x86nops + 1 + 2 + 3 + 4, 117 x86nops + 1 + 2 + 3 + 4 + 5, 118 x86nops + 1 + 2 + 3 + 4 + 5 + 6, 119 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 120 #ifdef CONFIG_64BIT 121 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 122 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9, 123 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10, 124 #endif 125 }; 126 127 /* 128 * Nomenclature for variable names to simplify and clarify this code and ease 129 * any potential staring at it: 130 * 131 * @instr: source address of the original instructions in the kernel text as 132 * generated by the compiler. 133 * 134 * @buf: temporary buffer on which the patching operates. This buffer is 135 * eventually text-poked into the kernel image. 136 * 137 * @replacement/@repl: pointer to the opcodes which are replacing @instr, located 138 * in the .altinstr_replacement section. 139 */ 140 141 /* 142 * Fill the buffer with a single effective instruction of size @len. 143 * 144 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info) 145 * for every single-byte NOP, try to generate the maximally available NOP of 146 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for 147 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and 148 * *jump* over instead of executing long and daft NOPs. 149 */ 150 static void add_nop(u8 *buf, unsigned int len) 151 { 152 u8 *target = buf + len; 153 154 if (!len) 155 return; 156 157 if (len <= ASM_NOP_MAX) { 158 memcpy(buf, x86_nops[len], len); 159 return; 160 } 161 162 if (len < 128) { 163 __text_gen_insn(buf, JMP8_INSN_OPCODE, buf, target, JMP8_INSN_SIZE); 164 buf += JMP8_INSN_SIZE; 165 } else { 166 __text_gen_insn(buf, JMP32_INSN_OPCODE, buf, target, JMP32_INSN_SIZE); 167 buf += JMP32_INSN_SIZE; 168 } 169 170 for (;buf < target; buf++) 171 *buf = INT3_INSN_OPCODE; 172 } 173 174 extern s32 __retpoline_sites[], __retpoline_sites_end[]; 175 extern s32 __return_sites[], __return_sites_end[]; 176 extern s32 __cfi_sites[], __cfi_sites_end[]; 177 extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[]; 178 extern s32 __smp_locks[], __smp_locks_end[]; 179 void text_poke_early(void *addr, const void *opcode, size_t len); 180 181 /* 182 * Matches NOP and NOPL, not any of the other possible NOPs. 183 */ 184 static bool insn_is_nop(struct insn *insn) 185 { 186 /* Anything NOP, but no REP NOP */ 187 if (insn->opcode.bytes[0] == 0x90 && 188 (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3)) 189 return true; 190 191 /* NOPL */ 192 if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F) 193 return true; 194 195 /* TODO: more nops */ 196 197 return false; 198 } 199 200 /* 201 * Find the offset of the first non-NOP instruction starting at @offset 202 * but no further than @len. 203 */ 204 static int skip_nops(u8 *buf, int offset, int len) 205 { 206 struct insn insn; 207 208 for (; offset < len; offset += insn.length) { 209 if (insn_decode_kernel(&insn, &buf[offset])) 210 break; 211 212 if (!insn_is_nop(&insn)) 213 break; 214 } 215 216 return offset; 217 } 218 219 /* 220 * "noinline" to cause control flow change and thus invalidate I$ and 221 * cause refetch after modification. 222 */ 223 static void noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len) 224 { 225 for (int next, i = 0; i < len; i = next) { 226 struct insn insn; 227 228 if (insn_decode_kernel(&insn, &buf[i])) 229 return; 230 231 next = i + insn.length; 232 233 if (insn_is_nop(&insn)) { 234 int nop = i; 235 236 /* Has the NOP already been optimized? */ 237 if (i + insn.length == len) 238 return; 239 240 next = skip_nops(buf, next, len); 241 242 add_nop(buf + nop, next - nop); 243 DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, next); 244 } 245 } 246 } 247 248 /* 249 * In this context, "source" is where the instructions are placed in the 250 * section .altinstr_replacement, for example during kernel build by the 251 * toolchain. 252 * "Destination" is where the instructions are being patched in by this 253 * machinery. 254 * 255 * The source offset is: 256 * 257 * src_imm = target - src_next_ip (1) 258 * 259 * and the target offset is: 260 * 261 * dst_imm = target - dst_next_ip (2) 262 * 263 * so rework (1) as an expression for target like: 264 * 265 * target = src_imm + src_next_ip (1a) 266 * 267 * and substitute in (2) to get: 268 * 269 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3) 270 * 271 * Now, since the instruction stream is 'identical' at src and dst (it 272 * is being copied after all) it can be stated that: 273 * 274 * src_next_ip = src + ip_offset 275 * dst_next_ip = dst + ip_offset (4) 276 * 277 * Substitute (4) in (3) and observe ip_offset being cancelled out to 278 * obtain: 279 * 280 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset) 281 * = src_imm + src - dst + ip_offset - ip_offset 282 * = src_imm + src - dst (5) 283 * 284 * IOW, only the relative displacement of the code block matters. 285 */ 286 287 #define apply_reloc_n(n_, p_, d_) \ 288 do { \ 289 s32 v = *(s##n_ *)(p_); \ 290 v += (d_); \ 291 BUG_ON((v >> 31) != (v >> (n_-1))); \ 292 *(s##n_ *)(p_) = (s##n_)v; \ 293 } while (0) 294 295 296 static __always_inline 297 void apply_reloc(int n, void *ptr, uintptr_t diff) 298 { 299 switch (n) { 300 case 1: apply_reloc_n(8, ptr, diff); break; 301 case 2: apply_reloc_n(16, ptr, diff); break; 302 case 4: apply_reloc_n(32, ptr, diff); break; 303 default: BUG(); 304 } 305 } 306 307 static __always_inline 308 bool need_reloc(unsigned long offset, u8 *src, size_t src_len) 309 { 310 u8 *target = src + offset; 311 /* 312 * If the target is inside the patched block, it's relative to the 313 * block itself and does not need relocation. 314 */ 315 return (target < src || target > src + src_len); 316 } 317 318 static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) 319 { 320 for (int next, i = 0; i < instrlen; i = next) { 321 struct insn insn; 322 323 if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i]))) 324 return; 325 326 next = i + insn.length; 327 328 switch (insn.opcode.bytes[0]) { 329 case 0x0f: 330 if (insn.opcode.bytes[1] < 0x80 || 331 insn.opcode.bytes[1] > 0x8f) 332 break; 333 334 fallthrough; /* Jcc.d32 */ 335 case 0x70 ... 0x7f: /* Jcc.d8 */ 336 case JMP8_INSN_OPCODE: 337 case JMP32_INSN_OPCODE: 338 case CALL_INSN_OPCODE: 339 if (need_reloc(next + insn.immediate.value, repl, repl_len)) { 340 apply_reloc(insn.immediate.nbytes, 341 buf + i + insn_offset_immediate(&insn), 342 repl - instr); 343 } 344 345 /* 346 * Where possible, convert JMP.d32 into JMP.d8. 347 */ 348 if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) { 349 s32 imm = insn.immediate.value; 350 imm += repl - instr; 351 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE; 352 if ((imm >> 31) == (imm >> 7)) { 353 buf[i+0] = JMP8_INSN_OPCODE; 354 buf[i+1] = (s8)imm; 355 356 memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2); 357 } 358 } 359 break; 360 } 361 362 if (insn_rip_relative(&insn)) { 363 if (need_reloc(next + insn.displacement.value, repl, repl_len)) { 364 apply_reloc(insn.displacement.nbytes, 365 buf + i + insn_offset_displacement(&insn), 366 repl - instr); 367 } 368 } 369 } 370 } 371 372 void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) 373 { 374 __apply_relocation(buf, instr, instrlen, repl, repl_len); 375 optimize_nops(instr, buf, instrlen); 376 } 377 378 /* Low-level backend functions usable from alternative code replacements. */ 379 DEFINE_ASM_FUNC(nop_func, "", .entry.text); 380 EXPORT_SYMBOL_GPL(nop_func); 381 382 noinstr void BUG_func(void) 383 { 384 BUG(); 385 } 386 EXPORT_SYMBOL(BUG_func); 387 388 #define CALL_RIP_REL_OPCODE 0xff 389 #define CALL_RIP_REL_MODRM 0x15 390 391 /* 392 * Rewrite the "call BUG_func" replacement to point to the target of the 393 * indirect pv_ops call "call *disp(%ip)". 394 */ 395 static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a) 396 { 397 void *target, *bug = &BUG_func; 398 s32 disp; 399 400 if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) { 401 pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n"); 402 BUG(); 403 } 404 405 if (a->instrlen != 6 || 406 instr[0] != CALL_RIP_REL_OPCODE || 407 instr[1] != CALL_RIP_REL_MODRM) { 408 pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n"); 409 BUG(); 410 } 411 412 /* Skip CALL_RIP_REL_OPCODE and CALL_RIP_REL_MODRM */ 413 disp = *(s32 *)(instr + 2); 414 #ifdef CONFIG_X86_64 415 /* ff 15 00 00 00 00 call *0x0(%rip) */ 416 /* target address is stored at "next instruction + disp". */ 417 target = *(void **)(instr + a->instrlen + disp); 418 #else 419 /* ff 15 00 00 00 00 call *0x0 */ 420 /* target address is stored at disp. */ 421 target = *(void **)disp; 422 #endif 423 if (!target) 424 target = bug; 425 426 /* (BUG_func - .) + (target - BUG_func) := target - . */ 427 *(s32 *)(insn_buff + 1) += target - bug; 428 429 if (target == &nop_func) 430 return 0; 431 432 return 5; 433 } 434 435 static inline u8 * instr_va(struct alt_instr *i) 436 { 437 return (u8 *)&i->instr_offset + i->instr_offset; 438 } 439 440 /* 441 * Replace instructions with better alternatives for this CPU type. This runs 442 * before SMP is initialized to avoid SMP problems with self modifying code. 443 * This implies that asymmetric systems where APs have less capabilities than 444 * the boot processor are not handled. Tough. Make sure you disable such 445 * features by hand. 446 * 447 * Marked "noinline" to cause control flow change and thus insn cache 448 * to refetch changed I$ lines. 449 */ 450 void __init_or_module noinline apply_alternatives(struct alt_instr *start, 451 struct alt_instr *end) 452 { 453 u8 insn_buff[MAX_PATCH_LEN]; 454 u8 *instr, *replacement; 455 struct alt_instr *a, *b; 456 457 DPRINTK(ALT, "alt table %px, -> %px", start, end); 458 459 /* 460 * In the case CONFIG_X86_5LEVEL=y, KASAN_SHADOW_START is defined using 461 * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here. 462 * During the process, KASAN becomes confused seeing partial LA57 463 * conversion and triggers a false-positive out-of-bound report. 464 * 465 * Disable KASAN until the patching is complete. 466 */ 467 kasan_disable_current(); 468 469 /* 470 * The scan order should be from start to end. A later scanned 471 * alternative code can overwrite previously scanned alternative code. 472 * Some kernel functions (e.g. memcpy, memset, etc) use this order to 473 * patch code. 474 * 475 * So be careful if you want to change the scan order to any other 476 * order. 477 */ 478 for (a = start; a < end; a++) { 479 int insn_buff_sz = 0; 480 481 /* 482 * In case of nested ALTERNATIVE()s the outer alternative might 483 * add more padding. To ensure consistent patching find the max 484 * padding for all alt_instr entries for this site (nested 485 * alternatives result in consecutive entries). 486 */ 487 for (b = a+1; b < end && instr_va(b) == instr_va(a); b++) { 488 u8 len = max(a->instrlen, b->instrlen); 489 a->instrlen = b->instrlen = len; 490 } 491 492 instr = instr_va(a); 493 replacement = (u8 *)&a->repl_offset + a->repl_offset; 494 BUG_ON(a->instrlen > sizeof(insn_buff)); 495 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); 496 497 /* 498 * Patch if either: 499 * - feature is present 500 * - feature not present but ALT_FLAG_NOT is set to mean, 501 * patch if feature is *NOT* present. 502 */ 503 if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) { 504 memcpy(insn_buff, instr, a->instrlen); 505 optimize_nops(instr, insn_buff, a->instrlen); 506 text_poke_early(instr, insn_buff, a->instrlen); 507 continue; 508 } 509 510 DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x", 511 a->cpuid >> 5, 512 a->cpuid & 0x1f, 513 instr, instr, a->instrlen, 514 replacement, a->replacementlen, a->flags); 515 516 memcpy(insn_buff, replacement, a->replacementlen); 517 insn_buff_sz = a->replacementlen; 518 519 if (a->flags & ALT_FLAG_DIRECT_CALL) { 520 insn_buff_sz = alt_replace_call(instr, insn_buff, a); 521 if (insn_buff_sz < 0) 522 continue; 523 } 524 525 for (; insn_buff_sz < a->instrlen; insn_buff_sz++) 526 insn_buff[insn_buff_sz] = 0x90; 527 528 apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen); 529 530 DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr); 531 DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement); 532 DUMP_BYTES(ALT, insn_buff, insn_buff_sz, "%px: final_insn: ", instr); 533 534 text_poke_early(instr, insn_buff, insn_buff_sz); 535 } 536 537 kasan_enable_current(); 538 } 539 540 static inline bool is_jcc32(struct insn *insn) 541 { 542 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ 543 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; 544 } 545 546 #if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL) 547 548 /* 549 * CALL/JMP *%\reg 550 */ 551 static int emit_indirect(int op, int reg, u8 *bytes) 552 { 553 int i = 0; 554 u8 modrm; 555 556 switch (op) { 557 case CALL_INSN_OPCODE: 558 modrm = 0x10; /* Reg = 2; CALL r/m */ 559 break; 560 561 case JMP32_INSN_OPCODE: 562 modrm = 0x20; /* Reg = 4; JMP r/m */ 563 break; 564 565 default: 566 WARN_ON_ONCE(1); 567 return -1; 568 } 569 570 if (reg >= 8) { 571 bytes[i++] = 0x41; /* REX.B prefix */ 572 reg -= 8; 573 } 574 575 modrm |= 0xc0; /* Mod = 3 */ 576 modrm += reg; 577 578 bytes[i++] = 0xff; /* opcode */ 579 bytes[i++] = modrm; 580 581 return i; 582 } 583 584 static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) 585 { 586 u8 op = insn->opcode.bytes[0]; 587 int i = 0; 588 589 /* 590 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional 591 * tail-calls. Deal with them. 592 */ 593 if (is_jcc32(insn)) { 594 bytes[i++] = op; 595 op = insn->opcode.bytes[1]; 596 goto clang_jcc; 597 } 598 599 if (insn->length == 6) 600 bytes[i++] = 0x2e; /* CS-prefix */ 601 602 switch (op) { 603 case CALL_INSN_OPCODE: 604 __text_gen_insn(bytes+i, op, addr+i, 605 __x86_indirect_call_thunk_array[reg], 606 CALL_INSN_SIZE); 607 i += CALL_INSN_SIZE; 608 break; 609 610 case JMP32_INSN_OPCODE: 611 clang_jcc: 612 __text_gen_insn(bytes+i, op, addr+i, 613 __x86_indirect_jump_thunk_array[reg], 614 JMP32_INSN_SIZE); 615 i += JMP32_INSN_SIZE; 616 break; 617 618 default: 619 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr); 620 return -1; 621 } 622 623 WARN_ON_ONCE(i != insn->length); 624 625 return i; 626 } 627 628 /* 629 * Rewrite the compiler generated retpoline thunk calls. 630 * 631 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate 632 * indirect instructions, avoiding the extra indirection. 633 * 634 * For example, convert: 635 * 636 * CALL __x86_indirect_thunk_\reg 637 * 638 * into: 639 * 640 * CALL *%\reg 641 * 642 * It also tries to inline spectre_v2=retpoline,lfence when size permits. 643 */ 644 static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) 645 { 646 retpoline_thunk_t *target; 647 int reg, ret, i = 0; 648 u8 op, cc; 649 650 target = addr + insn->length + insn->immediate.value; 651 reg = target - __x86_indirect_thunk_array; 652 653 if (WARN_ON_ONCE(reg & ~0xf)) 654 return -1; 655 656 /* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */ 657 BUG_ON(reg == 4); 658 659 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) && 660 !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { 661 if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 662 return emit_call_track_retpoline(addr, insn, reg, bytes); 663 664 return -1; 665 } 666 667 op = insn->opcode.bytes[0]; 668 669 /* 670 * Convert: 671 * 672 * Jcc.d32 __x86_indirect_thunk_\reg 673 * 674 * into: 675 * 676 * Jncc.d8 1f 677 * [ LFENCE ] 678 * JMP *%\reg 679 * [ NOP ] 680 * 1: 681 */ 682 if (is_jcc32(insn)) { 683 cc = insn->opcode.bytes[1] & 0xf; 684 cc ^= 1; /* invert condition */ 685 686 bytes[i++] = 0x70 + cc; /* Jcc.d8 */ 687 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */ 688 689 /* Continue as if: JMP.d32 __x86_indirect_thunk_\reg */ 690 op = JMP32_INSN_OPCODE; 691 } 692 693 /* 694 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE. 695 */ 696 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { 697 bytes[i++] = 0x0f; 698 bytes[i++] = 0xae; 699 bytes[i++] = 0xe8; /* LFENCE */ 700 } 701 702 ret = emit_indirect(op, reg, bytes + i); 703 if (ret < 0) 704 return ret; 705 i += ret; 706 707 /* 708 * The compiler is supposed to EMIT an INT3 after every unconditional 709 * JMP instruction due to AMD BTC. However, if the compiler is too old 710 * or MITIGATION_SLS isn't enabled, we still need an INT3 after 711 * indirect JMPs even on Intel. 712 */ 713 if (op == JMP32_INSN_OPCODE && i < insn->length) 714 bytes[i++] = INT3_INSN_OPCODE; 715 716 for (; i < insn->length;) 717 bytes[i++] = BYTES_NOP1; 718 719 return i; 720 } 721 722 /* 723 * Generated by 'objtool --retpoline'. 724 */ 725 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) 726 { 727 s32 *s; 728 729 for (s = start; s < end; s++) { 730 void *addr = (void *)s + *s; 731 struct insn insn; 732 int len, ret; 733 u8 bytes[16]; 734 u8 op1, op2; 735 736 ret = insn_decode_kernel(&insn, addr); 737 if (WARN_ON_ONCE(ret < 0)) 738 continue; 739 740 op1 = insn.opcode.bytes[0]; 741 op2 = insn.opcode.bytes[1]; 742 743 switch (op1) { 744 case 0x70 ... 0x7f: /* Jcc.d8 */ 745 /* See cfi_paranoid. */ 746 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT); 747 continue; 748 749 case CALL_INSN_OPCODE: 750 case JMP32_INSN_OPCODE: 751 break; 752 753 case 0x0f: /* escape */ 754 if (op2 >= 0x80 && op2 <= 0x8f) 755 break; 756 fallthrough; 757 default: 758 WARN_ON_ONCE(1); 759 continue; 760 } 761 762 DPRINTK(RETPOLINE, "retpoline at: %pS (%px) len: %d to: %pS", 763 addr, addr, insn.length, 764 addr + insn.length + insn.immediate.value); 765 766 len = patch_retpoline(addr, &insn, bytes); 767 if (len == insn.length) { 768 optimize_nops(addr, bytes, len); 769 DUMP_BYTES(RETPOLINE, ((u8*)addr), len, "%px: orig: ", addr); 770 DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr); 771 text_poke_early(addr, bytes, len); 772 } 773 } 774 } 775 776 #ifdef CONFIG_MITIGATION_RETHUNK 777 778 /* 779 * Rewrite the compiler generated return thunk tail-calls. 780 * 781 * For example, convert: 782 * 783 * JMP __x86_return_thunk 784 * 785 * into: 786 * 787 * RET 788 */ 789 static int patch_return(void *addr, struct insn *insn, u8 *bytes) 790 { 791 int i = 0; 792 793 /* Patch the custom return thunks... */ 794 if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) { 795 i = JMP32_INSN_SIZE; 796 __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i); 797 } else { 798 /* ... or patch them out if not needed. */ 799 bytes[i++] = RET_INSN_OPCODE; 800 } 801 802 for (; i < insn->length;) 803 bytes[i++] = INT3_INSN_OPCODE; 804 return i; 805 } 806 807 void __init_or_module noinline apply_returns(s32 *start, s32 *end) 808 { 809 s32 *s; 810 811 if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) 812 static_call_force_reinit(); 813 814 for (s = start; s < end; s++) { 815 void *dest = NULL, *addr = (void *)s + *s; 816 struct insn insn; 817 int len, ret; 818 u8 bytes[16]; 819 u8 op; 820 821 ret = insn_decode_kernel(&insn, addr); 822 if (WARN_ON_ONCE(ret < 0)) 823 continue; 824 825 op = insn.opcode.bytes[0]; 826 if (op == JMP32_INSN_OPCODE) 827 dest = addr + insn.length + insn.immediate.value; 828 829 if (__static_call_fixup(addr, op, dest) || 830 WARN_ONCE(dest != &__x86_return_thunk, 831 "missing return thunk: %pS-%pS: %*ph", 832 addr, dest, 5, addr)) 833 continue; 834 835 DPRINTK(RET, "return thunk at: %pS (%px) len: %d to: %pS", 836 addr, addr, insn.length, 837 addr + insn.length + insn.immediate.value); 838 839 len = patch_return(addr, &insn, bytes); 840 if (len == insn.length) { 841 DUMP_BYTES(RET, ((u8*)addr), len, "%px: orig: ", addr); 842 DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr); 843 text_poke_early(addr, bytes, len); 844 } 845 } 846 } 847 #else /* !CONFIG_MITIGATION_RETHUNK: */ 848 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { } 849 #endif /* !CONFIG_MITIGATION_RETHUNK */ 850 851 #else /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */ 852 853 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { } 854 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { } 855 856 #endif /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */ 857 858 #ifdef CONFIG_X86_KERNEL_IBT 859 860 __noendbr bool is_endbr(u32 *val) 861 { 862 u32 endbr; 863 864 __get_kernel_nofault(&endbr, val, u32, Efault); 865 return __is_endbr(endbr); 866 867 Efault: 868 return false; 869 } 870 871 #ifdef CONFIG_FINEIBT 872 873 static __noendbr bool exact_endbr(u32 *val) 874 { 875 u32 endbr; 876 877 __get_kernel_nofault(&endbr, val, u32, Efault); 878 return endbr == gen_endbr(); 879 880 Efault: 881 return false; 882 } 883 884 #endif 885 886 static void poison_cfi(void *addr); 887 888 static void __init_or_module poison_endbr(void *addr) 889 { 890 u32 poison = gen_endbr_poison(); 891 892 if (WARN_ON_ONCE(!is_endbr(addr))) 893 return; 894 895 DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr); 896 897 /* 898 * When we have IBT, the lack of ENDBR will trigger #CP 899 */ 900 DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr); 901 DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr); 902 text_poke_early(addr, &poison, 4); 903 } 904 905 /* 906 * Generated by: objtool --ibt 907 * 908 * Seal the functions for indirect calls by clobbering the ENDBR instructions 909 * and the kCFI hash value. 910 */ 911 void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) 912 { 913 s32 *s; 914 915 for (s = start; s < end; s++) { 916 void *addr = (void *)s + *s; 917 918 poison_endbr(addr); 919 if (IS_ENABLED(CONFIG_FINEIBT)) 920 poison_cfi(addr - 16); 921 } 922 } 923 924 #else /* !CONFIG_X86_KERNEL_IBT: */ 925 926 void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { } 927 928 #endif /* !CONFIG_X86_KERNEL_IBT */ 929 930 #ifdef CONFIG_CFI_AUTO_DEFAULT 931 # define __CFI_DEFAULT CFI_AUTO 932 #elif defined(CONFIG_CFI_CLANG) 933 # define __CFI_DEFAULT CFI_KCFI 934 #else 935 # define __CFI_DEFAULT CFI_OFF 936 #endif 937 938 enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT; 939 940 #ifdef CONFIG_FINEIBT_BHI 941 bool cfi_bhi __ro_after_init = false; 942 #endif 943 944 #ifdef CONFIG_CFI_CLANG 945 struct bpf_insn; 946 947 /* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */ 948 extern unsigned int __bpf_prog_runX(const void *ctx, 949 const struct bpf_insn *insn); 950 951 KCFI_REFERENCE(__bpf_prog_runX); 952 953 /* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ 954 asm ( 955 " .pushsection .data..ro_after_init,\"aw\",@progbits \n" 956 " .type cfi_bpf_hash,@object \n" 957 " .globl cfi_bpf_hash \n" 958 " .p2align 2, 0x0 \n" 959 "cfi_bpf_hash: \n" 960 " .long __kcfi_typeid___bpf_prog_runX \n" 961 " .size cfi_bpf_hash, 4 \n" 962 " .popsection \n" 963 ); 964 965 /* Must match bpf_callback_t */ 966 extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64); 967 968 KCFI_REFERENCE(__bpf_callback_fn); 969 970 /* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */ 971 asm ( 972 " .pushsection .data..ro_after_init,\"aw\",@progbits \n" 973 " .type cfi_bpf_subprog_hash,@object \n" 974 " .globl cfi_bpf_subprog_hash \n" 975 " .p2align 2, 0x0 \n" 976 "cfi_bpf_subprog_hash: \n" 977 " .long __kcfi_typeid___bpf_callback_fn \n" 978 " .size cfi_bpf_subprog_hash, 4 \n" 979 " .popsection \n" 980 ); 981 982 u32 cfi_get_func_hash(void *func) 983 { 984 u32 hash; 985 986 func -= cfi_get_offset(); 987 switch (cfi_mode) { 988 case CFI_FINEIBT: 989 func += 7; 990 break; 991 case CFI_KCFI: 992 func += 1; 993 break; 994 default: 995 return 0; 996 } 997 998 if (get_kernel_nofault(hash, func)) 999 return 0; 1000 1001 return hash; 1002 } 1003 1004 int cfi_get_func_arity(void *func) 1005 { 1006 bhi_thunk *target; 1007 s32 disp; 1008 1009 if (cfi_mode != CFI_FINEIBT && !cfi_bhi) 1010 return 0; 1011 1012 if (get_kernel_nofault(disp, func - 4)) 1013 return 0; 1014 1015 target = func + disp; 1016 return target - __bhi_args; 1017 } 1018 #endif 1019 1020 #ifdef CONFIG_FINEIBT 1021 1022 static bool cfi_rand __ro_after_init = true; 1023 static u32 cfi_seed __ro_after_init; 1024 1025 static bool cfi_paranoid __ro_after_init = false; 1026 1027 /* 1028 * Re-hash the CFI hash with a boot-time seed while making sure the result is 1029 * not a valid ENDBR instruction. 1030 */ 1031 static u32 cfi_rehash(u32 hash) 1032 { 1033 hash ^= cfi_seed; 1034 while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) { 1035 bool lsb = hash & 1; 1036 hash >>= 1; 1037 if (lsb) 1038 hash ^= 0x80200003; 1039 } 1040 return hash; 1041 } 1042 1043 static __init int cfi_parse_cmdline(char *str) 1044 { 1045 if (!str) 1046 return -EINVAL; 1047 1048 while (str) { 1049 char *next = strchr(str, ','); 1050 if (next) { 1051 *next = 0; 1052 next++; 1053 } 1054 1055 if (!strcmp(str, "auto")) { 1056 cfi_mode = CFI_AUTO; 1057 } else if (!strcmp(str, "off")) { 1058 cfi_mode = CFI_OFF; 1059 cfi_rand = false; 1060 } else if (!strcmp(str, "kcfi")) { 1061 cfi_mode = CFI_KCFI; 1062 } else if (!strcmp(str, "fineibt")) { 1063 cfi_mode = CFI_FINEIBT; 1064 } else if (!strcmp(str, "norand")) { 1065 cfi_rand = false; 1066 } else if (!strcmp(str, "warn")) { 1067 pr_alert("CFI mismatch non-fatal!\n"); 1068 cfi_warn = true; 1069 } else if (!strcmp(str, "paranoid")) { 1070 if (cfi_mode == CFI_FINEIBT) { 1071 cfi_paranoid = true; 1072 } else { 1073 pr_err("Ignoring paranoid; depends on fineibt.\n"); 1074 } 1075 } else if (!strcmp(str, "bhi")) { 1076 #ifdef CONFIG_FINEIBT_BHI 1077 if (cfi_mode == CFI_FINEIBT) { 1078 cfi_bhi = true; 1079 } else { 1080 pr_err("Ignoring bhi; depends on fineibt.\n"); 1081 } 1082 #else 1083 pr_err("Ignoring bhi; depends on FINEIBT_BHI=y.\n"); 1084 #endif 1085 } else { 1086 pr_err("Ignoring unknown cfi option (%s).", str); 1087 } 1088 1089 str = next; 1090 } 1091 1092 return 0; 1093 } 1094 early_param("cfi", cfi_parse_cmdline); 1095 1096 /* 1097 * kCFI FineIBT 1098 * 1099 * __cfi_\func: __cfi_\func: 1100 * movl $0x12345678,%eax // 5 endbr64 // 4 1101 * nop subl $0x12345678,%r10d // 7 1102 * nop jne __cfi_\func+6 // 2 1103 * nop nop3 // 3 1104 * nop 1105 * nop 1106 * nop 1107 * nop 1108 * nop 1109 * nop 1110 * nop 1111 * nop 1112 * 1113 * 1114 * caller: caller: 1115 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6 1116 * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4 1117 * je 1f // 2 nop4 // 4 1118 * ud2 // 2 1119 * 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6 1120 * 1121 */ 1122 1123 /* 1124 * <fineibt_preamble_start>: 1125 * 0: f3 0f 1e fa endbr64 1126 * 4: 41 81 <ea> 78 56 34 12 sub $0x12345678, %r10d 1127 * b: 75 f9 jne 6 <fineibt_preamble_start+0x6> 1128 * d: 0f 1f 00 nopl (%rax) 1129 * 1130 * Note that the JNE target is the 0xEA byte inside the SUB, this decodes as 1131 * (bad) on x86_64 and raises #UD. 1132 */ 1133 asm( ".pushsection .rodata \n" 1134 "fineibt_preamble_start: \n" 1135 " endbr64 \n" 1136 " subl $0x12345678, %r10d \n" 1137 "fineibt_preamble_bhi: \n" 1138 " jne fineibt_preamble_start+6 \n" 1139 ASM_NOP3 1140 "fineibt_preamble_end: \n" 1141 ".popsection\n" 1142 ); 1143 1144 extern u8 fineibt_preamble_start[]; 1145 extern u8 fineibt_preamble_bhi[]; 1146 extern u8 fineibt_preamble_end[]; 1147 1148 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) 1149 #define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) 1150 #define fineibt_preamble_ud 6 1151 #define fineibt_preamble_hash 7 1152 1153 /* 1154 * <fineibt_caller_start>: 1155 * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d 1156 * 6: 4d 8d 5b f0 lea -0x10(%r11), %r11 1157 * a: 0f 1f 40 00 nopl 0x0(%rax) 1158 */ 1159 asm( ".pushsection .rodata \n" 1160 "fineibt_caller_start: \n" 1161 " movl $0x12345678, %r10d \n" 1162 " lea -0x10(%r11), %r11 \n" 1163 ASM_NOP4 1164 "fineibt_caller_end: \n" 1165 ".popsection \n" 1166 ); 1167 1168 extern u8 fineibt_caller_start[]; 1169 extern u8 fineibt_caller_end[]; 1170 1171 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start) 1172 #define fineibt_caller_hash 2 1173 1174 #define fineibt_caller_jmp (fineibt_caller_size - 2) 1175 1176 /* 1177 * Since FineIBT does hash validation on the callee side it is prone to 1178 * circumvention attacks where a 'naked' ENDBR instruction exists that 1179 * is not part of the fineibt_preamble sequence. 1180 * 1181 * Notably the x86 entry points must be ENDBR and equally cannot be 1182 * fineibt_preamble. 1183 * 1184 * The fineibt_paranoid caller sequence adds additional caller side 1185 * hash validation. This stops such circumvention attacks dead, but at the cost 1186 * of adding a load. 1187 * 1188 * <fineibt_paranoid_start>: 1189 * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d 1190 * 6: 45 3b 53 f7 cmp -0x9(%r11), %r10d 1191 * a: 4d 8d 5b <f0> lea -0x10(%r11), %r11 1192 * e: 75 fd jne d <fineibt_paranoid_start+0xd> 1193 * 10: 41 ff d3 call *%r11 1194 * 13: 90 nop 1195 * 1196 * Notably LEA does not modify flags and can be reordered with the CMP, 1197 * avoiding a dependency. Again, using a non-taken (backwards) branch 1198 * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for the 1199 * Jcc.d8, causing #UD. 1200 */ 1201 asm( ".pushsection .rodata \n" 1202 "fineibt_paranoid_start: \n" 1203 " movl $0x12345678, %r10d \n" 1204 " cmpl -9(%r11), %r10d \n" 1205 " lea -0x10(%r11), %r11 \n" 1206 " jne fineibt_paranoid_start+0xd \n" 1207 "fineibt_paranoid_ind: \n" 1208 " call *%r11 \n" 1209 " nop \n" 1210 "fineibt_paranoid_end: \n" 1211 ".popsection \n" 1212 ); 1213 1214 extern u8 fineibt_paranoid_start[]; 1215 extern u8 fineibt_paranoid_ind[]; 1216 extern u8 fineibt_paranoid_end[]; 1217 1218 #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start) 1219 #define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_start) 1220 #define fineibt_paranoid_ud 0xd 1221 1222 static u32 decode_preamble_hash(void *addr, int *reg) 1223 { 1224 u8 *p = addr; 1225 1226 /* b8+reg 78 56 34 12 movl $0x12345678,\reg */ 1227 if (p[0] >= 0xb8 && p[0] < 0xc0) { 1228 if (reg) 1229 *reg = p[0] - 0xb8; 1230 return *(u32 *)(addr + 1); 1231 } 1232 1233 return 0; /* invalid hash value */ 1234 } 1235 1236 static u32 decode_caller_hash(void *addr) 1237 { 1238 u8 *p = addr; 1239 1240 /* 41 ba 88 a9 cb ed mov $(-0x12345678),%r10d */ 1241 if (p[0] == 0x41 && p[1] == 0xba) 1242 return -*(u32 *)(addr + 2); 1243 1244 /* e8 0c 88 a9 cb ed jmp.d8 +12 */ 1245 if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp) 1246 return -*(u32 *)(addr + 2); 1247 1248 return 0; /* invalid hash value */ 1249 } 1250 1251 /* .retpoline_sites */ 1252 static int cfi_disable_callers(s32 *start, s32 *end) 1253 { 1254 /* 1255 * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate 1256 * in tact for later usage. Also see decode_caller_hash() and 1257 * cfi_rewrite_callers(). 1258 */ 1259 const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp }; 1260 s32 *s; 1261 1262 for (s = start; s < end; s++) { 1263 void *addr = (void *)s + *s; 1264 u32 hash; 1265 1266 addr -= fineibt_caller_size; 1267 hash = decode_caller_hash(addr); 1268 if (!hash) /* nocfi callers */ 1269 continue; 1270 1271 text_poke_early(addr, jmp, 2); 1272 } 1273 1274 return 0; 1275 } 1276 1277 static int cfi_enable_callers(s32 *start, s32 *end) 1278 { 1279 /* 1280 * Re-enable kCFI, undo what cfi_disable_callers() did. 1281 */ 1282 const u8 mov[] = { 0x41, 0xba }; 1283 s32 *s; 1284 1285 for (s = start; s < end; s++) { 1286 void *addr = (void *)s + *s; 1287 u32 hash; 1288 1289 addr -= fineibt_caller_size; 1290 hash = decode_caller_hash(addr); 1291 if (!hash) /* nocfi callers */ 1292 continue; 1293 1294 text_poke_early(addr, mov, 2); 1295 } 1296 1297 return 0; 1298 } 1299 1300 /* .cfi_sites */ 1301 static int cfi_rand_preamble(s32 *start, s32 *end) 1302 { 1303 s32 *s; 1304 1305 for (s = start; s < end; s++) { 1306 void *addr = (void *)s + *s; 1307 u32 hash; 1308 1309 hash = decode_preamble_hash(addr, NULL); 1310 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", 1311 addr, addr, 5, addr)) 1312 return -EINVAL; 1313 1314 hash = cfi_rehash(hash); 1315 text_poke_early(addr + 1, &hash, 4); 1316 } 1317 1318 return 0; 1319 } 1320 1321 static void cfi_fineibt_bhi_preamble(void *addr, int arity) 1322 { 1323 if (!arity) 1324 return; 1325 1326 if (!cfi_warn && arity == 1) { 1327 /* 1328 * Crazy scheme to allow arity-1 inline: 1329 * 1330 * __cfi_foo: 1331 * 0: f3 0f 1e fa endbr64 1332 * 4: 41 81 <ea> 78 56 34 12 sub 0x12345678, %r10d 1333 * b: 49 0f 45 fa cmovne %r10, %rdi 1334 * f: 75 f5 jne __cfi_foo+6 1335 * 11: 0f 1f 00 nopl (%rax) 1336 * 1337 * Code that direct calls to foo()+0, decodes the tail end as: 1338 * 1339 * foo: 1340 * 0: f5 cmc 1341 * 1: 0f 1f 00 nopl (%rax) 1342 * 1343 * which clobbers CF, but does not affect anything ABI 1344 * wise. 1345 * 1346 * Notably, this scheme is incompatible with permissive CFI 1347 * because the CMOVcc is unconditional and RDI will have been 1348 * clobbered. 1349 */ 1350 const u8 magic[9] = { 1351 0x49, 0x0f, 0x45, 0xfa, 1352 0x75, 0xf5, 1353 BYTES_NOP3, 1354 }; 1355 1356 text_poke_early(addr + fineibt_preamble_bhi, magic, 9); 1357 1358 return; 1359 } 1360 1361 text_poke_early(addr + fineibt_preamble_bhi, 1362 text_gen_insn(CALL_INSN_OPCODE, 1363 addr + fineibt_preamble_bhi, 1364 __bhi_args[arity]), 1365 CALL_INSN_SIZE); 1366 } 1367 1368 static int cfi_rewrite_preamble(s32 *start, s32 *end) 1369 { 1370 s32 *s; 1371 1372 for (s = start; s < end; s++) { 1373 void *addr = (void *)s + *s; 1374 int arity; 1375 u32 hash; 1376 1377 /* 1378 * When the function doesn't start with ENDBR the compiler will 1379 * have determined there are no indirect calls to it and we 1380 * don't need no CFI either. 1381 */ 1382 if (!is_endbr(addr + 16)) 1383 continue; 1384 1385 hash = decode_preamble_hash(addr, &arity); 1386 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", 1387 addr, addr, 5, addr)) 1388 return -EINVAL; 1389 1390 text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); 1391 WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678); 1392 text_poke_early(addr + fineibt_preamble_hash, &hash, 4); 1393 1394 WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity, 1395 "kCFI preamble has wrong register at: %pS %*ph\n", 1396 addr, 5, addr); 1397 1398 if (cfi_bhi) 1399 cfi_fineibt_bhi_preamble(addr, arity); 1400 } 1401 1402 return 0; 1403 } 1404 1405 static void cfi_rewrite_endbr(s32 *start, s32 *end) 1406 { 1407 s32 *s; 1408 1409 for (s = start; s < end; s++) { 1410 void *addr = (void *)s + *s; 1411 1412 if (!exact_endbr(addr + 16)) 1413 continue; 1414 1415 poison_endbr(addr + 16); 1416 } 1417 } 1418 1419 /* .retpoline_sites */ 1420 static int cfi_rand_callers(s32 *start, s32 *end) 1421 { 1422 s32 *s; 1423 1424 for (s = start; s < end; s++) { 1425 void *addr = (void *)s + *s; 1426 u32 hash; 1427 1428 addr -= fineibt_caller_size; 1429 hash = decode_caller_hash(addr); 1430 if (hash) { 1431 hash = -cfi_rehash(hash); 1432 text_poke_early(addr + 2, &hash, 4); 1433 } 1434 } 1435 1436 return 0; 1437 } 1438 1439 static int cfi_rewrite_callers(s32 *start, s32 *end) 1440 { 1441 s32 *s; 1442 1443 BUG_ON(fineibt_paranoid_size != 20); 1444 1445 for (s = start; s < end; s++) { 1446 void *addr = (void *)s + *s; 1447 struct insn insn; 1448 u8 bytes[20]; 1449 u32 hash; 1450 int ret; 1451 u8 op; 1452 1453 addr -= fineibt_caller_size; 1454 hash = decode_caller_hash(addr); 1455 if (!hash) 1456 continue; 1457 1458 if (!cfi_paranoid) { 1459 text_poke_early(addr, fineibt_caller_start, fineibt_caller_size); 1460 WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678); 1461 text_poke_early(addr + fineibt_caller_hash, &hash, 4); 1462 /* rely on apply_retpolines() */ 1463 continue; 1464 } 1465 1466 /* cfi_paranoid */ 1467 ret = insn_decode_kernel(&insn, addr + fineibt_caller_size); 1468 if (WARN_ON_ONCE(ret < 0)) 1469 continue; 1470 1471 op = insn.opcode.bytes[0]; 1472 if (op != CALL_INSN_OPCODE && op != JMP32_INSN_OPCODE) { 1473 WARN_ON_ONCE(1); 1474 continue; 1475 } 1476 1477 memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size); 1478 memcpy(bytes + fineibt_caller_hash, &hash, 4); 1479 1480 ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind); 1481 if (WARN_ON_ONCE(ret != 3)) 1482 continue; 1483 1484 text_poke_early(addr, bytes, fineibt_paranoid_size); 1485 } 1486 1487 return 0; 1488 } 1489 1490 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 1491 s32 *start_cfi, s32 *end_cfi, bool builtin) 1492 { 1493 int ret; 1494 1495 if (WARN_ONCE(fineibt_preamble_size != 16, 1496 "FineIBT preamble wrong size: %ld", fineibt_preamble_size)) 1497 return; 1498 1499 if (cfi_mode == CFI_AUTO) { 1500 cfi_mode = CFI_KCFI; 1501 if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) { 1502 /* 1503 * FRED has much saner context on exception entry and 1504 * is less easy to take advantage of. 1505 */ 1506 if (!cpu_feature_enabled(X86_FEATURE_FRED)) 1507 cfi_paranoid = true; 1508 cfi_mode = CFI_FINEIBT; 1509 } 1510 } 1511 1512 /* 1513 * Rewrite the callers to not use the __cfi_ stubs, such that we might 1514 * rewrite them. This disables all CFI. If this succeeds but any of the 1515 * later stages fails, we're without CFI. 1516 */ 1517 ret = cfi_disable_callers(start_retpoline, end_retpoline); 1518 if (ret) 1519 goto err; 1520 1521 if (cfi_rand) { 1522 if (builtin) { 1523 cfi_seed = get_random_u32(); 1524 cfi_bpf_hash = cfi_rehash(cfi_bpf_hash); 1525 cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash); 1526 } 1527 1528 ret = cfi_rand_preamble(start_cfi, end_cfi); 1529 if (ret) 1530 goto err; 1531 1532 ret = cfi_rand_callers(start_retpoline, end_retpoline); 1533 if (ret) 1534 goto err; 1535 } 1536 1537 switch (cfi_mode) { 1538 case CFI_OFF: 1539 if (builtin) 1540 pr_info("Disabling CFI\n"); 1541 return; 1542 1543 case CFI_KCFI: 1544 ret = cfi_enable_callers(start_retpoline, end_retpoline); 1545 if (ret) 1546 goto err; 1547 1548 if (builtin) 1549 pr_info("Using kCFI\n"); 1550 return; 1551 1552 case CFI_FINEIBT: 1553 /* place the FineIBT preamble at func()-16 */ 1554 ret = cfi_rewrite_preamble(start_cfi, end_cfi); 1555 if (ret) 1556 goto err; 1557 1558 /* rewrite the callers to target func()-16 */ 1559 ret = cfi_rewrite_callers(start_retpoline, end_retpoline); 1560 if (ret) 1561 goto err; 1562 1563 /* now that nobody targets func()+0, remove ENDBR there */ 1564 cfi_rewrite_endbr(start_cfi, end_cfi); 1565 1566 if (builtin) { 1567 pr_info("Using %sFineIBT%s CFI\n", 1568 cfi_paranoid ? "paranoid " : "", 1569 cfi_bhi ? "+BHI" : ""); 1570 } 1571 return; 1572 1573 default: 1574 break; 1575 } 1576 1577 err: 1578 pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n"); 1579 } 1580 1581 static inline void poison_hash(void *addr) 1582 { 1583 *(u32 *)addr = 0; 1584 } 1585 1586 static void poison_cfi(void *addr) 1587 { 1588 /* 1589 * Compilers manage to be inconsistent with ENDBR vs __cfi prefixes, 1590 * some (static) functions for which they can determine the address 1591 * is never taken do not get a __cfi prefix, but *DO* get an ENDBR. 1592 * 1593 * As such, these functions will get sealed, but we need to be careful 1594 * to not unconditionally scribble the previous function. 1595 */ 1596 switch (cfi_mode) { 1597 case CFI_FINEIBT: 1598 /* 1599 * FineIBT prefix should start with an ENDBR. 1600 */ 1601 if (!is_endbr(addr)) 1602 break; 1603 1604 /* 1605 * __cfi_\func: 1606 * osp nopl (%rax) 1607 * subl $0, %r10d 1608 * jz 1f 1609 * ud2 1610 * 1: nop 1611 */ 1612 poison_endbr(addr); 1613 poison_hash(addr + fineibt_preamble_hash); 1614 break; 1615 1616 case CFI_KCFI: 1617 /* 1618 * kCFI prefix should start with a valid hash. 1619 */ 1620 if (!decode_preamble_hash(addr, NULL)) 1621 break; 1622 1623 /* 1624 * __cfi_\func: 1625 * movl $0, %eax 1626 * .skip 11, 0x90 1627 */ 1628 poison_hash(addr + 1); 1629 break; 1630 1631 default: 1632 break; 1633 } 1634 } 1635 1636 /* 1637 * When regs->ip points to a 0xEA byte in the FineIBT preamble, 1638 * return true and fill out target and type. 1639 * 1640 * We check the preamble by checking for the ENDBR instruction relative to the 1641 * 0xEA instruction. 1642 */ 1643 static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type) 1644 { 1645 unsigned long addr = regs->ip - fineibt_preamble_ud; 1646 u32 hash; 1647 1648 if (!exact_endbr((void *)addr)) 1649 return false; 1650 1651 *target = addr + fineibt_preamble_size; 1652 1653 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); 1654 *type = (u32)regs->r10 + hash; 1655 1656 /* 1657 * Since regs->ip points to the middle of an instruction; it cannot 1658 * continue with the normal fixup. 1659 */ 1660 regs->ip = *target; 1661 1662 return true; 1663 1664 Efault: 1665 return false; 1666 } 1667 1668 /* 1669 * regs->ip points to one of the UD2 in __bhi_args[]. 1670 */ 1671 static bool decode_fineibt_bhi(struct pt_regs *regs, unsigned long *target, u32 *type) 1672 { 1673 unsigned long addr; 1674 u32 hash; 1675 1676 if (!cfi_bhi) 1677 return false; 1678 1679 if (regs->ip < (unsigned long)__bhi_args || 1680 regs->ip >= (unsigned long)__bhi_args_end) 1681 return false; 1682 1683 /* 1684 * Fetch the return address from the stack, this points to the 1685 * FineIBT preamble. Since the CALL instruction is in the 5 last 1686 * bytes of the preamble, the return address is in fact the target 1687 * address. 1688 */ 1689 __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault); 1690 *target = addr; 1691 1692 addr -= fineibt_preamble_size; 1693 if (!exact_endbr((void *)addr)) 1694 return false; 1695 1696 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); 1697 *type = (u32)regs->r10 + hash; 1698 1699 /* 1700 * The UD2 sites are constructed with a RET immediately following, 1701 * as such the non-fatal case can use the regular fixup. 1702 */ 1703 return true; 1704 1705 Efault: 1706 return false; 1707 } 1708 1709 /* 1710 * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[] 1711 * sequence. 1712 */ 1713 static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type) 1714 { 1715 unsigned long addr = regs->ip - fineibt_paranoid_ud; 1716 u32 hash; 1717 1718 if (!cfi_paranoid || !is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) 1719 return false; 1720 1721 __get_kernel_nofault(&hash, addr + fineibt_caller_hash, u32, Efault); 1722 *target = regs->r11 + fineibt_preamble_size; 1723 *type = regs->r10; 1724 1725 /* 1726 * Since the trapping instruction is the exact, but LOCK prefixed, 1727 * Jcc.d8 that got us here, the normal fixup will work. 1728 */ 1729 return true; 1730 1731 Efault: 1732 return false; 1733 } 1734 1735 bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) 1736 { 1737 if (decode_fineibt_paranoid(regs, target, type)) 1738 return true; 1739 1740 if (decode_fineibt_bhi(regs, target, type)) 1741 return true; 1742 1743 return decode_fineibt_preamble(regs, target, type); 1744 } 1745 1746 #else /* !CONFIG_FINEIBT: */ 1747 1748 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 1749 s32 *start_cfi, s32 *end_cfi, bool builtin) 1750 { 1751 } 1752 1753 #ifdef CONFIG_X86_KERNEL_IBT 1754 static void poison_cfi(void *addr) { } 1755 #endif 1756 1757 #endif /* !CONFIG_FINEIBT */ 1758 1759 void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 1760 s32 *start_cfi, s32 *end_cfi) 1761 { 1762 return __apply_fineibt(start_retpoline, end_retpoline, 1763 start_cfi, end_cfi, 1764 /* .builtin = */ false); 1765 } 1766 1767 #ifdef CONFIG_SMP 1768 static void alternatives_smp_lock(const s32 *start, const s32 *end, 1769 u8 *text, u8 *text_end) 1770 { 1771 const s32 *poff; 1772 1773 for (poff = start; poff < end; poff++) { 1774 u8 *ptr = (u8 *)poff + *poff; 1775 1776 if (!*poff || ptr < text || ptr >= text_end) 1777 continue; 1778 /* turn DS segment override prefix into lock prefix */ 1779 if (*ptr == 0x3e) 1780 text_poke(ptr, ((unsigned char []){0xf0}), 1); 1781 } 1782 } 1783 1784 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 1785 u8 *text, u8 *text_end) 1786 { 1787 const s32 *poff; 1788 1789 for (poff = start; poff < end; poff++) { 1790 u8 *ptr = (u8 *)poff + *poff; 1791 1792 if (!*poff || ptr < text || ptr >= text_end) 1793 continue; 1794 /* turn lock prefix into DS segment override prefix */ 1795 if (*ptr == 0xf0) 1796 text_poke(ptr, ((unsigned char []){0x3E}), 1); 1797 } 1798 } 1799 1800 struct smp_alt_module { 1801 /* what is this ??? */ 1802 struct module *mod; 1803 char *name; 1804 1805 /* ptrs to lock prefixes */ 1806 const s32 *locks; 1807 const s32 *locks_end; 1808 1809 /* .text segment, needed to avoid patching init code ;) */ 1810 u8 *text; 1811 u8 *text_end; 1812 1813 struct list_head next; 1814 }; 1815 static LIST_HEAD(smp_alt_modules); 1816 static bool uniproc_patched = false; /* protected by text_mutex */ 1817 1818 void __init_or_module alternatives_smp_module_add(struct module *mod, 1819 char *name, 1820 void *locks, void *locks_end, 1821 void *text, void *text_end) 1822 { 1823 struct smp_alt_module *smp; 1824 1825 mutex_lock(&text_mutex); 1826 if (!uniproc_patched) 1827 goto unlock; 1828 1829 if (num_possible_cpus() == 1) 1830 /* Don't bother remembering, we'll never have to undo it. */ 1831 goto smp_unlock; 1832 1833 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 1834 if (NULL == smp) 1835 /* we'll run the (safe but slow) SMP code then ... */ 1836 goto unlock; 1837 1838 smp->mod = mod; 1839 smp->name = name; 1840 smp->locks = locks; 1841 smp->locks_end = locks_end; 1842 smp->text = text; 1843 smp->text_end = text_end; 1844 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", 1845 smp->locks, smp->locks_end, 1846 smp->text, smp->text_end, smp->name); 1847 1848 list_add_tail(&smp->next, &smp_alt_modules); 1849 smp_unlock: 1850 alternatives_smp_unlock(locks, locks_end, text, text_end); 1851 unlock: 1852 mutex_unlock(&text_mutex); 1853 } 1854 1855 void __init_or_module alternatives_smp_module_del(struct module *mod) 1856 { 1857 struct smp_alt_module *item; 1858 1859 mutex_lock(&text_mutex); 1860 list_for_each_entry(item, &smp_alt_modules, next) { 1861 if (mod != item->mod) 1862 continue; 1863 list_del(&item->next); 1864 kfree(item); 1865 break; 1866 } 1867 mutex_unlock(&text_mutex); 1868 } 1869 1870 void alternatives_enable_smp(void) 1871 { 1872 struct smp_alt_module *mod; 1873 1874 /* Why bother if there are no other CPUs? */ 1875 BUG_ON(num_possible_cpus() == 1); 1876 1877 mutex_lock(&text_mutex); 1878 1879 if (uniproc_patched) { 1880 pr_info("switching to SMP code\n"); 1881 BUG_ON(num_online_cpus() != 1); 1882 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 1883 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 1884 list_for_each_entry(mod, &smp_alt_modules, next) 1885 alternatives_smp_lock(mod->locks, mod->locks_end, 1886 mod->text, mod->text_end); 1887 uniproc_patched = false; 1888 } 1889 mutex_unlock(&text_mutex); 1890 } 1891 1892 /* 1893 * Return 1 if the address range is reserved for SMP-alternatives. 1894 * Must hold text_mutex. 1895 */ 1896 int alternatives_text_reserved(void *start, void *end) 1897 { 1898 struct smp_alt_module *mod; 1899 const s32 *poff; 1900 u8 *text_start = start; 1901 u8 *text_end = end; 1902 1903 lockdep_assert_held(&text_mutex); 1904 1905 list_for_each_entry(mod, &smp_alt_modules, next) { 1906 if (mod->text > text_end || mod->text_end < text_start) 1907 continue; 1908 for (poff = mod->locks; poff < mod->locks_end; poff++) { 1909 const u8 *ptr = (const u8 *)poff + *poff; 1910 1911 if (text_start <= ptr && text_end > ptr) 1912 return 1; 1913 } 1914 } 1915 1916 return 0; 1917 } 1918 #endif /* CONFIG_SMP */ 1919 1920 /* 1921 * Self-test for the INT3 based CALL emulation code. 1922 * 1923 * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up 1924 * properly and that there is a stack gap between the INT3 frame and the 1925 * previous context. Without this gap doing a virtual PUSH on the interrupted 1926 * stack would corrupt the INT3 IRET frame. 1927 * 1928 * See entry_{32,64}.S for more details. 1929 */ 1930 1931 /* 1932 * We define the int3_magic() function in assembly to control the calling 1933 * convention such that we can 'call' it from assembly. 1934 */ 1935 1936 extern void int3_magic(unsigned int *ptr); /* defined in asm */ 1937 1938 asm ( 1939 " .pushsection .init.text, \"ax\", @progbits\n" 1940 " .type int3_magic, @function\n" 1941 "int3_magic:\n" 1942 ANNOTATE_NOENDBR 1943 " movl $1, (%" _ASM_ARG1 ")\n" 1944 ASM_RET 1945 " .size int3_magic, .-int3_magic\n" 1946 " .popsection\n" 1947 ); 1948 1949 extern void int3_selftest_ip(void); /* defined in asm below */ 1950 1951 static int __init 1952 int3_exception_notify(struct notifier_block *self, unsigned long val, void *data) 1953 { 1954 unsigned long selftest = (unsigned long)&int3_selftest_ip; 1955 struct die_args *args = data; 1956 struct pt_regs *regs = args->regs; 1957 1958 OPTIMIZER_HIDE_VAR(selftest); 1959 1960 if (!regs || user_mode(regs)) 1961 return NOTIFY_DONE; 1962 1963 if (val != DIE_INT3) 1964 return NOTIFY_DONE; 1965 1966 if (regs->ip - INT3_INSN_SIZE != selftest) 1967 return NOTIFY_DONE; 1968 1969 int3_emulate_call(regs, (unsigned long)&int3_magic); 1970 return NOTIFY_STOP; 1971 } 1972 1973 /* Must be noinline to ensure uniqueness of int3_selftest_ip. */ 1974 static noinline void __init int3_selftest(void) 1975 { 1976 static __initdata struct notifier_block int3_exception_nb = { 1977 .notifier_call = int3_exception_notify, 1978 .priority = INT_MAX-1, /* last */ 1979 }; 1980 unsigned int val = 0; 1981 1982 BUG_ON(register_die_notifier(&int3_exception_nb)); 1983 1984 /* 1985 * Basically: int3_magic(&val); but really complicated :-) 1986 * 1987 * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb 1988 * notifier above will emulate CALL for us. 1989 */ 1990 asm volatile ("int3_selftest_ip:\n\t" 1991 ANNOTATE_NOENDBR 1992 " int3; nop; nop; nop; nop\n\t" 1993 : ASM_CALL_CONSTRAINT 1994 : __ASM_SEL_RAW(a, D) (&val) 1995 : "memory"); 1996 1997 BUG_ON(val != 1); 1998 1999 unregister_die_notifier(&int3_exception_nb); 2000 } 2001 2002 static __initdata int __alt_reloc_selftest_addr; 2003 2004 extern void __init __alt_reloc_selftest(void *arg); 2005 __visible noinline void __init __alt_reloc_selftest(void *arg) 2006 { 2007 WARN_ON(arg != &__alt_reloc_selftest_addr); 2008 } 2009 2010 static noinline void __init alt_reloc_selftest(void) 2011 { 2012 /* 2013 * Tests apply_relocation(). 2014 * 2015 * This has a relative immediate (CALL) in a place other than the first 2016 * instruction and additionally on x86_64 we get a RIP-relative LEA: 2017 * 2018 * lea 0x0(%rip),%rdi # 5d0: R_X86_64_PC32 .init.data+0x5566c 2019 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4 2020 * 2021 * Getting this wrong will either crash and burn or tickle the WARN 2022 * above. 2023 */ 2024 asm_inline volatile ( 2025 ALTERNATIVE("", "lea %[mem], %%" _ASM_ARG1 "; call __alt_reloc_selftest;", X86_FEATURE_ALWAYS) 2026 : ASM_CALL_CONSTRAINT 2027 : [mem] "m" (__alt_reloc_selftest_addr) 2028 : _ASM_ARG1 2029 ); 2030 } 2031 2032 void __init alternative_instructions(void) 2033 { 2034 int3_selftest(); 2035 2036 /* 2037 * The patching is not fully atomic, so try to avoid local 2038 * interruptions that might execute the to be patched code. 2039 * Other CPUs are not running. 2040 */ 2041 stop_nmi(); 2042 2043 /* 2044 * Don't stop machine check exceptions while patching. 2045 * MCEs only happen when something got corrupted and in this 2046 * case we must do something about the corruption. 2047 * Ignoring it is worse than an unlikely patching race. 2048 * Also machine checks tend to be broadcast and if one CPU 2049 * goes into machine check the others follow quickly, so we don't 2050 * expect a machine check to cause undue problems during to code 2051 * patching. 2052 */ 2053 2054 /* 2055 * Make sure to set (artificial) features depending on used paravirt 2056 * functions which can later influence alternative patching. 2057 */ 2058 paravirt_set_cap(); 2059 2060 __apply_fineibt(__retpoline_sites, __retpoline_sites_end, 2061 __cfi_sites, __cfi_sites_end, true); 2062 2063 /* 2064 * Rewrite the retpolines, must be done before alternatives since 2065 * those can rewrite the retpoline thunks. 2066 */ 2067 apply_retpolines(__retpoline_sites, __retpoline_sites_end); 2068 apply_returns(__return_sites, __return_sites_end); 2069 2070 /* 2071 * Adjust all CALL instructions to point to func()-10, including 2072 * those in .altinstr_replacement. 2073 */ 2074 callthunks_patch_builtin_calls(); 2075 2076 apply_alternatives(__alt_instructions, __alt_instructions_end); 2077 2078 /* 2079 * Seal all functions that do not have their address taken. 2080 */ 2081 apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); 2082 2083 #ifdef CONFIG_SMP 2084 /* Patch to UP if other cpus not imminent. */ 2085 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { 2086 uniproc_patched = true; 2087 alternatives_smp_module_add(NULL, "core kernel", 2088 __smp_locks, __smp_locks_end, 2089 _text, _etext); 2090 } 2091 2092 if (!uniproc_patched || num_possible_cpus() == 1) { 2093 free_init_pages("SMP alternatives", 2094 (unsigned long)__smp_locks, 2095 (unsigned long)__smp_locks_end); 2096 } 2097 #endif 2098 2099 restart_nmi(); 2100 alternatives_patched = 1; 2101 2102 alt_reloc_selftest(); 2103 } 2104 2105 /** 2106 * text_poke_early - Update instructions on a live kernel at boot time 2107 * @addr: address to modify 2108 * @opcode: source of the copy 2109 * @len: length to copy 2110 * 2111 * When you use this code to patch more than one byte of an instruction 2112 * you need to make sure that other CPUs cannot execute this code in parallel. 2113 * Also no thread must be currently preempted in the middle of these 2114 * instructions. And on the local CPU you need to be protected against NMI or 2115 * MCE handlers seeing an inconsistent instruction while you patch. 2116 */ 2117 void __init_or_module text_poke_early(void *addr, const void *opcode, 2118 size_t len) 2119 { 2120 unsigned long flags; 2121 2122 if (boot_cpu_has(X86_FEATURE_NX) && 2123 is_module_text_address((unsigned long)addr)) { 2124 /* 2125 * Modules text is marked initially as non-executable, so the 2126 * code cannot be running and speculative code-fetches are 2127 * prevented. Just change the code. 2128 */ 2129 memcpy(addr, opcode, len); 2130 } else { 2131 local_irq_save(flags); 2132 memcpy(addr, opcode, len); 2133 sync_core(); 2134 local_irq_restore(flags); 2135 2136 /* 2137 * Could also do a CLFLUSH here to speed up CPU recovery; but 2138 * that causes hangs on some VIA CPUs. 2139 */ 2140 } 2141 } 2142 2143 typedef struct { 2144 struct mm_struct *mm; 2145 } temp_mm_state_t; 2146 2147 /* 2148 * Using a temporary mm allows to set temporary mappings that are not accessible 2149 * by other CPUs. Such mappings are needed to perform sensitive memory writes 2150 * that override the kernel memory protections (e.g., W^X), without exposing the 2151 * temporary page-table mappings that are required for these write operations to 2152 * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the 2153 * mapping is torn down. 2154 * 2155 * Context: The temporary mm needs to be used exclusively by a single core. To 2156 * harden security IRQs must be disabled while the temporary mm is 2157 * loaded, thereby preventing interrupt handler bugs from overriding 2158 * the kernel memory protection. 2159 */ 2160 static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) 2161 { 2162 temp_mm_state_t temp_state; 2163 2164 lockdep_assert_irqs_disabled(); 2165 2166 /* 2167 * Make sure not to be in TLB lazy mode, as otherwise we'll end up 2168 * with a stale address space WITHOUT being in lazy mode after 2169 * restoring the previous mm. 2170 */ 2171 if (this_cpu_read(cpu_tlbstate_shared.is_lazy)) 2172 leave_mm(); 2173 2174 temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm); 2175 switch_mm_irqs_off(NULL, mm, current); 2176 2177 /* 2178 * If breakpoints are enabled, disable them while the temporary mm is 2179 * used. Userspace might set up watchpoints on addresses that are used 2180 * in the temporary mm, which would lead to wrong signals being sent or 2181 * crashes. 2182 * 2183 * Note that breakpoints are not disabled selectively, which also causes 2184 * kernel breakpoints (e.g., perf's) to be disabled. This might be 2185 * undesirable, but still seems reasonable as the code that runs in the 2186 * temporary mm should be short. 2187 */ 2188 if (hw_breakpoint_active()) 2189 hw_breakpoint_disable(); 2190 2191 return temp_state; 2192 } 2193 2194 __ro_after_init struct mm_struct *poking_mm; 2195 __ro_after_init unsigned long poking_addr; 2196 2197 static inline void unuse_temporary_mm(temp_mm_state_t prev_state) 2198 { 2199 lockdep_assert_irqs_disabled(); 2200 2201 switch_mm_irqs_off(NULL, prev_state.mm, current); 2202 2203 /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ 2204 cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(poking_mm)); 2205 2206 /* 2207 * Restore the breakpoints if they were disabled before the temporary mm 2208 * was loaded. 2209 */ 2210 if (hw_breakpoint_active()) 2211 hw_breakpoint_restore(); 2212 } 2213 2214 static void text_poke_memcpy(void *dst, const void *src, size_t len) 2215 { 2216 memcpy(dst, src, len); 2217 } 2218 2219 static void text_poke_memset(void *dst, const void *src, size_t len) 2220 { 2221 int c = *(const int *)src; 2222 2223 memset(dst, c, len); 2224 } 2225 2226 typedef void text_poke_f(void *dst, const void *src, size_t len); 2227 2228 static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) 2229 { 2230 bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; 2231 struct page *pages[2] = {NULL}; 2232 temp_mm_state_t prev; 2233 unsigned long flags; 2234 pte_t pte, *ptep; 2235 spinlock_t *ptl; 2236 pgprot_t pgprot; 2237 2238 /* 2239 * While boot memory allocator is running we cannot use struct pages as 2240 * they are not yet initialized. There is no way to recover. 2241 */ 2242 BUG_ON(!after_bootmem); 2243 2244 if (!core_kernel_text((unsigned long)addr)) { 2245 pages[0] = vmalloc_to_page(addr); 2246 if (cross_page_boundary) 2247 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 2248 } else { 2249 pages[0] = virt_to_page(addr); 2250 WARN_ON(!PageReserved(pages[0])); 2251 if (cross_page_boundary) 2252 pages[1] = virt_to_page(addr + PAGE_SIZE); 2253 } 2254 /* 2255 * If something went wrong, crash and burn since recovery paths are not 2256 * implemented. 2257 */ 2258 BUG_ON(!pages[0] || (cross_page_boundary && !pages[1])); 2259 2260 /* 2261 * Map the page without the global bit, as TLB flushing is done with 2262 * flush_tlb_mm_range(), which is intended for non-global PTEs. 2263 */ 2264 pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL); 2265 2266 /* 2267 * The lock is not really needed, but this allows to avoid open-coding. 2268 */ 2269 ptep = get_locked_pte(poking_mm, poking_addr, &ptl); 2270 2271 /* 2272 * This must not fail; preallocated in poking_init(). 2273 */ 2274 VM_BUG_ON(!ptep); 2275 2276 local_irq_save(flags); 2277 2278 pte = mk_pte(pages[0], pgprot); 2279 set_pte_at(poking_mm, poking_addr, ptep, pte); 2280 2281 if (cross_page_boundary) { 2282 pte = mk_pte(pages[1], pgprot); 2283 set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); 2284 } 2285 2286 /* 2287 * Loading the temporary mm behaves as a compiler barrier, which 2288 * guarantees that the PTE will be set at the time memcpy() is done. 2289 */ 2290 prev = use_temporary_mm(poking_mm); 2291 2292 kasan_disable_current(); 2293 func((u8 *)poking_addr + offset_in_page(addr), src, len); 2294 kasan_enable_current(); 2295 2296 /* 2297 * Ensure that the PTE is only cleared after the instructions of memcpy 2298 * were issued by using a compiler barrier. 2299 */ 2300 barrier(); 2301 2302 pte_clear(poking_mm, poking_addr, ptep); 2303 if (cross_page_boundary) 2304 pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1); 2305 2306 /* 2307 * Loading the previous page-table hierarchy requires a serializing 2308 * instruction that already allows the core to see the updated version. 2309 * Xen-PV is assumed to serialize execution in a similar manner. 2310 */ 2311 unuse_temporary_mm(prev); 2312 2313 /* 2314 * Flushing the TLB might involve IPIs, which would require enabled 2315 * IRQs, but not if the mm is not used, as it is in this point. 2316 */ 2317 flush_tlb_mm_range(poking_mm, poking_addr, poking_addr + 2318 (cross_page_boundary ? 2 : 1) * PAGE_SIZE, 2319 PAGE_SHIFT, false); 2320 2321 if (func == text_poke_memcpy) { 2322 /* 2323 * If the text does not match what we just wrote then something is 2324 * fundamentally screwy; there's nothing we can really do about that. 2325 */ 2326 BUG_ON(memcmp(addr, src, len)); 2327 } 2328 2329 local_irq_restore(flags); 2330 pte_unmap_unlock(ptep, ptl); 2331 return addr; 2332 } 2333 2334 /** 2335 * text_poke - Update instructions on a live kernel 2336 * @addr: address to modify 2337 * @opcode: source of the copy 2338 * @len: length to copy 2339 * 2340 * Only atomic text poke/set should be allowed when not doing early patching. 2341 * It means the size must be writable atomically and the address must be aligned 2342 * in a way that permits an atomic write. It also makes sure we fit on a single 2343 * page. 2344 * 2345 * Note that the caller must ensure that if the modified code is part of a 2346 * module, the module would not be removed during poking. This can be achieved 2347 * by registering a module notifier, and ordering module removal and patching 2348 * through a mutex. 2349 */ 2350 void *text_poke(void *addr, const void *opcode, size_t len) 2351 { 2352 lockdep_assert_held(&text_mutex); 2353 2354 return __text_poke(text_poke_memcpy, addr, opcode, len); 2355 } 2356 2357 /** 2358 * text_poke_kgdb - Update instructions on a live kernel by kgdb 2359 * @addr: address to modify 2360 * @opcode: source of the copy 2361 * @len: length to copy 2362 * 2363 * Only atomic text poke/set should be allowed when not doing early patching. 2364 * It means the size must be writable atomically and the address must be aligned 2365 * in a way that permits an atomic write. It also makes sure we fit on a single 2366 * page. 2367 * 2368 * Context: should only be used by kgdb, which ensures no other core is running, 2369 * despite the fact it does not hold the text_mutex. 2370 */ 2371 void *text_poke_kgdb(void *addr, const void *opcode, size_t len) 2372 { 2373 return __text_poke(text_poke_memcpy, addr, opcode, len); 2374 } 2375 2376 void *text_poke_copy_locked(void *addr, const void *opcode, size_t len, 2377 bool core_ok) 2378 { 2379 unsigned long start = (unsigned long)addr; 2380 size_t patched = 0; 2381 2382 if (WARN_ON_ONCE(!core_ok && core_kernel_text(start))) 2383 return NULL; 2384 2385 while (patched < len) { 2386 unsigned long ptr = start + patched; 2387 size_t s; 2388 2389 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); 2390 2391 __text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s); 2392 patched += s; 2393 } 2394 return addr; 2395 } 2396 2397 /** 2398 * text_poke_copy - Copy instructions into (an unused part of) RX memory 2399 * @addr: address to modify 2400 * @opcode: source of the copy 2401 * @len: length to copy, could be more than 2x PAGE_SIZE 2402 * 2403 * Not safe against concurrent execution; useful for JITs to dump 2404 * new code blocks into unused regions of RX memory. Can be used in 2405 * conjunction with synchronize_rcu_tasks() to wait for existing 2406 * execution to quiesce after having made sure no existing functions 2407 * pointers are live. 2408 */ 2409 void *text_poke_copy(void *addr, const void *opcode, size_t len) 2410 { 2411 mutex_lock(&text_mutex); 2412 addr = text_poke_copy_locked(addr, opcode, len, false); 2413 mutex_unlock(&text_mutex); 2414 return addr; 2415 } 2416 2417 /** 2418 * text_poke_set - memset into (an unused part of) RX memory 2419 * @addr: address to modify 2420 * @c: the byte to fill the area with 2421 * @len: length to copy, could be more than 2x PAGE_SIZE 2422 * 2423 * This is useful to overwrite unused regions of RX memory with illegal 2424 * instructions. 2425 */ 2426 void *text_poke_set(void *addr, int c, size_t len) 2427 { 2428 unsigned long start = (unsigned long)addr; 2429 size_t patched = 0; 2430 2431 if (WARN_ON_ONCE(core_kernel_text(start))) 2432 return NULL; 2433 2434 mutex_lock(&text_mutex); 2435 while (patched < len) { 2436 unsigned long ptr = start + patched; 2437 size_t s; 2438 2439 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); 2440 2441 __text_poke(text_poke_memset, (void *)ptr, (void *)&c, s); 2442 patched += s; 2443 } 2444 mutex_unlock(&text_mutex); 2445 return addr; 2446 } 2447 2448 static void do_sync_core(void *info) 2449 { 2450 sync_core(); 2451 } 2452 2453 void text_poke_sync(void) 2454 { 2455 on_each_cpu(do_sync_core, NULL, 1); 2456 } 2457 2458 /* 2459 * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of 2460 * this thing. When len == 6 everything is prefixed with 0x0f and we map 2461 * opcode to Jcc.d8, using len to distinguish. 2462 */ 2463 struct text_poke_loc { 2464 /* addr := _stext + rel_addr */ 2465 s32 rel_addr; 2466 s32 disp; 2467 u8 len; 2468 u8 opcode; 2469 const u8 text[POKE_MAX_OPCODE_SIZE]; 2470 /* see text_poke_bp_batch() */ 2471 u8 old; 2472 }; 2473 2474 struct bp_patching_desc { 2475 struct text_poke_loc *vec; 2476 int nr_entries; 2477 atomic_t refs; 2478 }; 2479 2480 static struct bp_patching_desc bp_desc; 2481 2482 static __always_inline 2483 struct bp_patching_desc *try_get_desc(void) 2484 { 2485 struct bp_patching_desc *desc = &bp_desc; 2486 2487 if (!raw_atomic_inc_not_zero(&desc->refs)) 2488 return NULL; 2489 2490 return desc; 2491 } 2492 2493 static __always_inline void put_desc(void) 2494 { 2495 struct bp_patching_desc *desc = &bp_desc; 2496 2497 smp_mb__before_atomic(); 2498 raw_atomic_dec(&desc->refs); 2499 } 2500 2501 static __always_inline void *text_poke_addr(struct text_poke_loc *tp) 2502 { 2503 return _stext + tp->rel_addr; 2504 } 2505 2506 static __always_inline int patch_cmp(const void *key, const void *elt) 2507 { 2508 struct text_poke_loc *tp = (struct text_poke_loc *) elt; 2509 2510 if (key < text_poke_addr(tp)) 2511 return -1; 2512 if (key > text_poke_addr(tp)) 2513 return 1; 2514 return 0; 2515 } 2516 2517 noinstr int poke_int3_handler(struct pt_regs *regs) 2518 { 2519 struct bp_patching_desc *desc; 2520 struct text_poke_loc *tp; 2521 int ret = 0; 2522 void *ip; 2523 2524 if (user_mode(regs)) 2525 return 0; 2526 2527 /* 2528 * Having observed our INT3 instruction, we now must observe 2529 * bp_desc with non-zero refcount: 2530 * 2531 * bp_desc.refs = 1 INT3 2532 * WMB RMB 2533 * write INT3 if (bp_desc.refs != 0) 2534 */ 2535 smp_rmb(); 2536 2537 desc = try_get_desc(); 2538 if (!desc) 2539 return 0; 2540 2541 /* 2542 * Discount the INT3. See text_poke_bp_batch(). 2543 */ 2544 ip = (void *) regs->ip - INT3_INSN_SIZE; 2545 2546 /* 2547 * Skip the binary search if there is a single member in the vector. 2548 */ 2549 if (unlikely(desc->nr_entries > 1)) { 2550 tp = __inline_bsearch(ip, desc->vec, desc->nr_entries, 2551 sizeof(struct text_poke_loc), 2552 patch_cmp); 2553 if (!tp) 2554 goto out_put; 2555 } else { 2556 tp = desc->vec; 2557 if (text_poke_addr(tp) != ip) 2558 goto out_put; 2559 } 2560 2561 ip += tp->len; 2562 2563 switch (tp->opcode) { 2564 case INT3_INSN_OPCODE: 2565 /* 2566 * Someone poked an explicit INT3, they'll want to handle it, 2567 * do not consume. 2568 */ 2569 goto out_put; 2570 2571 case RET_INSN_OPCODE: 2572 int3_emulate_ret(regs); 2573 break; 2574 2575 case CALL_INSN_OPCODE: 2576 int3_emulate_call(regs, (long)ip + tp->disp); 2577 break; 2578 2579 case JMP32_INSN_OPCODE: 2580 case JMP8_INSN_OPCODE: 2581 int3_emulate_jmp(regs, (long)ip + tp->disp); 2582 break; 2583 2584 case 0x70 ... 0x7f: /* Jcc */ 2585 int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp); 2586 break; 2587 2588 default: 2589 BUG(); 2590 } 2591 2592 ret = 1; 2593 2594 out_put: 2595 put_desc(); 2596 return ret; 2597 } 2598 2599 #define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc)) 2600 static struct text_poke_loc tp_vec[TP_VEC_MAX]; 2601 static int tp_vec_nr; 2602 2603 /** 2604 * text_poke_bp_batch() -- update instructions on live kernel on SMP 2605 * @tp: vector of instructions to patch 2606 * @nr_entries: number of entries in the vector 2607 * 2608 * Modify multi-byte instruction by using int3 breakpoint on SMP. 2609 * We completely avoid stop_machine() here, and achieve the 2610 * synchronization using int3 breakpoint. 2611 * 2612 * The way it is done: 2613 * - For each entry in the vector: 2614 * - add a int3 trap to the address that will be patched 2615 * - sync cores 2616 * - For each entry in the vector: 2617 * - update all but the first byte of the patched range 2618 * - sync cores 2619 * - For each entry in the vector: 2620 * - replace the first byte (int3) by the first byte of 2621 * replacing opcode 2622 * - sync cores 2623 */ 2624 static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries) 2625 { 2626 unsigned char int3 = INT3_INSN_OPCODE; 2627 unsigned int i; 2628 int do_sync; 2629 2630 lockdep_assert_held(&text_mutex); 2631 2632 bp_desc.vec = tp; 2633 bp_desc.nr_entries = nr_entries; 2634 2635 /* 2636 * Corresponds to the implicit memory barrier in try_get_desc() to 2637 * ensure reading a non-zero refcount provides up to date bp_desc data. 2638 */ 2639 atomic_set_release(&bp_desc.refs, 1); 2640 2641 /* 2642 * Function tracing can enable thousands of places that need to be 2643 * updated. This can take quite some time, and with full kernel debugging 2644 * enabled, this could cause the softlockup watchdog to trigger. 2645 * This function gets called every 256 entries added to be patched. 2646 * Call cond_resched() here to make sure that other tasks can get scheduled 2647 * while processing all the functions being patched. 2648 */ 2649 cond_resched(); 2650 2651 /* 2652 * Corresponding read barrier in int3 notifier for making sure the 2653 * nr_entries and handler are correctly ordered wrt. patching. 2654 */ 2655 smp_wmb(); 2656 2657 /* 2658 * First step: add a int3 trap to the address that will be patched. 2659 */ 2660 for (i = 0; i < nr_entries; i++) { 2661 tp[i].old = *(u8 *)text_poke_addr(&tp[i]); 2662 text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE); 2663 } 2664 2665 text_poke_sync(); 2666 2667 /* 2668 * Second step: update all but the first byte of the patched range. 2669 */ 2670 for (do_sync = 0, i = 0; i < nr_entries; i++) { 2671 u8 old[POKE_MAX_OPCODE_SIZE+1] = { tp[i].old, }; 2672 u8 _new[POKE_MAX_OPCODE_SIZE+1]; 2673 const u8 *new = tp[i].text; 2674 int len = tp[i].len; 2675 2676 if (len - INT3_INSN_SIZE > 0) { 2677 memcpy(old + INT3_INSN_SIZE, 2678 text_poke_addr(&tp[i]) + INT3_INSN_SIZE, 2679 len - INT3_INSN_SIZE); 2680 2681 if (len == 6) { 2682 _new[0] = 0x0f; 2683 memcpy(_new + 1, new, 5); 2684 new = _new; 2685 } 2686 2687 text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE, 2688 new + INT3_INSN_SIZE, 2689 len - INT3_INSN_SIZE); 2690 2691 do_sync++; 2692 } 2693 2694 /* 2695 * Emit a perf event to record the text poke, primarily to 2696 * support Intel PT decoding which must walk the executable code 2697 * to reconstruct the trace. The flow up to here is: 2698 * - write INT3 byte 2699 * - IPI-SYNC 2700 * - write instruction tail 2701 * At this point the actual control flow will be through the 2702 * INT3 and handler and not hit the old or new instruction. 2703 * Intel PT outputs FUP/TIP packets for the INT3, so the flow 2704 * can still be decoded. Subsequently: 2705 * - emit RECORD_TEXT_POKE with the new instruction 2706 * - IPI-SYNC 2707 * - write first byte 2708 * - IPI-SYNC 2709 * So before the text poke event timestamp, the decoder will see 2710 * either the old instruction flow or FUP/TIP of INT3. After the 2711 * text poke event timestamp, the decoder will see either the 2712 * new instruction flow or FUP/TIP of INT3. Thus decoders can 2713 * use the timestamp as the point at which to modify the 2714 * executable code. 2715 * The old instruction is recorded so that the event can be 2716 * processed forwards or backwards. 2717 */ 2718 perf_event_text_poke(text_poke_addr(&tp[i]), old, len, new, len); 2719 } 2720 2721 if (do_sync) { 2722 /* 2723 * According to Intel, this core syncing is very likely 2724 * not necessary and we'd be safe even without it. But 2725 * better safe than sorry (plus there's not only Intel). 2726 */ 2727 text_poke_sync(); 2728 } 2729 2730 /* 2731 * Third step: replace the first byte (int3) by the first byte of 2732 * replacing opcode. 2733 */ 2734 for (do_sync = 0, i = 0; i < nr_entries; i++) { 2735 u8 byte = tp[i].text[0]; 2736 2737 if (tp[i].len == 6) 2738 byte = 0x0f; 2739 2740 if (byte == INT3_INSN_OPCODE) 2741 continue; 2742 2743 text_poke(text_poke_addr(&tp[i]), &byte, INT3_INSN_SIZE); 2744 do_sync++; 2745 } 2746 2747 if (do_sync) 2748 text_poke_sync(); 2749 2750 /* 2751 * Remove and wait for refs to be zero. 2752 */ 2753 if (!atomic_dec_and_test(&bp_desc.refs)) 2754 atomic_cond_read_acquire(&bp_desc.refs, !VAL); 2755 } 2756 2757 static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, 2758 const void *opcode, size_t len, const void *emulate) 2759 { 2760 struct insn insn; 2761 int ret, i = 0; 2762 2763 if (len == 6) 2764 i = 1; 2765 memcpy((void *)tp->text, opcode+i, len-i); 2766 if (!emulate) 2767 emulate = opcode; 2768 2769 ret = insn_decode_kernel(&insn, emulate); 2770 BUG_ON(ret < 0); 2771 2772 tp->rel_addr = addr - (void *)_stext; 2773 tp->len = len; 2774 tp->opcode = insn.opcode.bytes[0]; 2775 2776 if (is_jcc32(&insn)) { 2777 /* 2778 * Map Jcc.d32 onto Jcc.d8 and use len to distinguish. 2779 */ 2780 tp->opcode = insn.opcode.bytes[1] - 0x10; 2781 } 2782 2783 switch (tp->opcode) { 2784 case RET_INSN_OPCODE: 2785 case JMP32_INSN_OPCODE: 2786 case JMP8_INSN_OPCODE: 2787 /* 2788 * Control flow instructions without implied execution of the 2789 * next instruction can be padded with INT3. 2790 */ 2791 for (i = insn.length; i < len; i++) 2792 BUG_ON(tp->text[i] != INT3_INSN_OPCODE); 2793 break; 2794 2795 default: 2796 BUG_ON(len != insn.length); 2797 } 2798 2799 switch (tp->opcode) { 2800 case INT3_INSN_OPCODE: 2801 case RET_INSN_OPCODE: 2802 break; 2803 2804 case CALL_INSN_OPCODE: 2805 case JMP32_INSN_OPCODE: 2806 case JMP8_INSN_OPCODE: 2807 case 0x70 ... 0x7f: /* Jcc */ 2808 tp->disp = insn.immediate.value; 2809 break; 2810 2811 default: /* assume NOP */ 2812 switch (len) { 2813 case 2: /* NOP2 -- emulate as JMP8+0 */ 2814 BUG_ON(memcmp(emulate, x86_nops[len], len)); 2815 tp->opcode = JMP8_INSN_OPCODE; 2816 tp->disp = 0; 2817 break; 2818 2819 case 5: /* NOP5 -- emulate as JMP32+0 */ 2820 BUG_ON(memcmp(emulate, x86_nops[len], len)); 2821 tp->opcode = JMP32_INSN_OPCODE; 2822 tp->disp = 0; 2823 break; 2824 2825 default: /* unknown instruction */ 2826 BUG(); 2827 } 2828 break; 2829 } 2830 } 2831 2832 /* 2833 * We hard rely on the tp_vec being ordered; ensure this is so by flushing 2834 * early if needed. 2835 */ 2836 static bool tp_order_fail(void *addr) 2837 { 2838 struct text_poke_loc *tp; 2839 2840 if (!tp_vec_nr) 2841 return false; 2842 2843 if (!addr) /* force */ 2844 return true; 2845 2846 tp = &tp_vec[tp_vec_nr - 1]; 2847 if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr) 2848 return true; 2849 2850 return false; 2851 } 2852 2853 static void text_poke_flush(void *addr) 2854 { 2855 if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) { 2856 text_poke_bp_batch(tp_vec, tp_vec_nr); 2857 tp_vec_nr = 0; 2858 } 2859 } 2860 2861 void text_poke_finish(void) 2862 { 2863 text_poke_flush(NULL); 2864 } 2865 2866 void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate) 2867 { 2868 struct text_poke_loc *tp; 2869 2870 text_poke_flush(addr); 2871 2872 tp = &tp_vec[tp_vec_nr++]; 2873 text_poke_loc_init(tp, addr, opcode, len, emulate); 2874 } 2875 2876 /** 2877 * text_poke_bp() -- update instructions on live kernel on SMP 2878 * @addr: address to patch 2879 * @opcode: opcode of new instruction 2880 * @len: length to copy 2881 * @emulate: instruction to be emulated 2882 * 2883 * Update a single instruction with the vector in the stack, avoiding 2884 * dynamically allocated memory. This function should be used when it is 2885 * not possible to allocate memory. 2886 */ 2887 void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate) 2888 { 2889 struct text_poke_loc tp; 2890 2891 text_poke_loc_init(&tp, addr, opcode, len, emulate); 2892 text_poke_bp_batch(&tp, 1); 2893 } 2894