1 // SPDX-License-Identifier: GPL-2.0-only 2 #define pr_fmt(fmt) "SMP alternatives: " fmt 3 4 #include <linux/mmu_context.h> 5 #include <linux/perf_event.h> 6 #include <linux/vmalloc.h> 7 #include <linux/memory.h> 8 #include <linux/execmem.h> 9 10 #include <asm/text-patching.h> 11 #include <asm/insn.h> 12 #include <asm/insn-eval.h> 13 #include <asm/ibt.h> 14 #include <asm/set_memory.h> 15 #include <asm/nmi.h> 16 17 int __read_mostly alternatives_patched; 18 19 EXPORT_SYMBOL_GPL(alternatives_patched); 20 21 #define MAX_PATCH_LEN (255-1) 22 23 #define DA_ALL (~0) 24 #define DA_ALT 0x01 25 #define DA_RET 0x02 26 #define DA_RETPOLINE 0x04 27 #define DA_ENDBR 0x08 28 #define DA_SMP 0x10 29 30 static unsigned int debug_alternative; 31 32 static int __init debug_alt(char *str) 33 { 34 if (str && *str == '=') 35 str++; 36 37 if (!str || kstrtouint(str, 0, &debug_alternative)) 38 debug_alternative = DA_ALL; 39 40 return 1; 41 } 42 __setup("debug-alternative", debug_alt); 43 44 static int noreplace_smp; 45 46 static int __init setup_noreplace_smp(char *str) 47 { 48 noreplace_smp = 1; 49 return 1; 50 } 51 __setup("noreplace-smp", setup_noreplace_smp); 52 53 #define DPRINTK(type, fmt, args...) \ 54 do { \ 55 if (debug_alternative & DA_##type) \ 56 printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \ 57 } while (0) 58 59 #define DUMP_BYTES(type, buf, len, fmt, args...) \ 60 do { \ 61 if (unlikely(debug_alternative & DA_##type)) { \ 62 int j; \ 63 \ 64 if (!(len)) \ 65 break; \ 66 \ 67 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 68 for (j = 0; j < (len) - 1; j++) \ 69 printk(KERN_CONT "%02hhx ", buf[j]); \ 70 printk(KERN_CONT "%02hhx\n", buf[j]); \ 71 } \ 72 } while (0) 73 74 static const unsigned char x86nops[] = 75 { 76 BYTES_NOP1, 77 BYTES_NOP2, 78 BYTES_NOP3, 79 BYTES_NOP4, 80 BYTES_NOP5, 81 BYTES_NOP6, 82 BYTES_NOP7, 83 BYTES_NOP8, 84 #ifdef CONFIG_64BIT 85 BYTES_NOP9, 86 BYTES_NOP10, 87 BYTES_NOP11, 88 #endif 89 }; 90 91 const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 92 { 93 NULL, 94 x86nops, 95 x86nops + 1, 96 x86nops + 1 + 2, 97 x86nops + 1 + 2 + 3, 98 x86nops + 1 + 2 + 3 + 4, 99 x86nops + 1 + 2 + 3 + 4 + 5, 100 x86nops + 1 + 2 + 3 + 4 + 5 + 6, 101 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 102 #ifdef CONFIG_64BIT 103 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 104 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9, 105 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10, 106 #endif 107 }; 108 109 #ifdef CONFIG_FINEIBT 110 static bool cfi_paranoid __ro_after_init; 111 #endif 112 113 #ifdef CONFIG_MITIGATION_ITS 114 115 #ifdef CONFIG_MODULES 116 static struct module *its_mod; 117 #endif 118 static void *its_page; 119 static unsigned int its_offset; 120 struct its_array its_pages; 121 122 static void *__its_alloc(struct its_array *pages) 123 { 124 void *page __free(execmem) = execmem_alloc_rw(EXECMEM_MODULE_TEXT, PAGE_SIZE); 125 if (!page) 126 return NULL; 127 128 void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *), 129 GFP_KERNEL); 130 if (!tmp) 131 return NULL; 132 133 pages->pages = tmp; 134 pages->pages[pages->num++] = page; 135 136 return no_free_ptr(page); 137 } 138 139 /* Initialize a thunk with the "jmp *reg; int3" instructions. */ 140 static void *its_init_thunk(void *thunk, int reg) 141 { 142 u8 *bytes = thunk; 143 int offset = 0; 144 int i = 0; 145 146 #ifdef CONFIG_FINEIBT 147 if (cfi_paranoid) { 148 /* 149 * When ITS uses indirect branch thunk the fineibt_paranoid 150 * caller sequence doesn't fit in the caller site. So put the 151 * remaining part of the sequence (UDB + JNE) into the ITS 152 * thunk. 153 */ 154 bytes[i++] = 0xd6; /* UDB */ 155 bytes[i++] = 0x75; /* JNE */ 156 bytes[i++] = 0xfd; 157 158 offset = 1; 159 } 160 #endif 161 162 if (reg >= 8) { 163 bytes[i++] = 0x41; /* REX.B prefix */ 164 reg -= 8; 165 } 166 bytes[i++] = 0xff; 167 bytes[i++] = 0xe0 + reg; /* JMP *reg */ 168 bytes[i++] = 0xcc; 169 170 return thunk + offset; 171 } 172 173 static void its_pages_protect(struct its_array *pages) 174 { 175 for (int i = 0; i < pages->num; i++) { 176 void *page = pages->pages[i]; 177 execmem_restore_rox(page, PAGE_SIZE); 178 } 179 } 180 181 static void its_fini_core(void) 182 { 183 if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 184 its_pages_protect(&its_pages); 185 kfree(its_pages.pages); 186 } 187 188 #ifdef CONFIG_MODULES 189 void its_init_mod(struct module *mod) 190 { 191 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 192 return; 193 194 mutex_lock(&text_mutex); 195 its_mod = mod; 196 its_page = NULL; 197 } 198 199 void its_fini_mod(struct module *mod) 200 { 201 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 202 return; 203 204 WARN_ON_ONCE(its_mod != mod); 205 206 its_mod = NULL; 207 its_page = NULL; 208 mutex_unlock(&text_mutex); 209 210 if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 211 its_pages_protect(&mod->arch.its_pages); 212 } 213 214 void its_free_mod(struct module *mod) 215 { 216 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 217 return; 218 219 for (int i = 0; i < mod->arch.its_pages.num; i++) { 220 void *page = mod->arch.its_pages.pages[i]; 221 execmem_free(page); 222 } 223 kfree(mod->arch.its_pages.pages); 224 } 225 #endif /* CONFIG_MODULES */ 226 227 static void *its_alloc(void) 228 { 229 struct its_array *pages = &its_pages; 230 void *page; 231 232 #ifdef CONFIG_MODULES 233 if (its_mod) 234 pages = &its_mod->arch.its_pages; 235 #endif 236 237 page = __its_alloc(pages); 238 if (!page) 239 return NULL; 240 241 if (pages == &its_pages) 242 set_memory_x((unsigned long)page, 1); 243 244 return page; 245 } 246 247 static void *its_allocate_thunk(int reg) 248 { 249 int size = 3 + (reg / 8); 250 void *thunk; 251 252 #ifdef CONFIG_FINEIBT 253 /* 254 * The ITS thunk contains an indirect jump and an int3 instruction so 255 * its size is 3 or 4 bytes depending on the register used. If CFI 256 * paranoid is used then 3 extra bytes are added in the ITS thunk to 257 * complete the fineibt_paranoid caller sequence. 258 */ 259 if (cfi_paranoid) 260 size += 3; 261 #endif 262 263 if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) { 264 its_page = its_alloc(); 265 if (!its_page) { 266 pr_err("ITS page allocation failed\n"); 267 return NULL; 268 } 269 memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE); 270 its_offset = 32; 271 } 272 273 /* 274 * If the indirect branch instruction will be in the lower half 275 * of a cacheline, then update the offset to reach the upper half. 276 */ 277 if ((its_offset + size - 1) % 64 < 32) 278 its_offset = ((its_offset - 1) | 0x3F) + 33; 279 280 thunk = its_page + its_offset; 281 its_offset += size; 282 283 return its_init_thunk(thunk, reg); 284 } 285 286 u8 *its_static_thunk(int reg) 287 { 288 u8 *thunk = __x86_indirect_its_thunk_array[reg]; 289 290 #ifdef CONFIG_FINEIBT 291 /* Paranoid thunk starts 2 bytes before */ 292 if (cfi_paranoid) 293 return thunk - 2; 294 #endif 295 return thunk; 296 } 297 298 #else 299 static inline void its_fini_core(void) {} 300 #endif /* CONFIG_MITIGATION_ITS */ 301 302 /* 303 * Nomenclature for variable names to simplify and clarify this code and ease 304 * any potential staring at it: 305 * 306 * @instr: source address of the original instructions in the kernel text as 307 * generated by the compiler. 308 * 309 * @buf: temporary buffer on which the patching operates. This buffer is 310 * eventually text-poked into the kernel image. 311 * 312 * @replacement/@repl: pointer to the opcodes which are replacing @instr, located 313 * in the .altinstr_replacement section. 314 */ 315 316 /* 317 * Fill the buffer with a single effective instruction of size @len. 318 * 319 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info) 320 * for every single-byte NOP, try to generate the maximally available NOP of 321 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for 322 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and 323 * *jump* over instead of executing long and daft NOPs. 324 */ 325 static void add_nop(u8 *buf, unsigned int len) 326 { 327 u8 *target = buf + len; 328 329 if (!len) 330 return; 331 332 if (len <= ASM_NOP_MAX) { 333 memcpy(buf, x86_nops[len], len); 334 return; 335 } 336 337 if (len < 128) { 338 __text_gen_insn(buf, JMP8_INSN_OPCODE, buf, target, JMP8_INSN_SIZE); 339 buf += JMP8_INSN_SIZE; 340 } else { 341 __text_gen_insn(buf, JMP32_INSN_OPCODE, buf, target, JMP32_INSN_SIZE); 342 buf += JMP32_INSN_SIZE; 343 } 344 345 for (;buf < target; buf++) 346 *buf = INT3_INSN_OPCODE; 347 } 348 349 /* 350 * Find the offset of the first non-NOP instruction starting at @offset 351 * but no further than @len. 352 */ 353 static int skip_nops(u8 *buf, int offset, int len) 354 { 355 struct insn insn; 356 357 for (; offset < len; offset += insn.length) { 358 if (insn_decode_kernel(&insn, &buf[offset])) 359 break; 360 361 if (!insn_is_nop(&insn)) 362 break; 363 } 364 365 return offset; 366 } 367 368 /* 369 * "noinline" to cause control flow change and thus invalidate I$ and 370 * cause refetch after modification. 371 */ 372 static void noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len) 373 { 374 for (int next, i = 0; i < len; i = next) { 375 struct insn insn; 376 377 if (insn_decode_kernel(&insn, &buf[i])) 378 return; 379 380 next = i + insn.length; 381 382 if (insn_is_nop(&insn)) { 383 int nop = i; 384 385 /* Has the NOP already been optimized? */ 386 if (i + insn.length == len) 387 return; 388 389 next = skip_nops(buf, next, len); 390 391 add_nop(buf + nop, next - nop); 392 DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, next); 393 } 394 } 395 } 396 397 /* 398 * In this context, "source" is where the instructions are placed in the 399 * section .altinstr_replacement, for example during kernel build by the 400 * toolchain. 401 * "Destination" is where the instructions are being patched in by this 402 * machinery. 403 * 404 * The source offset is: 405 * 406 * src_imm = target - src_next_ip (1) 407 * 408 * and the target offset is: 409 * 410 * dst_imm = target - dst_next_ip (2) 411 * 412 * so rework (1) as an expression for target like: 413 * 414 * target = src_imm + src_next_ip (1a) 415 * 416 * and substitute in (2) to get: 417 * 418 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3) 419 * 420 * Now, since the instruction stream is 'identical' at src and dst (it 421 * is being copied after all) it can be stated that: 422 * 423 * src_next_ip = src + ip_offset 424 * dst_next_ip = dst + ip_offset (4) 425 * 426 * Substitute (4) in (3) and observe ip_offset being cancelled out to 427 * obtain: 428 * 429 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset) 430 * = src_imm + src - dst + ip_offset - ip_offset 431 * = src_imm + src - dst (5) 432 * 433 * IOW, only the relative displacement of the code block matters. 434 */ 435 436 #define apply_reloc_n(n_, p_, d_) \ 437 do { \ 438 s32 v = *(s##n_ *)(p_); \ 439 v += (d_); \ 440 BUG_ON((v >> 31) != (v >> (n_-1))); \ 441 *(s##n_ *)(p_) = (s##n_)v; \ 442 } while (0) 443 444 445 static __always_inline 446 void apply_reloc(int n, void *ptr, uintptr_t diff) 447 { 448 switch (n) { 449 case 1: apply_reloc_n(8, ptr, diff); break; 450 case 2: apply_reloc_n(16, ptr, diff); break; 451 case 4: apply_reloc_n(32, ptr, diff); break; 452 default: BUG(); 453 } 454 } 455 456 static __always_inline 457 bool need_reloc(unsigned long offset, u8 *src, size_t src_len) 458 { 459 u8 *target = src + offset; 460 /* 461 * If the target is inside the patched block, it's relative to the 462 * block itself and does not need relocation. 463 */ 464 return (target < src || target > src + src_len); 465 } 466 467 static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) 468 { 469 for (int next, i = 0; i < instrlen; i = next) { 470 struct insn insn; 471 472 if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i]))) 473 return; 474 475 next = i + insn.length; 476 477 switch (insn.opcode.bytes[0]) { 478 case 0x0f: 479 if (insn.opcode.bytes[1] < 0x80 || 480 insn.opcode.bytes[1] > 0x8f) 481 break; 482 483 fallthrough; /* Jcc.d32 */ 484 case 0x70 ... 0x7f: /* Jcc.d8 */ 485 case JMP8_INSN_OPCODE: 486 case JMP32_INSN_OPCODE: 487 case CALL_INSN_OPCODE: 488 if (need_reloc(next + insn.immediate.value, repl, repl_len)) { 489 apply_reloc(insn.immediate.nbytes, 490 buf + i + insn_offset_immediate(&insn), 491 repl - instr); 492 } 493 494 /* 495 * Where possible, convert JMP.d32 into JMP.d8. 496 */ 497 if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) { 498 s32 imm = insn.immediate.value; 499 imm += repl - instr; 500 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE; 501 if ((imm >> 31) == (imm >> 7)) { 502 buf[i+0] = JMP8_INSN_OPCODE; 503 buf[i+1] = (s8)imm; 504 505 memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2); 506 } 507 } 508 break; 509 } 510 511 if (insn_rip_relative(&insn)) { 512 if (need_reloc(next + insn.displacement.value, repl, repl_len)) { 513 apply_reloc(insn.displacement.nbytes, 514 buf + i + insn_offset_displacement(&insn), 515 repl - instr); 516 } 517 } 518 } 519 } 520 521 void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) 522 { 523 __apply_relocation(buf, instr, instrlen, repl, repl_len); 524 optimize_nops(instr, buf, instrlen); 525 } 526 527 /* Low-level backend functions usable from alternative code replacements. */ 528 DEFINE_ASM_FUNC(nop_func, "", .entry.text); 529 EXPORT_SYMBOL_GPL(nop_func); 530 531 noinstr void BUG_func(void) 532 { 533 BUG(); 534 } 535 EXPORT_SYMBOL(BUG_func); 536 537 #define CALL_RIP_REL_OPCODE 0xff 538 #define CALL_RIP_REL_MODRM 0x15 539 540 /* 541 * Rewrite the "call BUG_func" replacement to point to the target of the 542 * indirect pv_ops call "call *disp(%ip)". 543 */ 544 static unsigned int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a) 545 { 546 void *target, *bug = &BUG_func; 547 s32 disp; 548 549 if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) { 550 pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n"); 551 BUG(); 552 } 553 554 if (a->instrlen != 6 || 555 instr[0] != CALL_RIP_REL_OPCODE || 556 instr[1] != CALL_RIP_REL_MODRM) { 557 pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n"); 558 BUG(); 559 } 560 561 /* Skip CALL_RIP_REL_OPCODE and CALL_RIP_REL_MODRM */ 562 disp = *(s32 *)(instr + 2); 563 #ifdef CONFIG_X86_64 564 /* ff 15 00 00 00 00 call *0x0(%rip) */ 565 /* target address is stored at "next instruction + disp". */ 566 target = *(void **)(instr + a->instrlen + disp); 567 #else 568 /* ff 15 00 00 00 00 call *0x0 */ 569 /* target address is stored at disp. */ 570 target = *(void **)disp; 571 #endif 572 if (!target) 573 target = bug; 574 575 /* (BUG_func - .) + (target - BUG_func) := target - . */ 576 *(s32 *)(insn_buff + 1) += target - bug; 577 578 if (target == &nop_func) 579 return 0; 580 581 return 5; 582 } 583 584 static inline u8 * instr_va(struct alt_instr *i) 585 { 586 return (u8 *)&i->instr_offset + i->instr_offset; 587 } 588 589 struct patch_site { 590 u8 *instr; 591 struct alt_instr *alt; 592 u8 buff[MAX_PATCH_LEN]; 593 u8 len; 594 }; 595 596 static struct alt_instr * __init_or_module analyze_patch_site(struct patch_site *ps, 597 struct alt_instr *start, 598 struct alt_instr *end) 599 { 600 struct alt_instr *alt = start; 601 602 ps->instr = instr_va(start); 603 604 /* 605 * In case of nested ALTERNATIVE()s the outer alternative might add 606 * more padding. To ensure consistent patching find the max padding for 607 * all alt_instr entries for this site (nested alternatives result in 608 * consecutive entries). 609 * Find the last alt_instr eligible for patching at the site. 610 */ 611 for (; alt < end && instr_va(alt) == ps->instr; alt++) { 612 ps->len = max(ps->len, alt->instrlen); 613 614 BUG_ON(alt->cpuid >= (NCAPINTS + NBUGINTS) * 32); 615 /* 616 * Patch if either: 617 * - feature is present 618 * - feature not present but ALT_FLAG_NOT is set to mean, 619 * patch if feature is *NOT* present. 620 */ 621 if (!boot_cpu_has(alt->cpuid) != !(alt->flags & ALT_FLAG_NOT)) 622 ps->alt = alt; 623 } 624 625 BUG_ON(ps->len > sizeof(ps->buff)); 626 627 return alt; 628 } 629 630 static void __init_or_module prep_patch_site(struct patch_site *ps) 631 { 632 struct alt_instr *alt = ps->alt; 633 u8 buff_sz; 634 u8 *repl; 635 636 if (!alt) { 637 /* Nothing to patch, use original instruction. */ 638 memcpy(ps->buff, ps->instr, ps->len); 639 return; 640 } 641 642 repl = (u8 *)&alt->repl_offset + alt->repl_offset; 643 DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x", 644 alt->cpuid >> 5, alt->cpuid & 0x1f, 645 ps->instr, ps->instr, ps->len, 646 repl, alt->replacementlen, alt->flags); 647 648 memcpy(ps->buff, repl, alt->replacementlen); 649 buff_sz = alt->replacementlen; 650 651 if (alt->flags & ALT_FLAG_DIRECT_CALL) 652 buff_sz = alt_replace_call(ps->instr, ps->buff, alt); 653 654 for (; buff_sz < ps->len; buff_sz++) 655 ps->buff[buff_sz] = 0x90; 656 657 __apply_relocation(ps->buff, ps->instr, ps->len, repl, alt->replacementlen); 658 659 DUMP_BYTES(ALT, ps->instr, ps->len, "%px: old_insn: ", ps->instr); 660 DUMP_BYTES(ALT, repl, alt->replacementlen, "%px: rpl_insn: ", repl); 661 DUMP_BYTES(ALT, ps->buff, ps->len, "%px: final_insn: ", ps->instr); 662 } 663 664 static void __init_or_module patch_site(struct patch_site *ps) 665 { 666 optimize_nops(ps->instr, ps->buff, ps->len); 667 text_poke_early(ps->instr, ps->buff, ps->len); 668 } 669 670 /* 671 * Replace instructions with better alternatives for this CPU type. This runs 672 * before SMP is initialized to avoid SMP problems with self modifying code. 673 * This implies that asymmetric systems where APs have less capabilities than 674 * the boot processor are not handled. Tough. Make sure you disable such 675 * features by hand. 676 * 677 * Marked "noinline" to cause control flow change and thus insn cache 678 * to refetch changed I$ lines. 679 */ 680 void __init_or_module noinline apply_alternatives(struct alt_instr *start, 681 struct alt_instr *end) 682 { 683 struct alt_instr *a; 684 685 DPRINTK(ALT, "alt table %px, -> %px", start, end); 686 687 /* 688 * KASAN_SHADOW_START is defined using 689 * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here. 690 * During the process, KASAN becomes confused seeing partial LA57 691 * conversion and triggers a false-positive out-of-bound report. 692 * 693 * Disable KASAN until the patching is complete. 694 */ 695 kasan_disable_current(); 696 697 /* 698 * The scan order should be from start to end. A later scanned 699 * alternative code can overwrite previously scanned alternative code. 700 * Some kernel functions (e.g. memcpy, memset, etc) use this order to 701 * patch code. 702 * 703 * So be careful if you want to change the scan order to any other 704 * order. 705 */ 706 a = start; 707 while (a < end) { 708 struct patch_site ps = { 709 .alt = NULL, 710 .len = 0 711 }; 712 713 a = analyze_patch_site(&ps, a, end); 714 prep_patch_site(&ps); 715 patch_site(&ps); 716 } 717 718 kasan_enable_current(); 719 } 720 721 static inline bool is_jcc32(struct insn *insn) 722 { 723 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ 724 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; 725 } 726 727 #if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL) 728 729 /* 730 * [CS]{,3} CALL/JMP *%\reg [INT3]* 731 */ 732 static int emit_indirect(int op, int reg, u8 *bytes, int len) 733 { 734 int cs = 0, bp = 0; 735 int i = 0; 736 u8 modrm; 737 738 /* 739 * Set @len to the excess bytes after writing the instruction. 740 */ 741 len -= 2 + (reg >= 8); 742 WARN_ON_ONCE(len < 0); 743 744 switch (op) { 745 case CALL_INSN_OPCODE: 746 modrm = 0x10; /* Reg = 2; CALL r/m */ 747 /* 748 * Additional NOP is better than prefix decode penalty. 749 */ 750 if (len <= 3) 751 cs = len; 752 break; 753 754 case JMP32_INSN_OPCODE: 755 modrm = 0x20; /* Reg = 4; JMP r/m */ 756 bp = len; 757 break; 758 759 default: 760 WARN_ON_ONCE(1); 761 return -1; 762 } 763 764 while (cs--) 765 bytes[i++] = 0x2e; /* CS-prefix */ 766 767 if (reg >= 8) { 768 bytes[i++] = 0x41; /* REX.B prefix */ 769 reg -= 8; 770 } 771 772 modrm |= 0xc0; /* Mod = 3 */ 773 modrm += reg; 774 775 bytes[i++] = 0xff; /* opcode */ 776 bytes[i++] = modrm; 777 778 while (bp--) 779 bytes[i++] = 0xcc; /* INT3 */ 780 781 return i; 782 } 783 784 static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes, 785 void *call_dest, void *jmp_dest) 786 { 787 u8 op = insn->opcode.bytes[0]; 788 int i = 0; 789 790 /* 791 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional 792 * tail-calls. Deal with them. 793 */ 794 if (is_jcc32(insn)) { 795 bytes[i++] = op; 796 op = insn->opcode.bytes[1]; 797 goto clang_jcc; 798 } 799 800 if (insn->length == 6) 801 bytes[i++] = 0x2e; /* CS-prefix */ 802 803 switch (op) { 804 case CALL_INSN_OPCODE: 805 __text_gen_insn(bytes+i, op, addr+i, 806 call_dest, 807 CALL_INSN_SIZE); 808 i += CALL_INSN_SIZE; 809 break; 810 811 case JMP32_INSN_OPCODE: 812 clang_jcc: 813 __text_gen_insn(bytes+i, op, addr+i, 814 jmp_dest, 815 JMP32_INSN_SIZE); 816 i += JMP32_INSN_SIZE; 817 break; 818 819 default: 820 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr); 821 return -1; 822 } 823 824 WARN_ON_ONCE(i != insn->length); 825 826 return i; 827 } 828 829 static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) 830 { 831 return __emit_trampoline(addr, insn, bytes, 832 __x86_indirect_call_thunk_array[reg], 833 __x86_indirect_jump_thunk_array[reg]); 834 } 835 836 #ifdef CONFIG_MITIGATION_ITS 837 static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) 838 { 839 u8 *thunk = __x86_indirect_its_thunk_array[reg]; 840 u8 *tmp = its_allocate_thunk(reg); 841 842 if (tmp) 843 thunk = tmp; 844 845 return __emit_trampoline(addr, insn, bytes, thunk, thunk); 846 } 847 848 /* Check if an indirect branch is at ITS-unsafe address */ 849 static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg) 850 { 851 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 852 return false; 853 854 /* Indirect branch opcode is 2 or 3 bytes depending on reg */ 855 addr += 1 + reg / 8; 856 857 /* Lower-half of the cacheline? */ 858 return !(addr & 0x20); 859 } 860 #else /* CONFIG_MITIGATION_ITS */ 861 862 #ifdef CONFIG_FINEIBT 863 static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg) 864 { 865 return false; 866 } 867 #endif 868 869 #endif /* CONFIG_MITIGATION_ITS */ 870 871 /* 872 * Rewrite the compiler generated retpoline thunk calls. 873 * 874 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate 875 * indirect instructions, avoiding the extra indirection. 876 * 877 * For example, convert: 878 * 879 * CALL __x86_indirect_thunk_\reg 880 * 881 * into: 882 * 883 * CALL *%\reg 884 * 885 * It also tries to inline spectre_v2=retpoline,lfence when size permits. 886 */ 887 static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) 888 { 889 retpoline_thunk_t *target; 890 int reg, ret, i = 0; 891 u8 op, cc; 892 893 target = addr + insn->length + insn->immediate.value; 894 reg = target - __x86_indirect_thunk_array; 895 896 if (WARN_ON_ONCE(reg & ~0xf)) 897 return -1; 898 899 /* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */ 900 BUG_ON(reg == 4); 901 902 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) && 903 !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { 904 if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 905 return emit_call_track_retpoline(addr, insn, reg, bytes); 906 907 return -1; 908 } 909 910 op = insn->opcode.bytes[0]; 911 912 /* 913 * Convert: 914 * 915 * Jcc.d32 __x86_indirect_thunk_\reg 916 * 917 * into: 918 * 919 * Jncc.d8 1f 920 * [ LFENCE ] 921 * JMP *%\reg 922 * [ NOP ] 923 * 1: 924 */ 925 if (is_jcc32(insn)) { 926 cc = insn->opcode.bytes[1] & 0xf; 927 cc ^= 1; /* invert condition */ 928 929 bytes[i++] = 0x70 + cc; /* Jcc.d8 */ 930 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */ 931 932 /* Continue as if: JMP.d32 __x86_indirect_thunk_\reg */ 933 op = JMP32_INSN_OPCODE; 934 } 935 936 /* 937 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE. 938 */ 939 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { 940 bytes[i++] = 0x0f; 941 bytes[i++] = 0xae; 942 bytes[i++] = 0xe8; /* LFENCE */ 943 } 944 945 #ifdef CONFIG_MITIGATION_ITS 946 /* 947 * Check if the address of last byte of emitted-indirect is in 948 * lower-half of the cacheline. Such branches need ITS mitigation. 949 */ 950 if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg)) 951 return emit_its_trampoline(addr, insn, reg, bytes); 952 #endif 953 954 ret = emit_indirect(op, reg, bytes + i, insn->length - i); 955 if (ret < 0) 956 return ret; 957 i += ret; 958 959 for (; i < insn->length;) 960 bytes[i++] = BYTES_NOP1; 961 962 return i; 963 } 964 965 /* 966 * Generated by 'objtool --retpoline'. 967 */ 968 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) 969 { 970 s32 *s; 971 972 for (s = start; s < end; s++) { 973 void *addr = (void *)s + *s; 974 struct insn insn; 975 int len, ret; 976 u8 bytes[16]; 977 u8 op1, op2; 978 u8 *dest; 979 980 ret = insn_decode_kernel(&insn, addr); 981 if (WARN_ON_ONCE(ret < 0)) 982 continue; 983 984 op1 = insn.opcode.bytes[0]; 985 op2 = insn.opcode.bytes[1]; 986 987 switch (op1) { 988 case 0x70 ... 0x7f: /* Jcc.d8 */ 989 /* See cfi_paranoid. */ 990 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT); 991 continue; 992 993 case CALL_INSN_OPCODE: 994 case JMP32_INSN_OPCODE: 995 /* Check for cfi_paranoid + ITS */ 996 dest = addr + insn.length + insn.immediate.value; 997 if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) { 998 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT); 999 continue; 1000 } 1001 break; 1002 1003 case 0x0f: /* escape */ 1004 if (op2 >= 0x80 && op2 <= 0x8f) 1005 break; 1006 fallthrough; 1007 default: 1008 WARN_ON_ONCE(1); 1009 continue; 1010 } 1011 1012 DPRINTK(RETPOLINE, "retpoline at: %pS (%px) len: %d to: %pS", 1013 addr, addr, insn.length, 1014 addr + insn.length + insn.immediate.value); 1015 1016 len = patch_retpoline(addr, &insn, bytes); 1017 if (len == insn.length) { 1018 optimize_nops(addr, bytes, len); 1019 DUMP_BYTES(RETPOLINE, ((u8*)addr), len, "%px: orig: ", addr); 1020 DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr); 1021 text_poke_early(addr, bytes, len); 1022 } 1023 } 1024 } 1025 1026 #ifdef CONFIG_MITIGATION_RETHUNK 1027 1028 bool cpu_wants_rethunk(void) 1029 { 1030 return cpu_feature_enabled(X86_FEATURE_RETHUNK); 1031 } 1032 1033 bool cpu_wants_rethunk_at(void *addr) 1034 { 1035 if (!cpu_feature_enabled(X86_FEATURE_RETHUNK)) 1036 return false; 1037 if (x86_return_thunk != its_return_thunk) 1038 return true; 1039 1040 return !((unsigned long)addr & 0x20); 1041 } 1042 1043 /* 1044 * Rewrite the compiler generated return thunk tail-calls. 1045 * 1046 * For example, convert: 1047 * 1048 * JMP __x86_return_thunk 1049 * 1050 * into: 1051 * 1052 * RET 1053 */ 1054 static int patch_return(void *addr, struct insn *insn, u8 *bytes) 1055 { 1056 int i = 0; 1057 1058 /* Patch the custom return thunks... */ 1059 if (cpu_wants_rethunk_at(addr)) { 1060 i = JMP32_INSN_SIZE; 1061 __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i); 1062 } else { 1063 /* ... or patch them out if not needed. */ 1064 bytes[i++] = RET_INSN_OPCODE; 1065 } 1066 1067 for (; i < insn->length;) 1068 bytes[i++] = INT3_INSN_OPCODE; 1069 return i; 1070 } 1071 1072 void __init_or_module noinline apply_returns(s32 *start, s32 *end) 1073 { 1074 s32 *s; 1075 1076 if (cpu_wants_rethunk()) 1077 static_call_force_reinit(); 1078 1079 for (s = start; s < end; s++) { 1080 void *dest = NULL, *addr = (void *)s + *s; 1081 struct insn insn; 1082 int len, ret; 1083 u8 bytes[16]; 1084 u8 op; 1085 1086 ret = insn_decode_kernel(&insn, addr); 1087 if (WARN_ON_ONCE(ret < 0)) 1088 continue; 1089 1090 op = insn.opcode.bytes[0]; 1091 if (op == JMP32_INSN_OPCODE) 1092 dest = addr + insn.length + insn.immediate.value; 1093 1094 if (__static_call_fixup(addr, op, dest) || 1095 WARN_ONCE(dest != &__x86_return_thunk, 1096 "missing return thunk: %pS-%pS: %*ph", 1097 addr, dest, 5, addr)) 1098 continue; 1099 1100 DPRINTK(RET, "return thunk at: %pS (%px) len: %d to: %pS", 1101 addr, addr, insn.length, 1102 addr + insn.length + insn.immediate.value); 1103 1104 len = patch_return(addr, &insn, bytes); 1105 if (len == insn.length) { 1106 DUMP_BYTES(RET, ((u8*)addr), len, "%px: orig: ", addr); 1107 DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr); 1108 text_poke_early(addr, bytes, len); 1109 } 1110 } 1111 } 1112 #else /* !CONFIG_MITIGATION_RETHUNK: */ 1113 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { } 1114 #endif /* !CONFIG_MITIGATION_RETHUNK */ 1115 1116 #else /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */ 1117 1118 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { } 1119 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { } 1120 1121 #endif /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */ 1122 1123 #ifdef CONFIG_X86_KERNEL_IBT 1124 1125 __noendbr bool is_endbr(u32 *val) 1126 { 1127 u32 endbr; 1128 1129 __get_kernel_nofault(&endbr, val, u32, Efault); 1130 return __is_endbr(endbr); 1131 1132 Efault: 1133 return false; 1134 } 1135 1136 #ifdef CONFIG_FINEIBT 1137 1138 static __noendbr bool exact_endbr(u32 *val) 1139 { 1140 u32 endbr; 1141 1142 __get_kernel_nofault(&endbr, val, u32, Efault); 1143 return endbr == gen_endbr(); 1144 1145 Efault: 1146 return false; 1147 } 1148 1149 #endif 1150 1151 static void poison_cfi(void *addr); 1152 1153 static void __init_or_module poison_endbr(void *addr) 1154 { 1155 u32 poison = gen_endbr_poison(); 1156 1157 if (WARN_ON_ONCE(!is_endbr(addr))) 1158 return; 1159 1160 DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr); 1161 1162 /* 1163 * When we have IBT, the lack of ENDBR will trigger #CP 1164 */ 1165 DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr); 1166 DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr); 1167 text_poke_early(addr, &poison, 4); 1168 } 1169 1170 /* 1171 * Generated by: objtool --ibt 1172 * 1173 * Seal the functions for indirect calls by clobbering the ENDBR instructions 1174 * and the kCFI hash value. 1175 */ 1176 void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) 1177 { 1178 s32 *s; 1179 1180 for (s = start; s < end; s++) { 1181 void *addr = (void *)s + *s; 1182 1183 poison_endbr(addr); 1184 if (IS_ENABLED(CONFIG_FINEIBT)) 1185 poison_cfi(addr - 16); 1186 } 1187 } 1188 1189 #else /* !CONFIG_X86_KERNEL_IBT: */ 1190 1191 void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { } 1192 1193 #endif /* !CONFIG_X86_KERNEL_IBT */ 1194 1195 #ifdef CONFIG_CFI_AUTO_DEFAULT 1196 # define __CFI_DEFAULT CFI_AUTO 1197 #elif defined(CONFIG_CFI) 1198 # define __CFI_DEFAULT CFI_KCFI 1199 #else 1200 # define __CFI_DEFAULT CFI_OFF 1201 #endif 1202 1203 enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT; 1204 static bool cfi_debug __ro_after_init; 1205 1206 #ifdef CONFIG_FINEIBT_BHI 1207 bool cfi_bhi __ro_after_init = false; 1208 #endif 1209 1210 #ifdef CONFIG_CFI 1211 u32 cfi_get_func_hash(void *func) 1212 { 1213 u32 hash; 1214 1215 func -= cfi_get_offset(); 1216 switch (cfi_mode) { 1217 case CFI_FINEIBT: 1218 func += 7; 1219 break; 1220 case CFI_KCFI: 1221 func += 1; 1222 break; 1223 default: 1224 return 0; 1225 } 1226 1227 if (get_kernel_nofault(hash, func)) 1228 return 0; 1229 1230 return hash; 1231 } 1232 1233 int cfi_get_func_arity(void *func) 1234 { 1235 bhi_thunk *target; 1236 s32 disp; 1237 1238 if (cfi_mode != CFI_FINEIBT && !cfi_bhi) 1239 return 0; 1240 1241 if (get_kernel_nofault(disp, func - 4)) 1242 return 0; 1243 1244 target = func + disp; 1245 return target - __bhi_args; 1246 } 1247 #endif 1248 1249 #ifdef CONFIG_FINEIBT 1250 1251 static bool cfi_rand __ro_after_init = true; 1252 static u32 cfi_seed __ro_after_init; 1253 1254 /* 1255 * Re-hash the CFI hash with a boot-time seed while making sure the result is 1256 * not a valid ENDBR instruction. 1257 */ 1258 static u32 cfi_rehash(u32 hash) 1259 { 1260 hash ^= cfi_seed; 1261 while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) { 1262 bool lsb = hash & 1; 1263 hash >>= 1; 1264 if (lsb) 1265 hash ^= 0x80200003; 1266 } 1267 return hash; 1268 } 1269 1270 static __init int cfi_parse_cmdline(char *str) 1271 { 1272 if (!str) 1273 return -EINVAL; 1274 1275 while (str) { 1276 char *next = strchr(str, ','); 1277 if (next) { 1278 *next = 0; 1279 next++; 1280 } 1281 1282 if (!strcmp(str, "auto")) { 1283 cfi_mode = CFI_AUTO; 1284 } else if (!strcmp(str, "off")) { 1285 cfi_mode = CFI_OFF; 1286 cfi_rand = false; 1287 } else if (!strcmp(str, "debug")) { 1288 cfi_debug = true; 1289 } else if (!strcmp(str, "kcfi")) { 1290 cfi_mode = CFI_KCFI; 1291 } else if (!strcmp(str, "fineibt")) { 1292 cfi_mode = CFI_FINEIBT; 1293 } else if (!strcmp(str, "norand")) { 1294 cfi_rand = false; 1295 } else if (!strcmp(str, "warn")) { 1296 pr_alert("CFI: mismatch non-fatal!\n"); 1297 cfi_warn = true; 1298 } else if (!strcmp(str, "paranoid")) { 1299 if (cfi_mode == CFI_FINEIBT) { 1300 cfi_paranoid = true; 1301 } else { 1302 pr_err("CFI: ignoring paranoid; depends on fineibt.\n"); 1303 } 1304 } else if (!strcmp(str, "bhi")) { 1305 #ifdef CONFIG_FINEIBT_BHI 1306 if (cfi_mode == CFI_FINEIBT) { 1307 cfi_bhi = true; 1308 } else { 1309 pr_err("CFI: ignoring bhi; depends on fineibt.\n"); 1310 } 1311 #else 1312 pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n"); 1313 #endif 1314 } else { 1315 pr_err("CFI: Ignoring unknown option (%s).", str); 1316 } 1317 1318 str = next; 1319 } 1320 1321 return 0; 1322 } 1323 early_param("cfi", cfi_parse_cmdline); 1324 1325 /* 1326 * kCFI FineIBT 1327 * 1328 * __cfi_\func: __cfi_\func: 1329 * movl $0x12345678,%eax // 5 endbr64 // 4 1330 * nop subl $0x12345678,%eax // 5 1331 * nop jne.d32,pn \func+3 // 7 1332 * nop 1333 * nop 1334 * nop 1335 * nop 1336 * nop 1337 * nop 1338 * nop 1339 * nop 1340 * nop 1341 * \func: \func: 1342 * endbr64 nopl -42(%rax) 1343 * 1344 * 1345 * caller: caller: 1346 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%eax // 5 1347 * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4 1348 * je 1f // 2 nop5 // 5 1349 * ud2 // 2 1350 * 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6 1351 * 1352 * 1353 * Notably, the FineIBT sequences are crafted such that branches are presumed 1354 * non-taken. This is based on Agner Fog's optimization manual, which states: 1355 * 1356 * "Make conditional jumps most often not taken: The efficiency and throughput 1357 * for not-taken branches is better than for taken branches on most 1358 * processors. Therefore, it is good to place the most frequent branch first" 1359 */ 1360 1361 /* 1362 * <fineibt_preamble_start>: 1363 * 0: f3 0f 1e fa endbr64 1364 * 4: 2d 78 56 34 12 sub $0x12345678, %eax 1365 * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13> 1366 * 10: 0f 1f 40 d6 nopl -0x2a(%rax) 1367 * 1368 * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as 1369 * UDB on x86_64 and raises #UD. 1370 */ 1371 asm( ".pushsection .rodata \n" 1372 "fineibt_preamble_start: \n" 1373 " endbr64 \n" 1374 " subl $0x12345678, %eax \n" 1375 "fineibt_preamble_bhi: \n" 1376 " cs jne.d32 fineibt_preamble_start+0x13 \n" 1377 "#fineibt_func: \n" 1378 " nopl -42(%rax) \n" 1379 "fineibt_preamble_end: \n" 1380 ".popsection\n" 1381 ); 1382 1383 extern u8 fineibt_preamble_start[]; 1384 extern u8 fineibt_preamble_bhi[]; 1385 extern u8 fineibt_preamble_end[]; 1386 1387 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) 1388 #define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) 1389 #define fineibt_preamble_ud 0x13 1390 #define fineibt_preamble_hash 5 1391 1392 /* 1393 * <fineibt_caller_start>: 1394 * 0: b8 78 56 34 12 mov $0x12345678, %eax 1395 * 5: 4d 8d 5b f0 lea -0x10(%r11), %r11 1396 * 9: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1397 */ 1398 asm( ".pushsection .rodata \n" 1399 "fineibt_caller_start: \n" 1400 " movl $0x12345678, %eax \n" 1401 " lea -0x10(%r11), %r11 \n" 1402 ASM_NOP5 1403 "fineibt_caller_end: \n" 1404 ".popsection \n" 1405 ); 1406 1407 extern u8 fineibt_caller_start[]; 1408 extern u8 fineibt_caller_end[]; 1409 1410 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start) 1411 #define fineibt_caller_hash 1 1412 1413 #define fineibt_caller_jmp (fineibt_caller_size - 2) 1414 1415 /* 1416 * Since FineIBT does hash validation on the callee side it is prone to 1417 * circumvention attacks where a 'naked' ENDBR instruction exists that 1418 * is not part of the fineibt_preamble sequence. 1419 * 1420 * Notably the x86 entry points must be ENDBR and equally cannot be 1421 * fineibt_preamble. 1422 * 1423 * The fineibt_paranoid caller sequence adds additional caller side 1424 * hash validation. This stops such circumvention attacks dead, but at the cost 1425 * of adding a load. 1426 * 1427 * <fineibt_paranoid_start>: 1428 * 0: b8 78 56 34 12 mov $0x12345678, %eax 1429 * 5: 41 3b 43 f5 cmp -0x11(%r11), %eax 1430 * 9: 2e 4d 8d 5b <f0> cs lea -0x10(%r11), %r11 1431 * e: 75 fd jne d <fineibt_paranoid_start+0xd> 1432 * 10: 41 ff d3 call *%r11 1433 * 13: 90 nop 1434 * 1435 * Notably LEA does not modify flags and can be reordered with the CMP, 1436 * avoiding a dependency. Again, using a non-taken (backwards) branch 1437 * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for the 1438 * Jcc.d8, causing #UD. 1439 */ 1440 asm( ".pushsection .rodata \n" 1441 "fineibt_paranoid_start: \n" 1442 " mov $0x12345678, %eax \n" 1443 " cmpl -11(%r11), %eax \n" 1444 " cs lea -0x10(%r11), %r11 \n" 1445 "#fineibt_caller_size: \n" 1446 " jne fineibt_paranoid_start+0xd \n" 1447 "fineibt_paranoid_ind: \n" 1448 " cs call *%r11 \n" 1449 "fineibt_paranoid_end: \n" 1450 ".popsection \n" 1451 ); 1452 1453 extern u8 fineibt_paranoid_start[]; 1454 extern u8 fineibt_paranoid_ind[]; 1455 extern u8 fineibt_paranoid_end[]; 1456 1457 #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start) 1458 #define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_start) 1459 #define fineibt_paranoid_ud 0xd 1460 1461 static u32 decode_preamble_hash(void *addr, int *reg) 1462 { 1463 u8 *p = addr; 1464 1465 /* b8+reg 78 56 34 12 movl $0x12345678,\reg */ 1466 if (p[0] >= 0xb8 && p[0] < 0xc0) { 1467 if (reg) 1468 *reg = p[0] - 0xb8; 1469 return *(u32 *)(addr + 1); 1470 } 1471 1472 return 0; /* invalid hash value */ 1473 } 1474 1475 static u32 decode_caller_hash(void *addr) 1476 { 1477 u8 *p = addr; 1478 1479 /* 41 ba 88 a9 cb ed mov $(-0x12345678),%r10d */ 1480 if (p[0] == 0x41 && p[1] == 0xba) 1481 return -*(u32 *)(addr + 2); 1482 1483 /* e8 0c 88 a9 cb ed jmp.d8 +12 */ 1484 if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp) 1485 return -*(u32 *)(addr + 2); 1486 1487 return 0; /* invalid hash value */ 1488 } 1489 1490 /* .retpoline_sites */ 1491 static int cfi_disable_callers(s32 *start, s32 *end) 1492 { 1493 /* 1494 * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate 1495 * in tact for later usage. Also see decode_caller_hash() and 1496 * cfi_rewrite_callers(). 1497 */ 1498 const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp }; 1499 s32 *s; 1500 1501 for (s = start; s < end; s++) { 1502 void *addr = (void *)s + *s; 1503 u32 hash; 1504 1505 addr -= fineibt_caller_size; 1506 hash = decode_caller_hash(addr); 1507 if (!hash) /* nocfi callers */ 1508 continue; 1509 1510 text_poke_early(addr, jmp, 2); 1511 } 1512 1513 return 0; 1514 } 1515 1516 static int cfi_enable_callers(s32 *start, s32 *end) 1517 { 1518 /* 1519 * Re-enable kCFI, undo what cfi_disable_callers() did. 1520 */ 1521 const u8 mov[] = { 0x41, 0xba }; 1522 s32 *s; 1523 1524 for (s = start; s < end; s++) { 1525 void *addr = (void *)s + *s; 1526 u32 hash; 1527 1528 addr -= fineibt_caller_size; 1529 hash = decode_caller_hash(addr); 1530 if (!hash) /* nocfi callers */ 1531 continue; 1532 1533 text_poke_early(addr, mov, 2); 1534 } 1535 1536 return 0; 1537 } 1538 1539 /* .cfi_sites */ 1540 static int cfi_rand_preamble(s32 *start, s32 *end) 1541 { 1542 s32 *s; 1543 1544 for (s = start; s < end; s++) { 1545 void *addr = (void *)s + *s; 1546 u32 hash; 1547 1548 hash = decode_preamble_hash(addr, NULL); 1549 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", 1550 addr, addr, 5, addr)) 1551 return -EINVAL; 1552 1553 hash = cfi_rehash(hash); 1554 text_poke_early(addr + 1, &hash, 4); 1555 } 1556 1557 return 0; 1558 } 1559 1560 /* 1561 * Inline the bhi-arity 1 case: 1562 * 1563 * __cfi_foo: 1564 * 0: f3 0f 1e fa endbr64 1565 * 4: 2d 78 56 34 12 sub $0x12345678, %eax 1566 * 9: 49 0f 45 fa cmovne %rax, %rdi 1567 * d: 2e 75 03 jne,pn foo+0x3 1568 * 1569 * foo: 1570 * 10: 0f 1f 40 <d6> nopl -42(%rax) 1571 * 1572 * Notably, this scheme is incompatible with permissive CFI 1573 * because the CMOVcc is unconditional and RDI will have been 1574 * clobbered. 1575 */ 1576 asm( ".pushsection .rodata \n" 1577 "fineibt_bhi1_start: \n" 1578 " cmovne %rax, %rdi \n" 1579 " cs jne fineibt_bhi1_func + 0x3 \n" 1580 "fineibt_bhi1_func: \n" 1581 " nopl -42(%rax) \n" 1582 "fineibt_bhi1_end: \n" 1583 ".popsection \n" 1584 ); 1585 1586 extern u8 fineibt_bhi1_start[]; 1587 extern u8 fineibt_bhi1_end[]; 1588 1589 #define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start) 1590 1591 static void cfi_fineibt_bhi_preamble(void *addr, int arity) 1592 { 1593 u8 bytes[MAX_INSN_SIZE]; 1594 1595 if (!arity) 1596 return; 1597 1598 if (!cfi_warn && arity == 1) { 1599 text_poke_early(addr + fineibt_preamble_bhi, 1600 fineibt_bhi1_start, fineibt_bhi1_size); 1601 return; 1602 } 1603 1604 /* 1605 * Replace the bytes at fineibt_preamble_bhi with a CALL instruction 1606 * that lines up exactly with the end of the preamble, such that the 1607 * return address will be foo+0. 1608 * 1609 * __cfi_foo: 1610 * 0: f3 0f 1e fa endbr64 1611 * 4: 2d 78 56 34 12 sub $0x12345678, %eax 1612 * 9: 2e 2e e8 DD DD DD DD cs cs call __bhi_args[arity] 1613 */ 1614 bytes[0] = 0x2e; 1615 bytes[1] = 0x2e; 1616 __text_gen_insn(bytes + 2, CALL_INSN_OPCODE, 1617 addr + fineibt_preamble_bhi + 2, 1618 __bhi_args[arity], CALL_INSN_SIZE); 1619 1620 text_poke_early(addr + fineibt_preamble_bhi, bytes, 7); 1621 } 1622 1623 static int cfi_rewrite_preamble(s32 *start, s32 *end) 1624 { 1625 s32 *s; 1626 1627 for (s = start; s < end; s++) { 1628 void *addr = (void *)s + *s; 1629 int arity; 1630 u32 hash; 1631 1632 /* 1633 * When the function doesn't start with ENDBR the compiler will 1634 * have determined there are no indirect calls to it and we 1635 * don't need no CFI either. 1636 */ 1637 if (!is_endbr(addr + 16)) 1638 continue; 1639 1640 hash = decode_preamble_hash(addr, &arity); 1641 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", 1642 addr, addr, 5, addr)) 1643 return -EINVAL; 1644 1645 text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); 1646 WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678); 1647 text_poke_early(addr + fineibt_preamble_hash, &hash, 4); 1648 1649 WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity, 1650 "kCFI preamble has wrong register at: %pS %*ph\n", 1651 addr, 5, addr); 1652 1653 if (cfi_bhi) 1654 cfi_fineibt_bhi_preamble(addr, arity); 1655 } 1656 1657 return 0; 1658 } 1659 1660 static void cfi_rewrite_endbr(s32 *start, s32 *end) 1661 { 1662 s32 *s; 1663 1664 for (s = start; s < end; s++) { 1665 void *addr = (void *)s + *s; 1666 1667 if (!exact_endbr(addr + 16)) 1668 continue; 1669 1670 poison_endbr(addr + 16); 1671 } 1672 } 1673 1674 /* .retpoline_sites */ 1675 static int cfi_rand_callers(s32 *start, s32 *end) 1676 { 1677 s32 *s; 1678 1679 for (s = start; s < end; s++) { 1680 void *addr = (void *)s + *s; 1681 u32 hash; 1682 1683 addr -= fineibt_caller_size; 1684 hash = decode_caller_hash(addr); 1685 if (hash) { 1686 hash = -cfi_rehash(hash); 1687 text_poke_early(addr + 2, &hash, 4); 1688 } 1689 } 1690 1691 return 0; 1692 } 1693 1694 static int emit_paranoid_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) 1695 { 1696 u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2; 1697 1698 #ifdef CONFIG_MITIGATION_ITS 1699 u8 *tmp = its_allocate_thunk(reg); 1700 if (tmp) 1701 thunk = tmp; 1702 #endif 1703 1704 return __emit_trampoline(addr, insn, bytes, thunk, thunk); 1705 } 1706 1707 static int cfi_rewrite_callers(s32 *start, s32 *end) 1708 { 1709 s32 *s; 1710 1711 for (s = start; s < end; s++) { 1712 void *addr = (void *)s + *s; 1713 struct insn insn; 1714 u8 bytes[20]; 1715 u32 hash; 1716 int ret; 1717 u8 op; 1718 1719 addr -= fineibt_caller_size; 1720 hash = decode_caller_hash(addr); 1721 if (!hash) 1722 continue; 1723 1724 if (!cfi_paranoid) { 1725 text_poke_early(addr, fineibt_caller_start, fineibt_caller_size); 1726 WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678); 1727 text_poke_early(addr + fineibt_caller_hash, &hash, 4); 1728 /* rely on apply_retpolines() */ 1729 continue; 1730 } 1731 1732 /* cfi_paranoid */ 1733 ret = insn_decode_kernel(&insn, addr + fineibt_caller_size); 1734 if (WARN_ON_ONCE(ret < 0)) 1735 continue; 1736 1737 op = insn.opcode.bytes[0]; 1738 if (op != CALL_INSN_OPCODE && op != JMP32_INSN_OPCODE) { 1739 WARN_ON_ONCE(1); 1740 continue; 1741 } 1742 1743 memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size); 1744 memcpy(bytes + fineibt_caller_hash, &hash, 4); 1745 1746 if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + fineibt_paranoid_ind, 11)) { 1747 emit_paranoid_trampoline(addr + fineibt_caller_size, 1748 &insn, 11, bytes + fineibt_caller_size); 1749 } else { 1750 int len = fineibt_paranoid_size - fineibt_paranoid_ind; 1751 ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len); 1752 if (WARN_ON_ONCE(ret != len)) 1753 continue; 1754 } 1755 1756 text_poke_early(addr, bytes, fineibt_paranoid_size); 1757 } 1758 1759 return 0; 1760 } 1761 1762 #define pr_cfi_debug(X...) if (cfi_debug) pr_info(X) 1763 1764 #define FINEIBT_WARN(_f, _v) \ 1765 WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v) 1766 1767 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 1768 s32 *start_cfi, s32 *end_cfi, bool builtin) 1769 { 1770 int ret; 1771 1772 if (FINEIBT_WARN(fineibt_preamble_size, 20) || 1773 FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) || 1774 FINEIBT_WARN(fineibt_caller_size, 14) || 1775 FINEIBT_WARN(fineibt_paranoid_size, 20)) 1776 return; 1777 1778 if (cfi_mode == CFI_AUTO) { 1779 cfi_mode = CFI_KCFI; 1780 if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) { 1781 /* 1782 * FRED has much saner context on exception entry and 1783 * is less easy to take advantage of. 1784 */ 1785 if (!cpu_feature_enabled(X86_FEATURE_FRED)) 1786 cfi_paranoid = true; 1787 cfi_mode = CFI_FINEIBT; 1788 } 1789 } 1790 1791 /* 1792 * Rewrite the callers to not use the __cfi_ stubs, such that we might 1793 * rewrite them. This disables all CFI. If this succeeds but any of the 1794 * later stages fails, we're without CFI. 1795 */ 1796 pr_cfi_debug("CFI: disabling all indirect call checking\n"); 1797 ret = cfi_disable_callers(start_retpoline, end_retpoline); 1798 if (ret) 1799 goto err; 1800 1801 if (cfi_rand) { 1802 if (builtin) { 1803 cfi_seed = get_random_u32(); 1804 cfi_bpf_hash = cfi_rehash(cfi_bpf_hash); 1805 cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash); 1806 } 1807 pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed); 1808 1809 pr_cfi_debug("CFI: rehashing all preambles\n"); 1810 ret = cfi_rand_preamble(start_cfi, end_cfi); 1811 if (ret) 1812 goto err; 1813 1814 pr_cfi_debug("CFI: rehashing all indirect calls\n"); 1815 ret = cfi_rand_callers(start_retpoline, end_retpoline); 1816 if (ret) 1817 goto err; 1818 } else { 1819 pr_cfi_debug("CFI: rehashing disabled\n"); 1820 } 1821 1822 switch (cfi_mode) { 1823 case CFI_OFF: 1824 if (builtin) 1825 pr_info("CFI: disabled\n"); 1826 return; 1827 1828 case CFI_KCFI: 1829 pr_cfi_debug("CFI: re-enabling all indirect call checking\n"); 1830 ret = cfi_enable_callers(start_retpoline, end_retpoline); 1831 if (ret) 1832 goto err; 1833 1834 if (builtin) 1835 pr_info("CFI: Using %sretpoline kCFI\n", 1836 cfi_rand ? "rehashed " : ""); 1837 return; 1838 1839 case CFI_FINEIBT: 1840 pr_cfi_debug("CFI: adding FineIBT to all preambles\n"); 1841 /* place the FineIBT preamble at func()-16 */ 1842 ret = cfi_rewrite_preamble(start_cfi, end_cfi); 1843 if (ret) 1844 goto err; 1845 1846 /* rewrite the callers to target func()-16 */ 1847 pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n"); 1848 ret = cfi_rewrite_callers(start_retpoline, end_retpoline); 1849 if (ret) 1850 goto err; 1851 1852 /* now that nobody targets func()+0, remove ENDBR there */ 1853 pr_cfi_debug("CFI: removing old endbr insns\n"); 1854 cfi_rewrite_endbr(start_cfi, end_cfi); 1855 1856 if (builtin) { 1857 pr_info("Using %sFineIBT%s CFI\n", 1858 cfi_paranoid ? "paranoid " : "", 1859 cfi_bhi ? "+BHI" : ""); 1860 } 1861 return; 1862 1863 default: 1864 break; 1865 } 1866 1867 err: 1868 pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n"); 1869 } 1870 1871 static inline void poison_hash(void *addr) 1872 { 1873 *(u32 *)addr = 0; 1874 } 1875 1876 static void poison_cfi(void *addr) 1877 { 1878 /* 1879 * Compilers manage to be inconsistent with ENDBR vs __cfi prefixes, 1880 * some (static) functions for which they can determine the address 1881 * is never taken do not get a __cfi prefix, but *DO* get an ENDBR. 1882 * 1883 * As such, these functions will get sealed, but we need to be careful 1884 * to not unconditionally scribble the previous function. 1885 */ 1886 switch (cfi_mode) { 1887 case CFI_FINEIBT: 1888 /* 1889 * FineIBT prefix should start with an ENDBR. 1890 */ 1891 if (!is_endbr(addr)) 1892 break; 1893 1894 /* 1895 * __cfi_\func: 1896 * nopl -42(%rax) 1897 * sub $0, %eax 1898 * jne \func+3 1899 * \func: 1900 * nopl -42(%rax) 1901 */ 1902 poison_endbr(addr); 1903 poison_hash(addr + fineibt_preamble_hash); 1904 break; 1905 1906 case CFI_KCFI: 1907 /* 1908 * kCFI prefix should start with a valid hash. 1909 */ 1910 if (!decode_preamble_hash(addr, NULL)) 1911 break; 1912 1913 /* 1914 * __cfi_\func: 1915 * movl $0, %eax 1916 * .skip 11, 0x90 1917 */ 1918 poison_hash(addr + 1); 1919 break; 1920 1921 default: 1922 break; 1923 } 1924 } 1925 1926 #define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) 1927 1928 /* 1929 * When regs->ip points to a 0xD6 byte in the FineIBT preamble, 1930 * return true and fill out target and type. 1931 * 1932 * We check the preamble by checking for the ENDBR instruction relative to the 1933 * UDB instruction. 1934 */ 1935 static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type) 1936 { 1937 unsigned long addr = regs->ip - fineibt_preamble_ud; 1938 u32 hash; 1939 1940 if (!exact_endbr((void *)addr)) 1941 return false; 1942 1943 *target = addr + fineibt_prefix_size; 1944 1945 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); 1946 *type = (u32)regs->ax + hash; 1947 1948 /* 1949 * Since regs->ip points to the middle of an instruction; it cannot 1950 * continue with the normal fixup. 1951 */ 1952 regs->ip = *target; 1953 1954 return true; 1955 1956 Efault: 1957 return false; 1958 } 1959 1960 /* 1961 * regs->ip points to one of the UD2 in __bhi_args[]. 1962 */ 1963 static bool decode_fineibt_bhi(struct pt_regs *regs, unsigned long *target, u32 *type) 1964 { 1965 unsigned long addr; 1966 u32 hash; 1967 1968 if (!cfi_bhi) 1969 return false; 1970 1971 if (regs->ip < (unsigned long)__bhi_args || 1972 regs->ip >= (unsigned long)__bhi_args_end) 1973 return false; 1974 1975 /* 1976 * Fetch the return address from the stack, this points to the 1977 * FineIBT preamble. Since the CALL instruction is in the 5 last 1978 * bytes of the preamble, the return address is in fact the target 1979 * address. 1980 */ 1981 __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault); 1982 *target = addr; 1983 1984 addr -= fineibt_prefix_size; 1985 if (!exact_endbr((void *)addr)) 1986 return false; 1987 1988 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); 1989 *type = (u32)regs->ax + hash; 1990 1991 /* 1992 * The UD2 sites are constructed with a RET immediately following, 1993 * as such the non-fatal case can use the regular fixup. 1994 */ 1995 return true; 1996 1997 Efault: 1998 return false; 1999 } 2000 2001 static bool is_paranoid_thunk(unsigned long addr) 2002 { 2003 u32 thunk; 2004 2005 __get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault); 2006 return (thunk & 0x00FFFFFF) == 0xfd75d6; 2007 2008 Efault: 2009 return false; 2010 } 2011 2012 /* 2013 * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[] 2014 * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk. 2015 */ 2016 static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type) 2017 { 2018 unsigned long addr = regs->ip - fineibt_paranoid_ud; 2019 2020 if (!cfi_paranoid) 2021 return false; 2022 2023 if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) { 2024 *target = regs->r11 + fineibt_prefix_size; 2025 *type = regs->ax; 2026 2027 /* 2028 * Since the trapping instruction is the exact, but LOCK prefixed, 2029 * Jcc.d8 that got us here, the normal fixup will work. 2030 */ 2031 return true; 2032 } 2033 2034 /* 2035 * The cfi_paranoid + ITS thunk combination results in: 2036 * 2037 * 0: b8 78 56 34 12 mov $0x12345678, %eax 2038 * 5: 41 3b 43 f7 cmp -11(%r11), %eax 2039 * a: 2e 3d 8d 5b f0 cs lea -0x10(%r11), %r11 2040 * e: 2e e8 XX XX XX XX cs call __x86_indirect_paranoid_thunk_r11 2041 * 2042 * Where the paranoid_thunk looks like: 2043 * 2044 * 1d: <d6> udb 2045 * __x86_indirect_paranoid_thunk_r11: 2046 * 1e: 75 fd jne 1d 2047 * __x86_indirect_its_thunk_r11: 2048 * 20: 41 ff eb jmp *%r11 2049 * 23: cc int3 2050 * 2051 */ 2052 if (is_paranoid_thunk(regs->ip)) { 2053 *target = regs->r11 + fineibt_prefix_size; 2054 *type = regs->ax; 2055 2056 regs->ip = *target; 2057 return true; 2058 } 2059 2060 return false; 2061 } 2062 2063 bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) 2064 { 2065 if (decode_fineibt_paranoid(regs, target, type)) 2066 return true; 2067 2068 if (decode_fineibt_bhi(regs, target, type)) 2069 return true; 2070 2071 return decode_fineibt_preamble(regs, target, type); 2072 } 2073 2074 #else /* !CONFIG_FINEIBT: */ 2075 2076 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 2077 s32 *start_cfi, s32 *end_cfi, bool builtin) 2078 { 2079 if (IS_ENABLED(CONFIG_CFI) && builtin) 2080 pr_info("CFI: Using standard kCFI\n"); 2081 } 2082 2083 #ifdef CONFIG_X86_KERNEL_IBT 2084 static void poison_cfi(void *addr) { } 2085 #endif 2086 2087 #endif /* !CONFIG_FINEIBT */ 2088 2089 void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 2090 s32 *start_cfi, s32 *end_cfi) 2091 { 2092 return __apply_fineibt(start_retpoline, end_retpoline, 2093 start_cfi, end_cfi, 2094 /* .builtin = */ false); 2095 } 2096 2097 #ifdef CONFIG_SMP 2098 static void alternatives_smp_lock(const s32 *start, const s32 *end, 2099 u8 *text, u8 *text_end) 2100 { 2101 const s32 *poff; 2102 2103 for (poff = start; poff < end; poff++) { 2104 u8 *ptr = (u8 *)poff + *poff; 2105 2106 if (!*poff || ptr < text || ptr >= text_end) 2107 continue; 2108 /* turn DS segment override prefix into lock prefix */ 2109 if (*ptr == 0x3e) 2110 text_poke(ptr, ((unsigned char []){0xf0}), 1); 2111 } 2112 } 2113 2114 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 2115 u8 *text, u8 *text_end) 2116 { 2117 const s32 *poff; 2118 2119 for (poff = start; poff < end; poff++) { 2120 u8 *ptr = (u8 *)poff + *poff; 2121 2122 if (!*poff || ptr < text || ptr >= text_end) 2123 continue; 2124 /* turn lock prefix into DS segment override prefix */ 2125 if (*ptr == 0xf0) 2126 text_poke(ptr, ((unsigned char []){0x3E}), 1); 2127 } 2128 } 2129 2130 struct smp_alt_module { 2131 /* what is this ??? */ 2132 struct module *mod; 2133 char *name; 2134 2135 /* ptrs to lock prefixes */ 2136 const s32 *locks; 2137 const s32 *locks_end; 2138 2139 /* .text segment, needed to avoid patching init code ;) */ 2140 u8 *text; 2141 u8 *text_end; 2142 2143 struct list_head next; 2144 }; 2145 static LIST_HEAD(smp_alt_modules); 2146 static bool uniproc_patched = false; /* protected by text_mutex */ 2147 2148 void __init_or_module alternatives_smp_module_add(struct module *mod, 2149 char *name, 2150 void *locks, void *locks_end, 2151 void *text, void *text_end) 2152 { 2153 struct smp_alt_module *smp; 2154 2155 mutex_lock(&text_mutex); 2156 if (!uniproc_patched) 2157 goto unlock; 2158 2159 if (num_possible_cpus() == 1) 2160 /* Don't bother remembering, we'll never have to undo it. */ 2161 goto smp_unlock; 2162 2163 smp = kzalloc_obj(*smp, GFP_KERNEL); 2164 if (NULL == smp) 2165 /* we'll run the (safe but slow) SMP code then ... */ 2166 goto unlock; 2167 2168 smp->mod = mod; 2169 smp->name = name; 2170 smp->locks = locks; 2171 smp->locks_end = locks_end; 2172 smp->text = text; 2173 smp->text_end = text_end; 2174 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", 2175 smp->locks, smp->locks_end, 2176 smp->text, smp->text_end, smp->name); 2177 2178 list_add_tail(&smp->next, &smp_alt_modules); 2179 smp_unlock: 2180 alternatives_smp_unlock(locks, locks_end, text, text_end); 2181 unlock: 2182 mutex_unlock(&text_mutex); 2183 } 2184 2185 void __init_or_module alternatives_smp_module_del(struct module *mod) 2186 { 2187 struct smp_alt_module *item; 2188 2189 mutex_lock(&text_mutex); 2190 list_for_each_entry(item, &smp_alt_modules, next) { 2191 if (mod != item->mod) 2192 continue; 2193 list_del(&item->next); 2194 kfree(item); 2195 break; 2196 } 2197 mutex_unlock(&text_mutex); 2198 } 2199 2200 void alternatives_enable_smp(void) 2201 { 2202 struct smp_alt_module *mod; 2203 2204 /* Why bother if there are no other CPUs? */ 2205 BUG_ON(num_possible_cpus() == 1); 2206 2207 mutex_lock(&text_mutex); 2208 2209 if (uniproc_patched) { 2210 pr_info("switching to SMP code\n"); 2211 BUG_ON(num_online_cpus() != 1); 2212 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 2213 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 2214 list_for_each_entry(mod, &smp_alt_modules, next) 2215 alternatives_smp_lock(mod->locks, mod->locks_end, 2216 mod->text, mod->text_end); 2217 uniproc_patched = false; 2218 } 2219 mutex_unlock(&text_mutex); 2220 } 2221 2222 /* 2223 * Return 1 if the address range is reserved for SMP-alternatives. 2224 * Must hold text_mutex. 2225 */ 2226 int alternatives_text_reserved(void *start, void *end) 2227 { 2228 struct smp_alt_module *mod; 2229 const s32 *poff; 2230 u8 *text_start = start; 2231 u8 *text_end = end; 2232 2233 lockdep_assert_held(&text_mutex); 2234 2235 list_for_each_entry(mod, &smp_alt_modules, next) { 2236 if (mod->text > text_end || mod->text_end < text_start) 2237 continue; 2238 for (poff = mod->locks; poff < mod->locks_end; poff++) { 2239 const u8 *ptr = (const u8 *)poff + *poff; 2240 2241 if (text_start <= ptr && text_end > ptr) 2242 return 1; 2243 } 2244 } 2245 2246 return 0; 2247 } 2248 #endif /* CONFIG_SMP */ 2249 2250 /* 2251 * Self-test for the INT3 based CALL emulation code. 2252 * 2253 * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up 2254 * properly and that there is a stack gap between the INT3 frame and the 2255 * previous context. Without this gap doing a virtual PUSH on the interrupted 2256 * stack would corrupt the INT3 IRET frame. 2257 * 2258 * See entry_{32,64}.S for more details. 2259 */ 2260 2261 extern void int3_selftest_asm(unsigned int *ptr); 2262 2263 asm ( 2264 " .pushsection .init.text, \"ax\", @progbits\n" 2265 " .type int3_selftest_asm, @function\n" 2266 "int3_selftest_asm:\n" 2267 ANNOTATE_NOENDBR "\n" 2268 /* 2269 * INT3 padded with NOP to CALL_INSN_SIZE. The INT3 triggers an 2270 * exception, then the int3_exception_nb notifier emulates a call to 2271 * int3_selftest_callee(). 2272 */ 2273 " int3; nop; nop; nop; nop\n" 2274 ASM_RET 2275 " .size int3_selftest_asm, . - int3_selftest_asm\n" 2276 " .popsection\n" 2277 ); 2278 2279 extern void int3_selftest_callee(unsigned int *ptr); 2280 2281 asm ( 2282 " .pushsection .init.text, \"ax\", @progbits\n" 2283 " .type int3_selftest_callee, @function\n" 2284 "int3_selftest_callee:\n" 2285 ANNOTATE_NOENDBR "\n" 2286 " movl $0x1234, (%" _ASM_ARG1 ")\n" 2287 ASM_RET 2288 " .size int3_selftest_callee, . - int3_selftest_callee\n" 2289 " .popsection\n" 2290 ); 2291 2292 extern void int3_selftest_ip(void); /* defined in asm below */ 2293 2294 static int __init 2295 int3_exception_notify(struct notifier_block *self, unsigned long val, void *data) 2296 { 2297 unsigned long selftest = (unsigned long)&int3_selftest_asm; 2298 struct die_args *args = data; 2299 struct pt_regs *regs = args->regs; 2300 2301 OPTIMIZER_HIDE_VAR(selftest); 2302 2303 if (!regs || user_mode(regs)) 2304 return NOTIFY_DONE; 2305 2306 if (val != DIE_INT3) 2307 return NOTIFY_DONE; 2308 2309 if (regs->ip - INT3_INSN_SIZE != selftest) 2310 return NOTIFY_DONE; 2311 2312 int3_emulate_call(regs, (unsigned long)&int3_selftest_callee); 2313 return NOTIFY_STOP; 2314 } 2315 2316 /* Must be noinline to ensure uniqueness of int3_selftest_ip. */ 2317 static noinline void __init int3_selftest(void) 2318 { 2319 static __initdata struct notifier_block int3_exception_nb = { 2320 .notifier_call = int3_exception_notify, 2321 .priority = INT_MAX-1, /* last */ 2322 }; 2323 unsigned int val = 0; 2324 2325 BUG_ON(register_die_notifier(&int3_exception_nb)); 2326 2327 /* 2328 * Basically: int3_selftest_callee(&val); but really complicated :-) 2329 */ 2330 int3_selftest_asm(&val); 2331 2332 BUG_ON(val != 0x1234); 2333 2334 unregister_die_notifier(&int3_exception_nb); 2335 } 2336 2337 static __initdata int __alt_reloc_selftest_addr; 2338 2339 extern void __init __alt_reloc_selftest(void *arg); 2340 __visible noinline void __init __alt_reloc_selftest(void *arg) 2341 { 2342 WARN_ON(arg != &__alt_reloc_selftest_addr); 2343 } 2344 2345 static noinline void __init alt_reloc_selftest(void) 2346 { 2347 /* 2348 * Tests text_poke_apply_relocation(). 2349 * 2350 * This has a relative immediate (CALL) in a place other than the first 2351 * instruction and additionally on x86_64 we get a RIP-relative LEA: 2352 * 2353 * lea 0x0(%rip),%rdi # 5d0: R_X86_64_PC32 .init.data+0x5566c 2354 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4 2355 * 2356 * Getting this wrong will either crash and burn or tickle the WARN 2357 * above. 2358 */ 2359 asm_inline volatile ( 2360 ALTERNATIVE("", "lea %[mem], %%" _ASM_ARG1 "; call __alt_reloc_selftest;", X86_FEATURE_ALWAYS) 2361 : ASM_CALL_CONSTRAINT 2362 : [mem] "m" (__alt_reloc_selftest_addr) 2363 : _ASM_ARG1 2364 ); 2365 } 2366 2367 void __init alternative_instructions(void) 2368 { 2369 u64 ibt; 2370 2371 int3_selftest(); 2372 2373 /* 2374 * The patching is not fully atomic, so try to avoid local 2375 * interruptions that might execute the to be patched code. 2376 * Other CPUs are not running. 2377 */ 2378 stop_nmi(); 2379 2380 /* 2381 * Don't stop machine check exceptions while patching. 2382 * MCEs only happen when something got corrupted and in this 2383 * case we must do something about the corruption. 2384 * Ignoring it is worse than an unlikely patching race. 2385 * Also machine checks tend to be broadcast and if one CPU 2386 * goes into machine check the others follow quickly, so we don't 2387 * expect a machine check to cause undue problems during to code 2388 * patching. 2389 */ 2390 2391 /* 2392 * Make sure to set (artificial) features depending on used paravirt 2393 * functions which can later influence alternative patching. 2394 */ 2395 paravirt_set_cap(); 2396 2397 /* Keep CET-IBT disabled until caller/callee are patched */ 2398 ibt = ibt_save(/*disable*/ true); 2399 2400 __apply_fineibt(__retpoline_sites, __retpoline_sites_end, 2401 __cfi_sites, __cfi_sites_end, true); 2402 cfi_debug = false; 2403 2404 /* 2405 * Rewrite the retpolines, must be done before alternatives since 2406 * those can rewrite the retpoline thunks. 2407 */ 2408 apply_retpolines(__retpoline_sites, __retpoline_sites_end); 2409 apply_returns(__return_sites, __return_sites_end); 2410 2411 its_fini_core(); 2412 2413 /* 2414 * Adjust all CALL instructions to point to func()-10, including 2415 * those in .altinstr_replacement. 2416 */ 2417 callthunks_patch_builtin_calls(); 2418 2419 apply_alternatives(__alt_instructions, __alt_instructions_end); 2420 2421 /* 2422 * Seal all functions that do not have their address taken. 2423 */ 2424 apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); 2425 2426 ibt_restore(ibt); 2427 2428 #ifdef CONFIG_SMP 2429 /* Patch to UP if other cpus not imminent. */ 2430 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { 2431 uniproc_patched = true; 2432 alternatives_smp_module_add(NULL, "core kernel", 2433 __smp_locks, __smp_locks_end, 2434 _text, _etext); 2435 } 2436 2437 if (!uniproc_patched || num_possible_cpus() == 1) { 2438 free_init_pages("SMP alternatives", 2439 (unsigned long)__smp_locks, 2440 (unsigned long)__smp_locks_end); 2441 } 2442 #endif 2443 2444 restart_nmi(); 2445 alternatives_patched = 1; 2446 2447 alt_reloc_selftest(); 2448 } 2449 2450 /** 2451 * text_poke_early - Update instructions on a live kernel at boot time 2452 * @addr: address to modify 2453 * @opcode: source of the copy 2454 * @len: length to copy 2455 * 2456 * When you use this code to patch more than one byte of an instruction 2457 * you need to make sure that other CPUs cannot execute this code in parallel. 2458 * Also no thread must be currently preempted in the middle of these 2459 * instructions. And on the local CPU you need to be protected against NMI or 2460 * MCE handlers seeing an inconsistent instruction while you patch. 2461 */ 2462 void __init_or_module text_poke_early(void *addr, const void *opcode, 2463 size_t len) 2464 { 2465 unsigned long flags; 2466 2467 if (boot_cpu_has(X86_FEATURE_NX) && 2468 is_module_text_address((unsigned long)addr)) { 2469 /* 2470 * Modules text is marked initially as non-executable, so the 2471 * code cannot be running and speculative code-fetches are 2472 * prevented. Just change the code. 2473 */ 2474 memcpy(addr, opcode, len); 2475 } else { 2476 local_irq_save(flags); 2477 memcpy(addr, opcode, len); 2478 sync_core(); 2479 local_irq_restore(flags); 2480 2481 /* 2482 * Could also do a CLFLUSH here to speed up CPU recovery; but 2483 * that causes hangs on some VIA CPUs. 2484 */ 2485 } 2486 } 2487 2488 __ro_after_init struct mm_struct *text_poke_mm; 2489 __ro_after_init unsigned long text_poke_mm_addr; 2490 2491 /* 2492 * Text poking creates and uses a mapping in the lower half of the 2493 * address space. Relax LASS enforcement when accessing the poking 2494 * address. 2495 * 2496 * objtool enforces a strict policy of "no function calls within AC=1 2497 * regions". Adhere to the policy by using inline versions of 2498 * memcpy()/memset() that will never result in a function call. 2499 */ 2500 2501 static void text_poke_memcpy(void *dst, const void *src, size_t len) 2502 { 2503 lass_stac(); 2504 __inline_memcpy(dst, src, len); 2505 lass_clac(); 2506 } 2507 2508 static void text_poke_memset(void *dst, const void *src, size_t len) 2509 { 2510 int c = *(const int *)src; 2511 2512 lass_stac(); 2513 __inline_memset(dst, c, len); 2514 lass_clac(); 2515 } 2516 2517 typedef void text_poke_f(void *dst, const void *src, size_t len); 2518 2519 static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) 2520 { 2521 bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; 2522 struct page *pages[2] = {NULL}; 2523 struct mm_struct *prev_mm; 2524 unsigned long flags; 2525 pte_t pte, *ptep; 2526 spinlock_t *ptl; 2527 pgprot_t pgprot; 2528 2529 /* 2530 * While boot memory allocator is running we cannot use struct pages as 2531 * they are not yet initialized. There is no way to recover. 2532 */ 2533 BUG_ON(!after_bootmem); 2534 2535 if (!core_kernel_text((unsigned long)addr)) { 2536 pages[0] = vmalloc_to_page(addr); 2537 if (cross_page_boundary) 2538 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 2539 } else { 2540 pages[0] = virt_to_page(addr); 2541 WARN_ON(!PageReserved(pages[0])); 2542 if (cross_page_boundary) 2543 pages[1] = virt_to_page(addr + PAGE_SIZE); 2544 } 2545 /* 2546 * If something went wrong, crash and burn since recovery paths are not 2547 * implemented. 2548 */ 2549 BUG_ON(!pages[0] || (cross_page_boundary && !pages[1])); 2550 2551 /* 2552 * Map the page without the global bit, as TLB flushing is done with 2553 * flush_tlb_mm_range(), which is intended for non-global PTEs. 2554 */ 2555 pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL); 2556 2557 /* 2558 * The lock is not really needed, but this allows to avoid open-coding. 2559 */ 2560 ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl); 2561 2562 /* 2563 * This must not fail; preallocated in poking_init(). 2564 */ 2565 VM_BUG_ON(!ptep); 2566 2567 local_irq_save(flags); 2568 2569 pte = mk_pte(pages[0], pgprot); 2570 set_pte_at(text_poke_mm, text_poke_mm_addr, ptep, pte); 2571 2572 if (cross_page_boundary) { 2573 pte = mk_pte(pages[1], pgprot); 2574 set_pte_at(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1, pte); 2575 } 2576 2577 /* 2578 * Loading the temporary mm behaves as a compiler barrier, which 2579 * guarantees that the PTE will be set at the time memcpy() is done. 2580 */ 2581 prev_mm = use_temporary_mm(text_poke_mm); 2582 2583 kasan_disable_current(); 2584 func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len); 2585 kasan_enable_current(); 2586 2587 /* 2588 * Ensure that the PTE is only cleared after the instructions of memcpy 2589 * were issued by using a compiler barrier. 2590 */ 2591 barrier(); 2592 2593 pte_clear(text_poke_mm, text_poke_mm_addr, ptep); 2594 if (cross_page_boundary) 2595 pte_clear(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1); 2596 2597 /* 2598 * Loading the previous page-table hierarchy requires a serializing 2599 * instruction that already allows the core to see the updated version. 2600 * Xen-PV is assumed to serialize execution in a similar manner. 2601 */ 2602 unuse_temporary_mm(prev_mm); 2603 2604 /* 2605 * Flushing the TLB might involve IPIs, which would require enabled 2606 * IRQs, but not if the mm is not used, as it is in this point. 2607 */ 2608 flush_tlb_mm_range(text_poke_mm, text_poke_mm_addr, text_poke_mm_addr + 2609 (cross_page_boundary ? 2 : 1) * PAGE_SIZE, 2610 PAGE_SHIFT, false); 2611 2612 if (func == text_poke_memcpy) { 2613 /* 2614 * If the text does not match what we just wrote then something is 2615 * fundamentally screwy; there's nothing we can really do about that. 2616 */ 2617 BUG_ON(memcmp(addr, src, len)); 2618 } 2619 2620 local_irq_restore(flags); 2621 pte_unmap_unlock(ptep, ptl); 2622 return addr; 2623 } 2624 2625 /** 2626 * text_poke - Update instructions on a live kernel 2627 * @addr: address to modify 2628 * @opcode: source of the copy 2629 * @len: length to copy 2630 * 2631 * Only atomic text poke/set should be allowed when not doing early patching. 2632 * It means the size must be writable atomically and the address must be aligned 2633 * in a way that permits an atomic write. It also makes sure we fit on a single 2634 * page. 2635 * 2636 * Note that the caller must ensure that if the modified code is part of a 2637 * module, the module would not be removed during poking. This can be achieved 2638 * by registering a module notifier, and ordering module removal and patching 2639 * through a mutex. 2640 */ 2641 void *text_poke(void *addr, const void *opcode, size_t len) 2642 { 2643 lockdep_assert_held(&text_mutex); 2644 2645 return __text_poke(text_poke_memcpy, addr, opcode, len); 2646 } 2647 2648 /** 2649 * text_poke_kgdb - Update instructions on a live kernel by kgdb 2650 * @addr: address to modify 2651 * @opcode: source of the copy 2652 * @len: length to copy 2653 * 2654 * Only atomic text poke/set should be allowed when not doing early patching. 2655 * It means the size must be writable atomically and the address must be aligned 2656 * in a way that permits an atomic write. It also makes sure we fit on a single 2657 * page. 2658 * 2659 * Context: should only be used by kgdb, which ensures no other core is running, 2660 * despite the fact it does not hold the text_mutex. 2661 */ 2662 void *text_poke_kgdb(void *addr, const void *opcode, size_t len) 2663 { 2664 return __text_poke(text_poke_memcpy, addr, opcode, len); 2665 } 2666 2667 void *text_poke_copy_locked(void *addr, const void *opcode, size_t len, 2668 bool core_ok) 2669 { 2670 unsigned long start = (unsigned long)addr; 2671 size_t patched = 0; 2672 2673 if (WARN_ON_ONCE(!core_ok && core_kernel_text(start))) 2674 return NULL; 2675 2676 while (patched < len) { 2677 unsigned long ptr = start + patched; 2678 size_t s; 2679 2680 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); 2681 2682 __text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s); 2683 patched += s; 2684 } 2685 return addr; 2686 } 2687 2688 /** 2689 * text_poke_copy - Copy instructions into (an unused part of) RX memory 2690 * @addr: address to modify 2691 * @opcode: source of the copy 2692 * @len: length to copy, could be more than 2x PAGE_SIZE 2693 * 2694 * Not safe against concurrent execution; useful for JITs to dump 2695 * new code blocks into unused regions of RX memory. Can be used in 2696 * conjunction with synchronize_rcu_tasks() to wait for existing 2697 * execution to quiesce after having made sure no existing functions 2698 * pointers are live. 2699 */ 2700 void *text_poke_copy(void *addr, const void *opcode, size_t len) 2701 { 2702 mutex_lock(&text_mutex); 2703 addr = text_poke_copy_locked(addr, opcode, len, false); 2704 mutex_unlock(&text_mutex); 2705 return addr; 2706 } 2707 2708 /** 2709 * text_poke_set - memset into (an unused part of) RX memory 2710 * @addr: address to modify 2711 * @c: the byte to fill the area with 2712 * @len: length to copy, could be more than 2x PAGE_SIZE 2713 * 2714 * This is useful to overwrite unused regions of RX memory with illegal 2715 * instructions. 2716 */ 2717 void *text_poke_set(void *addr, int c, size_t len) 2718 { 2719 unsigned long start = (unsigned long)addr; 2720 size_t patched = 0; 2721 2722 if (WARN_ON_ONCE(core_kernel_text(start))) 2723 return NULL; 2724 2725 mutex_lock(&text_mutex); 2726 while (patched < len) { 2727 unsigned long ptr = start + patched; 2728 size_t s; 2729 2730 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); 2731 2732 __text_poke(text_poke_memset, (void *)ptr, (void *)&c, s); 2733 patched += s; 2734 } 2735 mutex_unlock(&text_mutex); 2736 return addr; 2737 } 2738 2739 static void do_sync_core(void *info) 2740 { 2741 sync_core(); 2742 } 2743 2744 void smp_text_poke_sync_each_cpu(void) 2745 { 2746 on_each_cpu(do_sync_core, NULL, 1); 2747 } 2748 2749 /* 2750 * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of 2751 * this thing. When len == 6 everything is prefixed with 0x0f and we map 2752 * opcode to Jcc.d8, using len to distinguish. 2753 */ 2754 struct smp_text_poke_loc { 2755 /* addr := _stext + rel_addr */ 2756 s32 rel_addr; 2757 s32 disp; 2758 u8 len; 2759 u8 opcode; 2760 const u8 text[TEXT_POKE_MAX_OPCODE_SIZE]; 2761 /* see smp_text_poke_batch_finish() */ 2762 u8 old; 2763 }; 2764 2765 #define TEXT_POKE_ARRAY_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) 2766 2767 static struct smp_text_poke_array { 2768 struct smp_text_poke_loc vec[TEXT_POKE_ARRAY_MAX]; 2769 int nr_entries; 2770 } text_poke_array; 2771 2772 static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); 2773 2774 /* 2775 * These four __always_inline annotations imply noinstr, necessary 2776 * due to smp_text_poke_int3_handler() being noinstr: 2777 */ 2778 2779 static __always_inline bool try_get_text_poke_array(void) 2780 { 2781 atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); 2782 2783 if (!raw_atomic_inc_not_zero(refs)) 2784 return false; 2785 2786 return true; 2787 } 2788 2789 static __always_inline void put_text_poke_array(void) 2790 { 2791 atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); 2792 2793 smp_mb__before_atomic(); 2794 raw_atomic_dec(refs); 2795 } 2796 2797 static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tpl) 2798 { 2799 return _stext + tpl->rel_addr; 2800 } 2801 2802 static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b) 2803 { 2804 if (tpl_a < text_poke_addr(tpl_b)) 2805 return -1; 2806 if (tpl_a > text_poke_addr(tpl_b)) 2807 return 1; 2808 return 0; 2809 } 2810 2811 noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) 2812 { 2813 struct smp_text_poke_loc *tpl; 2814 int ret = 0; 2815 void *ip; 2816 2817 if (user_mode(regs)) 2818 return 0; 2819 2820 /* 2821 * Having observed our INT3 instruction, we now must observe 2822 * text_poke_array with non-zero refcount: 2823 * 2824 * text_poke_array_refs = 1 INT3 2825 * WMB RMB 2826 * write INT3 if (text_poke_array_refs != 0) 2827 */ 2828 smp_rmb(); 2829 2830 if (!try_get_text_poke_array()) 2831 return 0; 2832 2833 /* 2834 * Discount the INT3. See smp_text_poke_batch_finish(). 2835 */ 2836 ip = (void *) regs->ip - INT3_INSN_SIZE; 2837 2838 /* 2839 * Skip the binary search if there is a single member in the vector. 2840 */ 2841 if (unlikely(text_poke_array.nr_entries > 1)) { 2842 tpl = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries, 2843 sizeof(struct smp_text_poke_loc), 2844 patch_cmp); 2845 if (!tpl) 2846 goto out_put; 2847 } else { 2848 tpl = text_poke_array.vec; 2849 if (text_poke_addr(tpl) != ip) 2850 goto out_put; 2851 } 2852 2853 ip += tpl->len; 2854 2855 switch (tpl->opcode) { 2856 case INT3_INSN_OPCODE: 2857 /* 2858 * Someone poked an explicit INT3, they'll want to handle it, 2859 * do not consume. 2860 */ 2861 goto out_put; 2862 2863 case RET_INSN_OPCODE: 2864 int3_emulate_ret(regs); 2865 break; 2866 2867 case CALL_INSN_OPCODE: 2868 int3_emulate_call(regs, (long)ip + tpl->disp); 2869 break; 2870 2871 case JMP32_INSN_OPCODE: 2872 case JMP8_INSN_OPCODE: 2873 int3_emulate_jmp(regs, (long)ip + tpl->disp); 2874 break; 2875 2876 case 0x70 ... 0x7f: /* Jcc */ 2877 int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp); 2878 break; 2879 2880 default: 2881 BUG(); 2882 } 2883 2884 ret = 1; 2885 2886 out_put: 2887 put_text_poke_array(); 2888 return ret; 2889 } 2890 2891 /** 2892 * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP 2893 * 2894 * Input state: 2895 * text_poke_array.vec: vector of instructions to patch 2896 * text_poke_array.nr_entries: number of entries in the vector 2897 * 2898 * Modify multi-byte instructions by using INT3 breakpoints on SMP. 2899 * We completely avoid using stop_machine() here, and achieve the 2900 * synchronization using INT3 breakpoints and SMP cross-calls. 2901 * 2902 * The way it is done: 2903 * - For each entry in the vector: 2904 * - add an INT3 trap to the address that will be patched 2905 * - SMP sync all CPUs 2906 * - For each entry in the vector: 2907 * - update all but the first byte of the patched range 2908 * - SMP sync all CPUs 2909 * - For each entry in the vector: 2910 * - replace the first byte (INT3) by the first byte of the 2911 * replacing opcode 2912 * - SMP sync all CPUs 2913 */ 2914 void smp_text_poke_batch_finish(void) 2915 { 2916 unsigned char int3 = INT3_INSN_OPCODE; 2917 unsigned int i; 2918 int do_sync; 2919 2920 if (!text_poke_array.nr_entries) 2921 return; 2922 2923 lockdep_assert_held(&text_mutex); 2924 2925 /* 2926 * Corresponds to the implicit memory barrier in try_get_text_poke_array() to 2927 * ensure reading a non-zero refcount provides up to date text_poke_array data. 2928 */ 2929 for_each_possible_cpu(i) 2930 atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1); 2931 2932 /* 2933 * Function tracing can enable thousands of places that need to be 2934 * updated. This can take quite some time, and with full kernel debugging 2935 * enabled, this could cause the softlockup watchdog to trigger. 2936 * This function gets called every 256 entries added to be patched. 2937 * Call cond_resched() here to make sure that other tasks can get scheduled 2938 * while processing all the functions being patched. 2939 */ 2940 cond_resched(); 2941 2942 /* 2943 * Corresponding read barrier in INT3 notifier for making sure the 2944 * text_poke_array.nr_entries and handler are correctly ordered wrt. patching. 2945 */ 2946 smp_wmb(); 2947 2948 /* 2949 * First step: add a INT3 trap to the address that will be patched. 2950 */ 2951 for (i = 0; i < text_poke_array.nr_entries; i++) { 2952 text_poke_array.vec[i].old = *(u8 *)text_poke_addr(&text_poke_array.vec[i]); 2953 text_poke(text_poke_addr(&text_poke_array.vec[i]), &int3, INT3_INSN_SIZE); 2954 } 2955 2956 smp_text_poke_sync_each_cpu(); 2957 2958 /* 2959 * Second step: update all but the first byte of the patched range. 2960 */ 2961 for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { 2962 u8 old[TEXT_POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, }; 2963 u8 _new[TEXT_POKE_MAX_OPCODE_SIZE+1]; 2964 const u8 *new = text_poke_array.vec[i].text; 2965 int len = text_poke_array.vec[i].len; 2966 2967 if (len - INT3_INSN_SIZE > 0) { 2968 memcpy(old + INT3_INSN_SIZE, 2969 text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE, 2970 len - INT3_INSN_SIZE); 2971 2972 if (len == 6) { 2973 _new[0] = 0x0f; 2974 memcpy(_new + 1, new, 5); 2975 new = _new; 2976 } 2977 2978 text_poke(text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE, 2979 new + INT3_INSN_SIZE, 2980 len - INT3_INSN_SIZE); 2981 2982 do_sync++; 2983 } 2984 2985 /* 2986 * Emit a perf event to record the text poke, primarily to 2987 * support Intel PT decoding which must walk the executable code 2988 * to reconstruct the trace. The flow up to here is: 2989 * - write INT3 byte 2990 * - IPI-SYNC 2991 * - write instruction tail 2992 * At this point the actual control flow will be through the 2993 * INT3 and handler and not hit the old or new instruction. 2994 * Intel PT outputs FUP/TIP packets for the INT3, so the flow 2995 * can still be decoded. Subsequently: 2996 * - emit RECORD_TEXT_POKE with the new instruction 2997 * - IPI-SYNC 2998 * - write first byte 2999 * - IPI-SYNC 3000 * So before the text poke event timestamp, the decoder will see 3001 * either the old instruction flow or FUP/TIP of INT3. After the 3002 * text poke event timestamp, the decoder will see either the 3003 * new instruction flow or FUP/TIP of INT3. Thus decoders can 3004 * use the timestamp as the point at which to modify the 3005 * executable code. 3006 * The old instruction is recorded so that the event can be 3007 * processed forwards or backwards. 3008 */ 3009 perf_event_text_poke(text_poke_addr(&text_poke_array.vec[i]), old, len, new, len); 3010 } 3011 3012 if (do_sync) { 3013 /* 3014 * According to Intel, this core syncing is very likely 3015 * not necessary and we'd be safe even without it. But 3016 * better safe than sorry (plus there's not only Intel). 3017 */ 3018 smp_text_poke_sync_each_cpu(); 3019 } 3020 3021 /* 3022 * Third step: replace the first byte (INT3) by the first byte of the 3023 * replacing opcode. 3024 */ 3025 for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { 3026 u8 byte = text_poke_array.vec[i].text[0]; 3027 3028 if (text_poke_array.vec[i].len == 6) 3029 byte = 0x0f; 3030 3031 if (byte == INT3_INSN_OPCODE) 3032 continue; 3033 3034 text_poke(text_poke_addr(&text_poke_array.vec[i]), &byte, INT3_INSN_SIZE); 3035 do_sync++; 3036 } 3037 3038 if (do_sync) 3039 smp_text_poke_sync_each_cpu(); 3040 3041 /* 3042 * Remove and wait for refs to be zero. 3043 * 3044 * Notably, if after step-3 above the INT3 got removed, then the 3045 * smp_text_poke_sync_each_cpu() will have serialized against any running INT3 3046 * handlers and the below spin-wait will not happen. 3047 * 3048 * IOW. unless the replacement instruction is INT3, this case goes 3049 * unused. 3050 */ 3051 for_each_possible_cpu(i) { 3052 atomic_t *refs = per_cpu_ptr(&text_poke_array_refs, i); 3053 3054 if (unlikely(!atomic_dec_and_test(refs))) 3055 atomic_cond_read_acquire(refs, !VAL); 3056 } 3057 3058 /* They are all completed: */ 3059 text_poke_array.nr_entries = 0; 3060 } 3061 3062 static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) 3063 { 3064 struct smp_text_poke_loc *tpl; 3065 struct insn insn; 3066 int ret, i = 0; 3067 3068 tpl = &text_poke_array.vec[text_poke_array.nr_entries++]; 3069 3070 if (len == 6) 3071 i = 1; 3072 memcpy((void *)tpl->text, opcode+i, len-i); 3073 if (!emulate) 3074 emulate = opcode; 3075 3076 ret = insn_decode_kernel(&insn, emulate); 3077 BUG_ON(ret < 0); 3078 3079 tpl->rel_addr = addr - (void *)_stext; 3080 tpl->len = len; 3081 tpl->opcode = insn.opcode.bytes[0]; 3082 3083 if (is_jcc32(&insn)) { 3084 /* 3085 * Map Jcc.d32 onto Jcc.d8 and use len to distinguish. 3086 */ 3087 tpl->opcode = insn.opcode.bytes[1] - 0x10; 3088 } 3089 3090 switch (tpl->opcode) { 3091 case RET_INSN_OPCODE: 3092 case JMP32_INSN_OPCODE: 3093 case JMP8_INSN_OPCODE: 3094 /* 3095 * Control flow instructions without implied execution of the 3096 * next instruction can be padded with INT3. 3097 */ 3098 for (i = insn.length; i < len; i++) 3099 BUG_ON(tpl->text[i] != INT3_INSN_OPCODE); 3100 break; 3101 3102 default: 3103 BUG_ON(len != insn.length); 3104 } 3105 3106 switch (tpl->opcode) { 3107 case INT3_INSN_OPCODE: 3108 case RET_INSN_OPCODE: 3109 break; 3110 3111 case CALL_INSN_OPCODE: 3112 case JMP32_INSN_OPCODE: 3113 case JMP8_INSN_OPCODE: 3114 case 0x70 ... 0x7f: /* Jcc */ 3115 tpl->disp = insn.immediate.value; 3116 break; 3117 3118 default: /* assume NOP */ 3119 switch (len) { 3120 case 2: /* NOP2 -- emulate as JMP8+0 */ 3121 BUG_ON(memcmp(emulate, x86_nops[len], len)); 3122 tpl->opcode = JMP8_INSN_OPCODE; 3123 tpl->disp = 0; 3124 break; 3125 3126 case 5: /* NOP5 -- emulate as JMP32+0 */ 3127 BUG_ON(memcmp(emulate, x86_nops[len], len)); 3128 tpl->opcode = JMP32_INSN_OPCODE; 3129 tpl->disp = 0; 3130 break; 3131 3132 default: /* unknown instruction */ 3133 BUG(); 3134 } 3135 break; 3136 } 3137 } 3138 3139 /* 3140 * We hard rely on the text_poke_array.vec being ordered; ensure this is so by flushing 3141 * early if needed. 3142 */ 3143 static bool text_poke_addr_ordered(void *addr) 3144 { 3145 WARN_ON_ONCE(!addr); 3146 3147 if (!text_poke_array.nr_entries) 3148 return true; 3149 3150 /* 3151 * If the last current entry's address is higher than the 3152 * new entry's address we'd like to add, then ordering 3153 * is violated and we must first flush all pending patching 3154 * requests: 3155 */ 3156 if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr) 3157 return false; 3158 3159 return true; 3160 } 3161 3162 /** 3163 * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched 3164 * @addr: address to patch 3165 * @opcode: opcode of new instruction 3166 * @len: length to copy 3167 * @emulate: instruction to be emulated 3168 * 3169 * Add a new instruction to the current queue of to-be-patched instructions 3170 * the kernel maintains. The patching request will not be executed immediately, 3171 * but becomes part of an array of patching requests, optimized for batched 3172 * execution. All pending patching requests will be executed on the next 3173 * smp_text_poke_batch_finish() call. 3174 */ 3175 void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) 3176 { 3177 if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr)) 3178 smp_text_poke_batch_finish(); 3179 __smp_text_poke_batch_add(addr, opcode, len, emulate); 3180 } 3181 3182 /** 3183 * smp_text_poke_single() -- update instruction on live kernel on SMP immediately 3184 * @addr: address to patch 3185 * @opcode: opcode of new instruction 3186 * @len: length to copy 3187 * @emulate: instruction to be emulated 3188 * 3189 * Update a single instruction with the vector in the stack, avoiding 3190 * dynamically allocated memory. This function should be used when it is 3191 * not possible to allocate memory for a vector. The single instruction 3192 * is patched in immediately. 3193 */ 3194 void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) 3195 { 3196 smp_text_poke_batch_add(addr, opcode, len, emulate); 3197 smp_text_poke_batch_finish(); 3198 } 3199