1 // SPDX-License-Identifier: GPL-2.0-only 2 #define pr_fmt(fmt) "SMP alternatives: " fmt 3 4 #include <linux/mmu_context.h> 5 #include <linux/perf_event.h> 6 #include <linux/vmalloc.h> 7 #include <linux/memory.h> 8 #include <linux/execmem.h> 9 10 #include <asm/text-patching.h> 11 #include <asm/insn.h> 12 #include <asm/ibt.h> 13 #include <asm/set_memory.h> 14 #include <asm/nmi.h> 15 16 int __read_mostly alternatives_patched; 17 18 EXPORT_SYMBOL_GPL(alternatives_patched); 19 20 #define MAX_PATCH_LEN (255-1) 21 22 #define DA_ALL (~0) 23 #define DA_ALT 0x01 24 #define DA_RET 0x02 25 #define DA_RETPOLINE 0x04 26 #define DA_ENDBR 0x08 27 #define DA_SMP 0x10 28 29 static unsigned int debug_alternative; 30 31 static int __init debug_alt(char *str) 32 { 33 if (str && *str == '=') 34 str++; 35 36 if (!str || kstrtouint(str, 0, &debug_alternative)) 37 debug_alternative = DA_ALL; 38 39 return 1; 40 } 41 __setup("debug-alternative", debug_alt); 42 43 static int noreplace_smp; 44 45 static int __init setup_noreplace_smp(char *str) 46 { 47 noreplace_smp = 1; 48 return 1; 49 } 50 __setup("noreplace-smp", setup_noreplace_smp); 51 52 #define DPRINTK(type, fmt, args...) \ 53 do { \ 54 if (debug_alternative & DA_##type) \ 55 printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \ 56 } while (0) 57 58 #define DUMP_BYTES(type, buf, len, fmt, args...) \ 59 do { \ 60 if (unlikely(debug_alternative & DA_##type)) { \ 61 int j; \ 62 \ 63 if (!(len)) \ 64 break; \ 65 \ 66 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 67 for (j = 0; j < (len) - 1; j++) \ 68 printk(KERN_CONT "%02hhx ", buf[j]); \ 69 printk(KERN_CONT "%02hhx\n", buf[j]); \ 70 } \ 71 } while (0) 72 73 static const unsigned char x86nops[] = 74 { 75 BYTES_NOP1, 76 BYTES_NOP2, 77 BYTES_NOP3, 78 BYTES_NOP4, 79 BYTES_NOP5, 80 BYTES_NOP6, 81 BYTES_NOP7, 82 BYTES_NOP8, 83 #ifdef CONFIG_64BIT 84 BYTES_NOP9, 85 BYTES_NOP10, 86 BYTES_NOP11, 87 #endif 88 }; 89 90 const unsigned char * const x86_nops[ASM_NOP_MAX+1] = 91 { 92 NULL, 93 x86nops, 94 x86nops + 1, 95 x86nops + 1 + 2, 96 x86nops + 1 + 2 + 3, 97 x86nops + 1 + 2 + 3 + 4, 98 x86nops + 1 + 2 + 3 + 4 + 5, 99 x86nops + 1 + 2 + 3 + 4 + 5 + 6, 100 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 101 #ifdef CONFIG_64BIT 102 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 103 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9, 104 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10, 105 #endif 106 }; 107 108 #ifdef CONFIG_FINEIBT 109 static bool cfi_paranoid __ro_after_init; 110 #endif 111 112 #ifdef CONFIG_MITIGATION_ITS 113 114 #ifdef CONFIG_MODULES 115 static struct module *its_mod; 116 #endif 117 static void *its_page; 118 static unsigned int its_offset; 119 struct its_array its_pages; 120 121 static void *__its_alloc(struct its_array *pages) 122 { 123 void *page __free(execmem) = execmem_alloc_rw(EXECMEM_MODULE_TEXT, PAGE_SIZE); 124 if (!page) 125 return NULL; 126 127 void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *), 128 GFP_KERNEL); 129 if (!tmp) 130 return NULL; 131 132 pages->pages = tmp; 133 pages->pages[pages->num++] = page; 134 135 return no_free_ptr(page); 136 } 137 138 /* Initialize a thunk with the "jmp *reg; int3" instructions. */ 139 static void *its_init_thunk(void *thunk, int reg) 140 { 141 u8 *bytes = thunk; 142 int offset = 0; 143 int i = 0; 144 145 #ifdef CONFIG_FINEIBT 146 if (cfi_paranoid) { 147 /* 148 * When ITS uses indirect branch thunk the fineibt_paranoid 149 * caller sequence doesn't fit in the caller site. So put the 150 * remaining part of the sequence (UDB + JNE) into the ITS 151 * thunk. 152 */ 153 bytes[i++] = 0xd6; /* UDB */ 154 bytes[i++] = 0x75; /* JNE */ 155 bytes[i++] = 0xfd; 156 157 offset = 1; 158 } 159 #endif 160 161 if (reg >= 8) { 162 bytes[i++] = 0x41; /* REX.B prefix */ 163 reg -= 8; 164 } 165 bytes[i++] = 0xff; 166 bytes[i++] = 0xe0 + reg; /* JMP *reg */ 167 bytes[i++] = 0xcc; 168 169 return thunk + offset; 170 } 171 172 static void its_pages_protect(struct its_array *pages) 173 { 174 for (int i = 0; i < pages->num; i++) { 175 void *page = pages->pages[i]; 176 execmem_restore_rox(page, PAGE_SIZE); 177 } 178 } 179 180 static void its_fini_core(void) 181 { 182 if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 183 its_pages_protect(&its_pages); 184 kfree(its_pages.pages); 185 } 186 187 #ifdef CONFIG_MODULES 188 void its_init_mod(struct module *mod) 189 { 190 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 191 return; 192 193 mutex_lock(&text_mutex); 194 its_mod = mod; 195 its_page = NULL; 196 } 197 198 void its_fini_mod(struct module *mod) 199 { 200 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 201 return; 202 203 WARN_ON_ONCE(its_mod != mod); 204 205 its_mod = NULL; 206 its_page = NULL; 207 mutex_unlock(&text_mutex); 208 209 if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 210 its_pages_protect(&mod->arch.its_pages); 211 } 212 213 void its_free_mod(struct module *mod) 214 { 215 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 216 return; 217 218 for (int i = 0; i < mod->arch.its_pages.num; i++) { 219 void *page = mod->arch.its_pages.pages[i]; 220 execmem_free(page); 221 } 222 kfree(mod->arch.its_pages.pages); 223 } 224 #endif /* CONFIG_MODULES */ 225 226 static void *its_alloc(void) 227 { 228 struct its_array *pages = &its_pages; 229 void *page; 230 231 #ifdef CONFIG_MODULES 232 if (its_mod) 233 pages = &its_mod->arch.its_pages; 234 #endif 235 236 page = __its_alloc(pages); 237 if (!page) 238 return NULL; 239 240 if (pages == &its_pages) 241 set_memory_x((unsigned long)page, 1); 242 243 return page; 244 } 245 246 static void *its_allocate_thunk(int reg) 247 { 248 int size = 3 + (reg / 8); 249 void *thunk; 250 251 #ifdef CONFIG_FINEIBT 252 /* 253 * The ITS thunk contains an indirect jump and an int3 instruction so 254 * its size is 3 or 4 bytes depending on the register used. If CFI 255 * paranoid is used then 3 extra bytes are added in the ITS thunk to 256 * complete the fineibt_paranoid caller sequence. 257 */ 258 if (cfi_paranoid) 259 size += 3; 260 #endif 261 262 if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) { 263 its_page = its_alloc(); 264 if (!its_page) { 265 pr_err("ITS page allocation failed\n"); 266 return NULL; 267 } 268 memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE); 269 its_offset = 32; 270 } 271 272 /* 273 * If the indirect branch instruction will be in the lower half 274 * of a cacheline, then update the offset to reach the upper half. 275 */ 276 if ((its_offset + size - 1) % 64 < 32) 277 its_offset = ((its_offset - 1) | 0x3F) + 33; 278 279 thunk = its_page + its_offset; 280 its_offset += size; 281 282 return its_init_thunk(thunk, reg); 283 } 284 285 u8 *its_static_thunk(int reg) 286 { 287 u8 *thunk = __x86_indirect_its_thunk_array[reg]; 288 289 #ifdef CONFIG_FINEIBT 290 /* Paranoid thunk starts 2 bytes before */ 291 if (cfi_paranoid) 292 return thunk - 2; 293 #endif 294 return thunk; 295 } 296 297 #else 298 static inline void its_fini_core(void) {} 299 #endif /* CONFIG_MITIGATION_ITS */ 300 301 /* 302 * Nomenclature for variable names to simplify and clarify this code and ease 303 * any potential staring at it: 304 * 305 * @instr: source address of the original instructions in the kernel text as 306 * generated by the compiler. 307 * 308 * @buf: temporary buffer on which the patching operates. This buffer is 309 * eventually text-poked into the kernel image. 310 * 311 * @replacement/@repl: pointer to the opcodes which are replacing @instr, located 312 * in the .altinstr_replacement section. 313 */ 314 315 /* 316 * Fill the buffer with a single effective instruction of size @len. 317 * 318 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info) 319 * for every single-byte NOP, try to generate the maximally available NOP of 320 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for 321 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and 322 * *jump* over instead of executing long and daft NOPs. 323 */ 324 static void add_nop(u8 *buf, unsigned int len) 325 { 326 u8 *target = buf + len; 327 328 if (!len) 329 return; 330 331 if (len <= ASM_NOP_MAX) { 332 memcpy(buf, x86_nops[len], len); 333 return; 334 } 335 336 if (len < 128) { 337 __text_gen_insn(buf, JMP8_INSN_OPCODE, buf, target, JMP8_INSN_SIZE); 338 buf += JMP8_INSN_SIZE; 339 } else { 340 __text_gen_insn(buf, JMP32_INSN_OPCODE, buf, target, JMP32_INSN_SIZE); 341 buf += JMP32_INSN_SIZE; 342 } 343 344 for (;buf < target; buf++) 345 *buf = INT3_INSN_OPCODE; 346 } 347 348 /* 349 * Matches NOP and NOPL, not any of the other possible NOPs. 350 */ 351 static bool insn_is_nop(struct insn *insn) 352 { 353 /* Anything NOP, but no REP NOP */ 354 if (insn->opcode.bytes[0] == 0x90 && 355 (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3)) 356 return true; 357 358 /* NOPL */ 359 if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F) 360 return true; 361 362 /* TODO: more nops */ 363 364 return false; 365 } 366 367 /* 368 * Find the offset of the first non-NOP instruction starting at @offset 369 * but no further than @len. 370 */ 371 static int skip_nops(u8 *buf, int offset, int len) 372 { 373 struct insn insn; 374 375 for (; offset < len; offset += insn.length) { 376 if (insn_decode_kernel(&insn, &buf[offset])) 377 break; 378 379 if (!insn_is_nop(&insn)) 380 break; 381 } 382 383 return offset; 384 } 385 386 /* 387 * "noinline" to cause control flow change and thus invalidate I$ and 388 * cause refetch after modification. 389 */ 390 static void noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len) 391 { 392 for (int next, i = 0; i < len; i = next) { 393 struct insn insn; 394 395 if (insn_decode_kernel(&insn, &buf[i])) 396 return; 397 398 next = i + insn.length; 399 400 if (insn_is_nop(&insn)) { 401 int nop = i; 402 403 /* Has the NOP already been optimized? */ 404 if (i + insn.length == len) 405 return; 406 407 next = skip_nops(buf, next, len); 408 409 add_nop(buf + nop, next - nop); 410 DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, next); 411 } 412 } 413 } 414 415 /* 416 * In this context, "source" is where the instructions are placed in the 417 * section .altinstr_replacement, for example during kernel build by the 418 * toolchain. 419 * "Destination" is where the instructions are being patched in by this 420 * machinery. 421 * 422 * The source offset is: 423 * 424 * src_imm = target - src_next_ip (1) 425 * 426 * and the target offset is: 427 * 428 * dst_imm = target - dst_next_ip (2) 429 * 430 * so rework (1) as an expression for target like: 431 * 432 * target = src_imm + src_next_ip (1a) 433 * 434 * and substitute in (2) to get: 435 * 436 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3) 437 * 438 * Now, since the instruction stream is 'identical' at src and dst (it 439 * is being copied after all) it can be stated that: 440 * 441 * src_next_ip = src + ip_offset 442 * dst_next_ip = dst + ip_offset (4) 443 * 444 * Substitute (4) in (3) and observe ip_offset being cancelled out to 445 * obtain: 446 * 447 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset) 448 * = src_imm + src - dst + ip_offset - ip_offset 449 * = src_imm + src - dst (5) 450 * 451 * IOW, only the relative displacement of the code block matters. 452 */ 453 454 #define apply_reloc_n(n_, p_, d_) \ 455 do { \ 456 s32 v = *(s##n_ *)(p_); \ 457 v += (d_); \ 458 BUG_ON((v >> 31) != (v >> (n_-1))); \ 459 *(s##n_ *)(p_) = (s##n_)v; \ 460 } while (0) 461 462 463 static __always_inline 464 void apply_reloc(int n, void *ptr, uintptr_t diff) 465 { 466 switch (n) { 467 case 1: apply_reloc_n(8, ptr, diff); break; 468 case 2: apply_reloc_n(16, ptr, diff); break; 469 case 4: apply_reloc_n(32, ptr, diff); break; 470 default: BUG(); 471 } 472 } 473 474 static __always_inline 475 bool need_reloc(unsigned long offset, u8 *src, size_t src_len) 476 { 477 u8 *target = src + offset; 478 /* 479 * If the target is inside the patched block, it's relative to the 480 * block itself and does not need relocation. 481 */ 482 return (target < src || target > src + src_len); 483 } 484 485 static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) 486 { 487 for (int next, i = 0; i < instrlen; i = next) { 488 struct insn insn; 489 490 if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i]))) 491 return; 492 493 next = i + insn.length; 494 495 switch (insn.opcode.bytes[0]) { 496 case 0x0f: 497 if (insn.opcode.bytes[1] < 0x80 || 498 insn.opcode.bytes[1] > 0x8f) 499 break; 500 501 fallthrough; /* Jcc.d32 */ 502 case 0x70 ... 0x7f: /* Jcc.d8 */ 503 case JMP8_INSN_OPCODE: 504 case JMP32_INSN_OPCODE: 505 case CALL_INSN_OPCODE: 506 if (need_reloc(next + insn.immediate.value, repl, repl_len)) { 507 apply_reloc(insn.immediate.nbytes, 508 buf + i + insn_offset_immediate(&insn), 509 repl - instr); 510 } 511 512 /* 513 * Where possible, convert JMP.d32 into JMP.d8. 514 */ 515 if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) { 516 s32 imm = insn.immediate.value; 517 imm += repl - instr; 518 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE; 519 if ((imm >> 31) == (imm >> 7)) { 520 buf[i+0] = JMP8_INSN_OPCODE; 521 buf[i+1] = (s8)imm; 522 523 memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2); 524 } 525 } 526 break; 527 } 528 529 if (insn_rip_relative(&insn)) { 530 if (need_reloc(next + insn.displacement.value, repl, repl_len)) { 531 apply_reloc(insn.displacement.nbytes, 532 buf + i + insn_offset_displacement(&insn), 533 repl - instr); 534 } 535 } 536 } 537 } 538 539 void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) 540 { 541 __apply_relocation(buf, instr, instrlen, repl, repl_len); 542 optimize_nops(instr, buf, instrlen); 543 } 544 545 /* Low-level backend functions usable from alternative code replacements. */ 546 DEFINE_ASM_FUNC(nop_func, "", .entry.text); 547 EXPORT_SYMBOL_GPL(nop_func); 548 549 noinstr void BUG_func(void) 550 { 551 BUG(); 552 } 553 EXPORT_SYMBOL(BUG_func); 554 555 #define CALL_RIP_REL_OPCODE 0xff 556 #define CALL_RIP_REL_MODRM 0x15 557 558 /* 559 * Rewrite the "call BUG_func" replacement to point to the target of the 560 * indirect pv_ops call "call *disp(%ip)". 561 */ 562 static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a) 563 { 564 void *target, *bug = &BUG_func; 565 s32 disp; 566 567 if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) { 568 pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n"); 569 BUG(); 570 } 571 572 if (a->instrlen != 6 || 573 instr[0] != CALL_RIP_REL_OPCODE || 574 instr[1] != CALL_RIP_REL_MODRM) { 575 pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n"); 576 BUG(); 577 } 578 579 /* Skip CALL_RIP_REL_OPCODE and CALL_RIP_REL_MODRM */ 580 disp = *(s32 *)(instr + 2); 581 #ifdef CONFIG_X86_64 582 /* ff 15 00 00 00 00 call *0x0(%rip) */ 583 /* target address is stored at "next instruction + disp". */ 584 target = *(void **)(instr + a->instrlen + disp); 585 #else 586 /* ff 15 00 00 00 00 call *0x0 */ 587 /* target address is stored at disp. */ 588 target = *(void **)disp; 589 #endif 590 if (!target) 591 target = bug; 592 593 /* (BUG_func - .) + (target - BUG_func) := target - . */ 594 *(s32 *)(insn_buff + 1) += target - bug; 595 596 if (target == &nop_func) 597 return 0; 598 599 return 5; 600 } 601 602 static inline u8 * instr_va(struct alt_instr *i) 603 { 604 return (u8 *)&i->instr_offset + i->instr_offset; 605 } 606 607 /* 608 * Replace instructions with better alternatives for this CPU type. This runs 609 * before SMP is initialized to avoid SMP problems with self modifying code. 610 * This implies that asymmetric systems where APs have less capabilities than 611 * the boot processor are not handled. Tough. Make sure you disable such 612 * features by hand. 613 * 614 * Marked "noinline" to cause control flow change and thus insn cache 615 * to refetch changed I$ lines. 616 */ 617 void __init_or_module noinline apply_alternatives(struct alt_instr *start, 618 struct alt_instr *end) 619 { 620 u8 insn_buff[MAX_PATCH_LEN]; 621 u8 *instr, *replacement; 622 struct alt_instr *a, *b; 623 624 DPRINTK(ALT, "alt table %px, -> %px", start, end); 625 626 /* 627 * KASAN_SHADOW_START is defined using 628 * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here. 629 * During the process, KASAN becomes confused seeing partial LA57 630 * conversion and triggers a false-positive out-of-bound report. 631 * 632 * Disable KASAN until the patching is complete. 633 */ 634 kasan_disable_current(); 635 636 /* 637 * The scan order should be from start to end. A later scanned 638 * alternative code can overwrite previously scanned alternative code. 639 * Some kernel functions (e.g. memcpy, memset, etc) use this order to 640 * patch code. 641 * 642 * So be careful if you want to change the scan order to any other 643 * order. 644 */ 645 for (a = start; a < end; a++) { 646 int insn_buff_sz = 0; 647 648 /* 649 * In case of nested ALTERNATIVE()s the outer alternative might 650 * add more padding. To ensure consistent patching find the max 651 * padding for all alt_instr entries for this site (nested 652 * alternatives result in consecutive entries). 653 */ 654 for (b = a+1; b < end && instr_va(b) == instr_va(a); b++) { 655 u8 len = max(a->instrlen, b->instrlen); 656 a->instrlen = b->instrlen = len; 657 } 658 659 instr = instr_va(a); 660 replacement = (u8 *)&a->repl_offset + a->repl_offset; 661 BUG_ON(a->instrlen > sizeof(insn_buff)); 662 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); 663 664 /* 665 * Patch if either: 666 * - feature is present 667 * - feature not present but ALT_FLAG_NOT is set to mean, 668 * patch if feature is *NOT* present. 669 */ 670 if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) { 671 memcpy(insn_buff, instr, a->instrlen); 672 optimize_nops(instr, insn_buff, a->instrlen); 673 text_poke_early(instr, insn_buff, a->instrlen); 674 continue; 675 } 676 677 DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x", 678 a->cpuid >> 5, 679 a->cpuid & 0x1f, 680 instr, instr, a->instrlen, 681 replacement, a->replacementlen, a->flags); 682 683 memcpy(insn_buff, replacement, a->replacementlen); 684 insn_buff_sz = a->replacementlen; 685 686 if (a->flags & ALT_FLAG_DIRECT_CALL) { 687 insn_buff_sz = alt_replace_call(instr, insn_buff, a); 688 if (insn_buff_sz < 0) 689 continue; 690 } 691 692 for (; insn_buff_sz < a->instrlen; insn_buff_sz++) 693 insn_buff[insn_buff_sz] = 0x90; 694 695 text_poke_apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen); 696 697 DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr); 698 DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement); 699 DUMP_BYTES(ALT, insn_buff, insn_buff_sz, "%px: final_insn: ", instr); 700 701 text_poke_early(instr, insn_buff, insn_buff_sz); 702 } 703 704 kasan_enable_current(); 705 } 706 707 static inline bool is_jcc32(struct insn *insn) 708 { 709 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ 710 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; 711 } 712 713 #if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL) 714 715 /* 716 * [CS]{,3} CALL/JMP *%\reg [INT3]* 717 */ 718 static int emit_indirect(int op, int reg, u8 *bytes, int len) 719 { 720 int cs = 0, bp = 0; 721 int i = 0; 722 u8 modrm; 723 724 /* 725 * Set @len to the excess bytes after writing the instruction. 726 */ 727 len -= 2 + (reg >= 8); 728 WARN_ON_ONCE(len < 0); 729 730 switch (op) { 731 case CALL_INSN_OPCODE: 732 modrm = 0x10; /* Reg = 2; CALL r/m */ 733 /* 734 * Additional NOP is better than prefix decode penalty. 735 */ 736 if (len <= 3) 737 cs = len; 738 break; 739 740 case JMP32_INSN_OPCODE: 741 modrm = 0x20; /* Reg = 4; JMP r/m */ 742 bp = len; 743 break; 744 745 default: 746 WARN_ON_ONCE(1); 747 return -1; 748 } 749 750 while (cs--) 751 bytes[i++] = 0x2e; /* CS-prefix */ 752 753 if (reg >= 8) { 754 bytes[i++] = 0x41; /* REX.B prefix */ 755 reg -= 8; 756 } 757 758 modrm |= 0xc0; /* Mod = 3 */ 759 modrm += reg; 760 761 bytes[i++] = 0xff; /* opcode */ 762 bytes[i++] = modrm; 763 764 while (bp--) 765 bytes[i++] = 0xcc; /* INT3 */ 766 767 return i; 768 } 769 770 static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes, 771 void *call_dest, void *jmp_dest) 772 { 773 u8 op = insn->opcode.bytes[0]; 774 int i = 0; 775 776 /* 777 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional 778 * tail-calls. Deal with them. 779 */ 780 if (is_jcc32(insn)) { 781 bytes[i++] = op; 782 op = insn->opcode.bytes[1]; 783 goto clang_jcc; 784 } 785 786 if (insn->length == 6) 787 bytes[i++] = 0x2e; /* CS-prefix */ 788 789 switch (op) { 790 case CALL_INSN_OPCODE: 791 __text_gen_insn(bytes+i, op, addr+i, 792 call_dest, 793 CALL_INSN_SIZE); 794 i += CALL_INSN_SIZE; 795 break; 796 797 case JMP32_INSN_OPCODE: 798 clang_jcc: 799 __text_gen_insn(bytes+i, op, addr+i, 800 jmp_dest, 801 JMP32_INSN_SIZE); 802 i += JMP32_INSN_SIZE; 803 break; 804 805 default: 806 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr); 807 return -1; 808 } 809 810 WARN_ON_ONCE(i != insn->length); 811 812 return i; 813 } 814 815 static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) 816 { 817 return __emit_trampoline(addr, insn, bytes, 818 __x86_indirect_call_thunk_array[reg], 819 __x86_indirect_jump_thunk_array[reg]); 820 } 821 822 #ifdef CONFIG_MITIGATION_ITS 823 static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) 824 { 825 u8 *thunk = __x86_indirect_its_thunk_array[reg]; 826 u8 *tmp = its_allocate_thunk(reg); 827 828 if (tmp) 829 thunk = tmp; 830 831 return __emit_trampoline(addr, insn, bytes, thunk, thunk); 832 } 833 834 /* Check if an indirect branch is at ITS-unsafe address */ 835 static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg) 836 { 837 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) 838 return false; 839 840 /* Indirect branch opcode is 2 or 3 bytes depending on reg */ 841 addr += 1 + reg / 8; 842 843 /* Lower-half of the cacheline? */ 844 return !(addr & 0x20); 845 } 846 #else /* CONFIG_MITIGATION_ITS */ 847 848 #ifdef CONFIG_FINEIBT 849 static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg) 850 { 851 return false; 852 } 853 #endif 854 855 #endif /* CONFIG_MITIGATION_ITS */ 856 857 /* 858 * Rewrite the compiler generated retpoline thunk calls. 859 * 860 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate 861 * indirect instructions, avoiding the extra indirection. 862 * 863 * For example, convert: 864 * 865 * CALL __x86_indirect_thunk_\reg 866 * 867 * into: 868 * 869 * CALL *%\reg 870 * 871 * It also tries to inline spectre_v2=retpoline,lfence when size permits. 872 */ 873 static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) 874 { 875 retpoline_thunk_t *target; 876 int reg, ret, i = 0; 877 u8 op, cc; 878 879 target = addr + insn->length + insn->immediate.value; 880 reg = target - __x86_indirect_thunk_array; 881 882 if (WARN_ON_ONCE(reg & ~0xf)) 883 return -1; 884 885 /* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */ 886 BUG_ON(reg == 4); 887 888 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) && 889 !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { 890 if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 891 return emit_call_track_retpoline(addr, insn, reg, bytes); 892 893 return -1; 894 } 895 896 op = insn->opcode.bytes[0]; 897 898 /* 899 * Convert: 900 * 901 * Jcc.d32 __x86_indirect_thunk_\reg 902 * 903 * into: 904 * 905 * Jncc.d8 1f 906 * [ LFENCE ] 907 * JMP *%\reg 908 * [ NOP ] 909 * 1: 910 */ 911 if (is_jcc32(insn)) { 912 cc = insn->opcode.bytes[1] & 0xf; 913 cc ^= 1; /* invert condition */ 914 915 bytes[i++] = 0x70 + cc; /* Jcc.d8 */ 916 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */ 917 918 /* Continue as if: JMP.d32 __x86_indirect_thunk_\reg */ 919 op = JMP32_INSN_OPCODE; 920 } 921 922 /* 923 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE. 924 */ 925 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { 926 bytes[i++] = 0x0f; 927 bytes[i++] = 0xae; 928 bytes[i++] = 0xe8; /* LFENCE */ 929 } 930 931 #ifdef CONFIG_MITIGATION_ITS 932 /* 933 * Check if the address of last byte of emitted-indirect is in 934 * lower-half of the cacheline. Such branches need ITS mitigation. 935 */ 936 if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg)) 937 return emit_its_trampoline(addr, insn, reg, bytes); 938 #endif 939 940 ret = emit_indirect(op, reg, bytes + i, insn->length - i); 941 if (ret < 0) 942 return ret; 943 i += ret; 944 945 for (; i < insn->length;) 946 bytes[i++] = BYTES_NOP1; 947 948 return i; 949 } 950 951 /* 952 * Generated by 'objtool --retpoline'. 953 */ 954 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) 955 { 956 s32 *s; 957 958 for (s = start; s < end; s++) { 959 void *addr = (void *)s + *s; 960 struct insn insn; 961 int len, ret; 962 u8 bytes[16]; 963 u8 op1, op2; 964 u8 *dest; 965 966 ret = insn_decode_kernel(&insn, addr); 967 if (WARN_ON_ONCE(ret < 0)) 968 continue; 969 970 op1 = insn.opcode.bytes[0]; 971 op2 = insn.opcode.bytes[1]; 972 973 switch (op1) { 974 case 0x70 ... 0x7f: /* Jcc.d8 */ 975 /* See cfi_paranoid. */ 976 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT); 977 continue; 978 979 case CALL_INSN_OPCODE: 980 case JMP32_INSN_OPCODE: 981 /* Check for cfi_paranoid + ITS */ 982 dest = addr + insn.length + insn.immediate.value; 983 if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) { 984 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT); 985 continue; 986 } 987 break; 988 989 case 0x0f: /* escape */ 990 if (op2 >= 0x80 && op2 <= 0x8f) 991 break; 992 fallthrough; 993 default: 994 WARN_ON_ONCE(1); 995 continue; 996 } 997 998 DPRINTK(RETPOLINE, "retpoline at: %pS (%px) len: %d to: %pS", 999 addr, addr, insn.length, 1000 addr + insn.length + insn.immediate.value); 1001 1002 len = patch_retpoline(addr, &insn, bytes); 1003 if (len == insn.length) { 1004 optimize_nops(addr, bytes, len); 1005 DUMP_BYTES(RETPOLINE, ((u8*)addr), len, "%px: orig: ", addr); 1006 DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr); 1007 text_poke_early(addr, bytes, len); 1008 } 1009 } 1010 } 1011 1012 #ifdef CONFIG_MITIGATION_RETHUNK 1013 1014 bool cpu_wants_rethunk(void) 1015 { 1016 return cpu_feature_enabled(X86_FEATURE_RETHUNK); 1017 } 1018 1019 bool cpu_wants_rethunk_at(void *addr) 1020 { 1021 if (!cpu_feature_enabled(X86_FEATURE_RETHUNK)) 1022 return false; 1023 if (x86_return_thunk != its_return_thunk) 1024 return true; 1025 1026 return !((unsigned long)addr & 0x20); 1027 } 1028 1029 /* 1030 * Rewrite the compiler generated return thunk tail-calls. 1031 * 1032 * For example, convert: 1033 * 1034 * JMP __x86_return_thunk 1035 * 1036 * into: 1037 * 1038 * RET 1039 */ 1040 static int patch_return(void *addr, struct insn *insn, u8 *bytes) 1041 { 1042 int i = 0; 1043 1044 /* Patch the custom return thunks... */ 1045 if (cpu_wants_rethunk_at(addr)) { 1046 i = JMP32_INSN_SIZE; 1047 __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i); 1048 } else { 1049 /* ... or patch them out if not needed. */ 1050 bytes[i++] = RET_INSN_OPCODE; 1051 } 1052 1053 for (; i < insn->length;) 1054 bytes[i++] = INT3_INSN_OPCODE; 1055 return i; 1056 } 1057 1058 void __init_or_module noinline apply_returns(s32 *start, s32 *end) 1059 { 1060 s32 *s; 1061 1062 if (cpu_wants_rethunk()) 1063 static_call_force_reinit(); 1064 1065 for (s = start; s < end; s++) { 1066 void *dest = NULL, *addr = (void *)s + *s; 1067 struct insn insn; 1068 int len, ret; 1069 u8 bytes[16]; 1070 u8 op; 1071 1072 ret = insn_decode_kernel(&insn, addr); 1073 if (WARN_ON_ONCE(ret < 0)) 1074 continue; 1075 1076 op = insn.opcode.bytes[0]; 1077 if (op == JMP32_INSN_OPCODE) 1078 dest = addr + insn.length + insn.immediate.value; 1079 1080 if (__static_call_fixup(addr, op, dest) || 1081 WARN_ONCE(dest != &__x86_return_thunk, 1082 "missing return thunk: %pS-%pS: %*ph", 1083 addr, dest, 5, addr)) 1084 continue; 1085 1086 DPRINTK(RET, "return thunk at: %pS (%px) len: %d to: %pS", 1087 addr, addr, insn.length, 1088 addr + insn.length + insn.immediate.value); 1089 1090 len = patch_return(addr, &insn, bytes); 1091 if (len == insn.length) { 1092 DUMP_BYTES(RET, ((u8*)addr), len, "%px: orig: ", addr); 1093 DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr); 1094 text_poke_early(addr, bytes, len); 1095 } 1096 } 1097 } 1098 #else /* !CONFIG_MITIGATION_RETHUNK: */ 1099 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { } 1100 #endif /* !CONFIG_MITIGATION_RETHUNK */ 1101 1102 #else /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */ 1103 1104 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { } 1105 void __init_or_module noinline apply_returns(s32 *start, s32 *end) { } 1106 1107 #endif /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */ 1108 1109 #ifdef CONFIG_X86_KERNEL_IBT 1110 1111 __noendbr bool is_endbr(u32 *val) 1112 { 1113 u32 endbr; 1114 1115 __get_kernel_nofault(&endbr, val, u32, Efault); 1116 return __is_endbr(endbr); 1117 1118 Efault: 1119 return false; 1120 } 1121 1122 #ifdef CONFIG_FINEIBT 1123 1124 static __noendbr bool exact_endbr(u32 *val) 1125 { 1126 u32 endbr; 1127 1128 __get_kernel_nofault(&endbr, val, u32, Efault); 1129 return endbr == gen_endbr(); 1130 1131 Efault: 1132 return false; 1133 } 1134 1135 #endif 1136 1137 static void poison_cfi(void *addr); 1138 1139 static void __init_or_module poison_endbr(void *addr) 1140 { 1141 u32 poison = gen_endbr_poison(); 1142 1143 if (WARN_ON_ONCE(!is_endbr(addr))) 1144 return; 1145 1146 DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr); 1147 1148 /* 1149 * When we have IBT, the lack of ENDBR will trigger #CP 1150 */ 1151 DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr); 1152 DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr); 1153 text_poke_early(addr, &poison, 4); 1154 } 1155 1156 /* 1157 * Generated by: objtool --ibt 1158 * 1159 * Seal the functions for indirect calls by clobbering the ENDBR instructions 1160 * and the kCFI hash value. 1161 */ 1162 void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) 1163 { 1164 s32 *s; 1165 1166 for (s = start; s < end; s++) { 1167 void *addr = (void *)s + *s; 1168 1169 poison_endbr(addr); 1170 if (IS_ENABLED(CONFIG_FINEIBT)) 1171 poison_cfi(addr - 16); 1172 } 1173 } 1174 1175 #else /* !CONFIG_X86_KERNEL_IBT: */ 1176 1177 void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { } 1178 1179 #endif /* !CONFIG_X86_KERNEL_IBT */ 1180 1181 #ifdef CONFIG_CFI_AUTO_DEFAULT 1182 # define __CFI_DEFAULT CFI_AUTO 1183 #elif defined(CONFIG_CFI) 1184 # define __CFI_DEFAULT CFI_KCFI 1185 #else 1186 # define __CFI_DEFAULT CFI_OFF 1187 #endif 1188 1189 enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT; 1190 static bool cfi_debug __ro_after_init; 1191 1192 #ifdef CONFIG_FINEIBT_BHI 1193 bool cfi_bhi __ro_after_init = false; 1194 #endif 1195 1196 #ifdef CONFIG_CFI 1197 u32 cfi_get_func_hash(void *func) 1198 { 1199 u32 hash; 1200 1201 func -= cfi_get_offset(); 1202 switch (cfi_mode) { 1203 case CFI_FINEIBT: 1204 func += 7; 1205 break; 1206 case CFI_KCFI: 1207 func += 1; 1208 break; 1209 default: 1210 return 0; 1211 } 1212 1213 if (get_kernel_nofault(hash, func)) 1214 return 0; 1215 1216 return hash; 1217 } 1218 1219 int cfi_get_func_arity(void *func) 1220 { 1221 bhi_thunk *target; 1222 s32 disp; 1223 1224 if (cfi_mode != CFI_FINEIBT && !cfi_bhi) 1225 return 0; 1226 1227 if (get_kernel_nofault(disp, func - 4)) 1228 return 0; 1229 1230 target = func + disp; 1231 return target - __bhi_args; 1232 } 1233 #endif 1234 1235 #ifdef CONFIG_FINEIBT 1236 1237 static bool cfi_rand __ro_after_init = true; 1238 static u32 cfi_seed __ro_after_init; 1239 1240 /* 1241 * Re-hash the CFI hash with a boot-time seed while making sure the result is 1242 * not a valid ENDBR instruction. 1243 */ 1244 static u32 cfi_rehash(u32 hash) 1245 { 1246 hash ^= cfi_seed; 1247 while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) { 1248 bool lsb = hash & 1; 1249 hash >>= 1; 1250 if (lsb) 1251 hash ^= 0x80200003; 1252 } 1253 return hash; 1254 } 1255 1256 static __init int cfi_parse_cmdline(char *str) 1257 { 1258 if (!str) 1259 return -EINVAL; 1260 1261 while (str) { 1262 char *next = strchr(str, ','); 1263 if (next) { 1264 *next = 0; 1265 next++; 1266 } 1267 1268 if (!strcmp(str, "auto")) { 1269 cfi_mode = CFI_AUTO; 1270 } else if (!strcmp(str, "off")) { 1271 cfi_mode = CFI_OFF; 1272 cfi_rand = false; 1273 } else if (!strcmp(str, "debug")) { 1274 cfi_debug = true; 1275 } else if (!strcmp(str, "kcfi")) { 1276 cfi_mode = CFI_KCFI; 1277 } else if (!strcmp(str, "fineibt")) { 1278 cfi_mode = CFI_FINEIBT; 1279 } else if (!strcmp(str, "norand")) { 1280 cfi_rand = false; 1281 } else if (!strcmp(str, "warn")) { 1282 pr_alert("CFI: mismatch non-fatal!\n"); 1283 cfi_warn = true; 1284 } else if (!strcmp(str, "paranoid")) { 1285 if (cfi_mode == CFI_FINEIBT) { 1286 cfi_paranoid = true; 1287 } else { 1288 pr_err("CFI: ignoring paranoid; depends on fineibt.\n"); 1289 } 1290 } else if (!strcmp(str, "bhi")) { 1291 #ifdef CONFIG_FINEIBT_BHI 1292 if (cfi_mode == CFI_FINEIBT) { 1293 cfi_bhi = true; 1294 } else { 1295 pr_err("CFI: ignoring bhi; depends on fineibt.\n"); 1296 } 1297 #else 1298 pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n"); 1299 #endif 1300 } else { 1301 pr_err("CFI: Ignoring unknown option (%s).", str); 1302 } 1303 1304 str = next; 1305 } 1306 1307 return 0; 1308 } 1309 early_param("cfi", cfi_parse_cmdline); 1310 1311 /* 1312 * kCFI FineIBT 1313 * 1314 * __cfi_\func: __cfi_\func: 1315 * movl $0x12345678,%eax // 5 endbr64 // 4 1316 * nop subl $0x12345678,%eax // 5 1317 * nop jne.d32,pn \func+3 // 7 1318 * nop 1319 * nop 1320 * nop 1321 * nop 1322 * nop 1323 * nop 1324 * nop 1325 * nop 1326 * nop 1327 * \func: \func: 1328 * endbr64 nopl -42(%rax) 1329 * 1330 * 1331 * caller: caller: 1332 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%eax // 5 1333 * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4 1334 * je 1f // 2 nop5 // 5 1335 * ud2 // 2 1336 * 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6 1337 * 1338 * 1339 * Notably, the FineIBT sequences are crafted such that branches are presumed 1340 * non-taken. This is based on Agner Fog's optimization manual, which states: 1341 * 1342 * "Make conditional jumps most often not taken: The efficiency and throughput 1343 * for not-taken branches is better than for taken branches on most 1344 * processors. Therefore, it is good to place the most frequent branch first" 1345 */ 1346 1347 /* 1348 * <fineibt_preamble_start>: 1349 * 0: f3 0f 1e fa endbr64 1350 * 4: 2d 78 56 34 12 sub $0x12345678, %eax 1351 * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13> 1352 * 10: 0f 1f 40 d6 nopl -0x2a(%rax) 1353 * 1354 * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as 1355 * UDB on x86_64 and raises #UD. 1356 */ 1357 asm( ".pushsection .rodata \n" 1358 "fineibt_preamble_start: \n" 1359 " endbr64 \n" 1360 " subl $0x12345678, %eax \n" 1361 "fineibt_preamble_bhi: \n" 1362 " cs jne.d32 fineibt_preamble_start+0x13 \n" 1363 "#fineibt_func: \n" 1364 " nopl -42(%rax) \n" 1365 "fineibt_preamble_end: \n" 1366 ".popsection\n" 1367 ); 1368 1369 extern u8 fineibt_preamble_start[]; 1370 extern u8 fineibt_preamble_bhi[]; 1371 extern u8 fineibt_preamble_end[]; 1372 1373 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) 1374 #define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) 1375 #define fineibt_preamble_ud 0x13 1376 #define fineibt_preamble_hash 5 1377 1378 /* 1379 * <fineibt_caller_start>: 1380 * 0: b8 78 56 34 12 mov $0x12345678, %eax 1381 * 5: 4d 8d 5b f0 lea -0x10(%r11), %r11 1382 * 9: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1383 */ 1384 asm( ".pushsection .rodata \n" 1385 "fineibt_caller_start: \n" 1386 " movl $0x12345678, %eax \n" 1387 " lea -0x10(%r11), %r11 \n" 1388 ASM_NOP5 1389 "fineibt_caller_end: \n" 1390 ".popsection \n" 1391 ); 1392 1393 extern u8 fineibt_caller_start[]; 1394 extern u8 fineibt_caller_end[]; 1395 1396 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start) 1397 #define fineibt_caller_hash 1 1398 1399 #define fineibt_caller_jmp (fineibt_caller_size - 2) 1400 1401 /* 1402 * Since FineIBT does hash validation on the callee side it is prone to 1403 * circumvention attacks where a 'naked' ENDBR instruction exists that 1404 * is not part of the fineibt_preamble sequence. 1405 * 1406 * Notably the x86 entry points must be ENDBR and equally cannot be 1407 * fineibt_preamble. 1408 * 1409 * The fineibt_paranoid caller sequence adds additional caller side 1410 * hash validation. This stops such circumvention attacks dead, but at the cost 1411 * of adding a load. 1412 * 1413 * <fineibt_paranoid_start>: 1414 * 0: b8 78 56 34 12 mov $0x12345678, %eax 1415 * 5: 41 3b 43 f5 cmp -0x11(%r11), %eax 1416 * 9: 2e 4d 8d 5b <f0> cs lea -0x10(%r11), %r11 1417 * e: 75 fd jne d <fineibt_paranoid_start+0xd> 1418 * 10: 41 ff d3 call *%r11 1419 * 13: 90 nop 1420 * 1421 * Notably LEA does not modify flags and can be reordered with the CMP, 1422 * avoiding a dependency. Again, using a non-taken (backwards) branch 1423 * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for the 1424 * Jcc.d8, causing #UD. 1425 */ 1426 asm( ".pushsection .rodata \n" 1427 "fineibt_paranoid_start: \n" 1428 " mov $0x12345678, %eax \n" 1429 " cmpl -11(%r11), %eax \n" 1430 " cs lea -0x10(%r11), %r11 \n" 1431 "#fineibt_caller_size: \n" 1432 " jne fineibt_paranoid_start+0xd \n" 1433 "fineibt_paranoid_ind: \n" 1434 " cs call *%r11 \n" 1435 "fineibt_paranoid_end: \n" 1436 ".popsection \n" 1437 ); 1438 1439 extern u8 fineibt_paranoid_start[]; 1440 extern u8 fineibt_paranoid_ind[]; 1441 extern u8 fineibt_paranoid_end[]; 1442 1443 #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start) 1444 #define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_start) 1445 #define fineibt_paranoid_ud 0xd 1446 1447 static u32 decode_preamble_hash(void *addr, int *reg) 1448 { 1449 u8 *p = addr; 1450 1451 /* b8+reg 78 56 34 12 movl $0x12345678,\reg */ 1452 if (p[0] >= 0xb8 && p[0] < 0xc0) { 1453 if (reg) 1454 *reg = p[0] - 0xb8; 1455 return *(u32 *)(addr + 1); 1456 } 1457 1458 return 0; /* invalid hash value */ 1459 } 1460 1461 static u32 decode_caller_hash(void *addr) 1462 { 1463 u8 *p = addr; 1464 1465 /* 41 ba 88 a9 cb ed mov $(-0x12345678),%r10d */ 1466 if (p[0] == 0x41 && p[1] == 0xba) 1467 return -*(u32 *)(addr + 2); 1468 1469 /* e8 0c 88 a9 cb ed jmp.d8 +12 */ 1470 if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp) 1471 return -*(u32 *)(addr + 2); 1472 1473 return 0; /* invalid hash value */ 1474 } 1475 1476 /* .retpoline_sites */ 1477 static int cfi_disable_callers(s32 *start, s32 *end) 1478 { 1479 /* 1480 * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate 1481 * in tact for later usage. Also see decode_caller_hash() and 1482 * cfi_rewrite_callers(). 1483 */ 1484 const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp }; 1485 s32 *s; 1486 1487 for (s = start; s < end; s++) { 1488 void *addr = (void *)s + *s; 1489 u32 hash; 1490 1491 addr -= fineibt_caller_size; 1492 hash = decode_caller_hash(addr); 1493 if (!hash) /* nocfi callers */ 1494 continue; 1495 1496 text_poke_early(addr, jmp, 2); 1497 } 1498 1499 return 0; 1500 } 1501 1502 static int cfi_enable_callers(s32 *start, s32 *end) 1503 { 1504 /* 1505 * Re-enable kCFI, undo what cfi_disable_callers() did. 1506 */ 1507 const u8 mov[] = { 0x41, 0xba }; 1508 s32 *s; 1509 1510 for (s = start; s < end; s++) { 1511 void *addr = (void *)s + *s; 1512 u32 hash; 1513 1514 addr -= fineibt_caller_size; 1515 hash = decode_caller_hash(addr); 1516 if (!hash) /* nocfi callers */ 1517 continue; 1518 1519 text_poke_early(addr, mov, 2); 1520 } 1521 1522 return 0; 1523 } 1524 1525 /* .cfi_sites */ 1526 static int cfi_rand_preamble(s32 *start, s32 *end) 1527 { 1528 s32 *s; 1529 1530 for (s = start; s < end; s++) { 1531 void *addr = (void *)s + *s; 1532 u32 hash; 1533 1534 hash = decode_preamble_hash(addr, NULL); 1535 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", 1536 addr, addr, 5, addr)) 1537 return -EINVAL; 1538 1539 hash = cfi_rehash(hash); 1540 text_poke_early(addr + 1, &hash, 4); 1541 } 1542 1543 return 0; 1544 } 1545 1546 /* 1547 * Inline the bhi-arity 1 case: 1548 * 1549 * __cfi_foo: 1550 * 0: f3 0f 1e fa endbr64 1551 * 4: 2d 78 56 34 12 sub $0x12345678, %eax 1552 * 9: 49 0f 45 fa cmovne %rax, %rdi 1553 * d: 2e 75 03 jne,pn foo+0x3 1554 * 1555 * foo: 1556 * 10: 0f 1f 40 <d6> nopl -42(%rax) 1557 * 1558 * Notably, this scheme is incompatible with permissive CFI 1559 * because the CMOVcc is unconditional and RDI will have been 1560 * clobbered. 1561 */ 1562 asm( ".pushsection .rodata \n" 1563 "fineibt_bhi1_start: \n" 1564 " cmovne %rax, %rdi \n" 1565 " cs jne fineibt_bhi1_func + 0x3 \n" 1566 "fineibt_bhi1_func: \n" 1567 " nopl -42(%rax) \n" 1568 "fineibt_bhi1_end: \n" 1569 ".popsection \n" 1570 ); 1571 1572 extern u8 fineibt_bhi1_start[]; 1573 extern u8 fineibt_bhi1_end[]; 1574 1575 #define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start) 1576 1577 static void cfi_fineibt_bhi_preamble(void *addr, int arity) 1578 { 1579 u8 bytes[MAX_INSN_SIZE]; 1580 1581 if (!arity) 1582 return; 1583 1584 if (!cfi_warn && arity == 1) { 1585 text_poke_early(addr + fineibt_preamble_bhi, 1586 fineibt_bhi1_start, fineibt_bhi1_size); 1587 return; 1588 } 1589 1590 /* 1591 * Replace the bytes at fineibt_preamble_bhi with a CALL instruction 1592 * that lines up exactly with the end of the preamble, such that the 1593 * return address will be foo+0. 1594 * 1595 * __cfi_foo: 1596 * 0: f3 0f 1e fa endbr64 1597 * 4: 2d 78 56 34 12 sub $0x12345678, %eax 1598 * 9: 2e 2e e8 DD DD DD DD cs cs call __bhi_args[arity] 1599 */ 1600 bytes[0] = 0x2e; 1601 bytes[1] = 0x2e; 1602 __text_gen_insn(bytes + 2, CALL_INSN_OPCODE, 1603 addr + fineibt_preamble_bhi + 2, 1604 __bhi_args[arity], CALL_INSN_SIZE); 1605 1606 text_poke_early(addr + fineibt_preamble_bhi, bytes, 7); 1607 } 1608 1609 static int cfi_rewrite_preamble(s32 *start, s32 *end) 1610 { 1611 s32 *s; 1612 1613 for (s = start; s < end; s++) { 1614 void *addr = (void *)s + *s; 1615 int arity; 1616 u32 hash; 1617 1618 /* 1619 * When the function doesn't start with ENDBR the compiler will 1620 * have determined there are no indirect calls to it and we 1621 * don't need no CFI either. 1622 */ 1623 if (!is_endbr(addr + 16)) 1624 continue; 1625 1626 hash = decode_preamble_hash(addr, &arity); 1627 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", 1628 addr, addr, 5, addr)) 1629 return -EINVAL; 1630 1631 text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); 1632 WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678); 1633 text_poke_early(addr + fineibt_preamble_hash, &hash, 4); 1634 1635 WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity, 1636 "kCFI preamble has wrong register at: %pS %*ph\n", 1637 addr, 5, addr); 1638 1639 if (cfi_bhi) 1640 cfi_fineibt_bhi_preamble(addr, arity); 1641 } 1642 1643 return 0; 1644 } 1645 1646 static void cfi_rewrite_endbr(s32 *start, s32 *end) 1647 { 1648 s32 *s; 1649 1650 for (s = start; s < end; s++) { 1651 void *addr = (void *)s + *s; 1652 1653 if (!exact_endbr(addr + 16)) 1654 continue; 1655 1656 poison_endbr(addr + 16); 1657 } 1658 } 1659 1660 /* .retpoline_sites */ 1661 static int cfi_rand_callers(s32 *start, s32 *end) 1662 { 1663 s32 *s; 1664 1665 for (s = start; s < end; s++) { 1666 void *addr = (void *)s + *s; 1667 u32 hash; 1668 1669 addr -= fineibt_caller_size; 1670 hash = decode_caller_hash(addr); 1671 if (hash) { 1672 hash = -cfi_rehash(hash); 1673 text_poke_early(addr + 2, &hash, 4); 1674 } 1675 } 1676 1677 return 0; 1678 } 1679 1680 static int emit_paranoid_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes) 1681 { 1682 u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2; 1683 1684 #ifdef CONFIG_MITIGATION_ITS 1685 u8 *tmp = its_allocate_thunk(reg); 1686 if (tmp) 1687 thunk = tmp; 1688 #endif 1689 1690 return __emit_trampoline(addr, insn, bytes, thunk, thunk); 1691 } 1692 1693 static int cfi_rewrite_callers(s32 *start, s32 *end) 1694 { 1695 s32 *s; 1696 1697 for (s = start; s < end; s++) { 1698 void *addr = (void *)s + *s; 1699 struct insn insn; 1700 u8 bytes[20]; 1701 u32 hash; 1702 int ret; 1703 u8 op; 1704 1705 addr -= fineibt_caller_size; 1706 hash = decode_caller_hash(addr); 1707 if (!hash) 1708 continue; 1709 1710 if (!cfi_paranoid) { 1711 text_poke_early(addr, fineibt_caller_start, fineibt_caller_size); 1712 WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678); 1713 text_poke_early(addr + fineibt_caller_hash, &hash, 4); 1714 /* rely on apply_retpolines() */ 1715 continue; 1716 } 1717 1718 /* cfi_paranoid */ 1719 ret = insn_decode_kernel(&insn, addr + fineibt_caller_size); 1720 if (WARN_ON_ONCE(ret < 0)) 1721 continue; 1722 1723 op = insn.opcode.bytes[0]; 1724 if (op != CALL_INSN_OPCODE && op != JMP32_INSN_OPCODE) { 1725 WARN_ON_ONCE(1); 1726 continue; 1727 } 1728 1729 memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size); 1730 memcpy(bytes + fineibt_caller_hash, &hash, 4); 1731 1732 if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + fineibt_paranoid_ind, 11)) { 1733 emit_paranoid_trampoline(addr + fineibt_caller_size, 1734 &insn, 11, bytes + fineibt_caller_size); 1735 } else { 1736 int len = fineibt_paranoid_size - fineibt_paranoid_ind; 1737 ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len); 1738 if (WARN_ON_ONCE(ret != len)) 1739 continue; 1740 } 1741 1742 text_poke_early(addr, bytes, fineibt_paranoid_size); 1743 } 1744 1745 return 0; 1746 } 1747 1748 #define pr_cfi_debug(X...) if (cfi_debug) pr_info(X) 1749 1750 #define FINEIBT_WARN(_f, _v) \ 1751 WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v) 1752 1753 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 1754 s32 *start_cfi, s32 *end_cfi, bool builtin) 1755 { 1756 int ret; 1757 1758 if (FINEIBT_WARN(fineibt_preamble_size, 20) || 1759 FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) || 1760 FINEIBT_WARN(fineibt_caller_size, 14) || 1761 FINEIBT_WARN(fineibt_paranoid_size, 20)) 1762 return; 1763 1764 if (cfi_mode == CFI_AUTO) { 1765 cfi_mode = CFI_KCFI; 1766 if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) { 1767 /* 1768 * FRED has much saner context on exception entry and 1769 * is less easy to take advantage of. 1770 */ 1771 if (!cpu_feature_enabled(X86_FEATURE_FRED)) 1772 cfi_paranoid = true; 1773 cfi_mode = CFI_FINEIBT; 1774 } 1775 } 1776 1777 /* 1778 * Rewrite the callers to not use the __cfi_ stubs, such that we might 1779 * rewrite them. This disables all CFI. If this succeeds but any of the 1780 * later stages fails, we're without CFI. 1781 */ 1782 pr_cfi_debug("CFI: disabling all indirect call checking\n"); 1783 ret = cfi_disable_callers(start_retpoline, end_retpoline); 1784 if (ret) 1785 goto err; 1786 1787 if (cfi_rand) { 1788 if (builtin) { 1789 cfi_seed = get_random_u32(); 1790 cfi_bpf_hash = cfi_rehash(cfi_bpf_hash); 1791 cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash); 1792 } 1793 pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed); 1794 1795 pr_cfi_debug("CFI: rehashing all preambles\n"); 1796 ret = cfi_rand_preamble(start_cfi, end_cfi); 1797 if (ret) 1798 goto err; 1799 1800 pr_cfi_debug("CFI: rehashing all indirect calls\n"); 1801 ret = cfi_rand_callers(start_retpoline, end_retpoline); 1802 if (ret) 1803 goto err; 1804 } else { 1805 pr_cfi_debug("CFI: rehashing disabled\n"); 1806 } 1807 1808 switch (cfi_mode) { 1809 case CFI_OFF: 1810 if (builtin) 1811 pr_info("CFI: disabled\n"); 1812 return; 1813 1814 case CFI_KCFI: 1815 pr_cfi_debug("CFI: re-enabling all indirect call checking\n"); 1816 ret = cfi_enable_callers(start_retpoline, end_retpoline); 1817 if (ret) 1818 goto err; 1819 1820 if (builtin) 1821 pr_info("CFI: Using %sretpoline kCFI\n", 1822 cfi_rand ? "rehashed " : ""); 1823 return; 1824 1825 case CFI_FINEIBT: 1826 pr_cfi_debug("CFI: adding FineIBT to all preambles\n"); 1827 /* place the FineIBT preamble at func()-16 */ 1828 ret = cfi_rewrite_preamble(start_cfi, end_cfi); 1829 if (ret) 1830 goto err; 1831 1832 /* rewrite the callers to target func()-16 */ 1833 pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n"); 1834 ret = cfi_rewrite_callers(start_retpoline, end_retpoline); 1835 if (ret) 1836 goto err; 1837 1838 /* now that nobody targets func()+0, remove ENDBR there */ 1839 pr_cfi_debug("CFI: removing old endbr insns\n"); 1840 cfi_rewrite_endbr(start_cfi, end_cfi); 1841 1842 if (builtin) { 1843 pr_info("Using %sFineIBT%s CFI\n", 1844 cfi_paranoid ? "paranoid " : "", 1845 cfi_bhi ? "+BHI" : ""); 1846 } 1847 return; 1848 1849 default: 1850 break; 1851 } 1852 1853 err: 1854 pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n"); 1855 } 1856 1857 static inline void poison_hash(void *addr) 1858 { 1859 *(u32 *)addr = 0; 1860 } 1861 1862 static void poison_cfi(void *addr) 1863 { 1864 /* 1865 * Compilers manage to be inconsistent with ENDBR vs __cfi prefixes, 1866 * some (static) functions for which they can determine the address 1867 * is never taken do not get a __cfi prefix, but *DO* get an ENDBR. 1868 * 1869 * As such, these functions will get sealed, but we need to be careful 1870 * to not unconditionally scribble the previous function. 1871 */ 1872 switch (cfi_mode) { 1873 case CFI_FINEIBT: 1874 /* 1875 * FineIBT prefix should start with an ENDBR. 1876 */ 1877 if (!is_endbr(addr)) 1878 break; 1879 1880 /* 1881 * __cfi_\func: 1882 * nopl -42(%rax) 1883 * sub $0, %eax 1884 * jne \func+3 1885 * \func: 1886 * nopl -42(%rax) 1887 */ 1888 poison_endbr(addr); 1889 poison_hash(addr + fineibt_preamble_hash); 1890 break; 1891 1892 case CFI_KCFI: 1893 /* 1894 * kCFI prefix should start with a valid hash. 1895 */ 1896 if (!decode_preamble_hash(addr, NULL)) 1897 break; 1898 1899 /* 1900 * __cfi_\func: 1901 * movl $0, %eax 1902 * .skip 11, 0x90 1903 */ 1904 poison_hash(addr + 1); 1905 break; 1906 1907 default: 1908 break; 1909 } 1910 } 1911 1912 #define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) 1913 1914 /* 1915 * When regs->ip points to a 0xD6 byte in the FineIBT preamble, 1916 * return true and fill out target and type. 1917 * 1918 * We check the preamble by checking for the ENDBR instruction relative to the 1919 * UDB instruction. 1920 */ 1921 static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type) 1922 { 1923 unsigned long addr = regs->ip - fineibt_preamble_ud; 1924 u32 hash; 1925 1926 if (!exact_endbr((void *)addr)) 1927 return false; 1928 1929 *target = addr + fineibt_prefix_size; 1930 1931 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); 1932 *type = (u32)regs->ax + hash; 1933 1934 /* 1935 * Since regs->ip points to the middle of an instruction; it cannot 1936 * continue with the normal fixup. 1937 */ 1938 regs->ip = *target; 1939 1940 return true; 1941 1942 Efault: 1943 return false; 1944 } 1945 1946 /* 1947 * regs->ip points to one of the UD2 in __bhi_args[]. 1948 */ 1949 static bool decode_fineibt_bhi(struct pt_regs *regs, unsigned long *target, u32 *type) 1950 { 1951 unsigned long addr; 1952 u32 hash; 1953 1954 if (!cfi_bhi) 1955 return false; 1956 1957 if (regs->ip < (unsigned long)__bhi_args || 1958 regs->ip >= (unsigned long)__bhi_args_end) 1959 return false; 1960 1961 /* 1962 * Fetch the return address from the stack, this points to the 1963 * FineIBT preamble. Since the CALL instruction is in the 5 last 1964 * bytes of the preamble, the return address is in fact the target 1965 * address. 1966 */ 1967 __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault); 1968 *target = addr; 1969 1970 addr -= fineibt_prefix_size; 1971 if (!exact_endbr((void *)addr)) 1972 return false; 1973 1974 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); 1975 *type = (u32)regs->ax + hash; 1976 1977 /* 1978 * The UD2 sites are constructed with a RET immediately following, 1979 * as such the non-fatal case can use the regular fixup. 1980 */ 1981 return true; 1982 1983 Efault: 1984 return false; 1985 } 1986 1987 static bool is_paranoid_thunk(unsigned long addr) 1988 { 1989 u32 thunk; 1990 1991 __get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault); 1992 return (thunk & 0x00FFFFFF) == 0xfd75d6; 1993 1994 Efault: 1995 return false; 1996 } 1997 1998 /* 1999 * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[] 2000 * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk. 2001 */ 2002 static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type) 2003 { 2004 unsigned long addr = regs->ip - fineibt_paranoid_ud; 2005 2006 if (!cfi_paranoid) 2007 return false; 2008 2009 if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) { 2010 *target = regs->r11 + fineibt_prefix_size; 2011 *type = regs->ax; 2012 2013 /* 2014 * Since the trapping instruction is the exact, but LOCK prefixed, 2015 * Jcc.d8 that got us here, the normal fixup will work. 2016 */ 2017 return true; 2018 } 2019 2020 /* 2021 * The cfi_paranoid + ITS thunk combination results in: 2022 * 2023 * 0: b8 78 56 34 12 mov $0x12345678, %eax 2024 * 5: 41 3b 43 f7 cmp -11(%r11), %eax 2025 * a: 2e 3d 8d 5b f0 cs lea -0x10(%r11), %r11 2026 * e: 2e e8 XX XX XX XX cs call __x86_indirect_paranoid_thunk_r11 2027 * 2028 * Where the paranoid_thunk looks like: 2029 * 2030 * 1d: <d6> udb 2031 * __x86_indirect_paranoid_thunk_r11: 2032 * 1e: 75 fd jne 1d 2033 * __x86_indirect_its_thunk_r11: 2034 * 20: 41 ff eb jmp *%r11 2035 * 23: cc int3 2036 * 2037 */ 2038 if (is_paranoid_thunk(regs->ip)) { 2039 *target = regs->r11 + fineibt_prefix_size; 2040 *type = regs->ax; 2041 2042 regs->ip = *target; 2043 return true; 2044 } 2045 2046 return false; 2047 } 2048 2049 bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) 2050 { 2051 if (decode_fineibt_paranoid(regs, target, type)) 2052 return true; 2053 2054 if (decode_fineibt_bhi(regs, target, type)) 2055 return true; 2056 2057 return decode_fineibt_preamble(regs, target, type); 2058 } 2059 2060 #else /* !CONFIG_FINEIBT: */ 2061 2062 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 2063 s32 *start_cfi, s32 *end_cfi, bool builtin) 2064 { 2065 if (IS_ENABLED(CONFIG_CFI) && builtin) 2066 pr_info("CFI: Using standard kCFI\n"); 2067 } 2068 2069 #ifdef CONFIG_X86_KERNEL_IBT 2070 static void poison_cfi(void *addr) { } 2071 #endif 2072 2073 #endif /* !CONFIG_FINEIBT */ 2074 2075 void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, 2076 s32 *start_cfi, s32 *end_cfi) 2077 { 2078 return __apply_fineibt(start_retpoline, end_retpoline, 2079 start_cfi, end_cfi, 2080 /* .builtin = */ false); 2081 } 2082 2083 #ifdef CONFIG_SMP 2084 static void alternatives_smp_lock(const s32 *start, const s32 *end, 2085 u8 *text, u8 *text_end) 2086 { 2087 const s32 *poff; 2088 2089 for (poff = start; poff < end; poff++) { 2090 u8 *ptr = (u8 *)poff + *poff; 2091 2092 if (!*poff || ptr < text || ptr >= text_end) 2093 continue; 2094 /* turn DS segment override prefix into lock prefix */ 2095 if (*ptr == 0x3e) 2096 text_poke(ptr, ((unsigned char []){0xf0}), 1); 2097 } 2098 } 2099 2100 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 2101 u8 *text, u8 *text_end) 2102 { 2103 const s32 *poff; 2104 2105 for (poff = start; poff < end; poff++) { 2106 u8 *ptr = (u8 *)poff + *poff; 2107 2108 if (!*poff || ptr < text || ptr >= text_end) 2109 continue; 2110 /* turn lock prefix into DS segment override prefix */ 2111 if (*ptr == 0xf0) 2112 text_poke(ptr, ((unsigned char []){0x3E}), 1); 2113 } 2114 } 2115 2116 struct smp_alt_module { 2117 /* what is this ??? */ 2118 struct module *mod; 2119 char *name; 2120 2121 /* ptrs to lock prefixes */ 2122 const s32 *locks; 2123 const s32 *locks_end; 2124 2125 /* .text segment, needed to avoid patching init code ;) */ 2126 u8 *text; 2127 u8 *text_end; 2128 2129 struct list_head next; 2130 }; 2131 static LIST_HEAD(smp_alt_modules); 2132 static bool uniproc_patched = false; /* protected by text_mutex */ 2133 2134 void __init_or_module alternatives_smp_module_add(struct module *mod, 2135 char *name, 2136 void *locks, void *locks_end, 2137 void *text, void *text_end) 2138 { 2139 struct smp_alt_module *smp; 2140 2141 mutex_lock(&text_mutex); 2142 if (!uniproc_patched) 2143 goto unlock; 2144 2145 if (num_possible_cpus() == 1) 2146 /* Don't bother remembering, we'll never have to undo it. */ 2147 goto smp_unlock; 2148 2149 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 2150 if (NULL == smp) 2151 /* we'll run the (safe but slow) SMP code then ... */ 2152 goto unlock; 2153 2154 smp->mod = mod; 2155 smp->name = name; 2156 smp->locks = locks; 2157 smp->locks_end = locks_end; 2158 smp->text = text; 2159 smp->text_end = text_end; 2160 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", 2161 smp->locks, smp->locks_end, 2162 smp->text, smp->text_end, smp->name); 2163 2164 list_add_tail(&smp->next, &smp_alt_modules); 2165 smp_unlock: 2166 alternatives_smp_unlock(locks, locks_end, text, text_end); 2167 unlock: 2168 mutex_unlock(&text_mutex); 2169 } 2170 2171 void __init_or_module alternatives_smp_module_del(struct module *mod) 2172 { 2173 struct smp_alt_module *item; 2174 2175 mutex_lock(&text_mutex); 2176 list_for_each_entry(item, &smp_alt_modules, next) { 2177 if (mod != item->mod) 2178 continue; 2179 list_del(&item->next); 2180 kfree(item); 2181 break; 2182 } 2183 mutex_unlock(&text_mutex); 2184 } 2185 2186 void alternatives_enable_smp(void) 2187 { 2188 struct smp_alt_module *mod; 2189 2190 /* Why bother if there are no other CPUs? */ 2191 BUG_ON(num_possible_cpus() == 1); 2192 2193 mutex_lock(&text_mutex); 2194 2195 if (uniproc_patched) { 2196 pr_info("switching to SMP code\n"); 2197 BUG_ON(num_online_cpus() != 1); 2198 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 2199 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 2200 list_for_each_entry(mod, &smp_alt_modules, next) 2201 alternatives_smp_lock(mod->locks, mod->locks_end, 2202 mod->text, mod->text_end); 2203 uniproc_patched = false; 2204 } 2205 mutex_unlock(&text_mutex); 2206 } 2207 2208 /* 2209 * Return 1 if the address range is reserved for SMP-alternatives. 2210 * Must hold text_mutex. 2211 */ 2212 int alternatives_text_reserved(void *start, void *end) 2213 { 2214 struct smp_alt_module *mod; 2215 const s32 *poff; 2216 u8 *text_start = start; 2217 u8 *text_end = end; 2218 2219 lockdep_assert_held(&text_mutex); 2220 2221 list_for_each_entry(mod, &smp_alt_modules, next) { 2222 if (mod->text > text_end || mod->text_end < text_start) 2223 continue; 2224 for (poff = mod->locks; poff < mod->locks_end; poff++) { 2225 const u8 *ptr = (const u8 *)poff + *poff; 2226 2227 if (text_start <= ptr && text_end > ptr) 2228 return 1; 2229 } 2230 } 2231 2232 return 0; 2233 } 2234 #endif /* CONFIG_SMP */ 2235 2236 /* 2237 * Self-test for the INT3 based CALL emulation code. 2238 * 2239 * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up 2240 * properly and that there is a stack gap between the INT3 frame and the 2241 * previous context. Without this gap doing a virtual PUSH on the interrupted 2242 * stack would corrupt the INT3 IRET frame. 2243 * 2244 * See entry_{32,64}.S for more details. 2245 */ 2246 2247 /* 2248 * We define the int3_magic() function in assembly to control the calling 2249 * convention such that we can 'call' it from assembly. 2250 */ 2251 2252 extern void int3_magic(unsigned int *ptr); /* defined in asm */ 2253 2254 asm ( 2255 " .pushsection .init.text, \"ax\", @progbits\n" 2256 " .type int3_magic, @function\n" 2257 "int3_magic:\n" 2258 ANNOTATE_NOENDBR 2259 " movl $1, (%" _ASM_ARG1 ")\n" 2260 ASM_RET 2261 " .size int3_magic, .-int3_magic\n" 2262 " .popsection\n" 2263 ); 2264 2265 extern void int3_selftest_ip(void); /* defined in asm below */ 2266 2267 static int __init 2268 int3_exception_notify(struct notifier_block *self, unsigned long val, void *data) 2269 { 2270 unsigned long selftest = (unsigned long)&int3_selftest_ip; 2271 struct die_args *args = data; 2272 struct pt_regs *regs = args->regs; 2273 2274 OPTIMIZER_HIDE_VAR(selftest); 2275 2276 if (!regs || user_mode(regs)) 2277 return NOTIFY_DONE; 2278 2279 if (val != DIE_INT3) 2280 return NOTIFY_DONE; 2281 2282 if (regs->ip - INT3_INSN_SIZE != selftest) 2283 return NOTIFY_DONE; 2284 2285 int3_emulate_call(regs, (unsigned long)&int3_magic); 2286 return NOTIFY_STOP; 2287 } 2288 2289 /* Must be noinline to ensure uniqueness of int3_selftest_ip. */ 2290 static noinline void __init int3_selftest(void) 2291 { 2292 static __initdata struct notifier_block int3_exception_nb = { 2293 .notifier_call = int3_exception_notify, 2294 .priority = INT_MAX-1, /* last */ 2295 }; 2296 unsigned int val = 0; 2297 2298 BUG_ON(register_die_notifier(&int3_exception_nb)); 2299 2300 /* 2301 * Basically: int3_magic(&val); but really complicated :-) 2302 * 2303 * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb 2304 * notifier above will emulate CALL for us. 2305 */ 2306 asm volatile ("int3_selftest_ip:\n\t" 2307 ANNOTATE_NOENDBR 2308 " int3; nop; nop; nop; nop\n\t" 2309 : ASM_CALL_CONSTRAINT 2310 : __ASM_SEL_RAW(a, D) (&val) 2311 : "memory"); 2312 2313 BUG_ON(val != 1); 2314 2315 unregister_die_notifier(&int3_exception_nb); 2316 } 2317 2318 static __initdata int __alt_reloc_selftest_addr; 2319 2320 extern void __init __alt_reloc_selftest(void *arg); 2321 __visible noinline void __init __alt_reloc_selftest(void *arg) 2322 { 2323 WARN_ON(arg != &__alt_reloc_selftest_addr); 2324 } 2325 2326 static noinline void __init alt_reloc_selftest(void) 2327 { 2328 /* 2329 * Tests text_poke_apply_relocation(). 2330 * 2331 * This has a relative immediate (CALL) in a place other than the first 2332 * instruction and additionally on x86_64 we get a RIP-relative LEA: 2333 * 2334 * lea 0x0(%rip),%rdi # 5d0: R_X86_64_PC32 .init.data+0x5566c 2335 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4 2336 * 2337 * Getting this wrong will either crash and burn or tickle the WARN 2338 * above. 2339 */ 2340 asm_inline volatile ( 2341 ALTERNATIVE("", "lea %[mem], %%" _ASM_ARG1 "; call __alt_reloc_selftest;", X86_FEATURE_ALWAYS) 2342 : ASM_CALL_CONSTRAINT 2343 : [mem] "m" (__alt_reloc_selftest_addr) 2344 : _ASM_ARG1 2345 ); 2346 } 2347 2348 void __init alternative_instructions(void) 2349 { 2350 u64 ibt; 2351 2352 int3_selftest(); 2353 2354 /* 2355 * The patching is not fully atomic, so try to avoid local 2356 * interruptions that might execute the to be patched code. 2357 * Other CPUs are not running. 2358 */ 2359 stop_nmi(); 2360 2361 /* 2362 * Don't stop machine check exceptions while patching. 2363 * MCEs only happen when something got corrupted and in this 2364 * case we must do something about the corruption. 2365 * Ignoring it is worse than an unlikely patching race. 2366 * Also machine checks tend to be broadcast and if one CPU 2367 * goes into machine check the others follow quickly, so we don't 2368 * expect a machine check to cause undue problems during to code 2369 * patching. 2370 */ 2371 2372 /* 2373 * Make sure to set (artificial) features depending on used paravirt 2374 * functions which can later influence alternative patching. 2375 */ 2376 paravirt_set_cap(); 2377 2378 /* Keep CET-IBT disabled until caller/callee are patched */ 2379 ibt = ibt_save(/*disable*/ true); 2380 2381 __apply_fineibt(__retpoline_sites, __retpoline_sites_end, 2382 __cfi_sites, __cfi_sites_end, true); 2383 cfi_debug = false; 2384 2385 /* 2386 * Rewrite the retpolines, must be done before alternatives since 2387 * those can rewrite the retpoline thunks. 2388 */ 2389 apply_retpolines(__retpoline_sites, __retpoline_sites_end); 2390 apply_returns(__return_sites, __return_sites_end); 2391 2392 its_fini_core(); 2393 2394 /* 2395 * Adjust all CALL instructions to point to func()-10, including 2396 * those in .altinstr_replacement. 2397 */ 2398 callthunks_patch_builtin_calls(); 2399 2400 apply_alternatives(__alt_instructions, __alt_instructions_end); 2401 2402 /* 2403 * Seal all functions that do not have their address taken. 2404 */ 2405 apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); 2406 2407 ibt_restore(ibt); 2408 2409 #ifdef CONFIG_SMP 2410 /* Patch to UP if other cpus not imminent. */ 2411 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { 2412 uniproc_patched = true; 2413 alternatives_smp_module_add(NULL, "core kernel", 2414 __smp_locks, __smp_locks_end, 2415 _text, _etext); 2416 } 2417 2418 if (!uniproc_patched || num_possible_cpus() == 1) { 2419 free_init_pages("SMP alternatives", 2420 (unsigned long)__smp_locks, 2421 (unsigned long)__smp_locks_end); 2422 } 2423 #endif 2424 2425 restart_nmi(); 2426 alternatives_patched = 1; 2427 2428 alt_reloc_selftest(); 2429 } 2430 2431 /** 2432 * text_poke_early - Update instructions on a live kernel at boot time 2433 * @addr: address to modify 2434 * @opcode: source of the copy 2435 * @len: length to copy 2436 * 2437 * When you use this code to patch more than one byte of an instruction 2438 * you need to make sure that other CPUs cannot execute this code in parallel. 2439 * Also no thread must be currently preempted in the middle of these 2440 * instructions. And on the local CPU you need to be protected against NMI or 2441 * MCE handlers seeing an inconsistent instruction while you patch. 2442 */ 2443 void __init_or_module text_poke_early(void *addr, const void *opcode, 2444 size_t len) 2445 { 2446 unsigned long flags; 2447 2448 if (boot_cpu_has(X86_FEATURE_NX) && 2449 is_module_text_address((unsigned long)addr)) { 2450 /* 2451 * Modules text is marked initially as non-executable, so the 2452 * code cannot be running and speculative code-fetches are 2453 * prevented. Just change the code. 2454 */ 2455 memcpy(addr, opcode, len); 2456 } else { 2457 local_irq_save(flags); 2458 memcpy(addr, opcode, len); 2459 sync_core(); 2460 local_irq_restore(flags); 2461 2462 /* 2463 * Could also do a CLFLUSH here to speed up CPU recovery; but 2464 * that causes hangs on some VIA CPUs. 2465 */ 2466 } 2467 } 2468 2469 __ro_after_init struct mm_struct *text_poke_mm; 2470 __ro_after_init unsigned long text_poke_mm_addr; 2471 2472 static void text_poke_memcpy(void *dst, const void *src, size_t len) 2473 { 2474 memcpy(dst, src, len); 2475 } 2476 2477 static void text_poke_memset(void *dst, const void *src, size_t len) 2478 { 2479 int c = *(const int *)src; 2480 2481 memset(dst, c, len); 2482 } 2483 2484 typedef void text_poke_f(void *dst, const void *src, size_t len); 2485 2486 static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) 2487 { 2488 bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; 2489 struct page *pages[2] = {NULL}; 2490 struct mm_struct *prev_mm; 2491 unsigned long flags; 2492 pte_t pte, *ptep; 2493 spinlock_t *ptl; 2494 pgprot_t pgprot; 2495 2496 /* 2497 * While boot memory allocator is running we cannot use struct pages as 2498 * they are not yet initialized. There is no way to recover. 2499 */ 2500 BUG_ON(!after_bootmem); 2501 2502 if (!core_kernel_text((unsigned long)addr)) { 2503 pages[0] = vmalloc_to_page(addr); 2504 if (cross_page_boundary) 2505 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 2506 } else { 2507 pages[0] = virt_to_page(addr); 2508 WARN_ON(!PageReserved(pages[0])); 2509 if (cross_page_boundary) 2510 pages[1] = virt_to_page(addr + PAGE_SIZE); 2511 } 2512 /* 2513 * If something went wrong, crash and burn since recovery paths are not 2514 * implemented. 2515 */ 2516 BUG_ON(!pages[0] || (cross_page_boundary && !pages[1])); 2517 2518 /* 2519 * Map the page without the global bit, as TLB flushing is done with 2520 * flush_tlb_mm_range(), which is intended for non-global PTEs. 2521 */ 2522 pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL); 2523 2524 /* 2525 * The lock is not really needed, but this allows to avoid open-coding. 2526 */ 2527 ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl); 2528 2529 /* 2530 * This must not fail; preallocated in poking_init(). 2531 */ 2532 VM_BUG_ON(!ptep); 2533 2534 local_irq_save(flags); 2535 2536 pte = mk_pte(pages[0], pgprot); 2537 set_pte_at(text_poke_mm, text_poke_mm_addr, ptep, pte); 2538 2539 if (cross_page_boundary) { 2540 pte = mk_pte(pages[1], pgprot); 2541 set_pte_at(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1, pte); 2542 } 2543 2544 /* 2545 * Loading the temporary mm behaves as a compiler barrier, which 2546 * guarantees that the PTE will be set at the time memcpy() is done. 2547 */ 2548 prev_mm = use_temporary_mm(text_poke_mm); 2549 2550 kasan_disable_current(); 2551 func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len); 2552 kasan_enable_current(); 2553 2554 /* 2555 * Ensure that the PTE is only cleared after the instructions of memcpy 2556 * were issued by using a compiler barrier. 2557 */ 2558 barrier(); 2559 2560 pte_clear(text_poke_mm, text_poke_mm_addr, ptep); 2561 if (cross_page_boundary) 2562 pte_clear(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1); 2563 2564 /* 2565 * Loading the previous page-table hierarchy requires a serializing 2566 * instruction that already allows the core to see the updated version. 2567 * Xen-PV is assumed to serialize execution in a similar manner. 2568 */ 2569 unuse_temporary_mm(prev_mm); 2570 2571 /* 2572 * Flushing the TLB might involve IPIs, which would require enabled 2573 * IRQs, but not if the mm is not used, as it is in this point. 2574 */ 2575 flush_tlb_mm_range(text_poke_mm, text_poke_mm_addr, text_poke_mm_addr + 2576 (cross_page_boundary ? 2 : 1) * PAGE_SIZE, 2577 PAGE_SHIFT, false); 2578 2579 if (func == text_poke_memcpy) { 2580 /* 2581 * If the text does not match what we just wrote then something is 2582 * fundamentally screwy; there's nothing we can really do about that. 2583 */ 2584 BUG_ON(memcmp(addr, src, len)); 2585 } 2586 2587 local_irq_restore(flags); 2588 pte_unmap_unlock(ptep, ptl); 2589 return addr; 2590 } 2591 2592 /** 2593 * text_poke - Update instructions on a live kernel 2594 * @addr: address to modify 2595 * @opcode: source of the copy 2596 * @len: length to copy 2597 * 2598 * Only atomic text poke/set should be allowed when not doing early patching. 2599 * It means the size must be writable atomically and the address must be aligned 2600 * in a way that permits an atomic write. It also makes sure we fit on a single 2601 * page. 2602 * 2603 * Note that the caller must ensure that if the modified code is part of a 2604 * module, the module would not be removed during poking. This can be achieved 2605 * by registering a module notifier, and ordering module removal and patching 2606 * through a mutex. 2607 */ 2608 void *text_poke(void *addr, const void *opcode, size_t len) 2609 { 2610 lockdep_assert_held(&text_mutex); 2611 2612 return __text_poke(text_poke_memcpy, addr, opcode, len); 2613 } 2614 2615 /** 2616 * text_poke_kgdb - Update instructions on a live kernel by kgdb 2617 * @addr: address to modify 2618 * @opcode: source of the copy 2619 * @len: length to copy 2620 * 2621 * Only atomic text poke/set should be allowed when not doing early patching. 2622 * It means the size must be writable atomically and the address must be aligned 2623 * in a way that permits an atomic write. It also makes sure we fit on a single 2624 * page. 2625 * 2626 * Context: should only be used by kgdb, which ensures no other core is running, 2627 * despite the fact it does not hold the text_mutex. 2628 */ 2629 void *text_poke_kgdb(void *addr, const void *opcode, size_t len) 2630 { 2631 return __text_poke(text_poke_memcpy, addr, opcode, len); 2632 } 2633 2634 void *text_poke_copy_locked(void *addr, const void *opcode, size_t len, 2635 bool core_ok) 2636 { 2637 unsigned long start = (unsigned long)addr; 2638 size_t patched = 0; 2639 2640 if (WARN_ON_ONCE(!core_ok && core_kernel_text(start))) 2641 return NULL; 2642 2643 while (patched < len) { 2644 unsigned long ptr = start + patched; 2645 size_t s; 2646 2647 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); 2648 2649 __text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s); 2650 patched += s; 2651 } 2652 return addr; 2653 } 2654 2655 /** 2656 * text_poke_copy - Copy instructions into (an unused part of) RX memory 2657 * @addr: address to modify 2658 * @opcode: source of the copy 2659 * @len: length to copy, could be more than 2x PAGE_SIZE 2660 * 2661 * Not safe against concurrent execution; useful for JITs to dump 2662 * new code blocks into unused regions of RX memory. Can be used in 2663 * conjunction with synchronize_rcu_tasks() to wait for existing 2664 * execution to quiesce after having made sure no existing functions 2665 * pointers are live. 2666 */ 2667 void *text_poke_copy(void *addr, const void *opcode, size_t len) 2668 { 2669 mutex_lock(&text_mutex); 2670 addr = text_poke_copy_locked(addr, opcode, len, false); 2671 mutex_unlock(&text_mutex); 2672 return addr; 2673 } 2674 2675 /** 2676 * text_poke_set - memset into (an unused part of) RX memory 2677 * @addr: address to modify 2678 * @c: the byte to fill the area with 2679 * @len: length to copy, could be more than 2x PAGE_SIZE 2680 * 2681 * This is useful to overwrite unused regions of RX memory with illegal 2682 * instructions. 2683 */ 2684 void *text_poke_set(void *addr, int c, size_t len) 2685 { 2686 unsigned long start = (unsigned long)addr; 2687 size_t patched = 0; 2688 2689 if (WARN_ON_ONCE(core_kernel_text(start))) 2690 return NULL; 2691 2692 mutex_lock(&text_mutex); 2693 while (patched < len) { 2694 unsigned long ptr = start + patched; 2695 size_t s; 2696 2697 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); 2698 2699 __text_poke(text_poke_memset, (void *)ptr, (void *)&c, s); 2700 patched += s; 2701 } 2702 mutex_unlock(&text_mutex); 2703 return addr; 2704 } 2705 2706 static void do_sync_core(void *info) 2707 { 2708 sync_core(); 2709 } 2710 2711 void smp_text_poke_sync_each_cpu(void) 2712 { 2713 on_each_cpu(do_sync_core, NULL, 1); 2714 } 2715 2716 /* 2717 * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of 2718 * this thing. When len == 6 everything is prefixed with 0x0f and we map 2719 * opcode to Jcc.d8, using len to distinguish. 2720 */ 2721 struct smp_text_poke_loc { 2722 /* addr := _stext + rel_addr */ 2723 s32 rel_addr; 2724 s32 disp; 2725 u8 len; 2726 u8 opcode; 2727 const u8 text[TEXT_POKE_MAX_OPCODE_SIZE]; 2728 /* see smp_text_poke_batch_finish() */ 2729 u8 old; 2730 }; 2731 2732 #define TEXT_POKE_ARRAY_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) 2733 2734 static struct smp_text_poke_array { 2735 struct smp_text_poke_loc vec[TEXT_POKE_ARRAY_MAX]; 2736 int nr_entries; 2737 } text_poke_array; 2738 2739 static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); 2740 2741 /* 2742 * These four __always_inline annotations imply noinstr, necessary 2743 * due to smp_text_poke_int3_handler() being noinstr: 2744 */ 2745 2746 static __always_inline bool try_get_text_poke_array(void) 2747 { 2748 atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); 2749 2750 if (!raw_atomic_inc_not_zero(refs)) 2751 return false; 2752 2753 return true; 2754 } 2755 2756 static __always_inline void put_text_poke_array(void) 2757 { 2758 atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); 2759 2760 smp_mb__before_atomic(); 2761 raw_atomic_dec(refs); 2762 } 2763 2764 static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tpl) 2765 { 2766 return _stext + tpl->rel_addr; 2767 } 2768 2769 static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b) 2770 { 2771 if (tpl_a < text_poke_addr(tpl_b)) 2772 return -1; 2773 if (tpl_a > text_poke_addr(tpl_b)) 2774 return 1; 2775 return 0; 2776 } 2777 2778 noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) 2779 { 2780 struct smp_text_poke_loc *tpl; 2781 int ret = 0; 2782 void *ip; 2783 2784 if (user_mode(regs)) 2785 return 0; 2786 2787 /* 2788 * Having observed our INT3 instruction, we now must observe 2789 * text_poke_array with non-zero refcount: 2790 * 2791 * text_poke_array_refs = 1 INT3 2792 * WMB RMB 2793 * write INT3 if (text_poke_array_refs != 0) 2794 */ 2795 smp_rmb(); 2796 2797 if (!try_get_text_poke_array()) 2798 return 0; 2799 2800 /* 2801 * Discount the INT3. See smp_text_poke_batch_finish(). 2802 */ 2803 ip = (void *) regs->ip - INT3_INSN_SIZE; 2804 2805 /* 2806 * Skip the binary search if there is a single member in the vector. 2807 */ 2808 if (unlikely(text_poke_array.nr_entries > 1)) { 2809 tpl = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries, 2810 sizeof(struct smp_text_poke_loc), 2811 patch_cmp); 2812 if (!tpl) 2813 goto out_put; 2814 } else { 2815 tpl = text_poke_array.vec; 2816 if (text_poke_addr(tpl) != ip) 2817 goto out_put; 2818 } 2819 2820 ip += tpl->len; 2821 2822 switch (tpl->opcode) { 2823 case INT3_INSN_OPCODE: 2824 /* 2825 * Someone poked an explicit INT3, they'll want to handle it, 2826 * do not consume. 2827 */ 2828 goto out_put; 2829 2830 case RET_INSN_OPCODE: 2831 int3_emulate_ret(regs); 2832 break; 2833 2834 case CALL_INSN_OPCODE: 2835 int3_emulate_call(regs, (long)ip + tpl->disp); 2836 break; 2837 2838 case JMP32_INSN_OPCODE: 2839 case JMP8_INSN_OPCODE: 2840 int3_emulate_jmp(regs, (long)ip + tpl->disp); 2841 break; 2842 2843 case 0x70 ... 0x7f: /* Jcc */ 2844 int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp); 2845 break; 2846 2847 default: 2848 BUG(); 2849 } 2850 2851 ret = 1; 2852 2853 out_put: 2854 put_text_poke_array(); 2855 return ret; 2856 } 2857 2858 /** 2859 * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP 2860 * 2861 * Input state: 2862 * text_poke_array.vec: vector of instructions to patch 2863 * text_poke_array.nr_entries: number of entries in the vector 2864 * 2865 * Modify multi-byte instructions by using INT3 breakpoints on SMP. 2866 * We completely avoid using stop_machine() here, and achieve the 2867 * synchronization using INT3 breakpoints and SMP cross-calls. 2868 * 2869 * The way it is done: 2870 * - For each entry in the vector: 2871 * - add an INT3 trap to the address that will be patched 2872 * - SMP sync all CPUs 2873 * - For each entry in the vector: 2874 * - update all but the first byte of the patched range 2875 * - SMP sync all CPUs 2876 * - For each entry in the vector: 2877 * - replace the first byte (INT3) by the first byte of the 2878 * replacing opcode 2879 * - SMP sync all CPUs 2880 */ 2881 void smp_text_poke_batch_finish(void) 2882 { 2883 unsigned char int3 = INT3_INSN_OPCODE; 2884 unsigned int i; 2885 int do_sync; 2886 2887 if (!text_poke_array.nr_entries) 2888 return; 2889 2890 lockdep_assert_held(&text_mutex); 2891 2892 /* 2893 * Corresponds to the implicit memory barrier in try_get_text_poke_array() to 2894 * ensure reading a non-zero refcount provides up to date text_poke_array data. 2895 */ 2896 for_each_possible_cpu(i) 2897 atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1); 2898 2899 /* 2900 * Function tracing can enable thousands of places that need to be 2901 * updated. This can take quite some time, and with full kernel debugging 2902 * enabled, this could cause the softlockup watchdog to trigger. 2903 * This function gets called every 256 entries added to be patched. 2904 * Call cond_resched() here to make sure that other tasks can get scheduled 2905 * while processing all the functions being patched. 2906 */ 2907 cond_resched(); 2908 2909 /* 2910 * Corresponding read barrier in INT3 notifier for making sure the 2911 * text_poke_array.nr_entries and handler are correctly ordered wrt. patching. 2912 */ 2913 smp_wmb(); 2914 2915 /* 2916 * First step: add a INT3 trap to the address that will be patched. 2917 */ 2918 for (i = 0; i < text_poke_array.nr_entries; i++) { 2919 text_poke_array.vec[i].old = *(u8 *)text_poke_addr(&text_poke_array.vec[i]); 2920 text_poke(text_poke_addr(&text_poke_array.vec[i]), &int3, INT3_INSN_SIZE); 2921 } 2922 2923 smp_text_poke_sync_each_cpu(); 2924 2925 /* 2926 * Second step: update all but the first byte of the patched range. 2927 */ 2928 for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { 2929 u8 old[TEXT_POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, }; 2930 u8 _new[TEXT_POKE_MAX_OPCODE_SIZE+1]; 2931 const u8 *new = text_poke_array.vec[i].text; 2932 int len = text_poke_array.vec[i].len; 2933 2934 if (len - INT3_INSN_SIZE > 0) { 2935 memcpy(old + INT3_INSN_SIZE, 2936 text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE, 2937 len - INT3_INSN_SIZE); 2938 2939 if (len == 6) { 2940 _new[0] = 0x0f; 2941 memcpy(_new + 1, new, 5); 2942 new = _new; 2943 } 2944 2945 text_poke(text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE, 2946 new + INT3_INSN_SIZE, 2947 len - INT3_INSN_SIZE); 2948 2949 do_sync++; 2950 } 2951 2952 /* 2953 * Emit a perf event to record the text poke, primarily to 2954 * support Intel PT decoding which must walk the executable code 2955 * to reconstruct the trace. The flow up to here is: 2956 * - write INT3 byte 2957 * - IPI-SYNC 2958 * - write instruction tail 2959 * At this point the actual control flow will be through the 2960 * INT3 and handler and not hit the old or new instruction. 2961 * Intel PT outputs FUP/TIP packets for the INT3, so the flow 2962 * can still be decoded. Subsequently: 2963 * - emit RECORD_TEXT_POKE with the new instruction 2964 * - IPI-SYNC 2965 * - write first byte 2966 * - IPI-SYNC 2967 * So before the text poke event timestamp, the decoder will see 2968 * either the old instruction flow or FUP/TIP of INT3. After the 2969 * text poke event timestamp, the decoder will see either the 2970 * new instruction flow or FUP/TIP of INT3. Thus decoders can 2971 * use the timestamp as the point at which to modify the 2972 * executable code. 2973 * The old instruction is recorded so that the event can be 2974 * processed forwards or backwards. 2975 */ 2976 perf_event_text_poke(text_poke_addr(&text_poke_array.vec[i]), old, len, new, len); 2977 } 2978 2979 if (do_sync) { 2980 /* 2981 * According to Intel, this core syncing is very likely 2982 * not necessary and we'd be safe even without it. But 2983 * better safe than sorry (plus there's not only Intel). 2984 */ 2985 smp_text_poke_sync_each_cpu(); 2986 } 2987 2988 /* 2989 * Third step: replace the first byte (INT3) by the first byte of the 2990 * replacing opcode. 2991 */ 2992 for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { 2993 u8 byte = text_poke_array.vec[i].text[0]; 2994 2995 if (text_poke_array.vec[i].len == 6) 2996 byte = 0x0f; 2997 2998 if (byte == INT3_INSN_OPCODE) 2999 continue; 3000 3001 text_poke(text_poke_addr(&text_poke_array.vec[i]), &byte, INT3_INSN_SIZE); 3002 do_sync++; 3003 } 3004 3005 if (do_sync) 3006 smp_text_poke_sync_each_cpu(); 3007 3008 /* 3009 * Remove and wait for refs to be zero. 3010 * 3011 * Notably, if after step-3 above the INT3 got removed, then the 3012 * smp_text_poke_sync_each_cpu() will have serialized against any running INT3 3013 * handlers and the below spin-wait will not happen. 3014 * 3015 * IOW. unless the replacement instruction is INT3, this case goes 3016 * unused. 3017 */ 3018 for_each_possible_cpu(i) { 3019 atomic_t *refs = per_cpu_ptr(&text_poke_array_refs, i); 3020 3021 if (unlikely(!atomic_dec_and_test(refs))) 3022 atomic_cond_read_acquire(refs, !VAL); 3023 } 3024 3025 /* They are all completed: */ 3026 text_poke_array.nr_entries = 0; 3027 } 3028 3029 static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) 3030 { 3031 struct smp_text_poke_loc *tpl; 3032 struct insn insn; 3033 int ret, i = 0; 3034 3035 tpl = &text_poke_array.vec[text_poke_array.nr_entries++]; 3036 3037 if (len == 6) 3038 i = 1; 3039 memcpy((void *)tpl->text, opcode+i, len-i); 3040 if (!emulate) 3041 emulate = opcode; 3042 3043 ret = insn_decode_kernel(&insn, emulate); 3044 BUG_ON(ret < 0); 3045 3046 tpl->rel_addr = addr - (void *)_stext; 3047 tpl->len = len; 3048 tpl->opcode = insn.opcode.bytes[0]; 3049 3050 if (is_jcc32(&insn)) { 3051 /* 3052 * Map Jcc.d32 onto Jcc.d8 and use len to distinguish. 3053 */ 3054 tpl->opcode = insn.opcode.bytes[1] - 0x10; 3055 } 3056 3057 switch (tpl->opcode) { 3058 case RET_INSN_OPCODE: 3059 case JMP32_INSN_OPCODE: 3060 case JMP8_INSN_OPCODE: 3061 /* 3062 * Control flow instructions without implied execution of the 3063 * next instruction can be padded with INT3. 3064 */ 3065 for (i = insn.length; i < len; i++) 3066 BUG_ON(tpl->text[i] != INT3_INSN_OPCODE); 3067 break; 3068 3069 default: 3070 BUG_ON(len != insn.length); 3071 } 3072 3073 switch (tpl->opcode) { 3074 case INT3_INSN_OPCODE: 3075 case RET_INSN_OPCODE: 3076 break; 3077 3078 case CALL_INSN_OPCODE: 3079 case JMP32_INSN_OPCODE: 3080 case JMP8_INSN_OPCODE: 3081 case 0x70 ... 0x7f: /* Jcc */ 3082 tpl->disp = insn.immediate.value; 3083 break; 3084 3085 default: /* assume NOP */ 3086 switch (len) { 3087 case 2: /* NOP2 -- emulate as JMP8+0 */ 3088 BUG_ON(memcmp(emulate, x86_nops[len], len)); 3089 tpl->opcode = JMP8_INSN_OPCODE; 3090 tpl->disp = 0; 3091 break; 3092 3093 case 5: /* NOP5 -- emulate as JMP32+0 */ 3094 BUG_ON(memcmp(emulate, x86_nops[len], len)); 3095 tpl->opcode = JMP32_INSN_OPCODE; 3096 tpl->disp = 0; 3097 break; 3098 3099 default: /* unknown instruction */ 3100 BUG(); 3101 } 3102 break; 3103 } 3104 } 3105 3106 /* 3107 * We hard rely on the text_poke_array.vec being ordered; ensure this is so by flushing 3108 * early if needed. 3109 */ 3110 static bool text_poke_addr_ordered(void *addr) 3111 { 3112 WARN_ON_ONCE(!addr); 3113 3114 if (!text_poke_array.nr_entries) 3115 return true; 3116 3117 /* 3118 * If the last current entry's address is higher than the 3119 * new entry's address we'd like to add, then ordering 3120 * is violated and we must first flush all pending patching 3121 * requests: 3122 */ 3123 if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr) 3124 return false; 3125 3126 return true; 3127 } 3128 3129 /** 3130 * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched 3131 * @addr: address to patch 3132 * @opcode: opcode of new instruction 3133 * @len: length to copy 3134 * @emulate: instruction to be emulated 3135 * 3136 * Add a new instruction to the current queue of to-be-patched instructions 3137 * the kernel maintains. The patching request will not be executed immediately, 3138 * but becomes part of an array of patching requests, optimized for batched 3139 * execution. All pending patching requests will be executed on the next 3140 * smp_text_poke_batch_finish() call. 3141 */ 3142 void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) 3143 { 3144 if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr)) 3145 smp_text_poke_batch_finish(); 3146 __smp_text_poke_batch_add(addr, opcode, len, emulate); 3147 } 3148 3149 /** 3150 * smp_text_poke_single() -- update instruction on live kernel on SMP immediately 3151 * @addr: address to patch 3152 * @opcode: opcode of new instruction 3153 * @len: length to copy 3154 * @emulate: instruction to be emulated 3155 * 3156 * Update a single instruction with the vector in the stack, avoiding 3157 * dynamically allocated memory. This function should be used when it is 3158 * not possible to allocate memory for a vector. The single instruction 3159 * is patched in immediately. 3160 */ 3161 void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) 3162 { 3163 smp_text_poke_batch_add(addr, opcode, len, emulate); 3164 smp_text_poke_batch_finish(); 3165 } 3166