alternative.c (1260ed77798502de9c98020040d2995008de10cc) alternative.c (8754e67ad4ac692c67ff1f99c0d07156f04ae40c)
1// SPDX-License-Identifier: GPL-2.0-only
2#define pr_fmt(fmt) "SMP alternatives: " fmt
3
4#include <linux/module.h>
5#include <linux/sched.h>
6#include <linux/perf_event.h>
7#include <linux/mutex.h>
8#include <linux/list.h>

--- 567 unchanged lines hidden (view full) ---

576 modrm += reg;
577
578 bytes[i++] = 0xff; /* opcode */
579 bytes[i++] = modrm;
580
581 return i;
582}
583
1// SPDX-License-Identifier: GPL-2.0-only
2#define pr_fmt(fmt) "SMP alternatives: " fmt
3
4#include <linux/module.h>
5#include <linux/sched.h>
6#include <linux/perf_event.h>
7#include <linux/mutex.h>
8#include <linux/list.h>

--- 567 unchanged lines hidden (view full) ---

576 modrm += reg;
577
578 bytes[i++] = 0xff; /* opcode */
579 bytes[i++] = modrm;
580
581 return i;
582}
583
584static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
584static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
585 void *call_dest, void *jmp_dest)
585{
586 u8 op = insn->opcode.bytes[0];
587 int i = 0;
588
589 /*
590 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
591 * tail-calls. Deal with them.
592 */

--- 4 unchanged lines hidden (view full) ---

597 }
598
599 if (insn->length == 6)
600 bytes[i++] = 0x2e; /* CS-prefix */
601
602 switch (op) {
603 case CALL_INSN_OPCODE:
604 __text_gen_insn(bytes+i, op, addr+i,
586{
587 u8 op = insn->opcode.bytes[0];
588 int i = 0;
589
590 /*
591 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
592 * tail-calls. Deal with them.
593 */

--- 4 unchanged lines hidden (view full) ---

598 }
599
600 if (insn->length == 6)
601 bytes[i++] = 0x2e; /* CS-prefix */
602
603 switch (op) {
604 case CALL_INSN_OPCODE:
605 __text_gen_insn(bytes+i, op, addr+i,
605 __x86_indirect_call_thunk_array[reg],
606 call_dest,
606 CALL_INSN_SIZE);
607 i += CALL_INSN_SIZE;
608 break;
609
610 case JMP32_INSN_OPCODE:
611clang_jcc:
612 __text_gen_insn(bytes+i, op, addr+i,
607 CALL_INSN_SIZE);
608 i += CALL_INSN_SIZE;
609 break;
610
611 case JMP32_INSN_OPCODE:
612clang_jcc:
613 __text_gen_insn(bytes+i, op, addr+i,
613 __x86_indirect_jump_thunk_array[reg],
614 jmp_dest,
614 JMP32_INSN_SIZE);
615 i += JMP32_INSN_SIZE;
616 break;
617
618 default:
619 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
620 return -1;
621 }
622
623 WARN_ON_ONCE(i != insn->length);
624
625 return i;
626}
627
615 JMP32_INSN_SIZE);
616 i += JMP32_INSN_SIZE;
617 break;
618
619 default:
620 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
621 return -1;
622 }
623
624 WARN_ON_ONCE(i != insn->length);
625
626 return i;
627}
628
629static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
630{
631 return __emit_trampoline(addr, insn, bytes,
632 __x86_indirect_call_thunk_array[reg],
633 __x86_indirect_jump_thunk_array[reg]);
634}
635
636#ifdef CONFIG_MITIGATION_ITS
637static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
638{
639 return __emit_trampoline(addr, insn, bytes,
640 __x86_indirect_its_thunk_array[reg],
641 __x86_indirect_its_thunk_array[reg]);
642}
643
644/* Check if an indirect branch is at ITS-unsafe address */
645static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
646{
647 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
648 return false;
649
650 /* Indirect branch opcode is 2 or 3 bytes depending on reg */
651 addr += 1 + reg / 8;
652
653 /* Lower-half of the cacheline? */
654 return !(addr & 0x20);
655}
656#endif
657
628/*
629 * Rewrite the compiler generated retpoline thunk calls.
630 *
631 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate
632 * indirect instructions, avoiding the extra indirection.
633 *
634 * For example, convert:
635 *

--- 58 unchanged lines hidden (view full) ---

694 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
695 */
696 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
697 bytes[i++] = 0x0f;
698 bytes[i++] = 0xae;
699 bytes[i++] = 0xe8; /* LFENCE */
700 }
701
658/*
659 * Rewrite the compiler generated retpoline thunk calls.
660 *
661 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate
662 * indirect instructions, avoiding the extra indirection.
663 *
664 * For example, convert:
665 *

--- 58 unchanged lines hidden (view full) ---

724 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
725 */
726 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
727 bytes[i++] = 0x0f;
728 bytes[i++] = 0xae;
729 bytes[i++] = 0xe8; /* LFENCE */
730 }
731
732#ifdef CONFIG_MITIGATION_ITS
733 /*
734 * Check if the address of last byte of emitted-indirect is in
735 * lower-half of the cacheline. Such branches need ITS mitigation.
736 */
737 if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg))
738 return emit_its_trampoline(addr, insn, reg, bytes);
739#endif
740
702 ret = emit_indirect(op, reg, bytes + i);
703 if (ret < 0)
704 return ret;
705 i += ret;
706
707 /*
708 * The compiler is supposed to EMIT an INT3 after every unconditional
709 * JMP instruction due to AMD BTC. However, if the compiler is too old

--- 2184 unchanged lines hidden --- 		741 ret = emit_indirect(op, reg, bytes + i);
742 if (ret < 0)
743 return ret;
744 i += ret;
745
746 /*
747 * The compiler is supposed to EMIT an INT3 after every unconditional
748 * JMP instruction due to AMD BTC. However, if the compiler is too old

--- 2184 unchanged lines hidden ---