1f19fbd5eSMartin Schwidefsky // SPDX-License-Identifier: GPL-2.0
2f19fbd5eSMartin Schwidefsky #include <linux/module.h>
3d424986fSMartin Schwidefsky #include <linux/device.h>
40336e04aSJosh Poimboeuf #include <linux/cpu.h>
5f19fbd5eSMartin Schwidefsky #include <asm/nospec-branch.h>
6f19fbd5eSMartin Schwidefsky
7*47837a5cSHeiko Carstens int nobp = IS_ENABLED(CONFIG_KERNEL_NOBP);
8*47837a5cSHeiko Carstens
nobp_setup_early(char * str)9b2e2f43aSMartin Schwidefsky static int __init nobp_setup_early(char *str)
10b2e2f43aSMartin Schwidefsky {
11b2e2f43aSMartin Schwidefsky bool enabled;
12b2e2f43aSMartin Schwidefsky int rc;
13b2e2f43aSMartin Schwidefsky
14b2e2f43aSMartin Schwidefsky rc = kstrtobool(str, &enabled);
15b2e2f43aSMartin Schwidefsky if (rc)
16b2e2f43aSMartin Schwidefsky return rc;
176e179d64SMartin Schwidefsky if (enabled && test_facility(82)) {
186e179d64SMartin Schwidefsky /*
19cada938aSHeiko Carstens * The user explicitly requested nobp=1, enable it and
206e179d64SMartin Schwidefsky * disable the expoline support.
216e179d64SMartin Schwidefsky */
22*47837a5cSHeiko Carstens nobp = 1;
236e179d64SMartin Schwidefsky if (IS_ENABLED(CONFIG_EXPOLINE))
246e179d64SMartin Schwidefsky nospec_disable = 1;
256e179d64SMartin Schwidefsky } else {
26*47837a5cSHeiko Carstens nobp = 0;
276e179d64SMartin Schwidefsky }
28b2e2f43aSMartin Schwidefsky return 0;
29b2e2f43aSMartin Schwidefsky }
30b2e2f43aSMartin Schwidefsky early_param("nobp", nobp_setup_early);
31b2e2f43aSMartin Schwidefsky
nospec_setup_early(char * str)32b2e2f43aSMartin Schwidefsky static int __init nospec_setup_early(char *str)
33b2e2f43aSMartin Schwidefsky {
34*47837a5cSHeiko Carstens nobp = 0;
35b2e2f43aSMartin Schwidefsky return 0;
36b2e2f43aSMartin Schwidefsky }
37b2e2f43aSMartin Schwidefsky early_param("nospec", nospec_setup_early);
38b2e2f43aSMartin Schwidefsky
nospec_report(void)39bc035599SMartin Schwidefsky static int __init nospec_report(void)
40bc035599SMartin Schwidefsky {
41aeaf7002SMartin Schwidefsky if (test_facility(156))
42aeaf7002SMartin Schwidefsky pr_info("Spectre V2 mitigation: etokens\n");
435d17d4edSSven Schnelle if (nospec_uses_trampoline())
44b7e7f505SMartin Schwidefsky pr_info("Spectre V2 mitigation: execute trampolines\n");
45*47837a5cSHeiko Carstens if (nobp_enabled())
46b7e7f505SMartin Schwidefsky pr_info("Spectre V2 mitigation: limited branch prediction\n");
47bc035599SMartin Schwidefsky return 0;
48bc035599SMartin Schwidefsky }
49bc035599SMartin Schwidefsky arch_initcall(nospec_report);
50bc035599SMartin Schwidefsky
51b2e2f43aSMartin Schwidefsky #ifdef CONFIG_EXPOLINE
52b2e2f43aSMartin Schwidefsky
536e179d64SMartin Schwidefsky int nospec_disable = IS_ENABLED(CONFIG_EXPOLINE_OFF);
54f19fbd5eSMartin Schwidefsky
nospectre_v2_setup_early(char * str)55f19fbd5eSMartin Schwidefsky static int __init nospectre_v2_setup_early(char *str)
56f19fbd5eSMartin Schwidefsky {
576e179d64SMartin Schwidefsky nospec_disable = 1;
58f19fbd5eSMartin Schwidefsky return 0;
59f19fbd5eSMartin Schwidefsky }
60f19fbd5eSMartin Schwidefsky early_param("nospectre_v2", nospectre_v2_setup_early);
61f19fbd5eSMartin Schwidefsky
nospec_auto_detect(void)626a3d1e81SMartin Schwidefsky void __init nospec_auto_detect(void)
636e179d64SMartin Schwidefsky {
640336e04aSJosh Poimboeuf if (test_facility(156) || cpu_mitigations_off()) {
65aeaf7002SMartin Schwidefsky /*
66aeaf7002SMartin Schwidefsky * The machine supports etokens.
67aeaf7002SMartin Schwidefsky * Disable expolines and disable nobp.
68aeaf7002SMartin Schwidefsky */
69475c8e9eSJoe Perches if (__is_defined(CC_USING_EXPOLINE))
70aeaf7002SMartin Schwidefsky nospec_disable = 1;
71*47837a5cSHeiko Carstens nobp = 0;
72475c8e9eSJoe Perches } else if (__is_defined(CC_USING_EXPOLINE)) {
736e179d64SMartin Schwidefsky /*
746e179d64SMartin Schwidefsky * The kernel has been compiled with expolines.
756e179d64SMartin Schwidefsky * Keep expolines enabled and disable nobp.
766e179d64SMartin Schwidefsky */
776e179d64SMartin Schwidefsky nospec_disable = 0;
78*47837a5cSHeiko Carstens nobp = 0;
796e179d64SMartin Schwidefsky }
806e179d64SMartin Schwidefsky /*
816e179d64SMartin Schwidefsky * If the kernel has not been compiled with expolines the
826e179d64SMartin Schwidefsky * nobp setting decides what is done, this depends on the
836e179d64SMartin Schwidefsky * CONFIG_KERNEL_NP option and the nobp/nospec parameters.
846e179d64SMartin Schwidefsky */
856e179d64SMartin Schwidefsky }
866e179d64SMartin Schwidefsky
spectre_v2_setup_early(char * str)87f19fbd5eSMartin Schwidefsky static int __init spectre_v2_setup_early(char *str)
88f19fbd5eSMartin Schwidefsky {
89f19fbd5eSMartin Schwidefsky if (str && !strncmp(str, "on", 2)) {
906e179d64SMartin Schwidefsky nospec_disable = 0;
91*47837a5cSHeiko Carstens nobp = 0;
92f19fbd5eSMartin Schwidefsky }
936e179d64SMartin Schwidefsky if (str && !strncmp(str, "off", 3))
946e179d64SMartin Schwidefsky nospec_disable = 1;
956e179d64SMartin Schwidefsky if (str && !strncmp(str, "auto", 4))
966a3d1e81SMartin Schwidefsky nospec_auto_detect();
97f19fbd5eSMartin Schwidefsky return 0;
98f19fbd5eSMartin Schwidefsky }
99f19fbd5eSMartin Schwidefsky early_param("spectre_v2", spectre_v2_setup_early);
100f19fbd5eSMartin Schwidefsky
__nospec_revert(s32 * start,s32 * end)101f19fbd5eSMartin Schwidefsky static void __init_or_module __nospec_revert(s32 *start, s32 *end)
102f19fbd5eSMartin Schwidefsky {
103f19fbd5eSMartin Schwidefsky enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type;
104c74d3c18SKees Cook static const u8 branch[] = { 0x47, 0x00, 0x07, 0x00 };
105f19fbd5eSMartin Schwidefsky u8 *instr, *thunk, *br;
106f19fbd5eSMartin Schwidefsky u8 insnbuf[6];
107f19fbd5eSMartin Schwidefsky s32 *epo;
108f19fbd5eSMartin Schwidefsky
109f19fbd5eSMartin Schwidefsky /* Second part of the instruction replace is always a nop */
1102268169cSVasily Gorbik memcpy(insnbuf + 2, branch, sizeof(branch));
111f19fbd5eSMartin Schwidefsky for (epo = start; epo < end; epo++) {
112f19fbd5eSMartin Schwidefsky instr = (u8 *) epo + *epo;
113f19fbd5eSMartin Schwidefsky if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04)
114f19fbd5eSMartin Schwidefsky type = BRCL_EXPOLINE; /* brcl instruction */
115f19fbd5eSMartin Schwidefsky else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05)
116f19fbd5eSMartin Schwidefsky type = BRASL_EXPOLINE; /* brasl instruction */
117f19fbd5eSMartin Schwidefsky else
118f19fbd5eSMartin Schwidefsky continue;
119ea84f14dSVasily Gorbik thunk = instr + (long)(*(int *)(instr + 2)) * 2;
120f19fbd5eSMartin Schwidefsky if (thunk[0] == 0xc6 && thunk[1] == 0x00)
121f19fbd5eSMartin Schwidefsky /* exrl %r0,<target-br> */
122ea84f14dSVasily Gorbik br = thunk + (long)(*(int *)(thunk + 2)) * 2;
123f19fbd5eSMartin Schwidefsky else
124f19fbd5eSMartin Schwidefsky continue;
1252268169cSVasily Gorbik if (br[0] != 0x07 || (br[1] & 0xf0) != 0xf0)
126f19fbd5eSMartin Schwidefsky continue;
127f19fbd5eSMartin Schwidefsky switch (type) {
128f19fbd5eSMartin Schwidefsky case BRCL_EXPOLINE:
1292268169cSVasily Gorbik /* brcl to thunk, replace with br + nop */
130f19fbd5eSMartin Schwidefsky insnbuf[0] = br[0];
131f19fbd5eSMartin Schwidefsky insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
132f19fbd5eSMartin Schwidefsky break;
133f19fbd5eSMartin Schwidefsky case BRASL_EXPOLINE:
1342268169cSVasily Gorbik /* brasl to thunk, replace with basr + nop */
1356deaa3bbSMartin Schwidefsky insnbuf[0] = 0x0d;
1362268169cSVasily Gorbik insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
137f19fbd5eSMartin Schwidefsky break;
138f19fbd5eSMartin Schwidefsky }
139f19fbd5eSMartin Schwidefsky
140f19fbd5eSMartin Schwidefsky s390_kernel_write(instr, insnbuf, 6);
141f19fbd5eSMartin Schwidefsky }
142f19fbd5eSMartin Schwidefsky }
143f19fbd5eSMartin Schwidefsky
nospec_revert(s32 * start,s32 * end)1446e179d64SMartin Schwidefsky void __init_or_module nospec_revert(s32 *start, s32 *end)
145f19fbd5eSMartin Schwidefsky {
1466e179d64SMartin Schwidefsky if (nospec_disable)
147f19fbd5eSMartin Schwidefsky __nospec_revert(start, end);
148f19fbd5eSMartin Schwidefsky }
149f19fbd5eSMartin Schwidefsky
150f19fbd5eSMartin Schwidefsky extern s32 __nospec_call_start[], __nospec_call_end[];
151f19fbd5eSMartin Schwidefsky extern s32 __nospec_return_start[], __nospec_return_end[];
nospec_init_branches(void)152f19fbd5eSMartin Schwidefsky void __init nospec_init_branches(void)
153f19fbd5eSMartin Schwidefsky {
1546e179d64SMartin Schwidefsky nospec_revert(__nospec_call_start, __nospec_call_end);
1556e179d64SMartin Schwidefsky nospec_revert(__nospec_return_start, __nospec_return_end);
156f19fbd5eSMartin Schwidefsky }
157b2e2f43aSMartin Schwidefsky
158b2e2f43aSMartin Schwidefsky #endif /* CONFIG_EXPOLINE */
159