1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Support for Kernel relocation at boot time
4 *
5 * Copyright (C) 2023 Loongson Technology Corporation Limited
6 */
7
8 #include <linux/elf.h>
9 #include <linux/kernel.h>
10 #include <linux/printk.h>
11 #include <linux/panic_notifier.h>
12 #include <linux/start_kernel.h>
13 #include <asm/bootinfo.h>
14 #include <asm/early_ioremap.h>
15 #include <asm/inst.h>
16 #include <asm/io.h>
17 #include <asm/sections.h>
18 #include <asm/setup.h>
19
20 #define RELOCATED(x) ((void *)((long)x + reloc_offset))
21 #define RELOCATED_KASLR(x) ((void *)((long)x + random_offset))
22
23 static unsigned long reloc_offset;
24
relocate_relative(void)25 static inline void __init relocate_relative(void)
26 {
27 Elf64_Rela *rela, *rela_end;
28 rela = (Elf64_Rela *)&__rela_dyn_begin;
29 rela_end = (Elf64_Rela *)&__rela_dyn_end;
30
31 for ( ; rela < rela_end; rela++) {
32 Elf64_Addr addr = rela->r_offset;
33 Elf64_Addr relocated_addr = rela->r_addend;
34
35 if (rela->r_info != R_LARCH_RELATIVE)
36 continue;
37
38 relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr);
39 *(Elf64_Addr *)RELOCATED(addr) = relocated_addr;
40 }
41
42 #ifdef CONFIG_RELR
43 u64 *addr = NULL;
44 u64 *relr = (u64 *)&__relr_dyn_begin;
45 u64 *relr_end = (u64 *)&__relr_dyn_end;
46
47 for ( ; relr < relr_end; relr++) {
48 if ((*relr & 1) == 0) {
49 addr = (u64 *)(*relr + reloc_offset);
50 *addr++ += reloc_offset;
51 } else {
52 for (u64 *p = addr, r = *relr >> 1; r; p++, r >>= 1)
53 if (r & 1)
54 *p += reloc_offset;
55 addr += 63;
56 }
57 }
58 #endif
59 }
60
relocate_absolute(long random_offset)61 static inline void __init relocate_absolute(long random_offset)
62 {
63 void *begin, *end;
64 struct rela_la_abs *p;
65
66 begin = RELOCATED_KASLR(&__la_abs_begin);
67 end = RELOCATED_KASLR(&__la_abs_end);
68
69 for (p = begin; (void *)p < end; p++) {
70 long v = p->symvalue;
71 uint32_t lu12iw, ori;
72 #ifdef CONFIG_64BIT
73 uint32_t lu32id, lu52id;
74 #endif
75 union loongarch_instruction *insn = (void *)p->pc;
76
77 lu12iw = (v >> 12) & 0xfffff;
78 ori = v & 0xfff;
79 #ifdef CONFIG_64BIT
80 lu32id = (v >> 32) & 0xfffff;
81 lu52id = v >> 52;
82 #endif
83
84 insn[0].reg1i20_format.immediate = lu12iw;
85 insn[1].reg2i12_format.immediate = ori;
86 #ifdef CONFIG_64BIT
87 insn[2].reg1i20_format.immediate = lu32id;
88 insn[3].reg2i12_format.immediate = lu52id;
89 #endif
90 }
91 }
92
93 #ifdef CONFIG_RANDOMIZE_BASE
rotate_xor(unsigned long hash,const void * area,size_t size)94 static inline __init unsigned long rotate_xor(unsigned long hash,
95 const void *area, size_t size)
96 {
97 size_t i, diff;
98 const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash));
99
100 diff = (void *)ptr - area;
101 if (size < diff + sizeof(hash))
102 return hash;
103
104 size = ALIGN_DOWN(size - diff, sizeof(hash));
105
106 for (i = 0; i < size / sizeof(hash); i++) {
107 /* Rotate by odd number of bits and XOR. */
108 hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7);
109 hash ^= ptr[i];
110 }
111
112 return hash;
113 }
114
get_random_boot(void)115 static inline __init unsigned long get_random_boot(void)
116 {
117 unsigned long hash = 0;
118 unsigned long entropy = random_get_entropy();
119
120 /* Attempt to create a simple but unpredictable starting entropy. */
121 hash = rotate_xor(hash, linux_banner, strlen(linux_banner));
122
123 /* Add in any runtime entropy we can get */
124 hash = rotate_xor(hash, &entropy, sizeof(entropy));
125
126 return hash;
127 }
128
nokaslr(char * p)129 static int __init nokaslr(char *p)
130 {
131 return 0; /* Just silence the boot warning */
132 }
133 early_param("nokaslr", nokaslr);
134
135 #define KASLR_DISABLED_MESSAGE "KASLR is disabled by %s in %s cmdline.\n"
136
137 /*
138 * Note: strictly-defined KASLR means the kernel's final runtime address
139 * has a random offset from the kernel's load address, which is implemented
140 * in relocate.c; broadly-defined KALSR means the kernel's final runtime
141 * address has a random offset from the kernel's link address (a.k.a.
142 * VMLINUX_LOAD_ADDRESS), which also include the efistlub implementation,
143 * kexec_file implementation and QEMU direct kernel boot. kaslr_disabled()
144 * return true only means strictly-defined KASLR is disabled.
145 */
kaslr_disabled(void)146 static inline __init bool kaslr_disabled(void)
147 {
148 char *str;
149 const char *builtin_cmdline = CONFIG_CMDLINE;
150
151 if (kaslr_offset())
152 return true; /* KASLR is performed during early boot. */
153
154 str = strstr(builtin_cmdline, "nokaslr");
155 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) {
156 pr_info(KASLR_DISABLED_MESSAGE, "\'nokaslr\'", "built-in");
157 return true;
158 }
159
160 str = strstr(boot_command_line, "nokaslr");
161 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) {
162 pr_info(KASLR_DISABLED_MESSAGE, "\'nokaslr\'", "bootloader");
163 return true;
164 }
165
166 #ifdef CONFIG_HIBERNATION
167 str = strstr(builtin_cmdline, "nohibernate");
168 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
169 return false;
170
171 str = strstr(boot_command_line, "nohibernate");
172 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
173 return false;
174
175 str = strstr(builtin_cmdline, "noresume");
176 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
177 return false;
178
179 str = strstr(boot_command_line, "noresume");
180 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
181 return false;
182
183 str = strstr(builtin_cmdline, "resume=");
184 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) {
185 pr_info(KASLR_DISABLED_MESSAGE, "\'resume=\'", "built-in");
186 return true;
187 }
188
189 str = strstr(boot_command_line, "resume=");
190 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) {
191 pr_info(KASLR_DISABLED_MESSAGE, "\'resume=\'", "bootloader");
192 return true;
193 }
194 #endif
195
196 str = strstr(boot_command_line, "kexec_file");
197 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) {
198 pr_info(KASLR_DISABLED_MESSAGE, "\'kexec_file\'", "bootloader");
199 return true;
200 }
201
202 return false;
203 }
204
205 /* Choose a new address for the kernel */
determine_relocation_address(void)206 static inline void __init *determine_relocation_address(void)
207 {
208 unsigned long kernel_length;
209 unsigned long random_offset;
210 void *destination = _text;
211
212 if (kaslr_disabled())
213 return destination;
214
215 kernel_length = (unsigned long)_end - (unsigned long)_text;
216
217 random_offset = get_random_boot() << 16;
218 random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1);
219 if (random_offset < kernel_length)
220 random_offset += ALIGN(kernel_length, 0xffff);
221
222 return RELOCATED_KASLR(destination);
223 }
224
determine_initrd_address(unsigned long * size)225 static unsigned long __init determine_initrd_address(unsigned long *size)
226 {
227 unsigned long start = 0;
228 unsigned long key_length;
229 char *p, *endp, *key = "initrd=";
230
231 key_length = strlen(key);
232 p = strstr(boot_command_line, key);
233
234 if (!p) {
235 key = "initrdmem=";
236 key_length = strlen(key);
237 p = strstr(boot_command_line, key);
238 }
239
240 if (p == boot_command_line || (p > boot_command_line && *(p - 1) == ' ')) {
241 p += key_length;
242 start = memparse(p, &endp);
243 if (*endp == ',')
244 *size = memparse(endp + 1, NULL);
245 }
246
247 return start;
248 }
249
relocation_addr_valid(void * location_new)250 static inline int __init relocation_addr_valid(void *location_new)
251 {
252 unsigned long kernel_start, kernel_size;
253 unsigned long initrd_start, initrd_size = 0;
254
255 if ((unsigned long)location_new & 0x00000ffff)
256 return 0; /* Inappropriately aligned new location */
257
258 if ((unsigned long)location_new < (unsigned long)_end)
259 return 0; /* New location overlaps original kernel */
260
261 initrd_start = determine_initrd_address(&initrd_size);
262 if (initrd_start && initrd_size) {
263 kernel_start = PHYSADDR(location_new);
264 kernel_size = (unsigned long)_end - (unsigned long)_text;
265
266 if (kernel_start < (initrd_start + initrd_size) &&
267 initrd_start < (kernel_start + kernel_size))
268 return 0; /* initrd/initramfs overlaps kernel */
269 }
270
271 return 1;
272 }
273 #endif
274
update_reloc_offset(unsigned long * addr,long random_offset)275 static inline void __init update_reloc_offset(unsigned long *addr, long random_offset)
276 {
277 unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr);
278
279 *new_addr = (unsigned long)reloc_offset;
280 }
281
relocate_kernel(void)282 unsigned long __init relocate_kernel(void)
283 {
284 unsigned long kernel_length;
285 unsigned long random_offset = 0;
286 void *location_new = _text; /* Default to original kernel start */
287 char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */
288
289 strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE);
290
291 #ifdef CONFIG_RANDOMIZE_BASE
292 location_new = determine_relocation_address();
293
294 /* Sanity check relocation address */
295 if (relocation_addr_valid(location_new))
296 random_offset = (unsigned long)location_new - (unsigned long)(_text);
297 #endif
298 reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS;
299 early_memunmap(cmdline, COMMAND_LINE_SIZE);
300
301 if (random_offset) {
302 kernel_length = (unsigned long)(_end) - (unsigned long)(_text);
303
304 /* Copy the kernel to it's new location */
305 memcpy(location_new, _text, kernel_length);
306
307 /* Sync the caches ready for execution of new kernel */
308 __asm__ __volatile__ (
309 "ibar 0 \t\n"
310 "dbar 0 \t\n"
311 ::: "memory");
312
313 reloc_offset += random_offset;
314
315 /* The current thread is now within the relocated kernel */
316 __current_thread_info = RELOCATED_KASLR(__current_thread_info);
317
318 update_reloc_offset(&reloc_offset, random_offset);
319 }
320
321 if (reloc_offset)
322 relocate_relative();
323
324 relocate_absolute(random_offset);
325
326 return random_offset;
327 }
328
329 /*
330 * Show relocation information on panic.
331 */
show_kernel_relocation(const char * level)332 static void show_kernel_relocation(const char *level)
333 {
334 if (reloc_offset > 0) {
335 printk(level);
336 pr_cont("Kernel relocated by 0x%lx\n", reloc_offset);
337 pr_cont(" .text @ 0x%px\n", _text);
338 pr_cont(" .data @ 0x%px\n", _sdata);
339 pr_cont(" .bss @ 0x%px\n", __bss_start);
340 }
341 }
342
kernel_location_notifier_fn(struct notifier_block * self,unsigned long v,void * p)343 static int kernel_location_notifier_fn(struct notifier_block *self,
344 unsigned long v, void *p)
345 {
346 show_kernel_relocation(KERN_EMERG);
347 return NOTIFY_DONE;
348 }
349
350 static struct notifier_block kernel_location_notifier = {
351 .notifier_call = kernel_location_notifier_fn
352 };
353
register_kernel_offset_dumper(void)354 static int __init register_kernel_offset_dumper(void)
355 {
356 atomic_notifier_chain_register(&panic_notifier_list,
357 &kernel_location_notifier);
358 return 0;
359 }
360
361 arch_initcall(register_kernel_offset_dumper);
362