1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Support for Kernel relocation at boot time 4 * 5 * Copyright (C) 2023 Loongson Technology Corporation Limited 6 */ 7 8 #include <linux/elf.h> 9 #include <linux/kernel.h> 10 #include <linux/printk.h> 11 #include <linux/panic_notifier.h> 12 #include <linux/start_kernel.h> 13 #include <asm/bootinfo.h> 14 #include <asm/early_ioremap.h> 15 #include <asm/inst.h> 16 #include <asm/io.h> 17 #include <asm/sections.h> 18 #include <asm/setup.h> 19 20 #define RELOCATED(x) ((void *)((long)x + reloc_offset)) 21 #define RELOCATED_KASLR(x) ((void *)((long)x + random_offset)) 22 23 static unsigned long reloc_offset; 24 25 static inline void __init relocate_relative(void) 26 { 27 Elf64_Rela *rela, *rela_end; 28 rela = (Elf64_Rela *)&__rela_dyn_begin; 29 rela_end = (Elf64_Rela *)&__rela_dyn_end; 30 31 for ( ; rela < rela_end; rela++) { 32 Elf64_Addr addr = rela->r_offset; 33 Elf64_Addr relocated_addr = rela->r_addend; 34 35 if (rela->r_info != R_LARCH_RELATIVE) 36 continue; 37 38 relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr); 39 *(Elf64_Addr *)RELOCATED(addr) = relocated_addr; 40 } 41 42 #ifdef CONFIG_RELR 43 u64 *addr = NULL; 44 u64 *relr = (u64 *)&__relr_dyn_begin; 45 u64 *relr_end = (u64 *)&__relr_dyn_end; 46 47 for ( ; relr < relr_end; relr++) { 48 if ((*relr & 1) == 0) { 49 addr = (u64 *)(*relr + reloc_offset); 50 *addr++ += reloc_offset; 51 } else { 52 for (u64 *p = addr, r = *relr >> 1; r; p++, r >>= 1) 53 if (r & 1) 54 *p += reloc_offset; 55 addr += 63; 56 } 57 } 58 #endif 59 } 60 61 static inline void __init relocate_absolute(long random_offset) 62 { 63 void *begin, *end; 64 struct rela_la_abs *p; 65 66 begin = RELOCATED_KASLR(&__la_abs_begin); 67 end = RELOCATED_KASLR(&__la_abs_end); 68 69 for (p = begin; (void *)p < end; p++) { 70 long v = p->symvalue; 71 uint32_t lu12iw, ori; 72 #ifdef CONFIG_64BIT 73 uint32_t lu32id, lu52id; 74 #endif 75 union loongarch_instruction *insn = (void *)p->pc; 76 77 lu12iw = (v >> 12) & 0xfffff; 78 ori = v & 0xfff; 79 #ifdef CONFIG_64BIT 80 lu32id = (v >> 32) & 0xfffff; 81 lu52id = v >> 52; 82 #endif 83 84 insn[0].reg1i20_format.immediate = lu12iw; 85 insn[1].reg2i12_format.immediate = ori; 86 #ifdef CONFIG_64BIT 87 insn[2].reg1i20_format.immediate = lu32id; 88 insn[3].reg2i12_format.immediate = lu52id; 89 #endif 90 } 91 } 92 93 #ifdef CONFIG_RANDOMIZE_BASE 94 static inline __init unsigned long rotate_xor(unsigned long hash, 95 const void *area, size_t size) 96 { 97 size_t i, diff; 98 const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash)); 99 100 diff = (void *)ptr - area; 101 if (size < diff + sizeof(hash)) 102 return hash; 103 104 size = ALIGN_DOWN(size - diff, sizeof(hash)); 105 106 for (i = 0; i < size / sizeof(hash); i++) { 107 /* Rotate by odd number of bits and XOR. */ 108 hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); 109 hash ^= ptr[i]; 110 } 111 112 return hash; 113 } 114 115 static inline __init unsigned long get_random_boot(void) 116 { 117 unsigned long hash = 0; 118 unsigned long entropy = random_get_entropy(); 119 120 /* Attempt to create a simple but unpredictable starting entropy. */ 121 hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); 122 123 /* Add in any runtime entropy we can get */ 124 hash = rotate_xor(hash, &entropy, sizeof(entropy)); 125 126 return hash; 127 } 128 129 static int __init nokaslr(char *p) 130 { 131 return 0; /* Just silence the boot warning */ 132 } 133 early_param("nokaslr", nokaslr); 134 135 #define KASLR_DISABLED_MESSAGE "KASLR is disabled by %s in %s cmdline.\n" 136 137 /* 138 * Note: strictly-defined KASLR means the kernel's final runtime address 139 * has a random offset from the kernel's load address, which is implemented 140 * in relocate.c; broadly-defined KALSR means the kernel's final runtime 141 * address has a random offset from the kernel's link address (a.k.a. 142 * VMLINUX_LOAD_ADDRESS), which also include the efistlub implementation, 143 * kexec_file implementation and QEMU direct kernel boot. kaslr_disabled() 144 * return true only means strictly-defined KASLR is disabled. 145 */ 146 static inline __init bool kaslr_disabled(void) 147 { 148 char *str; 149 const char *builtin_cmdline = CONFIG_CMDLINE; 150 151 if (kaslr_offset()) 152 return true; /* KASLR is performed during early boot. */ 153 154 str = strstr(builtin_cmdline, "nokaslr"); 155 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) { 156 pr_info(KASLR_DISABLED_MESSAGE, "\'nokaslr\'", "built-in"); 157 return true; 158 } 159 160 str = strstr(boot_command_line, "nokaslr"); 161 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) { 162 pr_info(KASLR_DISABLED_MESSAGE, "\'nokaslr\'", "bootloader"); 163 return true; 164 } 165 166 #ifdef CONFIG_HIBERNATION 167 str = strstr(builtin_cmdline, "nohibernate"); 168 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) 169 return false; 170 171 str = strstr(boot_command_line, "nohibernate"); 172 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) 173 return false; 174 175 str = strstr(builtin_cmdline, "noresume"); 176 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) 177 return false; 178 179 str = strstr(boot_command_line, "noresume"); 180 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) 181 return false; 182 183 str = strstr(builtin_cmdline, "resume="); 184 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) { 185 pr_info(KASLR_DISABLED_MESSAGE, "\'resume=\'", "built-in"); 186 return true; 187 } 188 189 str = strstr(boot_command_line, "resume="); 190 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) { 191 pr_info(KASLR_DISABLED_MESSAGE, "\'resume=\'", "bootloader"); 192 return true; 193 } 194 #endif 195 196 str = strstr(boot_command_line, "kexec_file"); 197 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) { 198 pr_info(KASLR_DISABLED_MESSAGE, "\'kexec_file\'", "bootloader"); 199 return true; 200 } 201 202 return false; 203 } 204 205 /* Choose a new address for the kernel */ 206 static inline void __init *determine_relocation_address(void) 207 { 208 unsigned long kernel_length; 209 unsigned long random_offset; 210 void *destination = _text; 211 212 if (kaslr_disabled()) 213 return destination; 214 215 kernel_length = (unsigned long)_end - (unsigned long)_text; 216 217 random_offset = get_random_boot() << 16; 218 random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); 219 if (random_offset < kernel_length) 220 random_offset += ALIGN(kernel_length, 0xffff); 221 222 return RELOCATED_KASLR(destination); 223 } 224 225 static unsigned long __init determine_initrd_address(unsigned long *size) 226 { 227 unsigned long start = 0; 228 unsigned long key_length; 229 char *p, *endp, *key = "initrd="; 230 231 key_length = strlen(key); 232 p = strstr(boot_command_line, key); 233 234 if (!p) { 235 key = "initrdmem="; 236 key_length = strlen(key); 237 p = strstr(boot_command_line, key); 238 } 239 240 if (p == boot_command_line || (p > boot_command_line && *(p - 1) == ' ')) { 241 p += key_length; 242 start = memparse(p, &endp); 243 if (*endp == ',') 244 *size = memparse(endp + 1, NULL); 245 } 246 247 return start; 248 } 249 250 static inline int __init relocation_addr_valid(void *location_new) 251 { 252 unsigned long kernel_start, kernel_size; 253 unsigned long initrd_start, initrd_size = 0; 254 255 if ((unsigned long)location_new & 0x00000ffff) 256 return 0; /* Inappropriately aligned new location */ 257 258 if ((unsigned long)location_new < (unsigned long)_end) 259 return 0; /* New location overlaps original kernel */ 260 261 initrd_start = determine_initrd_address(&initrd_size); 262 if (initrd_start && initrd_size) { 263 kernel_start = PHYSADDR(location_new); 264 kernel_size = (unsigned long)_end - (unsigned long)_text; 265 266 if (kernel_start < (initrd_start + initrd_size) && 267 initrd_start < (kernel_start + kernel_size)) 268 return 0; /* initrd/initramfs overlaps kernel */ 269 } 270 271 return 1; 272 } 273 #endif 274 275 static inline void __init update_reloc_offset(unsigned long *addr, long random_offset) 276 { 277 unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr); 278 279 *new_addr = (unsigned long)reloc_offset; 280 } 281 282 unsigned long __init relocate_kernel(void) 283 { 284 unsigned long kernel_length; 285 unsigned long random_offset = 0; 286 void *location_new = _text; /* Default to original kernel start */ 287 char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */ 288 289 strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE); 290 291 #ifdef CONFIG_RANDOMIZE_BASE 292 location_new = determine_relocation_address(); 293 294 /* Sanity check relocation address */ 295 if (relocation_addr_valid(location_new)) 296 random_offset = (unsigned long)location_new - (unsigned long)(_text); 297 #endif 298 reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS; 299 early_memunmap(cmdline, COMMAND_LINE_SIZE); 300 301 if (random_offset) { 302 kernel_length = (unsigned long)(_end) - (unsigned long)(_text); 303 304 /* Copy the kernel to it's new location */ 305 memcpy(location_new, _text, kernel_length); 306 307 /* Sync the caches ready for execution of new kernel */ 308 __asm__ __volatile__ ( 309 "ibar 0 \t\n" 310 "dbar 0 \t\n" 311 ::: "memory"); 312 313 reloc_offset += random_offset; 314 315 /* The current thread is now within the relocated kernel */ 316 __current_thread_info = RELOCATED_KASLR(__current_thread_info); 317 318 update_reloc_offset(&reloc_offset, random_offset); 319 } 320 321 if (reloc_offset) 322 relocate_relative(); 323 324 relocate_absolute(random_offset); 325 326 return random_offset; 327 } 328 329 /* 330 * Show relocation information on panic. 331 */ 332 static void show_kernel_relocation(const char *level) 333 { 334 if (reloc_offset > 0) { 335 printk(level); 336 pr_cont("Kernel relocated by 0x%lx\n", reloc_offset); 337 pr_cont(" .text @ 0x%px\n", _text); 338 pr_cont(" .data @ 0x%px\n", _sdata); 339 pr_cont(" .bss @ 0x%px\n", __bss_start); 340 } 341 } 342 343 static int kernel_location_notifier_fn(struct notifier_block *self, 344 unsigned long v, void *p) 345 { 346 show_kernel_relocation(KERN_EMERG); 347 return NOTIFY_DONE; 348 } 349 350 static struct notifier_block kernel_location_notifier = { 351 .notifier_call = kernel_location_notifier_fn 352 }; 353 354 static int __init register_kernel_offset_dumper(void) 355 { 356 atomic_notifier_chain_register(&panic_notifier_list, 357 &kernel_location_notifier); 358 return 0; 359 } 360 361 arch_initcall(register_kernel_offset_dumper); 362