1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kexec_file for riscv, use vmlinux as the dump-capture kernel image. 4 * 5 * Copyright (C) 2021 Huawei Technologies Co, Ltd. 6 * 7 * Author: Liao Chang (liaochang1@huawei.com) 8 */ 9 #include <linux/kexec.h> 10 #include <linux/elf.h> 11 #include <linux/slab.h> 12 #include <linux/of.h> 13 #include <linux/libfdt.h> 14 #include <linux/types.h> 15 #include <linux/memblock.h> 16 #include <linux/vmalloc.h> 17 #include <asm/setup.h> 18 19 const struct kexec_file_ops * const kexec_file_loaders[] = { 20 &elf_kexec_ops, 21 &image_kexec_ops, 22 NULL 23 }; 24 25 int arch_kimage_file_post_load_cleanup(struct kimage *image) 26 { 27 kvfree(image->arch.fdt); 28 image->arch.fdt = NULL; 29 30 vfree(image->elf_headers); 31 image->elf_headers = NULL; 32 image->elf_headers_sz = 0; 33 34 return kexec_image_post_load_cleanup_default(image); 35 } 36 37 #ifdef CONFIG_CRASH_DUMP 38 static int get_nr_ram_ranges_callback(struct resource *res, void *arg) 39 { 40 unsigned int *nr_ranges = arg; 41 42 (*nr_ranges)++; 43 return 0; 44 } 45 46 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) 47 { 48 struct crash_mem *cmem = arg; 49 50 cmem->ranges[cmem->nr_ranges].start = res->start; 51 cmem->ranges[cmem->nr_ranges].end = res->end; 52 cmem->nr_ranges++; 53 54 return 0; 55 } 56 57 static int prepare_elf_headers(void **addr, unsigned long *sz) 58 { 59 struct crash_mem *cmem; 60 unsigned int nr_ranges; 61 int ret; 62 63 nr_ranges = 1; /* For exclusion of crashkernel region */ 64 walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); 65 66 cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL); 67 if (!cmem) 68 return -ENOMEM; 69 70 cmem->max_nr_ranges = nr_ranges; 71 cmem->nr_ranges = 0; 72 ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); 73 if (ret) 74 goto out; 75 76 /* Exclude crashkernel region */ 77 ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); 78 if (!ret) 79 ret = crash_prepare_elf64_headers(cmem, true, addr, sz); 80 81 out: 82 kfree(cmem); 83 return ret; 84 } 85 86 static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, 87 unsigned long cmdline_len) 88 { 89 int elfcorehdr_strlen; 90 char *cmdline_ptr; 91 92 cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL); 93 if (!cmdline_ptr) 94 return NULL; 95 96 elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ", 97 image->elf_load_addr); 98 99 if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) { 100 pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n"); 101 kfree(cmdline_ptr); 102 return NULL; 103 } 104 105 memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len); 106 /* Ensure it's nul terminated */ 107 cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0'; 108 return cmdline_ptr; 109 } 110 #endif 111 112 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 113 #define RISCV_IMM_BITS 12 114 #define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS) 115 #define RISCV_CONST_HIGH_PART(x) \ 116 (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1)) 117 #define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x)) 118 119 #define ENCODE_ITYPE_IMM(x) \ 120 (RV_X(x, 0, 12) << 20) 121 #define ENCODE_BTYPE_IMM(x) \ 122 ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \ 123 (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31)) 124 #define ENCODE_UTYPE_IMM(x) \ 125 (RV_X(x, 12, 20) << 12) 126 #define ENCODE_JTYPE_IMM(x) \ 127 ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \ 128 (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31)) 129 #define ENCODE_CBTYPE_IMM(x) \ 130 ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \ 131 (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12)) 132 #define ENCODE_CJTYPE_IMM(x) \ 133 ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \ 134 (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \ 135 (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12)) 136 #define ENCODE_UJTYPE_IMM(x) \ 137 (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \ 138 (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32)) 139 #define ENCODE_UITYPE_IMM(x) \ 140 (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32)) 141 142 #define CLEAN_IMM(type, x) \ 143 ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x)) 144 145 int arch_kexec_apply_relocations_add(struct purgatory_info *pi, 146 Elf_Shdr *section, 147 const Elf_Shdr *relsec, 148 const Elf_Shdr *symtab) 149 { 150 const char *strtab, *name, *shstrtab; 151 const Elf_Shdr *sechdrs; 152 Elf64_Rela *relas; 153 int i, r_type; 154 155 /* String & section header string table */ 156 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff; 157 strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset; 158 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset; 159 160 relas = (void *)pi->ehdr + relsec->sh_offset; 161 162 for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) { 163 const Elf_Sym *sym; /* symbol to relocate */ 164 unsigned long addr; /* final location after relocation */ 165 unsigned long val; /* relocated symbol value */ 166 unsigned long sec_base; /* relocated symbol value */ 167 void *loc; /* tmp location to modify */ 168 169 sym = (void *)pi->ehdr + symtab->sh_offset; 170 sym += ELF64_R_SYM(relas[i].r_info); 171 172 if (sym->st_name) 173 name = strtab + sym->st_name; 174 else 175 name = shstrtab + sechdrs[sym->st_shndx].sh_name; 176 177 loc = pi->purgatory_buf; 178 loc += section->sh_offset; 179 loc += relas[i].r_offset; 180 181 if (sym->st_shndx == SHN_ABS) 182 sec_base = 0; 183 else if (sym->st_shndx >= pi->ehdr->e_shnum) { 184 pr_err("Invalid section %d for symbol %s\n", 185 sym->st_shndx, name); 186 return -ENOEXEC; 187 } else 188 sec_base = pi->sechdrs[sym->st_shndx].sh_addr; 189 190 val = sym->st_value; 191 val += sec_base; 192 val += relas[i].r_addend; 193 194 addr = section->sh_addr + relas[i].r_offset; 195 196 r_type = ELF64_R_TYPE(relas[i].r_info); 197 198 switch (r_type) { 199 case R_RISCV_BRANCH: 200 *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) | 201 ENCODE_BTYPE_IMM(val - addr); 202 break; 203 case R_RISCV_JAL: 204 *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) | 205 ENCODE_JTYPE_IMM(val - addr); 206 break; 207 /* 208 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I 209 * sym is expected to be next to R_RISCV_PCREL_HI20 210 * in purgatory relsec. Handle it like R_RISCV_CALL 211 * sym, instead of searching the whole relsec. 212 */ 213 case R_RISCV_PCREL_HI20: 214 case R_RISCV_CALL_PLT: 215 case R_RISCV_CALL: 216 *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) | 217 ENCODE_UJTYPE_IMM(val - addr); 218 break; 219 case R_RISCV_RVC_BRANCH: 220 *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) | 221 ENCODE_CBTYPE_IMM(val - addr); 222 break; 223 case R_RISCV_RVC_JUMP: 224 *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) | 225 ENCODE_CJTYPE_IMM(val - addr); 226 break; 227 case R_RISCV_ADD16: 228 *(u16 *)loc += val; 229 break; 230 case R_RISCV_SUB16: 231 *(u16 *)loc -= val; 232 break; 233 case R_RISCV_ADD32: 234 *(u32 *)loc += val; 235 break; 236 case R_RISCV_SUB32: 237 *(u32 *)loc -= val; 238 break; 239 /* It has been applied by R_RISCV_PCREL_HI20 sym */ 240 case R_RISCV_PCREL_LO12_I: 241 case R_RISCV_ALIGN: 242 case R_RISCV_RELAX: 243 break; 244 case R_RISCV_64: 245 *(u64 *)loc = val; 246 break; 247 default: 248 pr_err("Unknown rela relocation: %d\n", r_type); 249 return -ENOEXEC; 250 } 251 } 252 return 0; 253 } 254 255 256 int load_extra_segments(struct kimage *image, unsigned long kernel_start, 257 unsigned long kernel_len, char *initrd, 258 unsigned long initrd_len, char *cmdline, 259 unsigned long cmdline_len) 260 { 261 int ret; 262 void *fdt; 263 unsigned long initrd_pbase = 0UL; 264 struct kexec_buf kbuf; 265 char *modified_cmdline = NULL; 266 267 kbuf.image = image; 268 kbuf.buf_min = kernel_start + kernel_len; 269 kbuf.buf_max = ULONG_MAX; 270 271 #ifdef CONFIG_CRASH_DUMP 272 /* Add elfcorehdr */ 273 if (image->type == KEXEC_TYPE_CRASH) { 274 void *headers; 275 unsigned long headers_sz; 276 ret = prepare_elf_headers(&headers, &headers_sz); 277 if (ret) { 278 pr_err("Preparing elf core header failed\n"); 279 goto out; 280 } 281 282 kbuf.buffer = headers; 283 kbuf.bufsz = headers_sz; 284 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 285 kbuf.memsz = headers_sz; 286 kbuf.buf_align = ELF_CORE_HEADER_ALIGN; 287 kbuf.top_down = true; 288 289 ret = kexec_add_buffer(&kbuf); 290 if (ret) { 291 vfree(headers); 292 goto out; 293 } 294 image->elf_headers = headers; 295 image->elf_load_addr = kbuf.mem; 296 image->elf_headers_sz = headers_sz; 297 298 kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n", 299 image->elf_load_addr, kbuf.bufsz, kbuf.memsz); 300 301 /* Setup cmdline for kdump kernel case */ 302 modified_cmdline = setup_kdump_cmdline(image, cmdline, 303 cmdline_len); 304 if (!modified_cmdline) { 305 pr_err("Setting up cmdline for kdump kernel failed\n"); 306 ret = -EINVAL; 307 goto out; 308 } 309 cmdline = modified_cmdline; 310 } 311 #endif 312 313 #ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY 314 /* Add purgatory to the image */ 315 kbuf.top_down = true; 316 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 317 ret = kexec_load_purgatory(image, &kbuf); 318 if (ret) { 319 pr_err("Error loading purgatory ret=%d\n", ret); 320 goto out; 321 } 322 kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem); 323 324 ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry", 325 &kernel_start, 326 sizeof(kernel_start), 0); 327 if (ret) 328 pr_err("Error update purgatory ret=%d\n", ret); 329 #endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */ 330 331 /* Add the initrd to the image */ 332 if (initrd != NULL) { 333 kbuf.buffer = initrd; 334 kbuf.bufsz = kbuf.memsz = initrd_len; 335 kbuf.buf_align = PAGE_SIZE; 336 kbuf.top_down = true; 337 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 338 ret = kexec_add_buffer(&kbuf); 339 if (ret) 340 goto out; 341 initrd_pbase = kbuf.mem; 342 kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase); 343 } 344 345 /* Add the DTB to the image */ 346 fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase, 347 initrd_len, cmdline, 0); 348 if (!fdt) { 349 pr_err("Error setting up the new device tree.\n"); 350 ret = -EINVAL; 351 goto out; 352 } 353 354 fdt_pack(fdt); 355 kbuf.buffer = fdt; 356 kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt); 357 kbuf.buf_align = PAGE_SIZE; 358 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 359 kbuf.top_down = true; 360 ret = kexec_add_buffer(&kbuf); 361 if (ret) { 362 pr_err("Error add DTB kbuf ret=%d\n", ret); 363 goto out_free_fdt; 364 } 365 /* Cache the fdt buffer address for memory cleanup */ 366 image->arch.fdt = fdt; 367 kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem); 368 goto out; 369 370 out_free_fdt: 371 kvfree(fdt); 372 out: 373 kfree(modified_cmdline); 374 return ret; 375 } 376