1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * powerpc code to implement the kexec_file_load syscall 4 * 5 * Copyright (C) 2004 Adam Litke (agl@us.ibm.com) 6 * Copyright (C) 2004 IBM Corp. 7 * Copyright (C) 2004,2005 Milton D Miller II, IBM Corporation 8 * Copyright (C) 2005 R Sharada (sharada@in.ibm.com) 9 * Copyright (C) 2006 Mohan Kumar M (mohan@in.ibm.com) 10 * Copyright (C) 2020 IBM Corporation 11 * 12 * Based on kexec-tools' kexec-ppc64.c, fs2dt.c. 13 * Heavily modified for the kernel by 14 * Hari Bathini, IBM Corporation. 15 */ 16 17 #define pr_fmt(fmt) "kexec ranges: " fmt 18 19 #include <linux/sort.h> 20 #include <linux/kexec.h> 21 #include <linux/of.h> 22 #include <linux/slab.h> 23 #include <linux/memblock.h> 24 #include <linux/minmax.h> 25 #include <linux/crash_core.h> 26 #include <asm/sections.h> 27 #include <asm/kexec_ranges.h> 28 #include <asm/crashdump-ppc64.h> 29 30 #if defined(CONFIG_KEXEC_FILE) || defined(CONFIG_CRASH_DUMP) 31 /** 32 * get_max_nr_ranges - Get the max no. of ranges crash_mem structure 33 * could hold, given the size allocated for it. 34 * @size: Allocation size of crash_mem structure. 35 * 36 * Returns the maximum no. of ranges. 37 */ 38 static inline unsigned int get_max_nr_ranges(size_t size) 39 { 40 return ((size - sizeof(struct crash_mem)) / 41 sizeof(struct range)); 42 } 43 44 /** 45 * get_mem_rngs_size - Get the allocated size of mem_rngs based on 46 * max_nr_ranges and chunk size. 47 * @mem_rngs: Memory ranges. 48 * 49 * Returns the maximum size of @mem_rngs. 50 */ 51 static inline size_t get_mem_rngs_size(struct crash_mem *mem_rngs) 52 { 53 size_t size; 54 55 if (!mem_rngs) 56 return 0; 57 58 size = (sizeof(struct crash_mem) + 59 (mem_rngs->max_nr_ranges * sizeof(struct range))); 60 61 /* 62 * Memory is allocated in size multiple of MEM_RANGE_CHUNK_SZ. 63 * So, align to get the actual length. 64 */ 65 return ALIGN(size, MEM_RANGE_CHUNK_SZ); 66 } 67 68 /** 69 * __add_mem_range - add a memory range to memory ranges list. 70 * @mem_ranges: Range list to add the memory range to. 71 * @base: Base address of the range to add. 72 * @size: Size of the memory range to add. 73 * 74 * (Re)allocates memory, if needed. 75 * 76 * Returns 0 on success, negative errno on error. 77 */ 78 static int __add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size) 79 { 80 struct crash_mem *mem_rngs = *mem_ranges; 81 82 if (!mem_rngs || (mem_rngs->nr_ranges == mem_rngs->max_nr_ranges)) { 83 mem_rngs = realloc_mem_ranges(mem_ranges); 84 if (!mem_rngs) 85 return -ENOMEM; 86 } 87 88 mem_rngs->ranges[mem_rngs->nr_ranges].start = base; 89 mem_rngs->ranges[mem_rngs->nr_ranges].end = base + size - 1; 90 pr_debug("Added memory range [%#016llx - %#016llx] at index %d\n", 91 base, base + size - 1, mem_rngs->nr_ranges); 92 mem_rngs->nr_ranges++; 93 return 0; 94 } 95 96 /** 97 * __merge_memory_ranges - Merges the given memory ranges list. 98 * @mem_rngs: Range list to merge. 99 * 100 * Assumes a sorted range list. 101 * 102 * Returns nothing. 103 */ 104 static void __merge_memory_ranges(struct crash_mem *mem_rngs) 105 { 106 struct range *ranges; 107 int i, idx; 108 109 if (!mem_rngs || mem_rngs->nr_ranges <= 1) 110 return; 111 112 idx = 0; 113 ranges = mem_rngs->ranges; 114 for (i = 1; i < mem_rngs->nr_ranges; i++) { 115 if (ranges[i].start <= (ranges[idx].end + 1)) 116 ranges[idx].end = max(ranges[idx].end, ranges[i].end); 117 else { 118 idx++; 119 ranges[idx] = ranges[i]; 120 } 121 } 122 mem_rngs->nr_ranges = idx + 1; 123 } 124 125 /* cmp_func_t callback to sort ranges with sort() */ 126 static int rngcmp(const void *_x, const void *_y) 127 { 128 const struct range *x = _x, *y = _y; 129 130 if (x->start > y->start) 131 return 1; 132 if (x->start < y->start) 133 return -1; 134 return 0; 135 } 136 137 /** 138 * sort_memory_ranges - Sorts the given memory ranges list. 139 * @mem_rngs: Range list to sort. 140 * @merge: If true, merge the list after sorting. 141 * 142 * Returns nothing. 143 */ 144 void sort_memory_ranges(struct crash_mem *mem_rngs, bool merge) 145 { 146 int i; 147 148 if (!mem_rngs) 149 return; 150 151 /* Sort the ranges in-place */ 152 sort(&(mem_rngs->ranges[0]), mem_rngs->nr_ranges, 153 sizeof(mem_rngs->ranges[0]), rngcmp, NULL); 154 155 if (merge) 156 __merge_memory_ranges(mem_rngs); 157 158 /* For debugging purpose */ 159 pr_debug("Memory ranges:\n"); 160 for (i = 0; i < mem_rngs->nr_ranges; i++) { 161 pr_debug("\t[%03d][%#016llx - %#016llx]\n", i, 162 mem_rngs->ranges[i].start, 163 mem_rngs->ranges[i].end); 164 } 165 } 166 167 /** 168 * realloc_mem_ranges - reallocate mem_ranges with size incremented 169 * by MEM_RANGE_CHUNK_SZ. Frees up the old memory, 170 * if memory allocation fails. 171 * @mem_ranges: Memory ranges to reallocate. 172 * 173 * Returns pointer to reallocated memory on success, NULL otherwise. 174 */ 175 struct crash_mem *realloc_mem_ranges(struct crash_mem **mem_ranges) 176 { 177 struct crash_mem *mem_rngs = *mem_ranges; 178 unsigned int nr_ranges; 179 size_t size; 180 181 size = get_mem_rngs_size(mem_rngs); 182 nr_ranges = mem_rngs ? mem_rngs->nr_ranges : 0; 183 184 size += MEM_RANGE_CHUNK_SZ; 185 mem_rngs = krealloc(*mem_ranges, size, GFP_KERNEL); 186 if (!mem_rngs) { 187 kfree(*mem_ranges); 188 *mem_ranges = NULL; 189 return NULL; 190 } 191 192 mem_rngs->nr_ranges = nr_ranges; 193 mem_rngs->max_nr_ranges = get_max_nr_ranges(size); 194 *mem_ranges = mem_rngs; 195 196 return mem_rngs; 197 } 198 199 /** 200 * add_mem_range - Updates existing memory range, if there is an overlap. 201 * Else, adds a new memory range. 202 * @mem_ranges: Range list to add the memory range to. 203 * @base: Base address of the range to add. 204 * @size: Size of the memory range to add. 205 * 206 * (Re)allocates memory, if needed. 207 * 208 * Returns 0 on success, negative errno on error. 209 */ 210 int add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size) 211 { 212 struct crash_mem *mem_rngs = *mem_ranges; 213 u64 mstart, mend, end; 214 unsigned int i; 215 216 if (!size) 217 return 0; 218 219 end = base + size - 1; 220 221 if (!mem_rngs || !(mem_rngs->nr_ranges)) 222 return __add_mem_range(mem_ranges, base, size); 223 224 for (i = 0; i < mem_rngs->nr_ranges; i++) { 225 mstart = mem_rngs->ranges[i].start; 226 mend = mem_rngs->ranges[i].end; 227 if (base < mend && end > mstart) { 228 if (base < mstart) 229 mem_rngs->ranges[i].start = base; 230 if (end > mend) 231 mem_rngs->ranges[i].end = end; 232 return 0; 233 } 234 } 235 236 return __add_mem_range(mem_ranges, base, size); 237 } 238 239 #endif /* CONFIG_KEXEC_FILE || CONFIG_CRASH_DUMP */ 240 241 #ifdef CONFIG_KEXEC_FILE 242 /** 243 * add_tce_mem_ranges - Adds tce-table range to the given memory ranges list. 244 * @mem_ranges: Range list to add the memory range(s) to. 245 * 246 * Returns 0 on success, negative errno on error. 247 */ 248 static int add_tce_mem_ranges(struct crash_mem **mem_ranges) 249 { 250 struct device_node *dn = NULL; 251 int ret = 0; 252 253 for_each_node_by_type(dn, "pci") { 254 u64 base; 255 u32 size; 256 257 ret = of_property_read_u64(dn, "linux,tce-base", &base); 258 ret |= of_property_read_u32(dn, "linux,tce-size", &size); 259 if (ret) { 260 /* 261 * It is ok to have pci nodes without tce. So, ignore 262 * property does not exist error. 263 */ 264 if (ret == -EINVAL) { 265 ret = 0; 266 continue; 267 } 268 break; 269 } 270 271 ret = add_mem_range(mem_ranges, base, size); 272 if (ret) 273 break; 274 } 275 276 of_node_put(dn); 277 return ret; 278 } 279 280 /** 281 * add_initrd_mem_range - Adds initrd range to the given memory ranges list, 282 * if the initrd was retained. 283 * @mem_ranges: Range list to add the memory range to. 284 * 285 * Returns 0 on success, negative errno on error. 286 */ 287 static int add_initrd_mem_range(struct crash_mem **mem_ranges) 288 { 289 u64 base, end; 290 int ret; 291 292 /* This range means something, only if initrd was retained */ 293 if (!strstr(saved_command_line, "retain_initrd")) 294 return 0; 295 296 ret = of_property_read_u64(of_chosen, "linux,initrd-start", &base); 297 ret |= of_property_read_u64(of_chosen, "linux,initrd-end", &end); 298 if (!ret) 299 ret = add_mem_range(mem_ranges, base, end - base + 1); 300 301 return ret; 302 } 303 304 /** 305 * add_htab_mem_range - Adds htab range to the given memory ranges list, 306 * if it exists 307 * @mem_ranges: Range list to add the memory range to. 308 * 309 * Returns 0 on success, negative errno on error. 310 */ 311 static int add_htab_mem_range(struct crash_mem **mem_ranges) 312 { 313 314 #ifdef CONFIG_PPC_64S_HASH_MMU 315 if (!htab_address) 316 return 0; 317 318 return add_mem_range(mem_ranges, __pa(htab_address), htab_size_bytes); 319 #else 320 return 0; 321 #endif 322 } 323 324 /** 325 * add_kernel_mem_range - Adds kernel text region to the given 326 * memory ranges list. 327 * @mem_ranges: Range list to add the memory range to. 328 * 329 * Returns 0 on success, negative errno on error. 330 */ 331 static int add_kernel_mem_range(struct crash_mem **mem_ranges) 332 { 333 return add_mem_range(mem_ranges, 0, __pa(_end)); 334 } 335 #endif /* CONFIG_KEXEC_FILE */ 336 337 #if defined(CONFIG_KEXEC_FILE) || defined(CONFIG_CRASH_DUMP) 338 /** 339 * add_rtas_mem_range - Adds RTAS region to the given memory ranges list. 340 * @mem_ranges: Range list to add the memory range to. 341 * 342 * Returns 0 on success, negative errno on error. 343 */ 344 static int add_rtas_mem_range(struct crash_mem **mem_ranges) 345 { 346 struct device_node *dn; 347 u32 base, size; 348 int ret = 0; 349 350 dn = of_find_node_by_path("/rtas"); 351 if (!dn) 352 return 0; 353 354 ret = of_property_read_u32(dn, "linux,rtas-base", &base); 355 ret |= of_property_read_u32(dn, "rtas-size", &size); 356 if (!ret) 357 ret = add_mem_range(mem_ranges, base, size); 358 359 of_node_put(dn); 360 return ret; 361 } 362 363 /** 364 * add_opal_mem_range - Adds OPAL region to the given memory ranges list. 365 * @mem_ranges: Range list to add the memory range to. 366 * 367 * Returns 0 on success, negative errno on error. 368 */ 369 static int add_opal_mem_range(struct crash_mem **mem_ranges) 370 { 371 struct device_node *dn; 372 u64 base, size; 373 int ret; 374 375 dn = of_find_node_by_path("/ibm,opal"); 376 if (!dn) 377 return 0; 378 379 ret = of_property_read_u64(dn, "opal-base-address", &base); 380 ret |= of_property_read_u64(dn, "opal-runtime-size", &size); 381 if (!ret) 382 ret = add_mem_range(mem_ranges, base, size); 383 384 of_node_put(dn); 385 return ret; 386 } 387 #endif /* CONFIG_KEXEC_FILE || CONFIG_CRASH_DUMP */ 388 389 #ifdef CONFIG_KEXEC_FILE 390 /** 391 * add_reserved_mem_ranges - Adds "/reserved-ranges" regions exported by f/w 392 * to the given memory ranges list. 393 * @mem_ranges: Range list to add the memory ranges to. 394 * 395 * Returns 0 on success, negative errno on error. 396 */ 397 static int add_reserved_mem_ranges(struct crash_mem **mem_ranges) 398 { 399 int n_mem_addr_cells, n_mem_size_cells, i, len, cells, ret = 0; 400 struct device_node *root = of_find_node_by_path("/"); 401 const __be32 *prop; 402 403 prop = of_get_property(root, "reserved-ranges", &len); 404 n_mem_addr_cells = of_n_addr_cells(root); 405 n_mem_size_cells = of_n_size_cells(root); 406 of_node_put(root); 407 if (!prop) 408 return 0; 409 410 cells = n_mem_addr_cells + n_mem_size_cells; 411 412 /* Each reserved range is an (address,size) pair */ 413 for (i = 0; i < (len / (sizeof(u32) * cells)); i++) { 414 u64 base, size; 415 416 base = of_read_number(prop + (i * cells), n_mem_addr_cells); 417 size = of_read_number(prop + (i * cells) + n_mem_addr_cells, 418 n_mem_size_cells); 419 420 ret = add_mem_range(mem_ranges, base, size); 421 if (ret) 422 break; 423 } 424 425 return ret; 426 } 427 428 /** 429 * get_reserved_memory_ranges - Get reserve memory ranges. This list includes 430 * memory regions that should be added to the 431 * memory reserve map to ensure the region is 432 * protected from any mischief. 433 * @mem_ranges: Range list to add the memory ranges to. 434 * 435 * Returns 0 on success, negative errno on error. 436 */ 437 int get_reserved_memory_ranges(struct crash_mem **mem_ranges) 438 { 439 int ret; 440 441 ret = add_rtas_mem_range(mem_ranges); 442 if (ret) 443 goto out; 444 445 ret = add_tce_mem_ranges(mem_ranges); 446 if (ret) 447 goto out; 448 449 ret = add_reserved_mem_ranges(mem_ranges); 450 out: 451 if (ret) 452 pr_err("Failed to setup reserved memory ranges\n"); 453 return ret; 454 } 455 456 /** 457 * get_exclude_memory_ranges - Get exclude memory ranges. This list includes 458 * regions like opal/rtas, tce-table, initrd, 459 * kernel, htab which should be avoided while 460 * setting up kexec load segments. 461 * @mem_ranges: Range list to add the memory ranges to. 462 * 463 * Returns 0 on success, negative errno on error. 464 */ 465 int get_exclude_memory_ranges(struct crash_mem **mem_ranges) 466 { 467 int ret; 468 469 ret = add_tce_mem_ranges(mem_ranges); 470 if (ret) 471 goto out; 472 473 ret = add_initrd_mem_range(mem_ranges); 474 if (ret) 475 goto out; 476 477 ret = add_htab_mem_range(mem_ranges); 478 if (ret) 479 goto out; 480 481 ret = add_kernel_mem_range(mem_ranges); 482 if (ret) 483 goto out; 484 485 ret = add_rtas_mem_range(mem_ranges); 486 if (ret) 487 goto out; 488 489 ret = add_opal_mem_range(mem_ranges); 490 if (ret) 491 goto out; 492 493 ret = add_reserved_mem_ranges(mem_ranges); 494 if (ret) 495 goto out; 496 497 /* exclude memory ranges should be sorted for easy lookup */ 498 sort_memory_ranges(*mem_ranges, true); 499 out: 500 if (ret) 501 pr_err("Failed to setup exclude memory ranges\n"); 502 return ret; 503 } 504 505 #ifdef CONFIG_CRASH_DUMP 506 /** 507 * get_usable_memory_ranges - Get usable memory ranges. This list includes 508 * regions like crashkernel, opal/rtas & tce-table, 509 * that kdump kernel could use. 510 * @mem_ranges: Range list to add the memory ranges to. 511 * 512 * Returns 0 on success, negative errno on error. 513 */ 514 int get_usable_memory_ranges(struct crash_mem **mem_ranges) 515 { 516 int ret, i; 517 518 /* 519 * Early boot failure observed on guests when low memory (first memory 520 * block?) is not added to usable memory. So, add [0, crashk_res.end] 521 * instead of [crashk_res.start, crashk_res.end] to workaround it. 522 * Also, crashed kernel's memory must be added to reserve map to 523 * avoid kdump kernel from using it. 524 */ 525 ret = add_mem_range(mem_ranges, 0, crashk_res.end + 1); 526 if (ret) 527 goto out; 528 529 for (i = 0; i < crashk_cma_cnt; i++) { 530 ret = add_mem_range(mem_ranges, crashk_cma_ranges[i].start, 531 crashk_cma_ranges[i].end - crashk_cma_ranges[i].start + 1); 532 if (ret) 533 goto out; 534 } 535 536 ret = add_rtas_mem_range(mem_ranges); 537 if (ret) 538 goto out; 539 540 ret = add_opal_mem_range(mem_ranges); 541 if (ret) 542 goto out; 543 544 ret = add_tce_mem_ranges(mem_ranges); 545 out: 546 if (ret) 547 pr_err("Failed to setup usable memory ranges\n"); 548 return ret; 549 } 550 #endif /* CONFIG_CRASH_DUMP */ 551 #endif /* CONFIG_KEXEC_FILE */ 552 553 #ifdef CONFIG_CRASH_DUMP 554 int arch_crash_exclude_mem_range(struct crash_mem **mem_ranges, 555 unsigned long long mstart, 556 unsigned long long mend) 557 { 558 struct crash_mem *tmem = *mem_ranges; 559 560 /* Reallocate memory ranges if there is no space to split ranges */ 561 if (tmem && (tmem->nr_ranges == tmem->max_nr_ranges)) { 562 tmem = realloc_mem_ranges(mem_ranges); 563 if (!tmem) 564 return -ENOMEM; 565 } 566 567 return crash_exclude_mem_range(tmem, mstart, mend); 568 } 569 570 /** 571 * get_crash_memory_ranges - Get crash memory ranges. This list includes 572 * first/crashing kernel's memory regions that 573 * would be exported via an elfcore. 574 * @mem_ranges: Range list to add the memory ranges to. 575 * 576 * Returns 0 on success, negative errno on error. 577 */ 578 int get_crash_memory_ranges(struct crash_mem **mem_ranges) 579 { 580 phys_addr_t base, end; 581 u64 i; 582 int ret; 583 584 for_each_mem_range(i, &base, &end) { 585 u64 size = end - base; 586 587 /* Skip backup memory region, which needs a separate entry */ 588 if (base == BACKUP_SRC_START) { 589 if (size > BACKUP_SRC_SIZE) { 590 base = BACKUP_SRC_END + 1; 591 size -= BACKUP_SRC_SIZE; 592 } else 593 continue; 594 } 595 596 ret = add_mem_range(mem_ranges, base, size); 597 if (ret) 598 goto out; 599 600 /* Try merging adjacent ranges before reallocation attempt */ 601 if ((*mem_ranges)->nr_ranges == (*mem_ranges)->max_nr_ranges) 602 sort_memory_ranges(*mem_ranges, true); 603 } 604 605 ret = crash_exclude_core_ranges(mem_ranges); 606 if (ret) 607 goto out; 608 609 /* 610 * FIXME: For now, stay in parity with kexec-tools but if RTAS/OPAL 611 * regions are exported to save their context at the time of 612 * crash, they should actually be backed up just like the 613 * first 64K bytes of memory. 614 */ 615 ret = add_rtas_mem_range(mem_ranges); 616 if (ret) 617 goto out; 618 619 ret = add_opal_mem_range(mem_ranges); 620 if (ret) 621 goto out; 622 623 /* create a separate program header for the backup region */ 624 ret = add_mem_range(mem_ranges, BACKUP_SRC_START, BACKUP_SRC_SIZE); 625 if (ret) 626 goto out; 627 628 sort_memory_ranges(*mem_ranges, false); 629 out: 630 if (ret) 631 pr_err("Failed to setup crash memory ranges\n"); 632 return ret; 633 } 634 #endif /* CONFIG_CRASH_DUMP */ 635