1 /* 2 * 3 * Common boot and setup code. 4 * 5 * Copyright (C) 2001 PPC64 Team, IBM Corp 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 13 #define DEBUG 14 15 #include <linux/export.h> 16 #include <linux/string.h> 17 #include <linux/sched.h> 18 #include <linux/init.h> 19 #include <linux/kernel.h> 20 #include <linux/reboot.h> 21 #include <linux/delay.h> 22 #include <linux/initrd.h> 23 #include <linux/seq_file.h> 24 #include <linux/ioport.h> 25 #include <linux/console.h> 26 #include <linux/utsname.h> 27 #include <linux/tty.h> 28 #include <linux/root_dev.h> 29 #include <linux/notifier.h> 30 #include <linux/cpu.h> 31 #include <linux/unistd.h> 32 #include <linux/serial.h> 33 #include <linux/serial_8250.h> 34 #include <linux/bootmem.h> 35 #include <linux/pci.h> 36 #include <linux/lockdep.h> 37 #include <linux/memblock.h> 38 #include <linux/memory.h> 39 #include <linux/nmi.h> 40 41 #include <asm/io.h> 42 #include <asm/kdump.h> 43 #include <asm/prom.h> 44 #include <asm/processor.h> 45 #include <asm/pgtable.h> 46 #include <asm/smp.h> 47 #include <asm/elf.h> 48 #include <asm/machdep.h> 49 #include <asm/paca.h> 50 #include <asm/time.h> 51 #include <asm/cputable.h> 52 #include <asm/sections.h> 53 #include <asm/btext.h> 54 #include <asm/nvram.h> 55 #include <asm/setup.h> 56 #include <asm/rtas.h> 57 #include <asm/iommu.h> 58 #include <asm/serial.h> 59 #include <asm/cache.h> 60 #include <asm/page.h> 61 #include <asm/mmu.h> 62 #include <asm/firmware.h> 63 #include <asm/xmon.h> 64 #include <asm/udbg.h> 65 #include <asm/kexec.h> 66 #include <asm/code-patching.h> 67 #include <asm/livepatch.h> 68 #include <asm/opal.h> 69 #include <asm/cputhreads.h> 70 71 #ifdef DEBUG 72 #define DBG(fmt...) udbg_printf(fmt) 73 #else 74 #define DBG(fmt...) 75 #endif 76 77 int spinning_secondaries; 78 u64 ppc64_pft_size; 79 80 /* Pick defaults since we might want to patch instructions 81 * before we've read this from the device tree. 82 */ 83 struct ppc64_caches ppc64_caches = { 84 .dline_size = 0x40, 85 .log_dline_size = 6, 86 .iline_size = 0x40, 87 .log_iline_size = 6 88 }; 89 EXPORT_SYMBOL_GPL(ppc64_caches); 90 91 /* 92 * These are used in binfmt_elf.c to put aux entries on the stack 93 * for each elf executable being started. 94 */ 95 int dcache_bsize; 96 int icache_bsize; 97 int ucache_bsize; 98 99 #if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP) 100 void __init setup_tlb_core_data(void) 101 { 102 int cpu; 103 104 BUILD_BUG_ON(offsetof(struct tlb_core_data, lock) != 0); 105 106 for_each_possible_cpu(cpu) { 107 int first = cpu_first_thread_sibling(cpu); 108 109 /* 110 * If we boot via kdump on a non-primary thread, 111 * make sure we point at the thread that actually 112 * set up this TLB. 113 */ 114 if (cpu_first_thread_sibling(boot_cpuid) == first) 115 first = boot_cpuid; 116 117 paca[cpu].tcd_ptr = &paca[first].tcd; 118 119 /* 120 * If we have threads, we need either tlbsrx. 121 * or e6500 tablewalk mode, or else TLB handlers 122 * will be racy and could produce duplicate entries. 123 */ 124 if (smt_enabled_at_boot >= 2 && 125 !mmu_has_feature(MMU_FTR_USE_TLBRSRV) && 126 book3e_htw_mode != PPC_HTW_E6500) { 127 /* Should we panic instead? */ 128 WARN_ONCE("%s: unsupported MMU configuration -- expect problems\n", 129 __func__); 130 } 131 } 132 } 133 #endif 134 135 #ifdef CONFIG_SMP 136 137 static char *smt_enabled_cmdline; 138 139 /* Look for ibm,smt-enabled OF option */ 140 void __init check_smt_enabled(void) 141 { 142 struct device_node *dn; 143 const char *smt_option; 144 145 /* Default to enabling all threads */ 146 smt_enabled_at_boot = threads_per_core; 147 148 /* Allow the command line to overrule the OF option */ 149 if (smt_enabled_cmdline) { 150 if (!strcmp(smt_enabled_cmdline, "on")) 151 smt_enabled_at_boot = threads_per_core; 152 else if (!strcmp(smt_enabled_cmdline, "off")) 153 smt_enabled_at_boot = 0; 154 else { 155 int smt; 156 int rc; 157 158 rc = kstrtoint(smt_enabled_cmdline, 10, &smt); 159 if (!rc) 160 smt_enabled_at_boot = 161 min(threads_per_core, smt); 162 } 163 } else { 164 dn = of_find_node_by_path("/options"); 165 if (dn) { 166 smt_option = of_get_property(dn, "ibm,smt-enabled", 167 NULL); 168 169 if (smt_option) { 170 if (!strcmp(smt_option, "on")) 171 smt_enabled_at_boot = threads_per_core; 172 else if (!strcmp(smt_option, "off")) 173 smt_enabled_at_boot = 0; 174 } 175 176 of_node_put(dn); 177 } 178 } 179 } 180 181 /* Look for smt-enabled= cmdline option */ 182 static int __init early_smt_enabled(char *p) 183 { 184 smt_enabled_cmdline = p; 185 return 0; 186 } 187 early_param("smt-enabled", early_smt_enabled); 188 189 #endif /* CONFIG_SMP */ 190 191 /** Fix up paca fields required for the boot cpu */ 192 static void __init fixup_boot_paca(void) 193 { 194 /* The boot cpu is started */ 195 get_paca()->cpu_start = 1; 196 /* Allow percpu accesses to work until we setup percpu data */ 197 get_paca()->data_offset = 0; 198 } 199 200 static void __init configure_exceptions(void) 201 { 202 /* 203 * Setup the trampolines from the lowmem exception vectors 204 * to the kdump kernel when not using a relocatable kernel. 205 */ 206 setup_kdump_trampoline(); 207 208 /* Under a PAPR hypervisor, we need hypercalls */ 209 if (firmware_has_feature(FW_FEATURE_SET_MODE)) { 210 /* Enable AIL if possible */ 211 pseries_enable_reloc_on_exc(); 212 213 /* 214 * Tell the hypervisor that we want our exceptions to 215 * be taken in little endian mode. 216 * 217 * We don't call this for big endian as our calling convention 218 * makes us always enter in BE, and the call may fail under 219 * some circumstances with kdump. 220 */ 221 #ifdef __LITTLE_ENDIAN__ 222 pseries_little_endian_exceptions(); 223 #endif 224 } else { 225 /* Set endian mode using OPAL */ 226 if (firmware_has_feature(FW_FEATURE_OPAL)) 227 opal_configure_cores(); 228 229 /* Enable AIL if supported, and we are in hypervisor mode */ 230 if (early_cpu_has_feature(CPU_FTR_HVMODE) && 231 early_cpu_has_feature(CPU_FTR_ARCH_207S)) { 232 unsigned long lpcr = mfspr(SPRN_LPCR); 233 mtspr(SPRN_LPCR, lpcr | LPCR_AIL_3); 234 } 235 } 236 } 237 238 static void cpu_ready_for_interrupts(void) 239 { 240 /* Set IR and DR in PACA MSR */ 241 get_paca()->kernel_msr = MSR_KERNEL; 242 } 243 244 /* 245 * Early initialization entry point. This is called by head.S 246 * with MMU translation disabled. We rely on the "feature" of 247 * the CPU that ignores the top 2 bits of the address in real 248 * mode so we can access kernel globals normally provided we 249 * only toy with things in the RMO region. From here, we do 250 * some early parsing of the device-tree to setup out MEMBLOCK 251 * data structures, and allocate & initialize the hash table 252 * and segment tables so we can start running with translation 253 * enabled. 254 * 255 * It is this function which will call the probe() callback of 256 * the various platform types and copy the matching one to the 257 * global ppc_md structure. Your platform can eventually do 258 * some very early initializations from the probe() routine, but 259 * this is not recommended, be very careful as, for example, the 260 * device-tree is not accessible via normal means at this point. 261 */ 262 263 void __init early_setup(unsigned long dt_ptr) 264 { 265 static __initdata struct paca_struct boot_paca; 266 267 /* -------- printk is _NOT_ safe to use here ! ------- */ 268 269 /* Identify CPU type */ 270 identify_cpu(0, mfspr(SPRN_PVR)); 271 272 /* Assume we're on cpu 0 for now. Don't write to the paca yet! */ 273 initialise_paca(&boot_paca, 0); 274 setup_paca(&boot_paca); 275 fixup_boot_paca(); 276 277 /* -------- printk is now safe to use ------- */ 278 279 /* Enable early debugging if any specified (see udbg.h) */ 280 udbg_early_init(); 281 282 DBG(" -> early_setup(), dt_ptr: 0x%lx\n", dt_ptr); 283 284 /* 285 * Do early initialization using the flattened device 286 * tree, such as retrieving the physical memory map or 287 * calculating/retrieving the hash table size. 288 */ 289 early_init_devtree(__va(dt_ptr)); 290 291 /* Now we know the logical id of our boot cpu, setup the paca. */ 292 setup_paca(&paca[boot_cpuid]); 293 fixup_boot_paca(); 294 295 /* 296 * Configure exception handlers. This include setting up trampolines 297 * if needed, setting exception endian mode, etc... 298 */ 299 configure_exceptions(); 300 301 /* Apply all the dynamic patching */ 302 apply_feature_fixups(); 303 304 /* Initialize the hash table or TLB handling */ 305 early_init_mmu(); 306 307 /* 308 * At this point, we can let interrupts switch to virtual mode 309 * (the MMU has been setup), so adjust the MSR in the PACA to 310 * have IR and DR set and enable AIL if it exists 311 */ 312 cpu_ready_for_interrupts(); 313 314 DBG(" <- early_setup()\n"); 315 316 #ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX 317 /* 318 * This needs to be done *last* (after the above DBG() even) 319 * 320 * Right after we return from this function, we turn on the MMU 321 * which means the real-mode access trick that btext does will 322 * no longer work, it needs to switch to using a real MMU 323 * mapping. This call will ensure that it does 324 */ 325 btext_map(); 326 #endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */ 327 } 328 329 #ifdef CONFIG_SMP 330 void early_setup_secondary(void) 331 { 332 /* Mark interrupts disabled in PACA */ 333 get_paca()->soft_enabled = 0; 334 335 /* Initialize the hash table or TLB handling */ 336 early_init_mmu_secondary(); 337 338 /* 339 * At this point, we can let interrupts switch to virtual mode 340 * (the MMU has been setup), so adjust the MSR in the PACA to 341 * have IR and DR set. 342 */ 343 cpu_ready_for_interrupts(); 344 } 345 346 #endif /* CONFIG_SMP */ 347 348 #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC) 349 static bool use_spinloop(void) 350 { 351 if (!IS_ENABLED(CONFIG_PPC_BOOK3E)) 352 return true; 353 354 /* 355 * When book3e boots from kexec, the ePAPR spin table does 356 * not get used. 357 */ 358 return of_property_read_bool(of_chosen, "linux,booted-from-kexec"); 359 } 360 361 void smp_release_cpus(void) 362 { 363 unsigned long *ptr; 364 int i; 365 366 if (!use_spinloop()) 367 return; 368 369 DBG(" -> smp_release_cpus()\n"); 370 371 /* All secondary cpus are spinning on a common spinloop, release them 372 * all now so they can start to spin on their individual paca 373 * spinloops. For non SMP kernels, the secondary cpus never get out 374 * of the common spinloop. 375 */ 376 377 ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop 378 - PHYSICAL_START); 379 *ptr = ppc_function_entry(generic_secondary_smp_init); 380 381 /* And wait a bit for them to catch up */ 382 for (i = 0; i < 100000; i++) { 383 mb(); 384 HMT_low(); 385 if (spinning_secondaries == 0) 386 break; 387 udelay(1); 388 } 389 DBG("spinning_secondaries = %d\n", spinning_secondaries); 390 391 DBG(" <- smp_release_cpus()\n"); 392 } 393 #endif /* CONFIG_SMP || CONFIG_KEXEC */ 394 395 /* 396 * Initialize some remaining members of the ppc64_caches and systemcfg 397 * structures 398 * (at least until we get rid of them completely). This is mostly some 399 * cache informations about the CPU that will be used by cache flush 400 * routines and/or provided to userland 401 */ 402 void __init initialize_cache_info(void) 403 { 404 struct device_node *np; 405 unsigned long num_cpus = 0; 406 407 DBG(" -> initialize_cache_info()\n"); 408 409 for_each_node_by_type(np, "cpu") { 410 num_cpus += 1; 411 412 /* 413 * We're assuming *all* of the CPUs have the same 414 * d-cache and i-cache sizes... -Peter 415 */ 416 if (num_cpus == 1) { 417 const __be32 *sizep, *lsizep; 418 u32 size, lsize; 419 420 size = 0; 421 lsize = cur_cpu_spec->dcache_bsize; 422 sizep = of_get_property(np, "d-cache-size", NULL); 423 if (sizep != NULL) 424 size = be32_to_cpu(*sizep); 425 lsizep = of_get_property(np, "d-cache-block-size", 426 NULL); 427 /* fallback if block size missing */ 428 if (lsizep == NULL) 429 lsizep = of_get_property(np, 430 "d-cache-line-size", 431 NULL); 432 if (lsizep != NULL) 433 lsize = be32_to_cpu(*lsizep); 434 if (sizep == NULL || lsizep == NULL) 435 DBG("Argh, can't find dcache properties ! " 436 "sizep: %p, lsizep: %p\n", sizep, lsizep); 437 438 ppc64_caches.dsize = size; 439 ppc64_caches.dline_size = lsize; 440 ppc64_caches.log_dline_size = __ilog2(lsize); 441 ppc64_caches.dlines_per_page = PAGE_SIZE / lsize; 442 443 size = 0; 444 lsize = cur_cpu_spec->icache_bsize; 445 sizep = of_get_property(np, "i-cache-size", NULL); 446 if (sizep != NULL) 447 size = be32_to_cpu(*sizep); 448 lsizep = of_get_property(np, "i-cache-block-size", 449 NULL); 450 if (lsizep == NULL) 451 lsizep = of_get_property(np, 452 "i-cache-line-size", 453 NULL); 454 if (lsizep != NULL) 455 lsize = be32_to_cpu(*lsizep); 456 if (sizep == NULL || lsizep == NULL) 457 DBG("Argh, can't find icache properties ! " 458 "sizep: %p, lsizep: %p\n", sizep, lsizep); 459 460 ppc64_caches.isize = size; 461 ppc64_caches.iline_size = lsize; 462 ppc64_caches.log_iline_size = __ilog2(lsize); 463 ppc64_caches.ilines_per_page = PAGE_SIZE / lsize; 464 } 465 } 466 467 /* For use by binfmt_elf */ 468 dcache_bsize = ppc64_caches.dline_size; 469 icache_bsize = ppc64_caches.iline_size; 470 471 DBG(" <- initialize_cache_info()\n"); 472 } 473 474 /* This returns the limit below which memory accesses to the linear 475 * mapping are guarnateed not to cause a TLB or SLB miss. This is 476 * used to allocate interrupt or emergency stacks for which our 477 * exception entry path doesn't deal with being interrupted. 478 */ 479 static __init u64 safe_stack_limit(void) 480 { 481 #ifdef CONFIG_PPC_BOOK3E 482 /* Freescale BookE bolts the entire linear mapping */ 483 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) 484 return linear_map_top; 485 /* Other BookE, we assume the first GB is bolted */ 486 return 1ul << 30; 487 #else 488 /* BookS, the first segment is bolted */ 489 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 490 return 1UL << SID_SHIFT_1T; 491 return 1UL << SID_SHIFT; 492 #endif 493 } 494 495 void __init irqstack_early_init(void) 496 { 497 u64 limit = safe_stack_limit(); 498 unsigned int i; 499 500 /* 501 * Interrupt stacks must be in the first segment since we 502 * cannot afford to take SLB misses on them. 503 */ 504 for_each_possible_cpu(i) { 505 softirq_ctx[i] = (struct thread_info *) 506 __va(memblock_alloc_base(THREAD_SIZE, 507 THREAD_SIZE, limit)); 508 hardirq_ctx[i] = (struct thread_info *) 509 __va(memblock_alloc_base(THREAD_SIZE, 510 THREAD_SIZE, limit)); 511 } 512 } 513 514 #ifdef CONFIG_PPC_BOOK3E 515 void __init exc_lvl_early_init(void) 516 { 517 unsigned int i; 518 unsigned long sp; 519 520 for_each_possible_cpu(i) { 521 sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); 522 critirq_ctx[i] = (struct thread_info *)__va(sp); 523 paca[i].crit_kstack = __va(sp + THREAD_SIZE); 524 525 sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); 526 dbgirq_ctx[i] = (struct thread_info *)__va(sp); 527 paca[i].dbg_kstack = __va(sp + THREAD_SIZE); 528 529 sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE); 530 mcheckirq_ctx[i] = (struct thread_info *)__va(sp); 531 paca[i].mc_kstack = __va(sp + THREAD_SIZE); 532 } 533 534 if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) 535 patch_exception(0x040, exc_debug_debug_book3e); 536 } 537 #endif 538 539 /* 540 * Stack space used when we detect a bad kernel stack pointer, and 541 * early in SMP boots before relocation is enabled. Exclusive emergency 542 * stack for machine checks. 543 */ 544 void __init emergency_stack_init(void) 545 { 546 u64 limit; 547 unsigned int i; 548 549 /* 550 * Emergency stacks must be under 256MB, we cannot afford to take 551 * SLB misses on them. The ABI also requires them to be 128-byte 552 * aligned. 553 * 554 * Since we use these as temporary stacks during secondary CPU 555 * bringup, we need to get at them in real mode. This means they 556 * must also be within the RMO region. 557 */ 558 limit = min(safe_stack_limit(), ppc64_rma_size); 559 560 for_each_possible_cpu(i) { 561 struct thread_info *ti; 562 ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); 563 klp_init_thread_info(ti); 564 paca[i].emergency_sp = (void *)ti + THREAD_SIZE; 565 566 #ifdef CONFIG_PPC_BOOK3S_64 567 /* emergency stack for machine check exception handling. */ 568 ti = __va(memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit)); 569 klp_init_thread_info(ti); 570 paca[i].mc_emergency_sp = (void *)ti + THREAD_SIZE; 571 #endif 572 } 573 } 574 575 #ifdef CONFIG_SMP 576 #define PCPU_DYN_SIZE () 577 578 static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align) 579 { 580 return __alloc_bootmem_node(NODE_DATA(cpu_to_node(cpu)), size, align, 581 __pa(MAX_DMA_ADDRESS)); 582 } 583 584 static void __init pcpu_fc_free(void *ptr, size_t size) 585 { 586 free_bootmem(__pa(ptr), size); 587 } 588 589 static int pcpu_cpu_distance(unsigned int from, unsigned int to) 590 { 591 if (cpu_to_node(from) == cpu_to_node(to)) 592 return LOCAL_DISTANCE; 593 else 594 return REMOTE_DISTANCE; 595 } 596 597 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 598 EXPORT_SYMBOL(__per_cpu_offset); 599 600 void __init setup_per_cpu_areas(void) 601 { 602 const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE; 603 size_t atom_size; 604 unsigned long delta; 605 unsigned int cpu; 606 int rc; 607 608 /* 609 * Linear mapping is one of 4K, 1M and 16M. For 4K, no need 610 * to group units. For larger mappings, use 1M atom which 611 * should be large enough to contain a number of units. 612 */ 613 if (mmu_linear_psize == MMU_PAGE_4K) 614 atom_size = PAGE_SIZE; 615 else 616 atom_size = 1 << 20; 617 618 rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance, 619 pcpu_fc_alloc, pcpu_fc_free); 620 if (rc < 0) 621 panic("cannot initialize percpu area (err=%d)", rc); 622 623 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; 624 for_each_possible_cpu(cpu) { 625 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; 626 paca[cpu].data_offset = __per_cpu_offset[cpu]; 627 } 628 } 629 #endif 630 631 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 632 unsigned long memory_block_size_bytes(void) 633 { 634 if (ppc_md.memory_block_size) 635 return ppc_md.memory_block_size(); 636 637 return MIN_MEMORY_BLOCK_SIZE; 638 } 639 #endif 640 641 #if defined(CONFIG_PPC_INDIRECT_PIO) || defined(CONFIG_PPC_INDIRECT_MMIO) 642 struct ppc_pci_io ppc_pci_io; 643 EXPORT_SYMBOL(ppc_pci_io); 644 #endif 645 646 #ifdef CONFIG_HARDLOCKUP_DETECTOR 647 u64 hw_nmi_get_sample_period(int watchdog_thresh) 648 { 649 return ppc_proc_freq * watchdog_thresh; 650 } 651 652 /* 653 * The hardlockup detector breaks PMU event based branches and is likely 654 * to get false positives in KVM guests, so disable it by default. 655 */ 656 static int __init disable_hardlockup_detector(void) 657 { 658 hardlockup_detector_disable(); 659 660 return 0; 661 } 662 early_initcall(disable_hardlockup_detector); 663 #endif 664