1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * This file contains miscellaneous low-level functions. 4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 5 * 6 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu) 7 * and Paul Mackerras. 8 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com) 9 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com) 10 */ 11 12#include <linux/sys.h> 13#include <asm/unistd.h> 14#include <asm/errno.h> 15#include <asm/processor.h> 16#include <asm/page.h> 17#include <asm/cache.h> 18#include <asm/ppc_asm.h> 19#include <asm/asm-offsets.h> 20#include <asm/cputable.h> 21#include <asm/thread_info.h> 22#include <asm/kexec.h> 23#include <asm/ptrace.h> 24#include <asm/mmu.h> 25#include <asm/export.h> 26#include <asm/feature-fixups.h> 27 28 .text 29 30_GLOBAL(call_do_softirq) 31 mflr r0 32 std r0,16(r1) 33 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 34 mr r1,r3 35 bl __do_softirq 36 ld r1,0(r1) 37 ld r0,16(r1) 38 mtlr r0 39 blr 40 41_GLOBAL(call_do_irq) 42 mflr r0 43 std r0,16(r1) 44 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4) 45 mr r1,r4 46 bl __do_irq 47 ld r1,0(r1) 48 ld r0,16(r1) 49 mtlr r0 50 blr 51 52_GLOBAL(__bswapdi2) 53EXPORT_SYMBOL(__bswapdi2) 54 srdi r8,r3,32 55 rlwinm r7,r3,8,0xffffffff 56 rlwimi r7,r3,24,0,7 57 rlwinm r9,r8,8,0xffffffff 58 rlwimi r7,r3,24,16,23 59 rlwimi r9,r8,24,0,7 60 rlwimi r9,r8,24,16,23 61 sldi r7,r7,32 62 or r3,r7,r9 63 blr 64 65 66#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX 67_GLOBAL(rmci_on) 68 sync 69 isync 70 li r3,0x100 71 rldicl r3,r3,32,0 72 mfspr r5,SPRN_HID4 73 or r5,r5,r3 74 sync 75 mtspr SPRN_HID4,r5 76 isync 77 slbia 78 isync 79 sync 80 blr 81 82_GLOBAL(rmci_off) 83 sync 84 isync 85 li r3,0x100 86 rldicl r3,r3,32,0 87 mfspr r5,SPRN_HID4 88 andc r5,r5,r3 89 sync 90 mtspr SPRN_HID4,r5 91 isync 92 slbia 93 isync 94 sync 95 blr 96#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */ 97 98#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) 99 100/* 101 * Do an IO access in real mode 102 */ 103_GLOBAL(real_readb) 104 mfmsr r7 105 ori r0,r7,MSR_DR 106 xori r0,r0,MSR_DR 107 sync 108 mtmsrd r0 109 sync 110 isync 111 mfspr r6,SPRN_HID4 112 rldicl r5,r6,32,0 113 ori r5,r5,0x100 114 rldicl r5,r5,32,0 115 sync 116 mtspr SPRN_HID4,r5 117 isync 118 slbia 119 isync 120 lbz r3,0(r3) 121 sync 122 mtspr SPRN_HID4,r6 123 isync 124 slbia 125 isync 126 mtmsrd r7 127 sync 128 isync 129 blr 130 131 /* 132 * Do an IO access in real mode 133 */ 134_GLOBAL(real_writeb) 135 mfmsr r7 136 ori r0,r7,MSR_DR 137 xori r0,r0,MSR_DR 138 sync 139 mtmsrd r0 140 sync 141 isync 142 mfspr r6,SPRN_HID4 143 rldicl r5,r6,32,0 144 ori r5,r5,0x100 145 rldicl r5,r5,32,0 146 sync 147 mtspr SPRN_HID4,r5 148 isync 149 slbia 150 isync 151 stb r3,0(r4) 152 sync 153 mtspr SPRN_HID4,r6 154 isync 155 slbia 156 isync 157 mtmsrd r7 158 sync 159 isync 160 blr 161#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */ 162 163#ifdef CONFIG_PPC_PASEMI 164 165_GLOBAL(real_205_readb) 166 mfmsr r7 167 ori r0,r7,MSR_DR 168 xori r0,r0,MSR_DR 169 sync 170 mtmsrd r0 171 sync 172 isync 173 LBZCIX(R3,R0,R3) 174 isync 175 mtmsrd r7 176 sync 177 isync 178 blr 179 180_GLOBAL(real_205_writeb) 181 mfmsr r7 182 ori r0,r7,MSR_DR 183 xori r0,r0,MSR_DR 184 sync 185 mtmsrd r0 186 sync 187 isync 188 STBCIX(R3,R0,R4) 189 isync 190 mtmsrd r7 191 sync 192 isync 193 blr 194 195#endif /* CONFIG_PPC_PASEMI */ 196 197 198#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE) 199/* 200 * SCOM access functions for 970 (FX only for now) 201 * 202 * unsigned long scom970_read(unsigned int address); 203 * void scom970_write(unsigned int address, unsigned long value); 204 * 205 * The address passed in is the 24 bits register address. This code 206 * is 970 specific and will not check the status bits, so you should 207 * know what you are doing. 208 */ 209_GLOBAL(scom970_read) 210 /* interrupts off */ 211 mfmsr r4 212 ori r0,r4,MSR_EE 213 xori r0,r0,MSR_EE 214 mtmsrd r0,1 215 216 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 217 * (including parity). On current CPUs they must be 0'd, 218 * and finally or in RW bit 219 */ 220 rlwinm r3,r3,8,0,15 221 ori r3,r3,0x8000 222 223 /* do the actual scom read */ 224 sync 225 mtspr SPRN_SCOMC,r3 226 isync 227 mfspr r3,SPRN_SCOMD 228 isync 229 mfspr r0,SPRN_SCOMC 230 isync 231 232 /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah 233 * that's the best we can do). Not implemented yet as we don't use 234 * the scom on any of the bogus CPUs yet, but may have to be done 235 * ultimately 236 */ 237 238 /* restore interrupts */ 239 mtmsrd r4,1 240 blr 241 242 243_GLOBAL(scom970_write) 244 /* interrupts off */ 245 mfmsr r5 246 ori r0,r5,MSR_EE 247 xori r0,r0,MSR_EE 248 mtmsrd r0,1 249 250 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 251 * (including parity). On current CPUs they must be 0'd. 252 */ 253 254 rlwinm r3,r3,8,0,15 255 256 sync 257 mtspr SPRN_SCOMD,r4 /* write data */ 258 isync 259 mtspr SPRN_SCOMC,r3 /* write command */ 260 isync 261 mfspr 3,SPRN_SCOMC 262 isync 263 264 /* restore interrupts */ 265 mtmsrd r5,1 266 blr 267#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */ 268 269/* kexec_wait(phys_cpu) 270 * 271 * wait for the flag to change, indicating this kernel is going away but 272 * the slave code for the next one is at addresses 0 to 100. 273 * 274 * This is used by all slaves, even those that did not find a matching 275 * paca in the secondary startup code. 276 * 277 * Physical (hardware) cpu id should be in r3. 278 */ 279_GLOBAL(kexec_wait) 280 bl 1f 2811: mflr r5 282 addi r5,r5,kexec_flag-1b 283 28499: HMT_LOW 285#ifdef CONFIG_KEXEC_CORE /* use no memory without kexec */ 286 lwz r4,0(r5) 287 cmpwi 0,r4,0 288 beq 99b 289#ifdef CONFIG_PPC_BOOK3S_64 290 li r10,0x60 291 mfmsr r11 292 clrrdi r11,r11,1 /* Clear MSR_LE */ 293 mtsrr0 r10 294 mtsrr1 r11 295 rfid 296#else 297 /* Create TLB entry in book3e_secondary_core_init */ 298 li r4,0 299 ba 0x60 300#endif 301#endif 302 303/* this can be in text because we won't change it until we are 304 * running in real anyways 305 */ 306kexec_flag: 307 .long 0 308 309 310#ifdef CONFIG_KEXEC_CORE 311#ifdef CONFIG_PPC_BOOK3E 312/* 313 * BOOK3E has no real MMU mode, so we have to setup the initial TLB 314 * for a core to identity map v:0 to p:0. This current implementation 315 * assumes that 1G is enough for kexec. 316 */ 317kexec_create_tlb: 318 /* 319 * Invalidate all non-IPROT TLB entries to avoid any TLB conflict. 320 * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict. 321 */ 322 PPC_TLBILX_ALL(0,R0) 323 sync 324 isync 325 326 mfspr r10,SPRN_TLB1CFG 327 andi. r10,r10,TLBnCFG_N_ENTRY /* Extract # entries */ 328 subi r10,r10,1 /* Last entry: no conflict with kernel text */ 329 lis r9,MAS0_TLBSEL(1)@h 330 rlwimi r9,r10,16,4,15 /* Setup MAS0 = TLBSEL | ESEL(r9) */ 331 332/* Set up a temp identity mapping v:0 to p:0 and return to it. */ 333 mtspr SPRN_MAS0,r9 334 335 lis r9,(MAS1_VALID|MAS1_IPROT)@h 336 ori r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l 337 mtspr SPRN_MAS1,r9 338 339 LOAD_REG_IMMEDIATE(r9, 0x0 | MAS2_M_IF_NEEDED) 340 mtspr SPRN_MAS2,r9 341 342 LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX) 343 mtspr SPRN_MAS3,r9 344 li r9,0 345 mtspr SPRN_MAS7,r9 346 347 tlbwe 348 isync 349 blr 350#endif 351 352/* kexec_smp_wait(void) 353 * 354 * call with interrupts off 355 * note: this is a terminal routine, it does not save lr 356 * 357 * get phys id from paca 358 * switch to real mode 359 * mark the paca as no longer used 360 * join other cpus in kexec_wait(phys_id) 361 */ 362_GLOBAL(kexec_smp_wait) 363 lhz r3,PACAHWCPUID(r13) 364 bl real_mode 365 366 li r4,KEXEC_STATE_REAL_MODE 367 stb r4,PACAKEXECSTATE(r13) 368 SYNC 369 370 b kexec_wait 371 372/* 373 * switch to real mode (turn mmu off) 374 * we use the early kernel trick that the hardware ignores bits 375 * 0 and 1 (big endian) of the effective address in real mode 376 * 377 * don't overwrite r3 here, it is live for kexec_wait above. 378 */ 379real_mode: /* assume normal blr return */ 380#ifdef CONFIG_PPC_BOOK3E 381 /* Create an identity mapping. */ 382 b kexec_create_tlb 383#else 3841: li r9,MSR_RI 385 li r10,MSR_DR|MSR_IR 386 mflr r11 /* return address to SRR0 */ 387 mfmsr r12 388 andc r9,r12,r9 389 andc r10,r12,r10 390 391 mtmsrd r9,1 392 mtspr SPRN_SRR1,r10 393 mtspr SPRN_SRR0,r11 394 rfid 395#endif 396 397/* 398 * kexec_sequence(newstack, start, image, control, clear_all(), 399 copy_with_mmu_off) 400 * 401 * does the grungy work with stack switching and real mode switches 402 * also does simple calls to other code 403 */ 404 405_GLOBAL(kexec_sequence) 406 mflr r0 407 std r0,16(r1) 408 409 /* switch stacks to newstack -- &kexec_stack.stack */ 410 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 411 mr r1,r3 412 413 li r0,0 414 std r0,16(r1) 415 416BEGIN_FTR_SECTION 417 /* 418 * This is the best time to turn AMR/IAMR off. 419 * key 0 is used in radix for supervisor<->user 420 * protection, but on hash key 0 is reserved 421 * ideally we want to enter with a clean state. 422 * NOTE, we rely on r0 being 0 from above. 423 */ 424 mtspr SPRN_IAMR,r0 425BEGIN_FTR_SECTION_NESTED(42) 426 mtspr SPRN_AMOR,r0 427END_FTR_SECTION_NESTED_IFSET(CPU_FTR_HVMODE, 42) 428END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 429 430 /* save regs for local vars on new stack. 431 * yes, we won't go back, but ... 432 */ 433 std r31,-8(r1) 434 std r30,-16(r1) 435 std r29,-24(r1) 436 std r28,-32(r1) 437 std r27,-40(r1) 438 std r26,-48(r1) 439 std r25,-56(r1) 440 441 stdu r1,-STACK_FRAME_OVERHEAD-64(r1) 442 443 /* save args into preserved regs */ 444 mr r31,r3 /* newstack (both) */ 445 mr r30,r4 /* start (real) */ 446 mr r29,r5 /* image (virt) */ 447 mr r28,r6 /* control, unused */ 448 mr r27,r7 /* clear_all() fn desc */ 449 mr r26,r8 /* copy_with_mmu_off */ 450 lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */ 451 452 /* disable interrupts, we are overwriting kernel data next */ 453#ifdef CONFIG_PPC_BOOK3E 454 wrteei 0 455#else 456 mfmsr r3 457 rlwinm r3,r3,0,17,15 458 mtmsrd r3,1 459#endif 460 461 /* We need to turn the MMU off unless we are in hash mode 462 * under a hypervisor 463 */ 464 cmpdi r26,0 465 beq 1f 466 bl real_mode 4671: 468 /* copy dest pages, flush whole dest image */ 469 mr r3,r29 470 bl kexec_copy_flush /* (image) */ 471 472 /* turn off mmu now if not done earlier */ 473 cmpdi r26,0 474 bne 1f 475 bl real_mode 476 477 /* copy 0x100 bytes starting at start to 0 */ 4781: li r3,0 479 mr r4,r30 /* start, aka phys mem offset */ 480 li r5,0x100 481 li r6,0 482 bl copy_and_flush /* (dest, src, copy limit, start offset) */ 4831: /* assume normal blr return */ 484 485 /* release other cpus to the new kernel secondary start at 0x60 */ 486 mflr r5 487 li r6,1 488 stw r6,kexec_flag-1b(5) 489 490 cmpdi r27,0 491 beq 1f 492 493 /* clear out hardware hash page table and tlb */ 494#ifdef PPC64_ELF_ABI_v1 495 ld r12,0(r27) /* deref function descriptor */ 496#else 497 mr r12,r27 498#endif 499 mtctr r12 500 bctrl /* mmu_hash_ops.hpte_clear_all(void); */ 501 502/* 503 * kexec image calling is: 504 * the first 0x100 bytes of the entry point are copied to 0 505 * 506 * all slaves branch to slave = 0x60 (absolute) 507 * slave(phys_cpu_id); 508 * 509 * master goes to start = entry point 510 * start(phys_cpu_id, start, 0); 511 * 512 * 513 * a wrapper is needed to call existing kernels, here is an approximate 514 * description of one method: 515 * 516 * v2: (2.6.10) 517 * start will be near the boot_block (maybe 0x100 bytes before it?) 518 * it will have a 0x60, which will b to boot_block, where it will wait 519 * and 0 will store phys into struct boot-block and load r3 from there, 520 * copy kernel 0-0x100 and tell slaves to back down to 0x60 again 521 * 522 * v1: (2.6.9) 523 * boot block will have all cpus scanning device tree to see if they 524 * are the boot cpu ????? 525 * other device tree differences (prop sizes, va vs pa, etc)... 526 */ 5271: mr r3,r25 # my phys cpu 528 mr r4,r30 # start, aka phys mem offset 529 mtlr 4 530 li r5,0 531 blr /* image->start(physid, image->start, 0); */ 532#endif /* CONFIG_KEXEC_CORE */ 533