1#include <asm/asm-offsets.h> 2#include <asm/bug.h> 3#ifdef CONFIG_PPC_BOOK3S 4#include <asm/exception-64s.h> 5#else 6#include <asm/exception-64e.h> 7#endif 8#include <asm/feature-fixups.h> 9#include <asm/head-64.h> 10#include <asm/hw_irq.h> 11#include <asm/kup.h> 12#include <asm/mmu.h> 13#include <asm/ppc_asm.h> 14#include <asm/ptrace.h> 15 16 .align 7 17 18.macro DEBUG_SRR_VALID srr 19#ifdef CONFIG_PPC_RFI_SRR_DEBUG 20 .ifc \srr,srr 21 mfspr r11,SPRN_SRR0 22 ld r12,_NIP(r1) 23 clrrdi r11,r11,2 24 clrrdi r12,r12,2 25100: tdne r11,r12 26 EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) 27 mfspr r11,SPRN_SRR1 28 ld r12,_MSR(r1) 29100: tdne r11,r12 30 EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) 31 .else 32 mfspr r11,SPRN_HSRR0 33 ld r12,_NIP(r1) 34 clrrdi r11,r11,2 35 clrrdi r12,r12,2 36100: tdne r11,r12 37 EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) 38 mfspr r11,SPRN_HSRR1 39 ld r12,_MSR(r1) 40100: tdne r11,r12 41 EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) 42 .endif 43#endif 44.endm 45 46#ifdef CONFIG_PPC_BOOK3S 47.macro system_call_vectored name trapnr 48 .globl system_call_vectored_\name 49system_call_vectored_\name: 50_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name) 51 SCV_INTERRUPT_TO_KERNEL 52 mr r10,r1 53 ld r1,PACAKSAVE(r13) 54 std r10,0(r1) 55 std r11,_LINK(r1) 56 std r11,_NIP(r1) /* Saved LR is also the next instruction */ 57 std r12,_MSR(r1) 58 std r0,GPR0(r1) 59 std r10,GPR1(r1) 60 std r2,GPR2(r1) 61 LOAD_PACA_TOC() 62 mfcr r12 63 li r11,0 64 /* Save syscall parameters in r3-r8 */ 65 SAVE_GPRS(3, 8, r1) 66 /* Zero r9-r12, this should only be required when restoring all GPRs */ 67 std r11,GPR9(r1) 68 std r11,GPR10(r1) 69 std r11,GPR11(r1) 70 std r11,GPR12(r1) 71 std r9,GPR13(r1) 72 SAVE_NVGPRS(r1) 73 std r11,_XER(r1) 74 std r11,_CTR(r1) 75 76 li r11,\trapnr 77 std r11,_TRAP(r1) 78 std r12,_CCR(r1) 79 std r3,ORIG_GPR3(r1) 80 LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) 81 std r11,STACK_INT_FRAME_MARKER(r1) /* "regs" marker */ 82 /* Calling convention has r3 = regs, r4 = orig r0 */ 83 addi r3,r1,STACK_INT_FRAME_REGS 84 mr r4,r0 85 86BEGIN_FTR_SECTION 87 HMT_MEDIUM 88END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 89 90 /* 91 * scv enters with MSR[EE]=1 and is immediately considered soft-masked. 92 * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED, 93 * and interrupts may be masked and pending already. 94 * system_call_exception() will call trace_hardirqs_off() which means 95 * interrupts could already have been blocked before trace_hardirqs_off, 96 * but this is the best we can do. 97 */ 98 99 /* 100 * Zero user registers to prevent influencing speculative execution 101 * state of kernel code. 102 */ 103 SANITIZE_SYSCALL_GPRS() 104 bl CFUNC(system_call_exception) 105 106.Lsyscall_vectored_\name\()_exit: 107 addi r4,r1,STACK_INT_FRAME_REGS 108 li r5,1 /* scv */ 109 bl CFUNC(syscall_exit_prepare) 110 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 111.Lsyscall_vectored_\name\()_rst_start: 112 lbz r11,PACAIRQHAPPENED(r13) 113 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l 114 bne- syscall_vectored_\name\()_restart 115 li r11,IRQS_ENABLED 116 stb r11,PACAIRQSOFTMASK(r13) 117 li r11,0 118 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS 119 120 ld r2,_CCR(r1) 121 ld r4,_NIP(r1) 122 ld r5,_MSR(r1) 123 124BEGIN_FTR_SECTION 125 stdcx. r0,0,r1 /* to clear the reservation */ 126END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 127 128BEGIN_FTR_SECTION 129 HMT_MEDIUM_LOW 130END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 131 132 SANITIZE_RESTORE_NVGPRS() 133 cmpdi r3,0 134 bne .Lsyscall_vectored_\name\()_restore_regs 135 136 /* rfscv returns with LR->NIA and CTR->MSR */ 137 mtlr r4 138 mtctr r5 139 140 /* Could zero these as per ABI, but we may consider a stricter ABI 141 * which preserves these if libc implementations can benefit, so 142 * restore them for now until further measurement is done. */ 143 REST_GPR(0, r1) 144 REST_GPRS(4, 8, r1) 145 /* Zero volatile regs that may contain sensitive kernel data */ 146 ZEROIZE_GPRS(9, 12) 147 mtspr SPRN_XER,r0 148 149 /* 150 * We don't need to restore AMR on the way back to userspace for KUAP. 151 * The value of AMR only matters while we're in the kernel. 152 */ 153 mtcr r2 154 REST_GPRS(2, 3, r1) 155 REST_GPR(13, r1) 156 REST_GPR(1, r1) 157 RFSCV_TO_USER 158 b . /* prevent speculative execution */ 159 160.Lsyscall_vectored_\name\()_restore_regs: 161 mtspr SPRN_SRR0,r4 162 mtspr SPRN_SRR1,r5 163 164 ld r3,_CTR(r1) 165 ld r4,_LINK(r1) 166 ld r5,_XER(r1) 167 168 HANDLER_RESTORE_NVGPRS() 169 REST_GPR(0, r1) 170 mtcr r2 171 mtctr r3 172 mtlr r4 173 mtspr SPRN_XER,r5 174 REST_GPRS(2, 13, r1) 175 REST_GPR(1, r1) 176 RFI_TO_USER 177.Lsyscall_vectored_\name\()_rst_end: 178 179syscall_vectored_\name\()_restart: 180_ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart) 181 GET_PACA(r13) 182 ld r1,PACA_EXIT_SAVE_R1(r13) 183 LOAD_PACA_TOC() 184 ld r3,RESULT(r1) 185 addi r4,r1,STACK_INT_FRAME_REGS 186 li r11,IRQS_ALL_DISABLED 187 stb r11,PACAIRQSOFTMASK(r13) 188 bl CFUNC(syscall_exit_restart) 189 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 190 b .Lsyscall_vectored_\name\()_rst_start 1911: 192 193SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b) 194RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart) 195 196.endm 197 198system_call_vectored common 0x3000 199 200/* 201 * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0 202 * which is tested by system_call_exception when r0 is -1 (as set by vector 203 * entry code). 204 */ 205system_call_vectored sigill 0x7ff0 206 207#endif /* CONFIG_PPC_BOOK3S */ 208 209 .balign IFETCH_ALIGN_BYTES 210 .globl system_call_common_real 211system_call_common_real: 212_ASM_NOKPROBE_SYMBOL(system_call_common_real) 213 ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 214 mtmsrd r10 215 216 .balign IFETCH_ALIGN_BYTES 217 .globl system_call_common 218system_call_common: 219_ASM_NOKPROBE_SYMBOL(system_call_common) 220 mr r10,r1 221 ld r1,PACAKSAVE(r13) 222 std r10,0(r1) 223 std r11,_NIP(r1) 224 std r12,_MSR(r1) 225 std r0,GPR0(r1) 226 std r10,GPR1(r1) 227 std r2,GPR2(r1) 228#ifdef CONFIG_PPC_E500 229START_BTB_FLUSH_SECTION 230 BTB_FLUSH(r10) 231END_BTB_FLUSH_SECTION 232#endif 233 LOAD_PACA_TOC() 234 mfcr r12 235 li r11,0 236 /* Save syscall parameters in r3-r8 */ 237 SAVE_GPRS(3, 8, r1) 238 /* Zero r9-r12, this should only be required when restoring all GPRs */ 239 std r11,GPR9(r1) 240 std r11,GPR10(r1) 241 std r11,GPR11(r1) 242 std r11,GPR12(r1) 243 std r9,GPR13(r1) 244 SAVE_NVGPRS(r1) 245 std r11,_XER(r1) 246 std r11,_CTR(r1) 247 mflr r10 248 249 /* 250 * This clears CR0.SO (bit 28), which is the error indication on 251 * return from this system call. 252 */ 253 rldimi r12,r11,28,(63-28) 254 li r11,0xc00 255 std r10,_LINK(r1) 256 std r11,_TRAP(r1) 257 std r12,_CCR(r1) 258 std r3,ORIG_GPR3(r1) 259 LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) 260 std r11,STACK_INT_FRAME_MARKER(r1) /* "regs" marker */ 261 /* Calling convention has r3 = regs, r4 = orig r0 */ 262 addi r3,r1,STACK_INT_FRAME_REGS 263 mr r4,r0 264 265#ifdef CONFIG_PPC_BOOK3S 266 li r11,1 267 stb r11,PACASRR_VALID(r13) 268#endif 269 270 /* 271 * We always enter kernel from userspace with irq soft-mask enabled and 272 * nothing pending. system_call_exception() will call 273 * trace_hardirqs_off(). 274 */ 275 li r11,IRQS_ALL_DISABLED 276 stb r11,PACAIRQSOFTMASK(r13) 277#ifdef CONFIG_PPC_BOOK3S 278 li r12,-1 /* Set MSR_EE and MSR_RI */ 279 mtmsrd r12,1 280#else 281 wrteei 1 282#endif 283 284 /* 285 * Zero user registers to prevent influencing speculative execution 286 * state of kernel code. 287 */ 288 SANITIZE_SYSCALL_GPRS() 289 bl CFUNC(system_call_exception) 290 291.Lsyscall_exit: 292 addi r4,r1,STACK_INT_FRAME_REGS 293 li r5,0 /* !scv */ 294 bl CFUNC(syscall_exit_prepare) 295 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 296#ifdef CONFIG_PPC_BOOK3S 297.Lsyscall_rst_start: 298 lbz r11,PACAIRQHAPPENED(r13) 299 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l 300 bne- syscall_restart 301#endif 302 li r11,IRQS_ENABLED 303 stb r11,PACAIRQSOFTMASK(r13) 304 li r11,0 305 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS 306 307 ld r2,_CCR(r1) 308 ld r6,_LINK(r1) 309 mtlr r6 310 311#ifdef CONFIG_PPC_BOOK3S 312 lbz r4,PACASRR_VALID(r13) 313 cmpdi r4,0 314 bne 1f 315 li r4,0 316 stb r4,PACASRR_VALID(r13) 317#endif 318 ld r4,_NIP(r1) 319 ld r5,_MSR(r1) 320 mtspr SPRN_SRR0,r4 321 mtspr SPRN_SRR1,r5 3221: 323 DEBUG_SRR_VALID srr 324 325BEGIN_FTR_SECTION 326 stdcx. r0,0,r1 /* to clear the reservation */ 327END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 328 329 SANITIZE_RESTORE_NVGPRS() 330 cmpdi r3,0 331 bne .Lsyscall_restore_regs 332 /* Zero volatile regs that may contain sensitive kernel data */ 333 ZEROIZE_GPR(0) 334 ZEROIZE_GPRS(4, 12) 335 mtctr r0 336 mtspr SPRN_XER,r0 337.Lsyscall_restore_regs_cont: 338 339BEGIN_FTR_SECTION 340 HMT_MEDIUM_LOW 341END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 342 343 /* 344 * We don't need to restore AMR on the way back to userspace for KUAP. 345 * The value of AMR only matters while we're in the kernel. 346 */ 347 mtcr r2 348 REST_GPRS(2, 3, r1) 349 REST_GPR(13, r1) 350 REST_GPR(1, r1) 351 RFI_TO_USER 352 b . /* prevent speculative execution */ 353 354.Lsyscall_restore_regs: 355 ld r3,_CTR(r1) 356 ld r4,_XER(r1) 357 HANDLER_RESTORE_NVGPRS() 358 mtctr r3 359 mtspr SPRN_XER,r4 360 REST_GPR(0, r1) 361 REST_GPRS(4, 12, r1) 362 b .Lsyscall_restore_regs_cont 363.Lsyscall_rst_end: 364 365#ifdef CONFIG_PPC_BOOK3S 366syscall_restart: 367_ASM_NOKPROBE_SYMBOL(syscall_restart) 368 GET_PACA(r13) 369 ld r1,PACA_EXIT_SAVE_R1(r13) 370 LOAD_PACA_TOC() 371 ld r3,RESULT(r1) 372 addi r4,r1,STACK_INT_FRAME_REGS 373 li r11,IRQS_ALL_DISABLED 374 stb r11,PACAIRQSOFTMASK(r13) 375 bl CFUNC(syscall_exit_restart) 376 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 377 b .Lsyscall_rst_start 3781: 379 380SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b) 381RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart) 382#endif 383 384 /* 385 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not 386 * touched, no exit work created, then this can be used. 387 */ 388 .balign IFETCH_ALIGN_BYTES 389 .globl fast_interrupt_return_srr 390fast_interrupt_return_srr: 391_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr) 392 kuap_check_amr r3, r4 393 ld r5,_MSR(r1) 394 andi. r0,r5,MSR_PR 395#ifdef CONFIG_PPC_BOOK3S 396 beq 1f 397 kuap_user_restore r3, r4 398 b .Lfast_user_interrupt_return_srr 3991: kuap_kernel_restore r3, r4 400 andi. r0,r5,MSR_RI 401 li r3,0 /* 0 return value, no EMULATE_STACK_STORE */ 402 bne+ .Lfast_kernel_interrupt_return_srr 403 addi r3,r1,STACK_INT_FRAME_REGS 404 bl CFUNC(unrecoverable_exception) 405 b . /* should not get here */ 406#else 407 bne .Lfast_user_interrupt_return_srr 408 b .Lfast_kernel_interrupt_return_srr 409#endif 410 411.macro interrupt_return_macro srr 412 .balign IFETCH_ALIGN_BYTES 413 .globl interrupt_return_\srr 414interrupt_return_\srr\(): 415_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()) 416 ld r4,_MSR(r1) 417 andi. r0,r4,MSR_PR 418 beq interrupt_return_\srr\()_kernel 419interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */ 420_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user) 421 addi r3,r1,STACK_INT_FRAME_REGS 422 bl CFUNC(interrupt_exit_user_prepare) 423#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS 424 cmpdi r3,0 425 bne- .Lrestore_nvgprs_\srr 426.Lrestore_nvgprs_\srr\()_cont: 427#endif 428 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 429#ifdef CONFIG_PPC_BOOK3S 430.Linterrupt_return_\srr\()_user_rst_start: 431 lbz r11,PACAIRQHAPPENED(r13) 432 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l 433 bne- interrupt_return_\srr\()_user_restart 434#endif 435 li r11,IRQS_ENABLED 436 stb r11,PACAIRQSOFTMASK(r13) 437 li r11,0 438 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS 439 440.Lfast_user_interrupt_return_\srr\(): 441 SANITIZE_RESTORE_NVGPRS() 442#ifdef CONFIG_PPC_BOOK3S 443 .ifc \srr,srr 444 lbz r4,PACASRR_VALID(r13) 445 .else 446 lbz r4,PACAHSRR_VALID(r13) 447 .endif 448 cmpdi r4,0 449 li r4,0 450 bne 1f 451#endif 452 ld r11,_NIP(r1) 453 ld r12,_MSR(r1) 454 .ifc \srr,srr 455 mtspr SPRN_SRR0,r11 456 mtspr SPRN_SRR1,r12 4571: 458#ifdef CONFIG_PPC_BOOK3S 459 stb r4,PACASRR_VALID(r13) 460#endif 461 .else 462 mtspr SPRN_HSRR0,r11 463 mtspr SPRN_HSRR1,r12 4641: 465#ifdef CONFIG_PPC_BOOK3S 466 stb r4,PACAHSRR_VALID(r13) 467#endif 468 .endif 469 DEBUG_SRR_VALID \srr 470 471#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG 472 lbz r4,PACAIRQSOFTMASK(r13) 473 tdnei r4,IRQS_ENABLED 474#endif 475 476BEGIN_FTR_SECTION 477 ld r10,_PPR(r1) 478 mtspr SPRN_PPR,r10 479END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 480 481BEGIN_FTR_SECTION 482 stdcx. r0,0,r1 /* to clear the reservation */ 483FTR_SECTION_ELSE 484 ldarx r0,0,r1 485ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 486 487 ld r3,_CCR(r1) 488 ld r4,_LINK(r1) 489 ld r5,_CTR(r1) 490 ld r6,_XER(r1) 491 li r0,0 492 493 REST_GPRS(7, 13, r1) 494 495 mtcr r3 496 mtlr r4 497 mtctr r5 498 mtspr SPRN_XER,r6 499 500 REST_GPRS(2, 6, r1) 501 REST_GPR(0, r1) 502 REST_GPR(1, r1) 503 .ifc \srr,srr 504 RFI_TO_USER 505 .else 506 HRFI_TO_USER 507 .endif 508 b . /* prevent speculative execution */ 509.Linterrupt_return_\srr\()_user_rst_end: 510 511#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS 512.Lrestore_nvgprs_\srr\(): 513 REST_NVGPRS(r1) 514 b .Lrestore_nvgprs_\srr\()_cont 515#endif 516 517#ifdef CONFIG_PPC_BOOK3S 518interrupt_return_\srr\()_user_restart: 519_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart) 520 GET_PACA(r13) 521 ld r1,PACA_EXIT_SAVE_R1(r13) 522 LOAD_PACA_TOC() 523 addi r3,r1,STACK_INT_FRAME_REGS 524 li r11,IRQS_ALL_DISABLED 525 stb r11,PACAIRQSOFTMASK(r13) 526 bl CFUNC(interrupt_exit_user_restart) 527 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 528 b .Linterrupt_return_\srr\()_user_rst_start 5291: 530 531SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b) 532RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart) 533#endif 534 535 .balign IFETCH_ALIGN_BYTES 536interrupt_return_\srr\()_kernel: 537_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel) 538 addi r3,r1,STACK_INT_FRAME_REGS 539 bl CFUNC(interrupt_exit_kernel_prepare) 540 541 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 542.Linterrupt_return_\srr\()_kernel_rst_start: 543 ld r11,SOFTE(r1) 544 cmpwi r11,IRQS_ENABLED 545 stb r11,PACAIRQSOFTMASK(r13) 546 beq .Linterrupt_return_\srr\()_soft_enabled 547 548 /* 549 * Returning to soft-disabled context. 550 * Check if a MUST_HARD_MASK interrupt has become pending, in which 551 * case we need to disable MSR[EE] in the return context. 552 * 553 * The MSR[EE] check catches among other things the short incoherency 554 * in hard_irq_disable() between clearing MSR[EE] and setting 555 * PACA_IRQ_HARD_DIS. 556 */ 557 ld r12,_MSR(r1) 558 andi. r10,r12,MSR_EE 559 beq .Lfast_kernel_interrupt_return_\srr\() // EE already disabled 560 lbz r11,PACAIRQHAPPENED(r13) 561 andi. r10,r11,PACA_IRQ_MUST_HARD_MASK 562 bne 1f // HARD_MASK is pending 563 // No HARD_MASK pending, clear possible HARD_DIS set by interrupt 564 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l 565 stb r11,PACAIRQHAPPENED(r13) 566 b .Lfast_kernel_interrupt_return_\srr\() 567 568 5691: /* Must clear MSR_EE from _MSR */ 570#ifdef CONFIG_PPC_BOOK3S 571 li r10,0 572 /* Clear valid before changing _MSR */ 573 .ifc \srr,srr 574 stb r10,PACASRR_VALID(r13) 575 .else 576 stb r10,PACAHSRR_VALID(r13) 577 .endif 578#endif 579 xori r12,r12,MSR_EE 580 std r12,_MSR(r1) 581 b .Lfast_kernel_interrupt_return_\srr\() 582 583.Linterrupt_return_\srr\()_soft_enabled: 584 /* 585 * In the soft-enabled case, need to double-check that we have no 586 * pending interrupts that might have come in before we reached the 587 * restart section of code, and restart the exit so those can be 588 * handled. 589 * 590 * If there are none, it is be possible that the interrupt still 591 * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the 592 * interrupted context. This clear will not clobber a new pending 593 * interrupt coming in, because we're in the restart section, so 594 * such would return to the restart location. 595 */ 596#ifdef CONFIG_PPC_BOOK3S 597 lbz r11,PACAIRQHAPPENED(r13) 598 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l 599 bne- interrupt_return_\srr\()_kernel_restart 600#endif 601 li r11,0 602 stb r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS 603 604.Lfast_kernel_interrupt_return_\srr\(): 605 SANITIZE_RESTORE_NVGPRS() 606 cmpdi cr1,r3,0 607#ifdef CONFIG_PPC_BOOK3S 608 .ifc \srr,srr 609 lbz r4,PACASRR_VALID(r13) 610 .else 611 lbz r4,PACAHSRR_VALID(r13) 612 .endif 613 cmpdi r4,0 614 li r4,0 615 bne 1f 616#endif 617 ld r11,_NIP(r1) 618 ld r12,_MSR(r1) 619 .ifc \srr,srr 620 mtspr SPRN_SRR0,r11 621 mtspr SPRN_SRR1,r12 6221: 623#ifdef CONFIG_PPC_BOOK3S 624 stb r4,PACASRR_VALID(r13) 625#endif 626 .else 627 mtspr SPRN_HSRR0,r11 628 mtspr SPRN_HSRR1,r12 6291: 630#ifdef CONFIG_PPC_BOOK3S 631 stb r4,PACAHSRR_VALID(r13) 632#endif 633 .endif 634 DEBUG_SRR_VALID \srr 635 636BEGIN_FTR_SECTION 637 stdcx. r0,0,r1 /* to clear the reservation */ 638FTR_SECTION_ELSE 639 ldarx r0,0,r1 640ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 641 642 ld r3,_LINK(r1) 643 ld r4,_CTR(r1) 644 ld r5,_XER(r1) 645 ld r6,_CCR(r1) 646 li r0,0 647 648 REST_GPRS(7, 12, r1) 649 650 mtlr r3 651 mtctr r4 652 mtspr SPRN_XER,r5 653 654 /* 655 * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse 656 * the reliable stack unwinder later on. Clear it. 657 */ 658 std r0,STACK_INT_FRAME_MARKER(r1) 659 660 REST_GPRS(2, 5, r1) 661 662 bne- cr1,1f /* emulate stack store */ 663 mtcr r6 664 REST_GPR(6, r1) 665 REST_GPR(0, r1) 666 REST_GPR(1, r1) 667 .ifc \srr,srr 668 RFI_TO_KERNEL 669 .else 670 HRFI_TO_KERNEL 671 .endif 672 b . /* prevent speculative execution */ 673 6741: /* 675 * Emulate stack store with update. New r1 value was already calculated 676 * and updated in our interrupt regs by emulate_loadstore, but we can't 677 * store the previous value of r1 to the stack before re-loading our 678 * registers from it, otherwise they could be clobbered. Use 679 * PACA_EXGEN as temporary storage to hold the store data, as 680 * interrupts are disabled here so it won't be clobbered. 681 */ 682 mtcr r6 683 std r9,PACA_EXGEN+0(r13) 684 addi r9,r1,INT_FRAME_SIZE /* get original r1 */ 685 REST_GPR(6, r1) 686 REST_GPR(0, r1) 687 REST_GPR(1, r1) 688 std r9,0(r1) /* perform store component of stdu */ 689 ld r9,PACA_EXGEN+0(r13) 690 691 .ifc \srr,srr 692 RFI_TO_KERNEL 693 .else 694 HRFI_TO_KERNEL 695 .endif 696 b . /* prevent speculative execution */ 697.Linterrupt_return_\srr\()_kernel_rst_end: 698 699#ifdef CONFIG_PPC_BOOK3S 700interrupt_return_\srr\()_kernel_restart: 701_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart) 702 GET_PACA(r13) 703 ld r1,PACA_EXIT_SAVE_R1(r13) 704 LOAD_PACA_TOC() 705 addi r3,r1,STACK_INT_FRAME_REGS 706 li r11,IRQS_ALL_DISABLED 707 stb r11,PACAIRQSOFTMASK(r13) 708 bl CFUNC(interrupt_exit_kernel_restart) 709 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */ 710 b .Linterrupt_return_\srr\()_kernel_rst_start 7111: 712 713SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b) 714RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart) 715#endif 716 717.endm 718 719interrupt_return_macro srr 720#ifdef CONFIG_PPC_BOOK3S 721interrupt_return_macro hsrr 722 723 .globl __end_soft_masked 724__end_soft_masked: 725DEFINE_FIXED_SYMBOL(__end_soft_masked, text) 726#endif /* CONFIG_PPC_BOOK3S */ 727 728#ifdef CONFIG_PPC_BOOK3S 729_GLOBAL(ret_from_fork_scv) 730 bl CFUNC(schedule_tail) 731 HANDLER_RESTORE_NVGPRS() 732 li r3,0 /* fork() return value */ 733 b .Lsyscall_vectored_common_exit 734#endif 735 736_GLOBAL(ret_from_fork) 737 bl CFUNC(schedule_tail) 738 HANDLER_RESTORE_NVGPRS() 739 li r3,0 /* fork() return value */ 740 b .Lsyscall_exit 741 742_GLOBAL(ret_from_kernel_user_thread) 743 bl CFUNC(schedule_tail) 744 mtctr r14 745 mr r3,r15 746#ifdef CONFIG_PPC64_ELF_ABI_V2 747 mr r12,r14 748#endif 749 bctrl 750 li r3,0 751 /* 752 * It does not matter whether this returns via the scv or sc path 753 * because it returns as execve() and therefore has no calling ABI 754 * (i.e., it sets registers according to the exec()ed entry point). 755 */ 756 b .Lsyscall_exit 757 758_GLOBAL(start_kernel_thread) 759 bl CFUNC(schedule_tail) 760 mtctr r14 761 mr r3,r15 762#ifdef CONFIG_PPC64_ELF_ABI_V2 763 mr r12,r14 764#endif 765 bctrl 766 /* 767 * This must not return. We actually want to BUG here, not WARN, 768 * because BUG will exit the process which is what the kernel thread 769 * should have done, which may give some hope of continuing. 770 */ 771100: trap 772 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 773