1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 167230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 178aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1846f52210SStephen Rothwell#include <asm/ptrace.h> 197cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 20da2bc464SMichael Ellerman#include <asm/head-64.h> 218aa34ab8SBenjamin Herrenschmidt 220ebc4cdaSBenjamin Herrenschmidt/* 2357f26649SNicholas Piggin * There are a few constraints to be concerned with. 2457f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 2557f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 2657f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 2757f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 2857f26649SNicholas Piggin * must be used. 2957f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 3057f26649SNicholas Piggin * virtual 0xc00... 3157f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 3257f26649SNicholas Piggin * 3357f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 3457f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 3557f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 3657f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 3757f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 3857f26649SNicholas Piggin * 3957f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 4057f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 4157f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 4257f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 4357f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 4457f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 4557f26649SNicholas Piggin * 4657f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 4757f26649SNicholas Piggin * 4857f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 4957f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 5057f26649SNicholas Piggin * 5157f26649SNicholas Piggin * 520ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 530ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 5457f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 5557f26649SNicholas Piggin * 0x1900 - 0x3fff : Real mode trampolines 5657f26649SNicholas Piggin * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 5757f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 580ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 5957f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 6057f26649SNicholas Piggin * setup code, rest of kernel. 61e0319829SNicholas Piggin * 62e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 63e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 64e0319829SNicholas Piggin * vectors there. 650ebc4cdaSBenjamin Herrenschmidt */ 6657f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 6757f26649SNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000) 6857f26649SNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900) 6957f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 7057f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 7157f26649SNicholas Piggin/* 7257f26649SNicholas Piggin * Data area reserved for FWNMI option. 7357f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 7457f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 7557f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 7657f26649SNicholas Piggin */ 7757f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 7857f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 7957f26649SNicholas Piggin#else 8057f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 8157f26649SNicholas Piggin#endif 8257f26649SNicholas Piggin 8357f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 8457f26649SNicholas Piggin 850ebc4cdaSBenjamin Herrenschmidt/* 860ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 870ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 880ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 890ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 900ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 910ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 920ebc4cdaSBenjamin Herrenschmidt */ 930ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 940ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 950ebc4cdaSBenjamin Herrenschmidt 96e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 971a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 98e0319829SNicholas Piggin 99fb479e44SNicholas Piggin 100fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 101fb479e44SNicholas Piggin /* 102fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 103ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 104ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 105ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 106ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 107ba6d334aSBenjamin Herrenschmidt * with mtmsr. 108fb479e44SNicholas Piggin */ 109fb479e44SNicholas Piggin#define IDLETEST(n) \ 110fb479e44SNicholas Piggin BEGIN_FTR_SECTION ; \ 111fb479e44SNicholas Piggin mfspr r10,SPRN_SRR1 ; \ 112fb479e44SNicholas Piggin rlwinm. r10,r10,47-31,30,31 ; \ 113fb479e44SNicholas Piggin beq- 1f ; \ 114fb479e44SNicholas Piggin cmpwi cr3,r10,2 ; \ 115b51351e2SNicholas Piggin BRANCH_TO_C000(r10, system_reset_idle_common) ; \ 116fb479e44SNicholas Piggin1: \ 1176de6638bSNicholas Piggin KVMTEST_PR(n) ; \ 118fb479e44SNicholas Piggin END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 119fb479e44SNicholas Piggin#else 120fb479e44SNicholas Piggin#define IDLETEST NOTEST 121fb479e44SNicholas Piggin#endif 122fb479e44SNicholas Piggin 1231a6822d1SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 124948cf67cSBenjamin Herrenschmidt SET_SCRATCH0(r13) 125c4f3b52cSNicholas Piggin /* 126c4f3b52cSNicholas Piggin * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 127c4f3b52cSNicholas Piggin * being used, so a nested NMI exception would corrupt it. 128c4f3b52cSNicholas Piggin */ 129c4f3b52cSNicholas Piggin EXCEPTION_PROLOG_PSERIES_NORI(PACA_EXNMI, system_reset_common, EXC_STD, 130fb479e44SNicholas Piggin IDLETEST, 0x100) 131371fefd6SPaul Mackerras 1321a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 1331a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 1346de6638bSNicholas PigginTRAMP_KVM(PACA_EXNMI, 0x100) 135fb479e44SNicholas Piggin 136fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 137fb479e44SNicholas PigginEXC_COMMON_BEGIN(system_reset_idle_common) 1389d292501SNicholas Piggin mfspr r12,SPRN_SRR1 139bf0153c1SNicholas Piggin b pnv_powersave_wakeup 140371fefd6SPaul Mackerras#endif 141371fefd6SPaul Mackerras 14215b4dd79SNicholas Piggin/* 14315b4dd79SNicholas Piggin * Set IRQS_ALL_DISABLED unconditionally so arch_irqs_disabled does 14415b4dd79SNicholas Piggin * the right thing. We do not want to reconcile because that goes 14515b4dd79SNicholas Piggin * through irq tracing which we don't want in NMI. 14615b4dd79SNicholas Piggin * 14715b4dd79SNicholas Piggin * Save PACAIRQHAPPENED because some code will do a hard disable 14815b4dd79SNicholas Piggin * (e.g., xmon). So we want to restore this back to where it was 14915b4dd79SNicholas Piggin * when we return. DAR is unused in the stack, so save it there. 15015b4dd79SNicholas Piggin */ 15115b4dd79SNicholas Piggin#define ADD_RECONCILE_NMI \ 15215b4dd79SNicholas Piggin li r10,IRQS_ALL_DISABLED; \ 15315b4dd79SNicholas Piggin stb r10,PACAIRQSOFTMASK(r13); \ 15415b4dd79SNicholas Piggin lbz r10,PACAIRQHAPPENED(r13); \ 15515b4dd79SNicholas Piggin std r10,_DAR(r1) 15615b4dd79SNicholas Piggin 157a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 158c4f3b52cSNicholas Piggin /* 159c4f3b52cSNicholas Piggin * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 160c4f3b52cSNicholas Piggin * to recover, but nested NMI will notice in_nmi and not recover 161c4f3b52cSNicholas Piggin * because of the use of the NMI stack. in_nmi reentrancy is tested in 162c4f3b52cSNicholas Piggin * system_reset_exception. 163c4f3b52cSNicholas Piggin */ 164c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 165c4f3b52cSNicholas Piggin addi r10,r10,1 166c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 167c4f3b52cSNicholas Piggin li r10,MSR_RI 168c4f3b52cSNicholas Piggin mtmsrd r10,1 169aca79d2bSVaidyanathan Srinivasan 170b1ee8a3dSNicholas Piggin mr r10,r1 171b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 172b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 173b1ee8a3dSNicholas Piggin EXCEPTION_COMMON_NORET_STACK(PACA_EXNMI, 0x100, 174b1ee8a3dSNicholas Piggin system_reset, system_reset_exception, 17515b4dd79SNicholas Piggin ADD_NVGPRS;ADD_RECONCILE_NMI) 17615b4dd79SNicholas Piggin 17715b4dd79SNicholas Piggin /* This (and MCE) can be simplified with mtmsrd L=1 */ 17815b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 17915b4dd79SNicholas Piggin li r0,MSR_RI 18015b4dd79SNicholas Piggin mfmsr r9 18115b4dd79SNicholas Piggin andc r9,r9,r0 18215b4dd79SNicholas Piggin mtmsrd r9,1 183c4f3b52cSNicholas Piggin 184c4f3b52cSNicholas Piggin /* 18515b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 186c4f3b52cSNicholas Piggin */ 187c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 188c4f3b52cSNicholas Piggin subi r10,r10,1 189c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 190c4f3b52cSNicholas Piggin 19115b4dd79SNicholas Piggin /* 19215b4dd79SNicholas Piggin * Restore soft mask settings. 19315b4dd79SNicholas Piggin */ 19415b4dd79SNicholas Piggin ld r10,_DAR(r1) 19515b4dd79SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 19615b4dd79SNicholas Piggin ld r10,SOFTE(r1) 19715b4dd79SNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 19815b4dd79SNicholas Piggin 19915b4dd79SNicholas Piggin /* 20015b4dd79SNicholas Piggin * Keep below code in synch with MACHINE_CHECK_HANDLER_WINDUP. 20115b4dd79SNicholas Piggin * Should share common bits... 20215b4dd79SNicholas Piggin */ 20315b4dd79SNicholas Piggin 20415b4dd79SNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 20515b4dd79SNicholas Piggin ld r9,_MSR(r1) 20615b4dd79SNicholas Piggin mtspr SPRN_SRR1,r9 20715b4dd79SNicholas Piggin ld r3,_NIP(r1) 20815b4dd79SNicholas Piggin mtspr SPRN_SRR0,r3 20915b4dd79SNicholas Piggin ld r9,_CTR(r1) 21015b4dd79SNicholas Piggin mtctr r9 21115b4dd79SNicholas Piggin ld r9,_XER(r1) 21215b4dd79SNicholas Piggin mtxer r9 21315b4dd79SNicholas Piggin ld r9,_LINK(r1) 21415b4dd79SNicholas Piggin mtlr r9 21515b4dd79SNicholas Piggin REST_GPR(0, r1) 21615b4dd79SNicholas Piggin REST_8GPRS(2, r1) 21715b4dd79SNicholas Piggin REST_GPR(10, r1) 21815b4dd79SNicholas Piggin ld r11,_CCR(r1) 21915b4dd79SNicholas Piggin mtcr r11 22015b4dd79SNicholas Piggin REST_GPR(11, r1) 22115b4dd79SNicholas Piggin REST_2GPRS(12, r1) 22215b4dd79SNicholas Piggin /* restore original r1. */ 22315b4dd79SNicholas Piggin ld r1,GPR1(r1) 22415b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 225582baf44SNicholas Piggin 226582baf44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 227582baf44SNicholas Piggin/* 228582baf44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 229582baf44SNicholas Piggin */ 230582baf44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 231582baf44SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 232c4f3b52cSNicholas Piggin /* See comment at system_reset exception */ 233c4f3b52cSNicholas Piggin EXCEPTION_PROLOG_PSERIES_NORI(PACA_EXNMI, system_reset_common, 234c4f3b52cSNicholas Piggin EXC_STD, NOTEST, 0x100) 235582baf44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 236582baf44SNicholas Piggin 2370ebc4cdaSBenjamin Herrenschmidt 2381a6822d1SNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 239b01c8b54SPaul Mackerras /* This is moved out of line as it can be patched by FW, but 240b01c8b54SPaul Mackerras * some code path might still want to branch into the original 241b01c8b54SPaul Mackerras * vector 242b01c8b54SPaul Mackerras */ 2431707dd16SPaul Mackerras SET_SCRATCH0(r13) /* save r13 */ 2441707dd16SPaul Mackerras EXCEPTION_PROLOG_0(PACA_EXMC) 2451e9b4507SMahesh SalgaonkarBEGIN_FTR_SECTION 2462513767dSMahesh Salgaonkar b machine_check_powernv_early 2471e9b4507SMahesh SalgaonkarFTR_SECTION_ELSE 2481707dd16SPaul Mackerras b machine_check_pSeries_0 2491e9b4507SMahesh SalgaonkarALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) 2501a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 2511a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 252afcf0095SNicholas PigginTRAMP_REAL_BEGIN(machine_check_powernv_early) 253afcf0095SNicholas PigginBEGIN_FTR_SECTION 254afcf0095SNicholas Piggin EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200) 255afcf0095SNicholas Piggin /* 256afcf0095SNicholas Piggin * Register contents: 257afcf0095SNicholas Piggin * R13 = PACA 258afcf0095SNicholas Piggin * R9 = CR 259afcf0095SNicholas Piggin * Original R9 to R13 is saved on PACA_EXMC 260afcf0095SNicholas Piggin * 261afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 262afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 263afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 264afcf0095SNicholas Piggin * 265afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 266afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 267afcf0095SNicholas Piggin * machine checks. 268afcf0095SNicholas Piggin * 269afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 270afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 271afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 272afcf0095SNicholas Piggin * 273afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 274afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 275afcf0095SNicholas Piggin * rfid with MSR_ME=1. 2761945bc45SNicholas Piggin * 2771945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 2781945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 279*2bf1071aSNicholas Piggin * to restore state. 280afcf0095SNicholas Piggin */ 281afcf0095SNicholas Piggin mr r11,r1 /* Save r1 */ 282afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 283afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 284afcf0095SNicholas Piggin bne 0f /* Yes, we are. */ 285afcf0095SNicholas Piggin /* First machine check entry */ 286afcf0095SNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 287afcf0095SNicholas Piggin0: subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 288afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 289afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 290afcf0095SNicholas Piggin /* Limit nested MCE to level 4 to avoid stack overflow */ 291ba41e1e1SBalbir Singh cmpwi r10,MAX_MCE_DEPTH 292afcf0095SNicholas Piggin bgt 2f /* Check if we hit limit of 4 */ 293afcf0095SNicholas Piggin std r11,GPR1(r1) /* Save r1 on the stack. */ 294afcf0095SNicholas Piggin std r11,0(r1) /* make stack chain pointer */ 295afcf0095SNicholas Piggin mfspr r11,SPRN_SRR0 /* Save SRR0 */ 296afcf0095SNicholas Piggin std r11,_NIP(r1) 297afcf0095SNicholas Piggin mfspr r11,SPRN_SRR1 /* Save SRR1 */ 298afcf0095SNicholas Piggin std r11,_MSR(r1) 299afcf0095SNicholas Piggin mfspr r11,SPRN_DAR /* Save DAR */ 300afcf0095SNicholas Piggin std r11,_DAR(r1) 301afcf0095SNicholas Piggin mfspr r11,SPRN_DSISR /* Save DSISR */ 302afcf0095SNicholas Piggin std r11,_DSISR(r1) 303afcf0095SNicholas Piggin std r9,_CCR(r1) /* Save CR in stackframe */ 304afcf0095SNicholas Piggin /* Save r9 through r13 from EXMC save area to stack frame. */ 305afcf0095SNicholas Piggin EXCEPTION_PROLOG_COMMON_2(PACA_EXMC) 306afcf0095SNicholas Piggin mfmsr r11 /* get MSR value */ 307afcf0095SNicholas Piggin ori r11,r11,MSR_ME /* turn on ME bit */ 308afcf0095SNicholas Piggin ori r11,r11,MSR_RI /* turn on RI bit */ 309afcf0095SNicholas Piggin LOAD_HANDLER(r12, machine_check_handle_early) 310afcf0095SNicholas Piggin1: mtspr SPRN_SRR0,r12 311afcf0095SNicholas Piggin mtspr SPRN_SRR1,r11 312222f20f1SNicholas Piggin RFI_TO_KERNEL 313afcf0095SNicholas Piggin b . /* prevent speculative execution */ 314afcf0095SNicholas Piggin2: 315afcf0095SNicholas Piggin /* Stack overflow. Stay on emergency stack and panic. 316afcf0095SNicholas Piggin * Keep the ME bit off while panic-ing, so that if we hit 317afcf0095SNicholas Piggin * another machine check we checkstop. 318afcf0095SNicholas Piggin */ 319afcf0095SNicholas Piggin addi r1,r1,INT_FRAME_SIZE /* go back to previous stack frame */ 320afcf0095SNicholas Piggin ld r11,PACAKMSR(r13) 321afcf0095SNicholas Piggin LOAD_HANDLER(r12, unrecover_mce) 322afcf0095SNicholas Piggin li r10,MSR_ME 323afcf0095SNicholas Piggin andc r11,r11,r10 /* Turn off MSR_ME */ 324afcf0095SNicholas Piggin b 1b 325afcf0095SNicholas Piggin b . /* prevent speculative execution */ 326afcf0095SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 327afcf0095SNicholas Piggin 328afcf0095SNicholas PigginTRAMP_REAL_BEGIN(machine_check_pSeries) 329afcf0095SNicholas Piggin .globl machine_check_fwnmi 330afcf0095SNicholas Pigginmachine_check_fwnmi: 331afcf0095SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 332afcf0095SNicholas Piggin EXCEPTION_PROLOG_0(PACA_EXMC) 333afcf0095SNicholas Pigginmachine_check_pSeries_0: 334afcf0095SNicholas Piggin EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST_PR, 0x200) 335afcf0095SNicholas Piggin /* 33683a980f7SNicholas Piggin * MSR_RI is not enabled, because PACA_EXMC is being used, so a 33783a980f7SNicholas Piggin * nested machine check corrupts it. machine_check_common enables 33883a980f7SNicholas Piggin * MSR_RI. 339afcf0095SNicholas Piggin */ 34083a980f7SNicholas Piggin EXCEPTION_PROLOG_PSERIES_1_NORI(machine_check_common, EXC_STD) 341afcf0095SNicholas Piggin 342afcf0095SNicholas PigginTRAMP_KVM_SKIP(PACA_EXMC, 0x200) 343afcf0095SNicholas Piggin 344afcf0095SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 345afcf0095SNicholas Piggin /* 346afcf0095SNicholas Piggin * Machine check is different because we use a different 347afcf0095SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 348afcf0095SNicholas Piggin */ 349afcf0095SNicholas Piggin mfspr r10,SPRN_DAR 350afcf0095SNicholas Piggin std r10,PACA_EXMC+EX_DAR(r13) 351afcf0095SNicholas Piggin mfspr r10,SPRN_DSISR 352afcf0095SNicholas Piggin stw r10,PACA_EXMC+EX_DSISR(r13) 353afcf0095SNicholas Piggin EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC) 354afcf0095SNicholas Piggin FINISH_NAP 355afcf0095SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 356afcf0095SNicholas Piggin ld r3,PACA_EXMC+EX_DAR(r13) 357afcf0095SNicholas Piggin lwz r4,PACA_EXMC+EX_DSISR(r13) 358afcf0095SNicholas Piggin /* Enable MSR_RI when finished with PACA_EXMC */ 359afcf0095SNicholas Piggin li r10,MSR_RI 360afcf0095SNicholas Piggin mtmsrd r10,1 361afcf0095SNicholas Piggin std r3,_DAR(r1) 362afcf0095SNicholas Piggin std r4,_DSISR(r1) 363afcf0095SNicholas Piggin bl save_nvgprs 364afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 365afcf0095SNicholas Piggin bl machine_check_exception 366afcf0095SNicholas Piggin b ret_from_except 367afcf0095SNicholas Piggin 368afcf0095SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 369afcf0095SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 370afcf0095SNicholas Piggin li r0,MSR_RI; \ 371afcf0095SNicholas Piggin mfmsr r9; /* get MSR value */ \ 372afcf0095SNicholas Piggin andc r9,r9,r0; \ 373afcf0095SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 374afcf0095SNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ \ 375afcf0095SNicholas Piggin ld r9,_MSR(r1); \ 376afcf0095SNicholas Piggin mtspr SPRN_SRR1,r9; \ 377afcf0095SNicholas Piggin ld r3,_NIP(r1); \ 378afcf0095SNicholas Piggin mtspr SPRN_SRR0,r3; \ 379afcf0095SNicholas Piggin ld r9,_CTR(r1); \ 380afcf0095SNicholas Piggin mtctr r9; \ 381afcf0095SNicholas Piggin ld r9,_XER(r1); \ 382afcf0095SNicholas Piggin mtxer r9; \ 383afcf0095SNicholas Piggin ld r9,_LINK(r1); \ 384afcf0095SNicholas Piggin mtlr r9; \ 385afcf0095SNicholas Piggin REST_GPR(0, r1); \ 386afcf0095SNicholas Piggin REST_8GPRS(2, r1); \ 387afcf0095SNicholas Piggin REST_GPR(10, r1); \ 388afcf0095SNicholas Piggin ld r11,_CCR(r1); \ 389afcf0095SNicholas Piggin mtcr r11; \ 390afcf0095SNicholas Piggin /* Decrement paca->in_mce. */ \ 391afcf0095SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 392afcf0095SNicholas Piggin subi r12,r12,1; \ 393afcf0095SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 394afcf0095SNicholas Piggin REST_GPR(11, r1); \ 395afcf0095SNicholas Piggin REST_2GPRS(12, r1); \ 396afcf0095SNicholas Piggin /* restore original r1. */ \ 397afcf0095SNicholas Piggin ld r1,GPR1(r1) 398afcf0095SNicholas Piggin 3991945bc45SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 4001945bc45SNicholas Piggin/* 4011945bc45SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 4021945bc45SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 4031945bc45SNicholas Piggin */ 4041945bc45SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 4051945bc45SNicholas Piggin bl machine_check_queue_event 4061945bc45SNicholas Piggin 4071945bc45SNicholas Piggin /* 4081945bc45SNicholas Piggin * We have not used any non-volatile GPRs here, and as a rule 4091945bc45SNicholas Piggin * most exception code including machine check does not. 4101945bc45SNicholas Piggin * Therefore PACA_NAPSTATELOST does not need to be set. Idle 4111945bc45SNicholas Piggin * wakeup will restore volatile registers. 4121945bc45SNicholas Piggin * 4131945bc45SNicholas Piggin * Load the original SRR1 into r3 for pnv_powersave_wakeup_mce. 4141945bc45SNicholas Piggin * 4151945bc45SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 4161945bc45SNicholas Piggin */ 4171945bc45SNicholas Piggin ld r3,_MSR(r1) 4181945bc45SNicholas Piggin 4191945bc45SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 4201945bc45SNicholas Piggin subi r11,r11,1 4211945bc45SNicholas Piggin sth r11,PACA_IN_MCE(r13) 4221945bc45SNicholas Piggin 4231945bc45SNicholas Piggin /* Turn off the RI bit because SRR1 is used by idle wakeup code. */ 4241945bc45SNicholas Piggin /* Recoverability could be improved by reducing the use of SRR1. */ 4251945bc45SNicholas Piggin li r11,0 4261945bc45SNicholas Piggin mtmsrd r11,1 4271945bc45SNicholas Piggin 4281945bc45SNicholas Piggin b pnv_powersave_wakeup_mce 4291945bc45SNicholas Piggin#endif 430afcf0095SNicholas Piggin /* 431afcf0095SNicholas Piggin * Handle machine check early in real mode. We come here with 432afcf0095SNicholas Piggin * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack. 433afcf0095SNicholas Piggin */ 434afcf0095SNicholas PigginEXC_COMMON_BEGIN(machine_check_handle_early) 435afcf0095SNicholas Piggin std r0,GPR0(r1) /* Save r0 */ 436afcf0095SNicholas Piggin EXCEPTION_PROLOG_COMMON_3(0x200) 437afcf0095SNicholas Piggin bl save_nvgprs 438afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 439afcf0095SNicholas Piggin bl machine_check_early 440afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 441afcf0095SNicholas Piggin ld r12,_MSR(r1) 4421945bc45SNicholas Piggin 443afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 444afcf0095SNicholas Piggin /* 445afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 446afcf0095SNicholas Piggin * of the following is true: 447afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 448afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 449afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 450afcf0095SNicholas Piggin * 451afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 452afcf0095SNicholas Piggin */ 4531945bc45SNicholas Piggin BEGIN_FTR_SECTION 4541945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 4556102c005SNicholas Piggin bne machine_check_idle_common 4561945bc45SNicholas Piggin END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 457afcf0095SNicholas Piggin#endif 4581945bc45SNicholas Piggin 459afcf0095SNicholas Piggin /* 460afcf0095SNicholas Piggin * Check if we are coming from hypervisor userspace. If yes then we 461afcf0095SNicholas Piggin * continue in host kernel in V mode to deliver the MC event. 462afcf0095SNicholas Piggin */ 463afcf0095SNicholas Piggin rldicl. r11,r12,4,63 /* See if MC hit while in HV mode. */ 464afcf0095SNicholas Piggin beq 5f 465afcf0095SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 466afcf0095SNicholas Piggin bne 9f /* continue in V mode if we are. */ 467afcf0095SNicholas Piggin 468afcf0095SNicholas Piggin5: 469afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 470afcf0095SNicholas Piggin /* 471afcf0095SNicholas Piggin * We are coming from kernel context. Check if we are coming from 472afcf0095SNicholas Piggin * guest. if yes, then we can continue. We will fall through 473afcf0095SNicholas Piggin * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest. 474afcf0095SNicholas Piggin */ 475afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 476afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 477afcf0095SNicholas Piggin bne 9f /* continue if we are. */ 478afcf0095SNicholas Piggin#endif 479afcf0095SNicholas Piggin /* 480afcf0095SNicholas Piggin * At this point we are not sure about what context we come from. 481afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 482afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 483afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 484afcf0095SNicholas Piggin */ 485afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 486afcf0095SNicholas Piggin bne 2f 487afcf0095SNicholas Piggin1: mfspr r11,SPRN_SRR0 488afcf0095SNicholas Piggin LOAD_HANDLER(r10,unrecover_mce) 489afcf0095SNicholas Piggin mtspr SPRN_SRR0,r10 490afcf0095SNicholas Piggin ld r10,PACAKMSR(r13) 491afcf0095SNicholas Piggin /* 492afcf0095SNicholas Piggin * We are going down. But there are chances that we might get hit by 493afcf0095SNicholas Piggin * another MCE during panic path and we may run into unstable state 494afcf0095SNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 495afcf0095SNicholas Piggin * when another MCE is hit during panic path, system will checkstop 496afcf0095SNicholas Piggin * and hypervisor will get restarted cleanly by SP. 497afcf0095SNicholas Piggin */ 498afcf0095SNicholas Piggin li r3,MSR_ME 499afcf0095SNicholas Piggin andc r10,r10,r3 /* Turn off MSR_ME */ 500afcf0095SNicholas Piggin mtspr SPRN_SRR1,r10 501222f20f1SNicholas Piggin RFI_TO_KERNEL 502afcf0095SNicholas Piggin b . 503afcf0095SNicholas Piggin2: 504afcf0095SNicholas Piggin /* 505afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 506afcf0095SNicholas Piggin * then stay on emergency stack and panic. 507afcf0095SNicholas Piggin */ 508afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 509afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 510afcf0095SNicholas Piggin 511afcf0095SNicholas Piggin beq 1b /* if !handled then panic */ 512afcf0095SNicholas Piggin /* 513afcf0095SNicholas Piggin * Return from MC interrupt. 514afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 515afcf0095SNicholas Piggin * returning from kernel or opal call. 516afcf0095SNicholas Piggin */ 517afcf0095SNicholas Piggin bl machine_check_queue_event 518afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 519222f20f1SNicholas Piggin RFI_TO_USER_OR_KERNEL 520afcf0095SNicholas Piggin9: 521afcf0095SNicholas Piggin /* Deliver the machine check to host kernel in V mode. */ 522afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 523afcf0095SNicholas Piggin b machine_check_pSeries 524afcf0095SNicholas Piggin 525afcf0095SNicholas PigginEXC_COMMON_BEGIN(unrecover_mce) 526afcf0095SNicholas Piggin /* Invoke machine_check_exception to print MCE event and panic. */ 527afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 528afcf0095SNicholas Piggin bl machine_check_exception 529afcf0095SNicholas Piggin /* 530afcf0095SNicholas Piggin * We will not reach here. Even if we did, there is no way out. Call 531afcf0095SNicholas Piggin * unrecoverable_exception and die. 532afcf0095SNicholas Piggin */ 533afcf0095SNicholas Piggin1: addi r3,r1,STACK_FRAME_OVERHEAD 534afcf0095SNicholas Piggin bl unrecoverable_exception 535afcf0095SNicholas Piggin b 1b 536afcf0095SNicholas Piggin 5370ebc4cdaSBenjamin Herrenschmidt 5381a6822d1SNicholas PigginEXC_REAL(data_access, 0x300, 0x80) 5391a6822d1SNicholas PigginEXC_VIRT(data_access, 0x4300, 0x80, 0x300) 54080795e6cSNicholas PigginTRAMP_KVM_SKIP(PACA_EXGEN, 0x300) 54180795e6cSNicholas Piggin 54280795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 54380795e6cSNicholas Piggin /* 54480795e6cSNicholas Piggin * Here r13 points to the paca, r9 contains the saved CR, 54580795e6cSNicholas Piggin * SRR0 and SRR1 are saved in r11 and r12, 54680795e6cSNicholas Piggin * r9 - r13 are saved in paca->exgen. 54780795e6cSNicholas Piggin */ 54880795e6cSNicholas Piggin mfspr r10,SPRN_DAR 54980795e6cSNicholas Piggin std r10,PACA_EXGEN+EX_DAR(r13) 55080795e6cSNicholas Piggin mfspr r10,SPRN_DSISR 55180795e6cSNicholas Piggin stw r10,PACA_EXGEN+EX_DSISR(r13) 55280795e6cSNicholas Piggin EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN) 55380795e6cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 55480795e6cSNicholas Piggin ld r12,_MSR(r1) 55580795e6cSNicholas Piggin ld r3,PACA_EXGEN+EX_DAR(r13) 55680795e6cSNicholas Piggin lwz r4,PACA_EXGEN+EX_DSISR(r13) 55780795e6cSNicholas Piggin li r5,0x300 55880795e6cSNicholas Piggin std r3,_DAR(r1) 55980795e6cSNicholas Piggin std r4,_DSISR(r1) 56080795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 56180795e6cSNicholas Piggin b do_hash_page /* Try to handle as hpte fault */ 56280795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 56380795e6cSNicholas Piggin b handle_page_fault 56480795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 56580795e6cSNicholas Piggin 5660ebc4cdaSBenjamin Herrenschmidt 5671a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 568673b189aSPaul Mackerras SET_SCRATCH0(r13) 5691707dd16SPaul Mackerras EXCEPTION_PROLOG_0(PACA_EXSLB) 570da2bc464SMichael Ellerman EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380) 5714d7cd3b9SNicholas Piggin mr r12,r3 /* save r3 */ 5720ebc4cdaSBenjamin Herrenschmidt mfspr r3,SPRN_DAR 5734d7cd3b9SNicholas Piggin mfspr r11,SPRN_SRR1 574f0f558b1SPaul Mackerras crset 4*cr6+eq 575442b6e8eSMichael Ellerman BRANCH_TO_COMMON(r10, slb_miss_common) 5761a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 5770ebc4cdaSBenjamin Herrenschmidt 5781a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 5792b9af6e4SNicholas Piggin SET_SCRATCH0(r13) 5802b9af6e4SNicholas Piggin EXCEPTION_PROLOG_0(PACA_EXSLB) 5812b9af6e4SNicholas Piggin EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380) 5824d7cd3b9SNicholas Piggin mr r12,r3 /* save r3 */ 5832b9af6e4SNicholas Piggin mfspr r3,SPRN_DAR 5844d7cd3b9SNicholas Piggin mfspr r11,SPRN_SRR1 5852b9af6e4SNicholas Piggin crset 4*cr6+eq 586442b6e8eSMichael Ellerman BRANCH_TO_COMMON(r10, slb_miss_common) 5871a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 5882b9af6e4SNicholas PigginTRAMP_KVM_SKIP(PACA_EXSLB, 0x380) 5892b9af6e4SNicholas Piggin 5902b9af6e4SNicholas Piggin 5911a6822d1SNicholas PigginEXC_REAL(instruction_access, 0x400, 0x80) 5921a6822d1SNicholas PigginEXC_VIRT(instruction_access, 0x4400, 0x80, 0x400) 59327ce77dfSNicholas PigginTRAMP_KVM(PACA_EXGEN, 0x400) 59427ce77dfSNicholas Piggin 59527ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 59627ce77dfSNicholas Piggin EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN) 59727ce77dfSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 59827ce77dfSNicholas Piggin ld r12,_MSR(r1) 59927ce77dfSNicholas Piggin ld r3,_NIP(r1) 600475b581fSMichael Ellerman andis. r4,r12,DSISR_SRR1_MATCH_64S@h 60127ce77dfSNicholas Piggin li r5,0x400 60227ce77dfSNicholas Piggin std r3,_DAR(r1) 60327ce77dfSNicholas Piggin std r4,_DSISR(r1) 60427ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 60527ce77dfSNicholas Piggin b do_hash_page /* Try to handle as hpte fault */ 60627ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 60727ce77dfSNicholas Piggin b handle_page_fault 60827ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 60927ce77dfSNicholas Piggin 6100ebc4cdaSBenjamin Herrenschmidt 6111a6822d1SNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 612673b189aSPaul Mackerras SET_SCRATCH0(r13) 6131707dd16SPaul Mackerras EXCEPTION_PROLOG_0(PACA_EXSLB) 614da2bc464SMichael Ellerman EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480) 6154d7cd3b9SNicholas Piggin mr r12,r3 /* save r3 */ 6160ebc4cdaSBenjamin Herrenschmidt mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ 6174d7cd3b9SNicholas Piggin mfspr r11,SPRN_SRR1 618f0f558b1SPaul Mackerras crclr 4*cr6+eq 619442b6e8eSMichael Ellerman BRANCH_TO_COMMON(r10, slb_miss_common) 6201a6822d1SNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 6210ebc4cdaSBenjamin Herrenschmidt 6221a6822d1SNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 6238d04631aSNicholas Piggin SET_SCRATCH0(r13) 6248d04631aSNicholas Piggin EXCEPTION_PROLOG_0(PACA_EXSLB) 6258d04631aSNicholas Piggin EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480) 6264d7cd3b9SNicholas Piggin mr r12,r3 /* save r3 */ 6278d04631aSNicholas Piggin mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ 6284d7cd3b9SNicholas Piggin mfspr r11,SPRN_SRR1 6298d04631aSNicholas Piggin crclr 4*cr6+eq 630442b6e8eSMichael Ellerman BRANCH_TO_COMMON(r10, slb_miss_common) 6311a6822d1SNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 6328d04631aSNicholas PigginTRAMP_KVM(PACA_EXSLB, 0x480) 6338d04631aSNicholas Piggin 6348d04631aSNicholas Piggin 635442b6e8eSMichael Ellerman/* 636442b6e8eSMichael Ellerman * This handler is used by the 0x380 and 0x480 SLB miss interrupts, as well as 637442b6e8eSMichael Ellerman * the virtual mode 0x4380 and 0x4480 interrupts if AIL is enabled. 638442b6e8eSMichael Ellerman */ 639442b6e8eSMichael EllermanEXC_COMMON_BEGIN(slb_miss_common) 6408d04631aSNicholas Piggin /* 6418d04631aSNicholas Piggin * r13 points to the PACA, r9 contains the saved CR, 6424d7cd3b9SNicholas Piggin * r12 contains the saved r3, 6434d7cd3b9SNicholas Piggin * r11 contain the saved SRR1, SRR0 is still ready for return 6448d04631aSNicholas Piggin * r3 has the faulting address 6458d04631aSNicholas Piggin * r9 - r13 are saved in paca->exslb. 6468d04631aSNicholas Piggin * cr6.eq is set for a D-SLB miss, clear for a I-SLB miss 6478d04631aSNicholas Piggin * We assume we aren't going to take any exceptions during this 6488d04631aSNicholas Piggin * procedure. 6498d04631aSNicholas Piggin */ 6508d04631aSNicholas Piggin mflr r10 6518d04631aSNicholas Piggin stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */ 6528d04631aSNicholas Piggin std r10,PACA_EXSLB+EX_LR(r13) /* save LR */ 6538d04631aSNicholas Piggin 654c7305645SNicholas Piggin andi. r9,r11,MSR_PR // Check for exception from userspace 655c7305645SNicholas Piggin cmpdi cr4,r9,MSR_PR // And save the result in CR4 for later 656c7305645SNicholas Piggin 6574d7cd3b9SNicholas Piggin /* 6584d7cd3b9SNicholas Piggin * Test MSR_RI before calling slb_allocate_realmode, because the 6594d7cd3b9SNicholas Piggin * MSR in r11 gets clobbered. However we still want to allocate 6604d7cd3b9SNicholas Piggin * SLB in case MSR_RI=0, to minimise the risk of getting stuck in 6614d7cd3b9SNicholas Piggin * recursive SLB faults. So use cr5 for this, which is preserved. 6624d7cd3b9SNicholas Piggin */ 6634d7cd3b9SNicholas Piggin andi. r11,r11,MSR_RI /* check for unrecoverable exception */ 6644d7cd3b9SNicholas Piggin cmpdi cr5,r11,MSR_RI 6654d7cd3b9SNicholas Piggin 6668d04631aSNicholas Piggin crset 4*cr0+eq 6674e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 6688d04631aSNicholas PigginBEGIN_MMU_FTR_SECTION 669fd88b945SMichael Ellerman bl slb_allocate 6708d04631aSNicholas PigginEND_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) 6718d04631aSNicholas Piggin#endif 6728d04631aSNicholas Piggin 6738d04631aSNicholas Piggin ld r10,PACA_EXSLB+EX_LR(r13) 6748d04631aSNicholas Piggin lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */ 6758d04631aSNicholas Piggin mtlr r10 6768d04631aSNicholas Piggin 677f384796cSAneesh Kumar K.V /* 678f384796cSAneesh Kumar K.V * Large address, check whether we have to allocate new contexts. 679f384796cSAneesh Kumar K.V */ 680f384796cSAneesh Kumar K.V beq- 8f 6818d04631aSNicholas Piggin 6824d7cd3b9SNicholas Piggin bne- cr5,2f /* if unrecoverable exception, oops */ 6838d04631aSNicholas Piggin 6848d04631aSNicholas Piggin /* All done -- return from exception. */ 6858d04631aSNicholas Piggin 686c7305645SNicholas Piggin bne cr4,1f /* returning to kernel */ 687c7305645SNicholas Piggin 6888d04631aSNicholas Piggin mtcrf 0x80,r9 689c7305645SNicholas Piggin mtcrf 0x08,r9 /* MSR[PR] indication is in cr4 */ 6904d7cd3b9SNicholas Piggin mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */ 6918d04631aSNicholas Piggin mtcrf 0x02,r9 /* I/D indication is in cr6 */ 6928d04631aSNicholas Piggin mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */ 6938d04631aSNicholas Piggin 694fe5482c0SNicholas Piggin RESTORE_CTR(r9, PACA_EXSLB) 6958d04631aSNicholas Piggin RESTORE_PPR_PACA(PACA_EXSLB, r9) 6964d7cd3b9SNicholas Piggin mr r3,r12 6978d04631aSNicholas Piggin ld r9,PACA_EXSLB+EX_R9(r13) 6988d04631aSNicholas Piggin ld r10,PACA_EXSLB+EX_R10(r13) 6998d04631aSNicholas Piggin ld r11,PACA_EXSLB+EX_R11(r13) 7008d04631aSNicholas Piggin ld r12,PACA_EXSLB+EX_R12(r13) 7018d04631aSNicholas Piggin ld r13,PACA_EXSLB+EX_R13(r13) 702c7305645SNicholas Piggin RFI_TO_USER 7038d04631aSNicholas Piggin b . /* prevent speculative execution */ 704c7305645SNicholas Piggin1: 705c7305645SNicholas Piggin mtcrf 0x80,r9 706c7305645SNicholas Piggin mtcrf 0x08,r9 /* MSR[PR] indication is in cr4 */ 707c7305645SNicholas Piggin mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */ 708c7305645SNicholas Piggin mtcrf 0x02,r9 /* I/D indication is in cr6 */ 709c7305645SNicholas Piggin mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */ 710c7305645SNicholas Piggin 711c7305645SNicholas Piggin RESTORE_CTR(r9, PACA_EXSLB) 712c7305645SNicholas Piggin RESTORE_PPR_PACA(PACA_EXSLB, r9) 713c7305645SNicholas Piggin mr r3,r12 714c7305645SNicholas Piggin ld r9,PACA_EXSLB+EX_R9(r13) 715c7305645SNicholas Piggin ld r10,PACA_EXSLB+EX_R10(r13) 716c7305645SNicholas Piggin ld r11,PACA_EXSLB+EX_R11(r13) 717c7305645SNicholas Piggin ld r12,PACA_EXSLB+EX_R12(r13) 718c7305645SNicholas Piggin ld r13,PACA_EXSLB+EX_R13(r13) 719c7305645SNicholas Piggin RFI_TO_KERNEL 720c7305645SNicholas Piggin b . /* prevent speculative execution */ 721c7305645SNicholas Piggin 7228d04631aSNicholas Piggin 7237c28f048SNicholas Piggin2: std r3,PACA_EXSLB+EX_DAR(r13) 7244d7cd3b9SNicholas Piggin mr r3,r12 7257c28f048SNicholas Piggin mfspr r11,SPRN_SRR0 7264d7cd3b9SNicholas Piggin mfspr r12,SPRN_SRR1 7278d04631aSNicholas Piggin LOAD_HANDLER(r10,unrecov_slb) 7288d04631aSNicholas Piggin mtspr SPRN_SRR0,r10 7298d04631aSNicholas Piggin ld r10,PACAKMSR(r13) 7308d04631aSNicholas Piggin mtspr SPRN_SRR1,r10 731222f20f1SNicholas Piggin RFI_TO_KERNEL 7328d04631aSNicholas Piggin b . 7338d04631aSNicholas Piggin 7347c28f048SNicholas Piggin8: std r3,PACA_EXSLB+EX_DAR(r13) 7354d7cd3b9SNicholas Piggin mr r3,r12 7367c28f048SNicholas Piggin mfspr r11,SPRN_SRR0 7374d7cd3b9SNicholas Piggin mfspr r12,SPRN_SRR1 738f384796cSAneesh Kumar K.V LOAD_HANDLER(r10, large_addr_slb) 7398d04631aSNicholas Piggin mtspr SPRN_SRR0,r10 7408d04631aSNicholas Piggin ld r10,PACAKMSR(r13) 7418d04631aSNicholas Piggin mtspr SPRN_SRR1,r10 742222f20f1SNicholas Piggin RFI_TO_KERNEL 7438d04631aSNicholas Piggin b . 7448d04631aSNicholas Piggin 7458d04631aSNicholas PigginEXC_COMMON_BEGIN(unrecov_slb) 7468d04631aSNicholas Piggin EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB) 7478d04631aSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 7488d04631aSNicholas Piggin bl save_nvgprs 7498d04631aSNicholas Piggin1: addi r3,r1,STACK_FRAME_OVERHEAD 7508d04631aSNicholas Piggin bl unrecoverable_exception 7518d04631aSNicholas Piggin b 1b 7528d04631aSNicholas Piggin 753f384796cSAneesh Kumar K.VEXC_COMMON_BEGIN(large_addr_slb) 7548d04631aSNicholas Piggin EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB) 7558d04631aSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 7568d04631aSNicholas Piggin ld r3, PACA_EXSLB+EX_DAR(r13) 7578d04631aSNicholas Piggin std r3, _DAR(r1) 7588d04631aSNicholas Piggin beq cr6, 2f 75952396500SNicholas Piggin li r10, 0x481 /* fix trap number for I-SLB miss */ 7608d04631aSNicholas Piggin std r10, _TRAP(r1) 7618d04631aSNicholas Piggin2: bl save_nvgprs 7628d04631aSNicholas Piggin addi r3, r1, STACK_FRAME_OVERHEAD 763f384796cSAneesh Kumar K.V bl slb_miss_large_addr 7648d04631aSNicholas Piggin b ret_from_except 7658d04631aSNicholas Piggin 7661a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 767b3e6b5dfSBenjamin Herrenschmidt .globl hardware_interrupt_hv; 768b3e6b5dfSBenjamin Herrenschmidthardware_interrupt_hv: 769a5d4f3adSBenjamin Herrenschmidt BEGIN_FTR_SECTION 770da2bc464SMichael Ellerman _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, 771f14e953bSMadhavan Srinivasan EXC_HV, SOFTEN_TEST_HV, 772f14e953bSMadhavan Srinivasan IRQS_DISABLED) 773de56a948SPaul Mackerras FTR_SECTION_ELSE 774da2bc464SMichael Ellerman _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, 775f14e953bSMadhavan Srinivasan EXC_STD, SOFTEN_TEST_PR, 776f14e953bSMadhavan Srinivasan IRQS_DISABLED) 777969391c5SPaul Mackerras ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 7781a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 779a5d4f3adSBenjamin Herrenschmidt 7801a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 781c138e588SNicholas Piggin .globl hardware_interrupt_relon_hv; 782c138e588SNicholas Pigginhardware_interrupt_relon_hv: 783c138e588SNicholas Piggin BEGIN_FTR_SECTION 784f14e953bSMadhavan Srinivasan _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, 785f14e953bSMadhavan Srinivasan EXC_HV, SOFTEN_TEST_HV, 786f14e953bSMadhavan Srinivasan IRQS_DISABLED) 787c138e588SNicholas Piggin FTR_SECTION_ELSE 788f14e953bSMadhavan Srinivasan _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, 789f14e953bSMadhavan Srinivasan EXC_STD, SOFTEN_TEST_PR, 790f14e953bSMadhavan Srinivasan IRQS_DISABLED) 791c138e588SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) 7921a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 793c138e588SNicholas Piggin 7947ede5317SNicholas PigginTRAMP_KVM(PACA_EXGEN, 0x500) 7957ede5317SNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0x500) 796c138e588SNicholas PigginEXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) 797c138e588SNicholas Piggin 798c138e588SNicholas Piggin 7991a6822d1SNicholas PigginEXC_REAL(alignment, 0x600, 0x100) 8001a6822d1SNicholas PigginEXC_VIRT(alignment, 0x4600, 0x100, 0x600) 801da2bc464SMichael EllermanTRAMP_KVM(PACA_EXGEN, 0x600) 802f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 803f9aa6714SNicholas Piggin mfspr r10,SPRN_DAR 804f9aa6714SNicholas Piggin std r10,PACA_EXGEN+EX_DAR(r13) 805f9aa6714SNicholas Piggin mfspr r10,SPRN_DSISR 806f9aa6714SNicholas Piggin stw r10,PACA_EXGEN+EX_DSISR(r13) 807f9aa6714SNicholas Piggin EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN) 808f9aa6714SNicholas Piggin ld r3,PACA_EXGEN+EX_DAR(r13) 809f9aa6714SNicholas Piggin lwz r4,PACA_EXGEN+EX_DSISR(r13) 810f9aa6714SNicholas Piggin std r3,_DAR(r1) 811f9aa6714SNicholas Piggin std r4,_DSISR(r1) 812f9aa6714SNicholas Piggin bl save_nvgprs 813f9aa6714SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 814f9aa6714SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 815f9aa6714SNicholas Piggin bl alignment_exception 816f9aa6714SNicholas Piggin b ret_from_except 817f9aa6714SNicholas Piggin 818b01c8b54SPaul Mackerras 8191a6822d1SNicholas PigginEXC_REAL(program_check, 0x700, 0x100) 8201a6822d1SNicholas PigginEXC_VIRT(program_check, 0x4700, 0x100, 0x700) 821da2bc464SMichael EllermanTRAMP_KVM(PACA_EXGEN, 0x700) 82211e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 823265e60a1SCyril Bur /* 824265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 825265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 826265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 827265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 828265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 829265e60a1SCyril Bur * the kernel. 830265e60a1SCyril Bur */ 831265e60a1SCyril Bur li r10,MSR_PR /* Build a mask of MSR_PR .. */ 832265e60a1SCyril Bur oris r10,r10,0x200000@h /* .. and SRR1_PROGTM */ 833265e60a1SCyril Bur and r10,r10,r12 /* Mask SRR1 with that. */ 834265e60a1SCyril Bur srdi r10,r10,8 /* Shift it so we can compare */ 835265e60a1SCyril Bur cmpldi r10,(0x200000 >> 8) /* .. with an immediate. */ 836265e60a1SCyril Bur bne 1f /* If != go to normal path. */ 837265e60a1SCyril Bur 838265e60a1SCyril Bur /* SRR1 had PR=0 and SRR1_PROGTM=1, so use the emergency stack */ 839265e60a1SCyril Bur andi. r10,r12,MSR_PR; /* Set CR0 correctly for label */ 840265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 841265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 842265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 843265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 844265e60a1SCyril Bur b 3f /* Jump into the macro !! */ 845265e60a1SCyril Bur1: EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN) 84611e87346SNicholas Piggin bl save_nvgprs 84711e87346SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 84811e87346SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 84911e87346SNicholas Piggin bl program_check_exception 85011e87346SNicholas Piggin b ret_from_except 85111e87346SNicholas Piggin 852a485c709SPaul Mackerras 8531a6822d1SNicholas PigginEXC_REAL(fp_unavailable, 0x800, 0x100) 8541a6822d1SNicholas PigginEXC_VIRT(fp_unavailable, 0x4800, 0x100, 0x800) 855da2bc464SMichael EllermanTRAMP_KVM(PACA_EXGEN, 0x800) 856c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 857c78d9b97SNicholas Piggin EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN) 858c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 859c78d9b97SNicholas Piggin bl save_nvgprs 860c78d9b97SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 861c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 862c78d9b97SNicholas Piggin bl kernel_fp_unavailable_exception 863c78d9b97SNicholas Piggin BUG_OPCODE 864c78d9b97SNicholas Piggin1: 865c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 866c78d9b97SNicholas PigginBEGIN_FTR_SECTION 867c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 868c78d9b97SNicholas Piggin * transaction), go do TM stuff 869c78d9b97SNicholas Piggin */ 870c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 871c78d9b97SNicholas Piggin bne- 2f 872c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 873c78d9b97SNicholas Piggin#endif 874c78d9b97SNicholas Piggin bl load_up_fpu 875c78d9b97SNicholas Piggin b fast_exception_return 876c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 877c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 878c78d9b97SNicholas Piggin bl save_nvgprs 879c78d9b97SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 880c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 881c78d9b97SNicholas Piggin bl fp_unavailable_tm 882c78d9b97SNicholas Piggin b ret_from_except 883c78d9b97SNicholas Piggin#endif 884c78d9b97SNicholas Piggin 885b01c8b54SPaul Mackerras 886a048a07dSNicholas PigginEXC_REAL_OOL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED) 887f14e953bSMadhavan SrinivasanEXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED) 88839c0da57SNicholas PigginTRAMP_KVM(PACA_EXGEN, 0x900) 88939c0da57SNicholas PigginEXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt) 89039c0da57SNicholas Piggin 8910ebc4cdaSBenjamin Herrenschmidt 8921a6822d1SNicholas PigginEXC_REAL_HV(hdecrementer, 0x980, 0x80) 8931a6822d1SNicholas PigginEXC_VIRT_HV(hdecrementer, 0x4980, 0x80, 0x980) 894facc6d74SNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0x980) 895facc6d74SNicholas PigginEXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt) 896facc6d74SNicholas Piggin 897da2bc464SMichael Ellerman 898f14e953bSMadhavan SrinivasanEXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100, IRQS_DISABLED) 899f14e953bSMadhavan SrinivasanEXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00, IRQS_DISABLED) 900da2bc464SMichael EllermanTRAMP_KVM(PACA_EXGEN, 0xa00) 901ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 902ca243163SNicholas PigginEXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception) 903ca243163SNicholas Piggin#else 904ca243163SNicholas PigginEXC_COMMON_ASYNC(doorbell_super_common, 0xa00, unknown_exception) 905ca243163SNicholas Piggin#endif 906ca243163SNicholas Piggin 907da2bc464SMichael Ellerman 9081a6822d1SNicholas PigginEXC_REAL(trap_0b, 0xb00, 0x100) 9091a6822d1SNicholas PigginEXC_VIRT(trap_0b, 0x4b00, 0x100, 0xb00) 910da2bc464SMichael EllermanTRAMP_KVM(PACA_EXGEN, 0xb00) 911341215dcSNicholas PigginEXC_COMMON(trap_0b_common, 0xb00, unknown_exception) 912341215dcSNicholas Piggin 913acd7d8ceSNicholas Piggin/* 914acd7d8ceSNicholas Piggin * system call / hypercall (0xc00, 0x4c00) 915acd7d8ceSNicholas Piggin * 916acd7d8ceSNicholas Piggin * The system call exception is invoked with "sc 0" and does not alter HV bit. 917acd7d8ceSNicholas Piggin * There is support for kernel code to invoke system calls but there are no 918acd7d8ceSNicholas Piggin * in-tree users. 919acd7d8ceSNicholas Piggin * 920acd7d8ceSNicholas Piggin * The hypercall is invoked with "sc 1" and sets HV=1. 921acd7d8ceSNicholas Piggin * 922acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 923acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 924acd7d8ceSNicholas Piggin * 925acd7d8ceSNicholas Piggin * Call convention: 926acd7d8ceSNicholas Piggin * 927acd7d8ceSNicholas Piggin * syscall register convention is in Documentation/powerpc/syscall64-abi.txt 928acd7d8ceSNicholas Piggin * 929acd7d8ceSNicholas Piggin * For hypercalls, the register convention is as follows: 930acd7d8ceSNicholas Piggin * r0 volatile 931acd7d8ceSNicholas Piggin * r1-2 nonvolatile 932acd7d8ceSNicholas Piggin * r3 volatile parameter and return value for status 933acd7d8ceSNicholas Piggin * r4-r10 volatile input and output value 934acd7d8ceSNicholas Piggin * r11 volatile hypercall number and output value 93576fc0cfcSNicholas Piggin * r12 volatile input and output value 936acd7d8ceSNicholas Piggin * r13-r31 nonvolatile 937acd7d8ceSNicholas Piggin * LR nonvolatile 938acd7d8ceSNicholas Piggin * CTR volatile 939acd7d8ceSNicholas Piggin * XER volatile 940acd7d8ceSNicholas Piggin * CR0-1 CR5-7 volatile 941acd7d8ceSNicholas Piggin * CR2-4 nonvolatile 942acd7d8ceSNicholas Piggin * Other registers nonvolatile 943acd7d8ceSNicholas Piggin * 944acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 94576fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 94676fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 94776fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 948acd7d8ceSNicholas Piggin */ 949bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 950bc355125SPaul Mackerras /* 951acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 95276fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 95376fc0cfcSNicholas Piggin * register. 954acd7d8ceSNicholas Piggin * 955acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 956acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 957bc355125SPaul Mackerras */ 958bc355125SPaul Mackerras#define SYSCALL_KVMTEST \ 95976fc0cfcSNicholas Piggin mtctr r13; \ 960bc355125SPaul Mackerras GET_PACA(r13); \ 96176fc0cfcSNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13); \ 962a048a07dSNicholas Piggin INTERRUPT_TO_KERNEL; \ 963acd7d8ceSNicholas Piggin KVMTEST_PR(0xc00); /* uses r10, branch to do_kvm_0xc00_system_call */ \ 964bc355125SPaul Mackerras HMT_MEDIUM; \ 96576fc0cfcSNicholas Piggin mfctr r9; 966bc355125SPaul Mackerras 967bc355125SPaul Mackerras#else 968bc355125SPaul Mackerras#define SYSCALL_KVMTEST \ 969acd7d8ceSNicholas Piggin HMT_MEDIUM; \ 970acd7d8ceSNicholas Piggin mr r9,r13; \ 971a048a07dSNicholas Piggin GET_PACA(r13); \ 972a048a07dSNicholas Piggin INTERRUPT_TO_KERNEL; 973bc355125SPaul Mackerras#endif 974bc355125SPaul Mackerras 975d807ad37SNicholas Piggin#define LOAD_SYSCALL_HANDLER(reg) \ 976fb479e44SNicholas Piggin __LOAD_HANDLER(reg, system_call_common) 977d807ad37SNicholas Piggin 978acd7d8ceSNicholas Piggin/* 979acd7d8ceSNicholas Piggin * After SYSCALL_KVMTEST, we reach here with PACA in r13, r13 in r9, 980acd7d8ceSNicholas Piggin * and HMT_MEDIUM. 981acd7d8ceSNicholas Piggin */ 982acd7d8ceSNicholas Piggin#define SYSCALL_REAL \ 983acd7d8ceSNicholas Piggin mfspr r11,SPRN_SRR0 ; \ 984d807ad37SNicholas Piggin mfspr r12,SPRN_SRR1 ; \ 985d807ad37SNicholas Piggin LOAD_SYSCALL_HANDLER(r10) ; \ 986d807ad37SNicholas Piggin mtspr SPRN_SRR0,r10 ; \ 987d807ad37SNicholas Piggin ld r10,PACAKMSR(r13) ; \ 988d807ad37SNicholas Piggin mtspr SPRN_SRR1,r10 ; \ 989222f20f1SNicholas Piggin RFI_TO_KERNEL ; \ 990d807ad37SNicholas Piggin b . ; /* prevent speculative execution */ 991d807ad37SNicholas Piggin 992727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 9935c2511bfSMichael Ellerman#define SYSCALL_FASTENDIAN_TEST \ 9945c2511bfSMichael EllermanBEGIN_FTR_SECTION \ 9955c2511bfSMichael Ellerman cmpdi r0,0x1ebe ; \ 9965c2511bfSMichael Ellerman beq- 1f ; \ 9975c2511bfSMichael EllermanEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \ 9985c2511bfSMichael Ellerman 999acd7d8ceSNicholas Piggin#define SYSCALL_FASTENDIAN \ 1000d807ad37SNicholas Piggin /* Fast LE/BE switch system call */ \ 1001d807ad37SNicholas Piggin1: mfspr r12,SPRN_SRR1 ; \ 1002d807ad37SNicholas Piggin xori r12,r12,MSR_LE ; \ 1003d807ad37SNicholas Piggin mtspr SPRN_SRR1,r12 ; \ 1004acd7d8ceSNicholas Piggin mr r13,r9 ; \ 1005222f20f1SNicholas Piggin RFI_TO_USER ; /* return to userspace */ \ 1006d807ad37SNicholas Piggin b . ; /* prevent speculative execution */ 1007727f1361SMichael Ellerman#else 1008727f1361SMichael Ellerman#define SYSCALL_FASTENDIAN_TEST 1009727f1361SMichael Ellerman#define SYSCALL_FASTENDIAN 1010727f1361SMichael Ellerman#endif /* CONFIG_PPC_FAST_ENDIAN_SWITCH */ 1011d807ad37SNicholas Piggin 1012d807ad37SNicholas Piggin#if defined(CONFIG_RELOCATABLE) 1013d807ad37SNicholas Piggin /* 1014d807ad37SNicholas Piggin * We can't branch directly so we do it via the CTR which 1015d807ad37SNicholas Piggin * is volatile across system calls. 1016d807ad37SNicholas Piggin */ 1017acd7d8ceSNicholas Piggin#define SYSCALL_VIRT \ 1018acd7d8ceSNicholas Piggin LOAD_SYSCALL_HANDLER(r10) ; \ 1019acd7d8ceSNicholas Piggin mtctr r10 ; \ 1020acd7d8ceSNicholas Piggin mfspr r11,SPRN_SRR0 ; \ 1021d807ad37SNicholas Piggin mfspr r12,SPRN_SRR1 ; \ 1022d807ad37SNicholas Piggin li r10,MSR_RI ; \ 1023d807ad37SNicholas Piggin mtmsrd r10,1 ; \ 1024d807ad37SNicholas Piggin bctr ; 1025d807ad37SNicholas Piggin#else 1026d807ad37SNicholas Piggin /* We can branch directly */ 1027acd7d8ceSNicholas Piggin#define SYSCALL_VIRT \ 1028acd7d8ceSNicholas Piggin mfspr r11,SPRN_SRR0 ; \ 1029d807ad37SNicholas Piggin mfspr r12,SPRN_SRR1 ; \ 1030d807ad37SNicholas Piggin li r10,MSR_RI ; \ 1031d807ad37SNicholas Piggin mtmsrd r10,1 ; /* Set RI (EE=0) */ \ 1032d807ad37SNicholas Piggin b system_call_common ; 1033d807ad37SNicholas Piggin#endif 1034d807ad37SNicholas Piggin 10351a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 1036acd7d8ceSNicholas Piggin SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */ 1037acd7d8ceSNicholas Piggin SYSCALL_FASTENDIAN_TEST 1038acd7d8ceSNicholas Piggin SYSCALL_REAL 1039acd7d8ceSNicholas Piggin SYSCALL_FASTENDIAN 10401a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 1041b01c8b54SPaul Mackerras 10421a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 1043acd7d8ceSNicholas Piggin SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */ 1044acd7d8ceSNicholas Piggin SYSCALL_FASTENDIAN_TEST 1045acd7d8ceSNicholas Piggin SYSCALL_VIRT 1046acd7d8ceSNicholas Piggin SYSCALL_FASTENDIAN 10471a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 1048d807ad37SNicholas Piggin 1049acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1050acd7d8ceSNicholas Piggin /* 1051acd7d8ceSNicholas Piggin * This is a hcall, so register convention is as above, with these 1052acd7d8ceSNicholas Piggin * differences: 1053acd7d8ceSNicholas Piggin * r13 = PACA 105476fc0cfcSNicholas Piggin * ctr = orig r13 105576fc0cfcSNicholas Piggin * orig r10 saved in PACA 1056acd7d8ceSNicholas Piggin */ 1057acd7d8ceSNicholas PigginTRAMP_KVM_BEGIN(do_kvm_0xc00) 1058acd7d8ceSNicholas Piggin /* 1059acd7d8ceSNicholas Piggin * Save the PPR (on systems that support it) before changing to 1060acd7d8ceSNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 1061acd7d8ceSNicholas Piggin * guest state (it is the guest's PPR value). 1062acd7d8ceSNicholas Piggin */ 106376fc0cfcSNicholas Piggin OPT_GET_SPR(r10, SPRN_PPR, CPU_FTR_HAS_PPR) 1064acd7d8ceSNicholas Piggin HMT_MEDIUM 106576fc0cfcSNicholas Piggin OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r10, CPU_FTR_HAS_PPR) 1066acd7d8ceSNicholas Piggin mfctr r10 106776fc0cfcSNicholas Piggin SET_SCRATCH0(r10) 1068acd7d8ceSNicholas Piggin std r9,PACA_EXGEN+EX_R9(r13) 1069acd7d8ceSNicholas Piggin mfcr r9 1070acd7d8ceSNicholas Piggin KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00) 1071acd7d8ceSNicholas Piggin#endif 1072da2bc464SMichael Ellerman 1073d807ad37SNicholas Piggin 10741a6822d1SNicholas PigginEXC_REAL(single_step, 0xd00, 0x100) 10751a6822d1SNicholas PigginEXC_VIRT(single_step, 0x4d00, 0x100, 0xd00) 1076da2bc464SMichael EllermanTRAMP_KVM(PACA_EXGEN, 0xd00) 1077bc6675c6SNicholas PigginEXC_COMMON(single_step_common, 0xd00, single_step_exception) 1078da2bc464SMichael Ellerman 10791a6822d1SNicholas PigginEXC_REAL_OOL_HV(h_data_storage, 0xe00, 0x20) 1080da0e7e62SMichael EllermanEXC_VIRT_OOL_HV(h_data_storage, 0x4e00, 0x20, 0xe00) 1081f5c32c1dSNicholas PigginTRAMP_KVM_HV_SKIP(PACA_EXGEN, 0xe00) 1082f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 1083f5c32c1dSNicholas Piggin mfspr r10,SPRN_HDAR 1084f5c32c1dSNicholas Piggin std r10,PACA_EXGEN+EX_DAR(r13) 1085f5c32c1dSNicholas Piggin mfspr r10,SPRN_HDSISR 1086f5c32c1dSNicholas Piggin stw r10,PACA_EXGEN+EX_DSISR(r13) 1087f5c32c1dSNicholas Piggin EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN) 1088f5c32c1dSNicholas Piggin bl save_nvgprs 1089f5c32c1dSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1090f5c32c1dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1091f5c32c1dSNicholas Piggin bl unknown_exception 1092f5c32c1dSNicholas Piggin b ret_from_except 1093f5c32c1dSNicholas Piggin 10941707dd16SPaul Mackerras 10951a6822d1SNicholas PigginEXC_REAL_OOL_HV(h_instr_storage, 0xe20, 0x20) 1096da0e7e62SMichael EllermanEXC_VIRT_OOL_HV(h_instr_storage, 0x4e20, 0x20, 0xe20) 109782517cabSNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0xe20) 109882517cabSNicholas PigginEXC_COMMON(h_instr_storage_common, 0xe20, unknown_exception) 109982517cabSNicholas Piggin 11001707dd16SPaul Mackerras 11011a6822d1SNicholas PigginEXC_REAL_OOL_HV(emulation_assist, 0xe40, 0x20) 11021a6822d1SNicholas PigginEXC_VIRT_OOL_HV(emulation_assist, 0x4e40, 0x20, 0xe40) 1103031b4026SNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0xe40) 1104031b4026SNicholas PigginEXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt) 1105031b4026SNicholas Piggin 11061707dd16SPaul Mackerras 1107e0319829SNicholas Piggin/* 1108e0319829SNicholas Piggin * hmi_exception trampoline is a special case. It jumps to hmi_exception_early 1109e0319829SNicholas Piggin * first, and then eventaully from there to the trampoline to get into virtual 1110e0319829SNicholas Piggin * mode. 1111e0319829SNicholas Piggin */ 11121a6822d1SNicholas Piggin__EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0x20, hmi_exception_early) 1113f14e953bSMadhavan Srinivasan__TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) 11141a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 111562f9b03bSNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0xe60) 111662f9b03bSNicholas PigginTRAMP_REAL_BEGIN(hmi_exception_early) 111762f9b03bSNicholas Piggin EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, 0xe60) 111862f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 1119a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 112062f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 112162f9b03bSNicholas Piggin mfspr r11,SPRN_HSRR0 /* Save HSRR0 */ 1122a4087a4dSNicholas Piggin mfspr r12,SPRN_HSRR1 /* Save HSRR1 */ 1123a4087a4dSNicholas Piggin EXCEPTION_PROLOG_COMMON_1() 112462f9b03bSNicholas Piggin EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN) 112562f9b03bSNicholas Piggin EXCEPTION_PROLOG_COMMON_3(0xe60) 112662f9b03bSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1127be5c5e84SMichael Ellerman BRANCH_LINK_TO_FAR(hmi_exception_realmode) /* Function call ABI */ 11285080332cSMichael Neuling cmpdi cr0,r3,0 11295080332cSMichael Neuling 113062f9b03bSNicholas Piggin /* Windup the stack. */ 113162f9b03bSNicholas Piggin /* Move original HSRR0 and HSRR1 into the respective regs */ 113262f9b03bSNicholas Piggin ld r9,_MSR(r1) 113362f9b03bSNicholas Piggin mtspr SPRN_HSRR1,r9 113462f9b03bSNicholas Piggin ld r3,_NIP(r1) 113562f9b03bSNicholas Piggin mtspr SPRN_HSRR0,r3 113662f9b03bSNicholas Piggin ld r9,_CTR(r1) 113762f9b03bSNicholas Piggin mtctr r9 113862f9b03bSNicholas Piggin ld r9,_XER(r1) 113962f9b03bSNicholas Piggin mtxer r9 114062f9b03bSNicholas Piggin ld r9,_LINK(r1) 114162f9b03bSNicholas Piggin mtlr r9 114262f9b03bSNicholas Piggin REST_GPR(0, r1) 114362f9b03bSNicholas Piggin REST_8GPRS(2, r1) 114462f9b03bSNicholas Piggin REST_GPR(10, r1) 114562f9b03bSNicholas Piggin ld r11,_CCR(r1) 11465080332cSMichael Neuling REST_2GPRS(12, r1) 11475080332cSMichael Neuling bne 1f 114862f9b03bSNicholas Piggin mtcr r11 114962f9b03bSNicholas Piggin REST_GPR(11, r1) 11505080332cSMichael Neuling ld r1,GPR1(r1) 1151222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 11525080332cSMichael Neuling 11535080332cSMichael Neuling1: mtcr r11 11545080332cSMichael Neuling REST_GPR(11, r1) 115562f9b03bSNicholas Piggin ld r1,GPR1(r1) 115662f9b03bSNicholas Piggin 115762f9b03bSNicholas Piggin /* 115862f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 115962f9b03bSNicholas Piggin * firmware. 116062f9b03bSNicholas Piggin */ 116162f9b03bSNicholas Piggin .globl hmi_exception_after_realmode 116262f9b03bSNicholas Pigginhmi_exception_after_realmode: 116362f9b03bSNicholas Piggin SET_SCRATCH0(r13) 116462f9b03bSNicholas Piggin EXCEPTION_PROLOG_0(PACA_EXGEN) 116562f9b03bSNicholas Piggin b tramp_real_hmi_exception 116662f9b03bSNicholas Piggin 11675080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 11685080332cSMichael NeulingEXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, handle_hmi_exception, 11695080332cSMichael Neuling ret_from_except, FINISH_NAP;ADD_NVGPRS;ADD_RECONCILE;RUNLATCH_ON) 11701707dd16SPaul Mackerras 1171f14e953bSMadhavan SrinivasanEXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20, IRQS_DISABLED) 1172f14e953bSMadhavan SrinivasanEXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80, IRQS_DISABLED) 11739bcb81bfSNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0xe80) 11749bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 11759bcb81bfSNicholas PigginEXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception) 11769bcb81bfSNicholas Piggin#else 11779bcb81bfSNicholas PigginEXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception) 11789bcb81bfSNicholas Piggin#endif 11799bcb81bfSNicholas Piggin 11800ebc4cdaSBenjamin Herrenschmidt 1181f14e953bSMadhavan SrinivasanEXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20, IRQS_DISABLED) 1182f14e953bSMadhavan SrinivasanEXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0, IRQS_DISABLED) 118374408776SNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0xea0) 118474408776SNicholas PigginEXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ) 118574408776SNicholas Piggin 11869baaef0aSBenjamin Herrenschmidt 11871a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 11881a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 11891a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 11901a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 1191bda7fea2SNicholas Piggin 11920ebc4cdaSBenjamin Herrenschmidt 1193f442d004SMadhavan SrinivasanEXC_REAL_OOL_MASKABLE(performance_monitor, 0xf00, 0x20, IRQS_PMI_DISABLED) 1194f442d004SMadhavan SrinivasanEXC_VIRT_OOL_MASKABLE(performance_monitor, 0x4f00, 0x20, 0xf00, IRQS_PMI_DISABLED) 1195b1c7f150SNicholas PigginTRAMP_KVM(PACA_EXGEN, 0xf00) 1196b1c7f150SNicholas PigginEXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception) 1197b1c7f150SNicholas Piggin 11980ebc4cdaSBenjamin Herrenschmidt 11991a6822d1SNicholas PigginEXC_REAL_OOL(altivec_unavailable, 0xf20, 0x20) 12001a6822d1SNicholas PigginEXC_VIRT_OOL(altivec_unavailable, 0x4f20, 0x20, 0xf20) 1201d1a0ca9cSNicholas PigginTRAMP_KVM(PACA_EXGEN, 0xf20) 1202d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 1203d1a0ca9cSNicholas Piggin EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN) 1204d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 1205d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 1206d1a0ca9cSNicholas Piggin beq 1f 1207d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1208d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 1209d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1210d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 1211d1a0ca9cSNicholas Piggin */ 1212d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1213d1a0ca9cSNicholas Piggin bne- 2f 1214d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 1215d1a0ca9cSNicholas Piggin#endif 1216d1a0ca9cSNicholas Piggin bl load_up_altivec 1217d1a0ca9cSNicholas Piggin b fast_exception_return 1218d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1219d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 1220d1a0ca9cSNicholas Piggin bl save_nvgprs 1221d1a0ca9cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1222d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1223d1a0ca9cSNicholas Piggin bl altivec_unavailable_tm 1224d1a0ca9cSNicholas Piggin b ret_from_except 1225d1a0ca9cSNicholas Piggin#endif 1226d1a0ca9cSNicholas Piggin1: 1227d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 1228d1a0ca9cSNicholas Piggin#endif 1229d1a0ca9cSNicholas Piggin bl save_nvgprs 1230d1a0ca9cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1231d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1232d1a0ca9cSNicholas Piggin bl altivec_unavailable_exception 1233d1a0ca9cSNicholas Piggin b ret_from_except 1234d1a0ca9cSNicholas Piggin 12350ebc4cdaSBenjamin Herrenschmidt 12361a6822d1SNicholas PigginEXC_REAL_OOL(vsx_unavailable, 0xf40, 0x20) 12371a6822d1SNicholas PigginEXC_VIRT_OOL(vsx_unavailable, 0x4f40, 0x20, 0xf40) 1238792cbdddSNicholas PigginTRAMP_KVM(PACA_EXGEN, 0xf40) 1239792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 1240792cbdddSNicholas Piggin EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN) 1241792cbdddSNicholas Piggin#ifdef CONFIG_VSX 1242792cbdddSNicholas PigginBEGIN_FTR_SECTION 1243792cbdddSNicholas Piggin beq 1f 1244792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1245792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 1246792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1247792cbdddSNicholas Piggin * transaction), go do TM stuff 1248792cbdddSNicholas Piggin */ 1249792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1250792cbdddSNicholas Piggin bne- 2f 1251792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 1252792cbdddSNicholas Piggin#endif 1253792cbdddSNicholas Piggin b load_up_vsx 1254792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1255792cbdddSNicholas Piggin2: /* User process was in a transaction */ 1256792cbdddSNicholas Piggin bl save_nvgprs 1257792cbdddSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1258792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1259792cbdddSNicholas Piggin bl vsx_unavailable_tm 1260792cbdddSNicholas Piggin b ret_from_except 1261792cbdddSNicholas Piggin#endif 1262792cbdddSNicholas Piggin1: 1263792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 1264792cbdddSNicholas Piggin#endif 1265792cbdddSNicholas Piggin bl save_nvgprs 1266792cbdddSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1267792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1268792cbdddSNicholas Piggin bl vsx_unavailable_exception 1269792cbdddSNicholas Piggin b ret_from_except 1270792cbdddSNicholas Piggin 1271d0c0c9a1SMichael Neuling 12721a6822d1SNicholas PigginEXC_REAL_OOL(facility_unavailable, 0xf60, 0x20) 12731a6822d1SNicholas PigginEXC_VIRT_OOL(facility_unavailable, 0x4f60, 0x20, 0xf60) 12741134713cSNicholas PigginTRAMP_KVM(PACA_EXGEN, 0xf60) 12751134713cSNicholas PigginEXC_COMMON(facility_unavailable_common, 0xf60, facility_unavailable_exception) 12761134713cSNicholas Piggin 1277da2bc464SMichael Ellerman 12781a6822d1SNicholas PigginEXC_REAL_OOL_HV(h_facility_unavailable, 0xf80, 0x20) 12791a6822d1SNicholas PigginEXC_VIRT_OOL_HV(h_facility_unavailable, 0x4f80, 0x20, 0xf80) 128014b0072cSNicholas PigginTRAMP_KVM_HV(PACA_EXGEN, 0xf80) 128114b0072cSNicholas PigginEXC_COMMON(h_facility_unavailable_common, 0xf80, facility_unavailable_exception) 128214b0072cSNicholas Piggin 1283da2bc464SMichael Ellerman 12841a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 12851a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 12861a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 12871a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 12881a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 12891a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 12901a6822d1SNicholas Piggin 12911a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 12921a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 12931a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 12941a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 1295da2bc464SMichael Ellerman 12960ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 12971a6822d1SNicholas PigginEXC_REAL_HV(cbe_system_error, 0x1200, 0x100) 12981a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 1299da2bc464SMichael EllermanTRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1200) 1300ff1b3206SNicholas PigginEXC_COMMON(cbe_system_error_common, 0x1200, cbe_system_error_exception) 1301da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 13021a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 13031a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 1304da2bc464SMichael Ellerman#endif 1305da2bc464SMichael Ellerman 1306ff1b3206SNicholas Piggin 13071a6822d1SNicholas PigginEXC_REAL(instruction_breakpoint, 0x1300, 0x100) 13081a6822d1SNicholas PigginEXC_VIRT(instruction_breakpoint, 0x5300, 0x100, 0x1300) 1309da2bc464SMichael EllermanTRAMP_KVM_SKIP(PACA_EXGEN, 0x1300) 13104e96dbbfSNicholas PigginEXC_COMMON(instruction_breakpoint_common, 0x1300, instruction_breakpoint_exception) 13114e96dbbfSNicholas Piggin 13121a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 13131a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 1314da2bc464SMichael Ellerman 13151a6822d1SNicholas PigginEXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) 1316b92a66a6SMichael Neuling mtspr SPRN_SPRG_HSCRATCH0,r13 13171707dd16SPaul Mackerras EXCEPTION_PROLOG_0(PACA_EXGEN) 1318630573c1SPaul Mackerras EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500) 1319b92a66a6SMichael Neuling 1320b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 1321b92a66a6SMichael Neuling mfspr r10,SPRN_HSRR1 1322b92a66a6SMichael Neuling mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 1323b92a66a6SMichael Neuling andis. r10,r10,(HSRR1_DENORM)@h /* denorm? */ 1324b92a66a6SMichael Neuling addi r11,r11,-4 /* HSRR0 is next instruction */ 1325b92a66a6SMichael Neuling bne+ denorm_assist 1326b92a66a6SMichael Neuling#endif 1327b92a66a6SMichael Neuling 13284bb3c7a0SPaul Mackerras KVMTEST_HV(0x1500) 1329b92a66a6SMichael Neuling EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV) 13301a6822d1SNicholas PigginEXC_REAL_END(denorm_exception_hv, 0x1500, 0x100) 1331da2bc464SMichael Ellerman 1332d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 13331a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 1334d7e89849SNicholas Piggin b exc_real_0x1500_denorm_exception_hv 13351a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 1336d7e89849SNicholas Piggin#else 13371a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 1338d7e89849SNicholas Piggin#endif 1339d7e89849SNicholas Piggin 13404bb3c7a0SPaul MackerrasTRAMP_KVM_HV(PACA_EXGEN, 0x1500) 1341b92a66a6SMichael Neuling 1342b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 1343da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 1344b92a66a6SMichael NeulingBEGIN_FTR_SECTION 1345b92a66a6SMichael Neuling/* 1346b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 1347b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 1348b92a66a6SMichael Neuling */ 1349b92a66a6SMichael Neuling mfmsr r10 1350b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 1351b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 1352b92a66a6SMichael Neuling mtmsrd r10 1353b92a66a6SMichael Neuling sync 1354d7c67fb1SMichael Neuling 1355d7c67fb1SMichael Neuling#define FMR2(n) fmr (n), (n) ; fmr n+1, n+1 1356d7c67fb1SMichael Neuling#define FMR4(n) FMR2(n) ; FMR2(n+2) 1357d7c67fb1SMichael Neuling#define FMR8(n) FMR4(n) ; FMR4(n+4) 1358d7c67fb1SMichael Neuling#define FMR16(n) FMR8(n) ; FMR8(n+8) 1359d7c67fb1SMichael Neuling#define FMR32(n) FMR16(n) ; FMR16(n+16) 1360d7c67fb1SMichael Neuling FMR32(0) 1361d7c67fb1SMichael Neuling 1362b92a66a6SMichael NeulingFTR_SECTION_ELSE 1363b92a66a6SMichael Neuling/* 1364b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 1365b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 1366b92a66a6SMichael Neuling */ 1367b92a66a6SMichael Neuling mfmsr r10 1368b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 1369b92a66a6SMichael Neuling mtmsrd r10 1370b92a66a6SMichael Neuling sync 1371d7c67fb1SMichael Neuling 1372d7c67fb1SMichael Neuling#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1) 1373d7c67fb1SMichael Neuling#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2) 1374d7c67fb1SMichael Neuling#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4) 1375d7c67fb1SMichael Neuling#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8) 1376d7c67fb1SMichael Neuling#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16) 1377d7c67fb1SMichael Neuling XVCPSGNDP32(0) 1378d7c67fb1SMichael Neuling 1379b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 1380fb0fce3eSMichael Neuling 1381fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 1382fb0fce3eSMichael Neuling b denorm_done 1383fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1384fb0fce3eSMichael Neuling/* 1385fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 1386fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 1387fb0fce3eSMichael Neuling */ 1388fb0fce3eSMichael Neuling XVCPSGNDP32(32) 1389fb0fce3eSMichael Neulingdenorm_done: 1390b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 1391b92a66a6SMichael Neuling mtcrf 0x80,r9 1392b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 139344e9309fSHaren Myneni RESTORE_PPR_PACA(PACA_EXGEN, r10) 1394630573c1SPaul MackerrasBEGIN_FTR_SECTION 1395630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 1396630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 1397630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1398b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 1399b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 1400b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 1401b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 1402222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 1403b92a66a6SMichael Neuling b . 1404b92a66a6SMichael Neuling#endif 1405b92a66a6SMichael Neuling 1406872e2ae4SBenjamin HerrenschmidtEXC_COMMON(denorm_common, 0x1500, unknown_exception) 1407d7e89849SNicholas Piggin 1408d7e89849SNicholas Piggin 1409d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 14101a6822d1SNicholas PigginEXC_REAL_HV(cbe_maintenance, 0x1600, 0x100) 14111a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 1412d7e89849SNicholas PigginTRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1600) 141369a79344SNicholas PigginEXC_COMMON(cbe_maintenance_common, 0x1600, cbe_maintenance_exception) 1414d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 14151a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 14161a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 1417d7e89849SNicholas Piggin#endif 1418d7e89849SNicholas Piggin 141969a79344SNicholas Piggin 14201a6822d1SNicholas PigginEXC_REAL(altivec_assist, 0x1700, 0x100) 14211a6822d1SNicholas PigginEXC_VIRT(altivec_assist, 0x5700, 0x100, 0x1700) 1422d7e89849SNicholas PigginTRAMP_KVM(PACA_EXGEN, 0x1700) 1423b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 1424b51c079eSNicholas PigginEXC_COMMON(altivec_assist_common, 0x1700, altivec_assist_exception) 1425b51c079eSNicholas Piggin#else 1426b51c079eSNicholas PigginEXC_COMMON(altivec_assist_common, 0x1700, unknown_exception) 1427b51c079eSNicholas Piggin#endif 1428b51c079eSNicholas Piggin 1429d7e89849SNicholas Piggin 1430d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 14311a6822d1SNicholas PigginEXC_REAL_HV(cbe_thermal, 0x1800, 0x100) 14321a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 1433d7e89849SNicholas PigginTRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1800) 14343965f8abSNicholas PigginEXC_COMMON(cbe_thermal_common, 0x1800, cbe_thermal_exception) 1435d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 14361a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 14371a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 1438d7e89849SNicholas Piggin#endif 1439d7e89849SNicholas Piggin 144075eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 14412104180aSNicholas Piggin 14422104180aSNicholas Piggin#define MASKED_DEC_HANDLER_LABEL 3f 14432104180aSNicholas Piggin 14442104180aSNicholas Piggin#define MASKED_DEC_HANDLER(_H) \ 14452104180aSNicholas Piggin3: /* soft-nmi */ \ 14462104180aSNicholas Piggin std r12,PACA_EXGEN+EX_R12(r13); \ 14472104180aSNicholas Piggin GET_SCRATCH0(r10); \ 14482104180aSNicholas Piggin std r10,PACA_EXGEN+EX_R13(r13); \ 14492104180aSNicholas Piggin EXCEPTION_PROLOG_PSERIES_1(soft_nmi_common, _H) 14502104180aSNicholas Piggin 1451cc491f1dSNicholas Piggin/* 1452cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 1453cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 1454cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 1455cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 1456cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 1457cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 1458cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 1459cc491f1dSNicholas Piggin */ 14602104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 14612104180aSNicholas Piggin mr r10,r1 14622104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 14632104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 14642104180aSNicholas Piggin EXCEPTION_COMMON_NORET_STACK(PACA_EXGEN, 0x900, 14652104180aSNicholas Piggin system_reset, soft_nmi_interrupt, 14662104180aSNicholas Piggin ADD_NVGPRS;ADD_RECONCILE) 14672104180aSNicholas Piggin b ret_from_except 14682104180aSNicholas Piggin 146975eb767eSNicholas Piggin#else /* CONFIG_PPC_WATCHDOG */ 14702104180aSNicholas Piggin#define MASKED_DEC_HANDLER_LABEL 2f /* normal return */ 14712104180aSNicholas Piggin#define MASKED_DEC_HANDLER(_H) 147275eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 1473d7e89849SNicholas Piggin 14740ebc4cdaSBenjamin Herrenschmidt/* 1475fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 1476fe9e1d54SIan Munsie * - If it was a decrementer interrupt, we bump the dec to max and and return. 1477fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 1478fe9e1d54SIan Munsie * triggered and won't automatically refire. 14790869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 14800869b6fdSMahesh Salgaonkar * and it won't refire. 14816cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 1482fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 14830ebc4cdaSBenjamin Herrenschmidt */ 14847230c564SBenjamin Herrenschmidt#define MASKED_INTERRUPT(_H) \ 14857230c564SBenjamin Herrenschmidtmasked_##_H##interrupt: \ 14867230c564SBenjamin Herrenschmidt std r11,PACA_EXGEN+EX_R11(r13); \ 14877230c564SBenjamin Herrenschmidt lbz r11,PACAIRQHAPPENED(r13); \ 14887230c564SBenjamin Herrenschmidt or r11,r11,r10; \ 14897230c564SBenjamin Herrenschmidt stb r11,PACAIRQHAPPENED(r13); \ 1490fe9e1d54SIan Munsie cmpwi r10,PACA_IRQ_DEC; \ 1491fe9e1d54SIan Munsie bne 1f; \ 14927230c564SBenjamin Herrenschmidt lis r10,0x7fff; \ 14937230c564SBenjamin Herrenschmidt ori r10,r10,0xffff; \ 14947230c564SBenjamin Herrenschmidt mtspr SPRN_DEC,r10; \ 14952104180aSNicholas Piggin b MASKED_DEC_HANDLER_LABEL; \ 14966cc3f91bSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK; \ 14976cc3f91bSNicholas Piggin beq 2f; \ 1498fe9e1d54SIan Munsie mfspr r10,SPRN_##_H##SRR1; \ 14996e9a2f6eSNicholas Piggin xori r10,r10,MSR_EE; /* clear MSR_EE */ \ 15007230c564SBenjamin Herrenschmidt mtspr SPRN_##_H##SRR1,r10; \ 15017230c564SBenjamin Herrenschmidt2: mtcrf 0x80,r9; \ 15027b08729cSMichael Ellerman std r1,PACAR1(r13); \ 15037230c564SBenjamin Herrenschmidt ld r9,PACA_EXGEN+EX_R9(r13); \ 15047230c564SBenjamin Herrenschmidt ld r10,PACA_EXGEN+EX_R10(r13); \ 15057230c564SBenjamin Herrenschmidt ld r11,PACA_EXGEN+EX_R11(r13); \ 1506c05f0be8SNicholas Piggin /* returns to kernel where r13 must be set up, so don't restore it */ \ 1507222f20f1SNicholas Piggin ##_H##RFI_TO_KERNEL; \ 15082104180aSNicholas Piggin b .; \ 15092104180aSNicholas Piggin MASKED_DEC_HANDLER(_H) 15100ebc4cdaSBenjamin Herrenschmidt 1511a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 1512a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 1513a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 1514a048a07dSNicholas Piggin sync 1515a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 1516a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 1517a048a07dSNicholas Piggin ori 31,31,0 1518a048a07dSNicholas Piggin .rept 14 1519a048a07dSNicholas Piggin b 1f 1520a048a07dSNicholas Piggin1: 1521a048a07dSNicholas Piggin .endr 1522a048a07dSNicholas Piggin blr 1523a048a07dSNicholas Piggin 1524aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 1525aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 1526aa8a5e00SMichael Ellerman GET_PACA(r13); 1527aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 1528aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 1529aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 1530aa8a5e00SMichael Ellerman mfctr r9 1531aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 1532bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 1533bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 1534aa8a5e00SMichael Ellerman mtctr r11 153515a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 1536aa8a5e00SMichael Ellerman 1537aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 1538aa8a5e00SMichael Ellerman sync 1539bdcb1aefSNicholas Piggin 1540bdcb1aefSNicholas Piggin /* 1541bdcb1aefSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 1542bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 1543bdcb1aefSNicholas Piggin * hurt). 1544bdcb1aefSNicholas Piggin */ 1545bdcb1aefSNicholas Piggin1: 1546bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 1547bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 1548bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 1549bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 1550bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 1551bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 1552bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 1553bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 1554bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 1555aa8a5e00SMichael Ellerman bdnz 1b 1556aa8a5e00SMichael Ellerman 1557aa8a5e00SMichael Ellerman mtctr r9 1558aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 1559aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 1560aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 1561aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 1562aa8a5e00SMichael Ellerman rfid 1563aa8a5e00SMichael Ellerman 1564aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 1565aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 1566aa8a5e00SMichael Ellerman GET_PACA(r13); 1567aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 1568aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 1569aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 1570aa8a5e00SMichael Ellerman mfctr r9 1571aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 1572bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 1573bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 1574aa8a5e00SMichael Ellerman mtctr r11 157515a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 1576aa8a5e00SMichael Ellerman 1577aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 1578aa8a5e00SMichael Ellerman sync 1579bdcb1aefSNicholas Piggin 1580bdcb1aefSNicholas Piggin /* 1581bdcb1aefSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 1582bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 1583bdcb1aefSNicholas Piggin * hurt). 1584bdcb1aefSNicholas Piggin */ 1585bdcb1aefSNicholas Piggin1: 1586bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 1587bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 1588bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 1589bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 1590bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 1591bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 1592bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 1593bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 1594bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 1595aa8a5e00SMichael Ellerman bdnz 1b 1596aa8a5e00SMichael Ellerman 1597aa8a5e00SMichael Ellerman mtctr r9 1598aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 1599aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 1600aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 1601aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 1602aa8a5e00SMichael Ellerman hrfid 1603aa8a5e00SMichael Ellerman 160457f26649SNicholas Piggin/* 160557f26649SNicholas Piggin * Real mode exceptions actually use this too, but alternate 160657f26649SNicholas Piggin * instruction code patches (which end up in the common .text area) 160757f26649SNicholas Piggin * cannot reach these if they are put there. 160857f26649SNicholas Piggin */ 160957f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 16107230c564SBenjamin Herrenschmidt MASKED_INTERRUPT() 16117230c564SBenjamin Herrenschmidt MASKED_INTERRUPT(H) 16127230c564SBenjamin Herrenschmidt 16134f6c11dbSPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1614da2bc464SMichael EllermanTRAMP_REAL_BEGIN(kvmppc_skip_interrupt) 16154f6c11dbSPaul Mackerras /* 16164f6c11dbSPaul Mackerras * Here all GPRs are unchanged from when the interrupt happened 16174f6c11dbSPaul Mackerras * except for r13, which is saved in SPRG_SCRATCH0. 16184f6c11dbSPaul Mackerras */ 16194f6c11dbSPaul Mackerras mfspr r13, SPRN_SRR0 16204f6c11dbSPaul Mackerras addi r13, r13, 4 16214f6c11dbSPaul Mackerras mtspr SPRN_SRR0, r13 16224f6c11dbSPaul Mackerras GET_SCRATCH0(r13) 1623222f20f1SNicholas Piggin RFI_TO_KERNEL 16244f6c11dbSPaul Mackerras b . 16254f6c11dbSPaul Mackerras 1626da2bc464SMichael EllermanTRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt) 16274f6c11dbSPaul Mackerras /* 16284f6c11dbSPaul Mackerras * Here all GPRs are unchanged from when the interrupt happened 16294f6c11dbSPaul Mackerras * except for r13, which is saved in SPRG_SCRATCH0. 16304f6c11dbSPaul Mackerras */ 16314f6c11dbSPaul Mackerras mfspr r13, SPRN_HSRR0 16324f6c11dbSPaul Mackerras addi r13, r13, 4 16334f6c11dbSPaul Mackerras mtspr SPRN_HSRR0, r13 16344f6c11dbSPaul Mackerras GET_SCRATCH0(r13) 1635222f20f1SNicholas Piggin HRFI_TO_KERNEL 16364f6c11dbSPaul Mackerras b . 16374f6c11dbSPaul Mackerras#endif 16384f6c11dbSPaul Mackerras 16390ebc4cdaSBenjamin Herrenschmidt/* 1640057b6d7eSHari Bathini * Ensure that any handlers that get invoked from the exception prologs 1641057b6d7eSHari Bathini * above are below the first 64KB (0x10000) of the kernel image because 1642057b6d7eSHari Bathini * the prologs assemble the addresses of these handlers using the 1643057b6d7eSHari Bathini * LOAD_HANDLER macro, which uses an ori instruction. 16440ebc4cdaSBenjamin Herrenschmidt */ 16450ebc4cdaSBenjamin Herrenschmidt 16460ebc4cdaSBenjamin Herrenschmidt/*** Common interrupt handlers ***/ 16470ebc4cdaSBenjamin Herrenschmidt 16480ebc4cdaSBenjamin Herrenschmidt 1649c1fb6816SMichael Neuling /* 1650c1fb6816SMichael Neuling * Relocation-on interrupts: A subset of the interrupts can be delivered 1651c1fb6816SMichael Neuling * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 1652c1fb6816SMichael Neuling * it. Addresses are the same as the original interrupt addresses, but 1653c1fb6816SMichael Neuling * offset by 0xc000000000004000. 1654c1fb6816SMichael Neuling * It's impossible to receive interrupts below 0x300 via this mechanism. 1655c1fb6816SMichael Neuling * KVM: None of these traps are from the guest ; anything that escalated 1656c1fb6816SMichael Neuling * to HV=1 from HV=0 is delivered via real mode handlers. 1657c1fb6816SMichael Neuling */ 1658c1fb6816SMichael Neuling 1659c1fb6816SMichael Neuling /* 1660c1fb6816SMichael Neuling * This uses the standard macro, since the original 0x300 vector 1661c1fb6816SMichael Neuling * only has extra guff for STAB-based processors -- which never 1662c1fb6816SMichael Neuling * come here. 1663c1fb6816SMichael Neuling */ 1664da2bc464SMichael Ellerman 166557f26649SNicholas PigginEXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) 1666b1576fecSAnton Blanchard b __ppc64_runlatch_on 1667fe1952fcSBenjamin Herrenschmidt 166857f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 16698ed8ab40SHari Bathini /* 16708ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 16718ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 16728ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 16738ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 16748ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 16758ed8ab40SHari Bathini */ 16768ed8ab40SHari Bathini .align 7 16778ed8ab40SHari Bathini .globl __end_interrupts 16788ed8ab40SHari Bathini__end_interrupts: 167957f26649SNicholas PigginDEFINE_FIXED_SYMBOL(__end_interrupts) 168061383407SBenjamin Herrenschmidt 1681087aa036SChen Gang#ifdef CONFIG_PPC_970_NAP 16827c8cb4b5SNicholas PigginEXC_COMMON_BEGIN(power4_fixup_nap) 1683087aa036SChen Gang andc r9,r9,r10 1684087aa036SChen Gang std r9,TI_LOCAL_FLAGS(r11) 1685087aa036SChen Gang ld r10,_LINK(r1) /* make idle task do the */ 1686087aa036SChen Gang std r10,_NIP(r1) /* equivalent of a blr */ 1687087aa036SChen Gang blr 1688087aa036SChen Gang#endif 1689087aa036SChen Gang 169057f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 169157f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 169257f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 169357f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 169457f26649SNicholas Piggin 169557f26649SNicholas PigginUSE_TEXT_SECTION() 169657f26649SNicholas Piggin 1697087aa036SChen Gang/* 16980ebc4cdaSBenjamin Herrenschmidt * Hash table stuff 16990ebc4cdaSBenjamin Herrenschmidt */ 1700f4329f2eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 17016a3bab90SAnton Blancharddo_hash_page: 17024e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 1703e6c2a479SRam Pai lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h 1704398a719dSBenjamin Herrenschmidt ori r0,r0,DSISR_BAD_FAULT_64S@l 1705398a719dSBenjamin Herrenschmidt and. r0,r4,r0 /* weird error? */ 17060ebc4cdaSBenjamin Herrenschmidt bne- handle_page_fault /* if not, try to insert a HPTE */ 17079778b696SStuart Yoder CURRENT_THREAD_INFO(r11, r1) 17089c1e1052SPaul Mackerras lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */ 17099c1e1052SPaul Mackerras andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */ 17109c1e1052SPaul Mackerras bne 77f /* then don't call hash_page now */ 17110ebc4cdaSBenjamin Herrenschmidt 17120ebc4cdaSBenjamin Herrenschmidt /* 17130ebc4cdaSBenjamin Herrenschmidt * r3 contains the faulting address 1714106713a1SAneesh Kumar K.V * r4 msr 17150ebc4cdaSBenjamin Herrenschmidt * r5 contains the trap number 1716aefa5688SAneesh Kumar K.V * r6 contains dsisr 17170ebc4cdaSBenjamin Herrenschmidt * 17187230c564SBenjamin Herrenschmidt * at return r3 = 0 for success, 1 for page fault, negative for error 17190ebc4cdaSBenjamin Herrenschmidt */ 1720106713a1SAneesh Kumar K.V mr r4,r12 1721aefa5688SAneesh Kumar K.V ld r6,_DSISR(r1) 1722106713a1SAneesh Kumar K.V bl __hash_page /* build HPTE if possible */ 1723106713a1SAneesh Kumar K.V cmpdi r3,0 /* see if __hash_page succeeded */ 17240ebc4cdaSBenjamin Herrenschmidt 17257230c564SBenjamin Herrenschmidt /* Success */ 17260ebc4cdaSBenjamin Herrenschmidt beq fast_exc_return_irq /* Return from exception on success */ 17270ebc4cdaSBenjamin Herrenschmidt 17287230c564SBenjamin Herrenschmidt /* Error */ 17297230c564SBenjamin Herrenschmidt blt- 13f 1730d89ba535SNaveen N. Rao 1731d89ba535SNaveen N. Rao /* Reload DSISR into r4 for the DABR check below */ 1732d89ba535SNaveen N. Rao ld r4,_DSISR(r1) 17334e003747SMichael Ellerman#endif /* CONFIG_PPC_BOOK3S_64 */ 17340ebc4cdaSBenjamin Herrenschmidt 1735a546498fSBenjamin Herrenschmidt/* Here we have a page fault that hash_page can't handle. */ 1736a546498fSBenjamin Herrenschmidthandle_page_fault: 1737d89ba535SNaveen N. Rao11: andis. r0,r4,DSISR_DABRMATCH@h 1738d89ba535SNaveen N. Rao bne- handle_dabr_fault 1739d89ba535SNaveen N. Rao ld r4,_DAR(r1) 1740a546498fSBenjamin Herrenschmidt ld r5,_DSISR(r1) 1741a546498fSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 1742b1576fecSAnton Blanchard bl do_page_fault 1743a546498fSBenjamin Herrenschmidt cmpdi r3,0 1744a546498fSBenjamin Herrenschmidt beq+ 12f 1745b1576fecSAnton Blanchard bl save_nvgprs 1746a546498fSBenjamin Herrenschmidt mr r5,r3 1747a546498fSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 1748a546498fSBenjamin Herrenschmidt lwz r4,_DAR(r1) 1749b1576fecSAnton Blanchard bl bad_page_fault 1750b1576fecSAnton Blanchard b ret_from_except 17510ebc4cdaSBenjamin Herrenschmidt 17529c7cc234SK.Prasad/* We have a data breakpoint exception - handle it */ 17539c7cc234SK.Prasadhandle_dabr_fault: 1754b1576fecSAnton Blanchard bl save_nvgprs 17559c7cc234SK.Prasad ld r4,_DAR(r1) 17569c7cc234SK.Prasad ld r5,_DSISR(r1) 17579c7cc234SK.Prasad addi r3,r1,STACK_FRAME_OVERHEAD 1758b1576fecSAnton Blanchard bl do_break 1759b1576fecSAnton Blanchard12: b ret_from_except_lite 17609c7cc234SK.Prasad 17610ebc4cdaSBenjamin Herrenschmidt 17624e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 17630ebc4cdaSBenjamin Herrenschmidt/* We have a page fault that hash_page could handle but HV refused 17640ebc4cdaSBenjamin Herrenschmidt * the PTE insertion 17650ebc4cdaSBenjamin Herrenschmidt */ 1766b1576fecSAnton Blanchard13: bl save_nvgprs 17670ebc4cdaSBenjamin Herrenschmidt mr r5,r3 17680ebc4cdaSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 17690ebc4cdaSBenjamin Herrenschmidt ld r4,_DAR(r1) 1770b1576fecSAnton Blanchard bl low_hash_fault 1771b1576fecSAnton Blanchard b ret_from_except 1772caca285eSAneesh Kumar K.V#endif 17730ebc4cdaSBenjamin Herrenschmidt 17749c1e1052SPaul Mackerras/* 17759c1e1052SPaul Mackerras * We come here as a result of a DSI at a point where we don't want 17769c1e1052SPaul Mackerras * to call hash_page, such as when we are accessing memory (possibly 17779c1e1052SPaul Mackerras * user memory) inside a PMU interrupt that occurred while interrupts 17789c1e1052SPaul Mackerras * were soft-disabled. We want to invoke the exception handler for 17799c1e1052SPaul Mackerras * the access, or panic if there isn't a handler. 17809c1e1052SPaul Mackerras */ 1781b1576fecSAnton Blanchard77: bl save_nvgprs 17829c1e1052SPaul Mackerras mr r4,r3 17839c1e1052SPaul Mackerras addi r3,r1,STACK_FRAME_OVERHEAD 17849c1e1052SPaul Mackerras li r5,SIGSEGV 1785b1576fecSAnton Blanchard bl bad_page_fault 1786b1576fecSAnton Blanchard b ret_from_except 17874e2bf01bSMichael Ellerman 17884e2bf01bSMichael Ellerman/* 17894e2bf01bSMichael Ellerman * Here we have detected that the kernel stack pointer is bad. 17904e2bf01bSMichael Ellerman * R9 contains the saved CR, r13 points to the paca, 17914e2bf01bSMichael Ellerman * r10 contains the (bad) kernel stack pointer, 17924e2bf01bSMichael Ellerman * r11 and r12 contain the saved SRR0 and SRR1. 17934e2bf01bSMichael Ellerman * We switch to using an emergency stack, save the registers there, 17944e2bf01bSMichael Ellerman * and call kernel_bad_stack(), which panics. 17954e2bf01bSMichael Ellerman */ 17964e2bf01bSMichael Ellermanbad_stack: 17974e2bf01bSMichael Ellerman ld r1,PACAEMERGSP(r13) 17984e2bf01bSMichael Ellerman subi r1,r1,64+INT_FRAME_SIZE 17994e2bf01bSMichael Ellerman std r9,_CCR(r1) 18004e2bf01bSMichael Ellerman std r10,GPR1(r1) 18014e2bf01bSMichael Ellerman std r11,_NIP(r1) 18024e2bf01bSMichael Ellerman std r12,_MSR(r1) 18034e2bf01bSMichael Ellerman mfspr r11,SPRN_DAR 18044e2bf01bSMichael Ellerman mfspr r12,SPRN_DSISR 18054e2bf01bSMichael Ellerman std r11,_DAR(r1) 18064e2bf01bSMichael Ellerman std r12,_DSISR(r1) 18074e2bf01bSMichael Ellerman mflr r10 18084e2bf01bSMichael Ellerman mfctr r11 18094e2bf01bSMichael Ellerman mfxer r12 18104e2bf01bSMichael Ellerman std r10,_LINK(r1) 18114e2bf01bSMichael Ellerman std r11,_CTR(r1) 18124e2bf01bSMichael Ellerman std r12,_XER(r1) 18134e2bf01bSMichael Ellerman SAVE_GPR(0,r1) 18144e2bf01bSMichael Ellerman SAVE_GPR(2,r1) 18154e2bf01bSMichael Ellerman ld r10,EX_R3(r3) 18164e2bf01bSMichael Ellerman std r10,GPR3(r1) 18174e2bf01bSMichael Ellerman SAVE_GPR(4,r1) 18184e2bf01bSMichael Ellerman SAVE_4GPRS(5,r1) 18194e2bf01bSMichael Ellerman ld r9,EX_R9(r3) 18204e2bf01bSMichael Ellerman ld r10,EX_R10(r3) 18214e2bf01bSMichael Ellerman SAVE_2GPRS(9,r1) 18224e2bf01bSMichael Ellerman ld r9,EX_R11(r3) 18234e2bf01bSMichael Ellerman ld r10,EX_R12(r3) 18244e2bf01bSMichael Ellerman ld r11,EX_R13(r3) 18254e2bf01bSMichael Ellerman std r9,GPR11(r1) 18264e2bf01bSMichael Ellerman std r10,GPR12(r1) 18274e2bf01bSMichael Ellerman std r11,GPR13(r1) 18284e2bf01bSMichael EllermanBEGIN_FTR_SECTION 18294e2bf01bSMichael Ellerman ld r10,EX_CFAR(r3) 18304e2bf01bSMichael Ellerman std r10,ORIG_GPR3(r1) 18314e2bf01bSMichael EllermanEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 18324e2bf01bSMichael Ellerman SAVE_8GPRS(14,r1) 18334e2bf01bSMichael Ellerman SAVE_10GPRS(22,r1) 18344e2bf01bSMichael Ellerman lhz r12,PACA_TRAP_SAVE(r13) 18354e2bf01bSMichael Ellerman std r12,_TRAP(r1) 18364e2bf01bSMichael Ellerman addi r11,r1,INT_FRAME_SIZE 18374e2bf01bSMichael Ellerman std r11,0(r1) 18384e2bf01bSMichael Ellerman li r12,0 18394e2bf01bSMichael Ellerman std r12,0(r11) 18404e2bf01bSMichael Ellerman ld r2,PACATOC(r13) 18414e2bf01bSMichael Ellerman ld r11,exception_marker@toc(r2) 18424e2bf01bSMichael Ellerman std r12,RESULT(r1) 18434e2bf01bSMichael Ellerman std r11,STACK_FRAME_OVERHEAD-16(r1) 18444e2bf01bSMichael Ellerman1: addi r3,r1,STACK_FRAME_OVERHEAD 18454e2bf01bSMichael Ellerman bl kernel_bad_stack 18464e2bf01bSMichael Ellerman b 1b 184715770a13SNaveen N. Rao_ASM_NOKPROBE_SYMBOL(bad_stack); 18480f0c6ca1SNicholas Piggin 18490f0c6ca1SNicholas Piggin/* 1850a9af97aaSNicholas Piggin * When doorbell is triggered from system reset wakeup, the message is 1851a9af97aaSNicholas Piggin * not cleared, so it would fire again when EE is enabled. 1852a9af97aaSNicholas Piggin * 1853a9af97aaSNicholas Piggin * When coming from local_irq_enable, there may be the same problem if 1854a9af97aaSNicholas Piggin * we were hard disabled. 1855a9af97aaSNicholas Piggin * 1856a9af97aaSNicholas Piggin * Execute msgclr to clear pending exceptions before handling it. 1857a9af97aaSNicholas Piggin */ 1858a9af97aaSNicholas Pigginh_doorbell_common_msgclr: 1859a9af97aaSNicholas Piggin LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36)) 1860a9af97aaSNicholas Piggin PPC_MSGCLR(3) 1861a9af97aaSNicholas Piggin b h_doorbell_common 1862a9af97aaSNicholas Piggin 1863a9af97aaSNicholas Piggindoorbell_super_common_msgclr: 1864a9af97aaSNicholas Piggin LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36)) 1865a9af97aaSNicholas Piggin PPC_MSGCLRP(3) 1866a9af97aaSNicholas Piggin b doorbell_super_common 1867a9af97aaSNicholas Piggin 1868a9af97aaSNicholas Piggin/* 18690f0c6ca1SNicholas Piggin * Called from arch_local_irq_enable when an interrupt needs 18700f0c6ca1SNicholas Piggin * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate 18710f0c6ca1SNicholas Piggin * which kind of interrupt. MSR:EE is already off. We generate a 18720f0c6ca1SNicholas Piggin * stackframe like if a real interrupt had happened. 18730f0c6ca1SNicholas Piggin * 18740f0c6ca1SNicholas Piggin * Note: While MSR:EE is off, we need to make sure that _MSR 18750f0c6ca1SNicholas Piggin * in the generated frame has EE set to 1 or the exception 18760f0c6ca1SNicholas Piggin * handler will not properly re-enable them. 1877b48bbb82SNicholas Piggin * 1878b48bbb82SNicholas Piggin * Note that we don't specify LR as the NIP (return address) for 1879b48bbb82SNicholas Piggin * the interrupt because that would unbalance the return branch 1880b48bbb82SNicholas Piggin * predictor. 18810f0c6ca1SNicholas Piggin */ 18820f0c6ca1SNicholas Piggin_GLOBAL(__replay_interrupt) 18830f0c6ca1SNicholas Piggin /* We are going to jump to the exception common code which 18840f0c6ca1SNicholas Piggin * will retrieve various register values from the PACA which 18850f0c6ca1SNicholas Piggin * we don't give a damn about, so we don't bother storing them. 18860f0c6ca1SNicholas Piggin */ 18870f0c6ca1SNicholas Piggin mfmsr r12 18883e23a12bSMichael Ellerman LOAD_REG_ADDR(r11, replay_interrupt_return) 18890f0c6ca1SNicholas Piggin mfcr r9 18900f0c6ca1SNicholas Piggin ori r12,r12,MSR_EE 18910f0c6ca1SNicholas Piggin cmpwi r3,0x900 18920f0c6ca1SNicholas Piggin beq decrementer_common 18930f0c6ca1SNicholas Piggin cmpwi r3,0x500 1894e6c1203dSNicholas PigginBEGIN_FTR_SECTION 1895e6c1203dSNicholas Piggin beq h_virt_irq_common 1896e6c1203dSNicholas PigginFTR_SECTION_ELSE 18970f0c6ca1SNicholas Piggin beq hardware_interrupt_common 1898e6c1203dSNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300) 1899f442d004SMadhavan Srinivasan cmpwi r3,0xf00 1900f442d004SMadhavan Srinivasan beq performance_monitor_common 19010f0c6ca1SNicholas PigginBEGIN_FTR_SECTION 1902d6f73fc6SNicholas Piggin cmpwi r3,0xa00 1903a9af97aaSNicholas Piggin beq h_doorbell_common_msgclr 19040f0c6ca1SNicholas Piggin cmpwi r3,0xe60 19050f0c6ca1SNicholas Piggin beq hmi_exception_common 19060f0c6ca1SNicholas PigginFTR_SECTION_ELSE 19070f0c6ca1SNicholas Piggin cmpwi r3,0xa00 1908a9af97aaSNicholas Piggin beq doorbell_super_common_msgclr 19090f0c6ca1SNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) 19103e23a12bSMichael Ellermanreplay_interrupt_return: 19110f0c6ca1SNicholas Piggin blr 1912b48bbb82SNicholas Piggin 191315770a13SNaveen N. Rao_ASM_NOKPROBE_SYMBOL(__replay_interrupt) 1914