1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 167230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 178aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1846f52210SStephen Rothwell#include <asm/ptrace.h> 197cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 20da2bc464SMichael Ellerman#include <asm/head-64.h> 212c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 22890274c2SMichael Ellerman#include <asm/kup.h> 238aa34ab8SBenjamin Herrenschmidt 2415820091SNicholas Piggin/* PACA save area offsets (exgen, exmc, etc) */ 2515820091SNicholas Piggin#define EX_R9 0 2615820091SNicholas Piggin#define EX_R10 8 2715820091SNicholas Piggin#define EX_R11 16 2815820091SNicholas Piggin#define EX_R12 24 2915820091SNicholas Piggin#define EX_R13 32 3015820091SNicholas Piggin#define EX_DAR 40 3115820091SNicholas Piggin#define EX_DSISR 48 3215820091SNicholas Piggin#define EX_CCR 52 3315820091SNicholas Piggin#define EX_CFAR 56 3415820091SNicholas Piggin#define EX_PPR 64 3515820091SNicholas Piggin#define EX_CTR 72 3615820091SNicholas Piggin.if EX_SIZE != 10 3715820091SNicholas Piggin .error "EX_SIZE is wrong" 3815820091SNicholas Piggin.endif 3915820091SNicholas Piggin 407299417cSNicholas Piggin/* 417299417cSNicholas Piggin * Following are fixed section helper macros. 427299417cSNicholas Piggin * 437299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 447299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 457299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 467299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 477299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 487299417cSNicholas Piggin */ 497299417cSNicholas Piggin 50a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 51a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 52a2432811SNicholas Piggin 53a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 54a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 55a2432811SNicholas Piggin 56a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 57a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 58a2432811SNicholas Piggin 59a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 60a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 61a2432811SNicholas Piggin 62a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 63a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 64a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 65a2432811SNicholas Piggin .global name; \ 66a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 67a2432811SNicholas Piggin DEFINE_FIXED_SYMBOL(name); \ 68a2432811SNicholas Pigginname: 69a2432811SNicholas Piggin 70a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 71a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 72a2432811SNicholas Piggin 73a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 74a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 75a2432811SNicholas Piggin 76a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 77a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 78a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 79a2432811SNicholas Piggin 80a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 81a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 82a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 83a2432811SNicholas Piggin 840ebc4cdaSBenjamin Herrenschmidt/* 8512a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 8612a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 8712a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 8812a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 8912a04809SNicholas Piggin * that kernelbase be 64K aligned. 9012a04809SNicholas Piggin */ 9112a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 9212a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 9312a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 9412a04809SNicholas Piggin 9512a04809SNicholas Piggin#define __LOAD_HANDLER(reg, label) \ 9612a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 9712a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l 9812a04809SNicholas Piggin 9912a04809SNicholas Piggin/* 10012a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 10112a04809SNicholas Piggin * head-y require >64K offsets. 10212a04809SNicholas Piggin */ 10312a04809SNicholas Piggin#define __LOAD_FAR_HANDLER(reg, label) \ 10412a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 10512a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l; \ 10612a04809SNicholas Piggin addis reg,reg,(ABS_ADDR(label))@h 10712a04809SNicholas Piggin 10812a04809SNicholas Piggin/* 10912a04809SNicholas Piggin * Branch to label using its 0xC000 address. This results in instruction 11012a04809SNicholas Piggin * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned 11112a04809SNicholas Piggin * on using mtmsr rather than rfid. 11212a04809SNicholas Piggin * 11312a04809SNicholas Piggin * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than 11412a04809SNicholas Piggin * load KBASE for a slight optimisation. 11512a04809SNicholas Piggin */ 11612a04809SNicholas Piggin#define BRANCH_TO_C000(reg, label) \ 1170e10be2bSNicholas Piggin __LOAD_FAR_HANDLER(reg, label); \ 11812a04809SNicholas Piggin mtctr reg; \ 11912a04809SNicholas Piggin bctr 12012a04809SNicholas Piggin 121a42a239dSNicholas Piggin/* 122a42a239dSNicholas Piggin * Interrupt code generation macros 123a42a239dSNicholas Piggin */ 12494325357SNicholas Piggin#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 12594325357SNicholas Piggin#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 12694325357SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 12794325357SNicholas Piggin#define IAREA .L_IAREA_\name\() /* PACA save area */ 12894325357SNicholas Piggin#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 12994325357SNicholas Piggin#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 13094325357SNicholas Piggin#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 13194325357SNicholas Piggin#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 13294325357SNicholas Piggin#define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */ 13394325357SNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 13494325357SNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 13594325357SNicholas Piggin#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 13694325357SNicholas Piggin#define IKVM_SKIP .L_IKVM_SKIP_\name\() /* Generate KVM skip handler */ 13794325357SNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 1384f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 13994325357SNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 14094325357SNicholas Piggin#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 1414f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 14294325357SNicholas Piggin#define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */ 14394325357SNicholas Piggin#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 144a42a239dSNicholas Piggin 145a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 146a42a239dSNicholas Piggin.macro int_define_ ## n name 147a42a239dSNicholas Piggin 148a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 149a42a239dSNicholas Piggin.endm ; \ 150a42a239dSNicholas Pigginint_define_ ## n n ; \ 151a42a239dSNicholas Piggindo_define_int n 152a42a239dSNicholas Piggin 153a42a239dSNicholas Piggin.macro do_define_int name 154a42a239dSNicholas Piggin .ifndef IVEC 155a42a239dSNicholas Piggin .error "IVEC not defined" 156a42a239dSNicholas Piggin .endif 157a42a239dSNicholas Piggin .ifndef IHSRR 1583f7fbd97SNicholas Piggin IHSRR=0 1593f7fbd97SNicholas Piggin .endif 1603f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 1613f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 162a42a239dSNicholas Piggin .endif 163a42a239dSNicholas Piggin .ifndef IAREA 164a42a239dSNicholas Piggin IAREA=PACA_EXGEN 165a42a239dSNicholas Piggin .endif 1668729c26eSNicholas Piggin .ifndef IVIRT 1678729c26eSNicholas Piggin IVIRT=1 1688729c26eSNicholas Piggin .endif 169a3cd35beSNicholas Piggin .ifndef IISIDE 170a3cd35beSNicholas Piggin IISIDE=0 171a3cd35beSNicholas Piggin .endif 172a42a239dSNicholas Piggin .ifndef IDAR 173a42a239dSNicholas Piggin IDAR=0 174a42a239dSNicholas Piggin .endif 175a42a239dSNicholas Piggin .ifndef IDSISR 176a42a239dSNicholas Piggin IDSISR=0 177a42a239dSNicholas Piggin .endif 178a42a239dSNicholas Piggin .ifndef ISET_RI 179a42a239dSNicholas Piggin ISET_RI=1 180a42a239dSNicholas Piggin .endif 181d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 182d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 183d73a10cbSNicholas Piggin .endif 184d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 185d73a10cbSNicholas Piggin IREALMODE_COMMON=0 186d73a10cbSNicholas Piggin .else 187d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 188d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 189d73a10cbSNicholas Piggin .endif 190a42a239dSNicholas Piggin .endif 191a42a239dSNicholas Piggin .ifndef IMASK 192a42a239dSNicholas Piggin IMASK=0 193a42a239dSNicholas Piggin .endif 194d52fd3d3SNicholas Piggin .ifndef IKVM_SKIP 195d52fd3d3SNicholas Piggin IKVM_SKIP=0 196d52fd3d3SNicholas Piggin .endif 197a42a239dSNicholas Piggin .ifndef IKVM_REAL 198a42a239dSNicholas Piggin IKVM_REAL=0 199a42a239dSNicholas Piggin .endif 200a42a239dSNicholas Piggin .ifndef IKVM_VIRT 201a42a239dSNicholas Piggin IKVM_VIRT=0 202a42a239dSNicholas Piggin .endif 2037cb3a1a0SNicholas Piggin .ifndef ISTACK 2047cb3a1a0SNicholas Piggin ISTACK=1 2057cb3a1a0SNicholas Piggin .endif 2067cb3a1a0SNicholas Piggin .ifndef IRECONCILE 2077cb3a1a0SNicholas Piggin IRECONCILE=1 2087cb3a1a0SNicholas Piggin .endif 2097cb3a1a0SNicholas Piggin .ifndef IKUAP 2107cb3a1a0SNicholas Piggin IKUAP=1 2117cb3a1a0SNicholas Piggin .endif 212a42a239dSNicholas Piggin.endm 213a42a239dSNicholas Piggin 21412a04809SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 21512a04809SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 21612a04809SNicholas Piggin/* 2172284ffeaSNicholas Piggin * All interrupts which set HSRR registers, as well as SRESET and MCE and 2182284ffeaSNicholas Piggin * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 2192284ffeaSNicholas Piggin * so they all generally need to test whether they were taken in guest context. 2202284ffeaSNicholas Piggin * 2212284ffeaSNicholas Piggin * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 2222284ffeaSNicholas Piggin * taken with MSR[HV]=0. 2232284ffeaSNicholas Piggin * 2242284ffeaSNicholas Piggin * Interrupts which set SRR registers (with the above exceptions) do not 2252284ffeaSNicholas Piggin * elevate to MSR[HV]=1 mode, though most can be taken when running with 2262284ffeaSNicholas Piggin * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 2272284ffeaSNicholas Piggin * not need to test whether a guest is running because they get delivered to 2282284ffeaSNicholas Piggin * the guest directly, including nested HV KVM guests. 2292284ffeaSNicholas Piggin * 2302284ffeaSNicholas Piggin * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 2312284ffeaSNicholas Piggin * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 2322284ffeaSNicholas Piggin * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 2332284ffeaSNicholas Piggin * delivered to the real-mode entry point, therefore such interrupts only test 2342284ffeaSNicholas Piggin * KVM in their real mode handlers, and only when PR KVM is possible. 2352284ffeaSNicholas Piggin * 2362284ffeaSNicholas Piggin * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 2372284ffeaSNicholas Piggin * delivered in real-mode when the MMU is in hash mode because the MMU 2382284ffeaSNicholas Piggin * registers are not set appropriately to translate host addresses. In nested 2392284ffeaSNicholas Piggin * radix mode these can be delivered in virt-mode as the host translations are 2402284ffeaSNicholas Piggin * used implicitly (see: effective LPID, effective PID). 2412284ffeaSNicholas Piggin */ 2422284ffeaSNicholas Piggin 2432284ffeaSNicholas Piggin/* 2442284ffeaSNicholas Piggin * If an interrupt is taken while a guest is running, it is immediately routed 2452284ffeaSNicholas Piggin * to KVM to handle. If both HV and PR KVM arepossible, KVM interrupts go first 2462284ffeaSNicholas Piggin * to kvmppc_interrupt_hv, which handles the PR guest case. 24712a04809SNicholas Piggin */ 24812a04809SNicholas Piggin#define kvmppc_interrupt kvmppc_interrupt_hv 24912a04809SNicholas Piggin#else 25012a04809SNicholas Piggin#define kvmppc_interrupt kvmppc_interrupt_pr 25112a04809SNicholas Piggin#endif 25212a04809SNicholas Piggin 2539d598f93SNicholas Piggin.macro KVMTEST name 25412a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 25512a04809SNicholas Piggin cmpwi r10,0 25605f97d94SNicholas Piggin bne \name\()_kvm 25712a04809SNicholas Piggin.endm 25812a04809SNicholas Piggin 259b177ae2fSNicholas Piggin.macro GEN_KVM name 2609600f261SNicholas Piggin .balign IFETCH_ALIGN_BYTES 2619600f261SNicholas Piggin\name\()_kvm: 2629600f261SNicholas Piggin 263b177ae2fSNicholas Piggin .if IKVM_SKIP 26412a04809SNicholas Piggin cmpwi r10,KVM_GUEST_MODE_SKIP 26512a04809SNicholas Piggin beq 89f 26612a04809SNicholas Piggin .else 267931dc86bSNicholas PigginBEGIN_FTR_SECTION 268b177ae2fSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 26912a04809SNicholas Piggin std r10,HSTATE_CFAR(r13) 270931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 27112a04809SNicholas Piggin .endif 27212a04809SNicholas Piggin 2739600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 2749600f261SNicholas Piggin mtctr r10 275931dc86bSNicholas PigginBEGIN_FTR_SECTION 276b177ae2fSNicholas Piggin ld r10,IAREA+EX_PPR(r13) 27712a04809SNicholas Piggin std r10,HSTATE_PPR(r13) 278931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2799600f261SNicholas Piggin ld r11,IAREA+EX_R11(r13) 2809600f261SNicholas Piggin ld r12,IAREA+EX_R12(r13) 28112a04809SNicholas Piggin std r12,HSTATE_SCRATCH0(r13) 28212a04809SNicholas Piggin sldi r12,r9,32 2839600f261SNicholas Piggin ld r9,IAREA+EX_R9(r13) 2849600f261SNicholas Piggin ld r10,IAREA+EX_R10(r13) 28512a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 2863f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 287def0db4fSNicholas Piggin BEGIN_FTR_SECTION 288b177ae2fSNicholas Piggin ori r12,r12,(IVEC + 0x2) 289def0db4fSNicholas Piggin FTR_SECTION_ELSE 290b177ae2fSNicholas Piggin ori r12,r12,(IVEC) 291def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 292b177ae2fSNicholas Piggin .elseif IHSRR 293b177ae2fSNicholas Piggin ori r12,r12,(IVEC+ 0x2) 29412a04809SNicholas Piggin .else 295b177ae2fSNicholas Piggin ori r12,r12,(IVEC) 29612a04809SNicholas Piggin .endif 29764e41351SNicholas Piggin b kvmppc_interrupt 29812a04809SNicholas Piggin 299b177ae2fSNicholas Piggin .if IKVM_SKIP 30012a04809SNicholas Piggin89: mtocrf 0x80,r9 3019600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 3029600f261SNicholas Piggin mtctr r10 303b177ae2fSNicholas Piggin ld r9,IAREA+EX_R9(r13) 304b177ae2fSNicholas Piggin ld r10,IAREA+EX_R10(r13) 3059600f261SNicholas Piggin ld r11,IAREA+EX_R11(r13) 3069600f261SNicholas Piggin ld r12,IAREA+EX_R12(r13) 3073f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 308def0db4fSNicholas Piggin BEGIN_FTR_SECTION 309def0db4fSNicholas Piggin b kvmppc_skip_Hinterrupt 310def0db4fSNicholas Piggin FTR_SECTION_ELSE 311def0db4fSNicholas Piggin b kvmppc_skip_interrupt 312def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 313b177ae2fSNicholas Piggin .elseif IHSRR 31412a04809SNicholas Piggin b kvmppc_skip_Hinterrupt 31512a04809SNicholas Piggin .else 31612a04809SNicholas Piggin b kvmppc_skip_interrupt 31712a04809SNicholas Piggin .endif 31812a04809SNicholas Piggin .endif 31912a04809SNicholas Piggin.endm 32012a04809SNicholas Piggin 32112a04809SNicholas Piggin#else 3229d598f93SNicholas Piggin.macro KVMTEST name 32312a04809SNicholas Piggin.endm 324b177ae2fSNicholas Piggin.macro GEN_KVM name 32512a04809SNicholas Piggin.endm 32612a04809SNicholas Piggin#endif 32712a04809SNicholas Piggin 328c7c5cbb4SNicholas Piggin/* 329c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 330c7c5cbb4SNicholas Piggin * 331c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 332c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 333c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 334c7c5cbb4SNicholas Piggin * the kernel. 335c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 336c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 337c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 338c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 339c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 340c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 341c7c5cbb4SNicholas Piggin * irq_soft_mask set. 342c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 343c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 344c7c5cbb4SNicholas Piggin * mode. 345c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 346c7c5cbb4SNicholas Piggin * This is done if early=2. 347c7c5cbb4SNicholas Piggin */ 3488729c26eSNicholas Piggin 3498729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 350d73a10cbSNicholas Piggin .if IREALMODE_COMMON 351d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 352d73a10cbSNicholas Piggin mtctr r10 353d73a10cbSNicholas Piggin bctr 354d73a10cbSNicholas Piggin .else 3558729c26eSNicholas Piggin .if \virt 3568729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 3578729c26eSNicholas Piggin b \name\()_common_virt 3588729c26eSNicholas Piggin#else 3598729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 3608729c26eSNicholas Piggin mtctr r10 3618729c26eSNicholas Piggin bctr 3628729c26eSNicholas Piggin#endif 3638729c26eSNicholas Piggin .else 3648729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 3658729c26eSNicholas Piggin mtctr r10 3668729c26eSNicholas Piggin bctr 3678729c26eSNicholas Piggin .endif 368d73a10cbSNicholas Piggin .endif 3698729c26eSNicholas Piggin.endm 3708729c26eSNicholas Piggin 371fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 372c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 373c7c5cbb4SNicholas Piggin GET_PACA(r13) 374fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 375931dc86bSNicholas PigginBEGIN_FTR_SECTION 376931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 377931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 378c7c5cbb4SNicholas Piggin HMT_MEDIUM 379fc589ee4SNicholas Piggin std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 380931dc86bSNicholas PigginBEGIN_FTR_SECTION 381931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 382931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 383c7c5cbb4SNicholas Piggin .if \ool 384c7c5cbb4SNicholas Piggin .if !\virt 385c7c5cbb4SNicholas Piggin b tramp_real_\name 386c7c5cbb4SNicholas Piggin .pushsection .text 387c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 388c7c5cbb4SNicholas Piggin .else 389c7c5cbb4SNicholas Piggin b tramp_virt_\name 390c7c5cbb4SNicholas Piggin .pushsection .text 391c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 392c7c5cbb4SNicholas Piggin .endif 393c7c5cbb4SNicholas Piggin .endif 394c7c5cbb4SNicholas Piggin 395931dc86bSNicholas PigginBEGIN_FTR_SECTION 396931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 397931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 398931dc86bSNicholas PigginBEGIN_FTR_SECTION 399931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 400931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 401c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 4028729c26eSNicholas Piggin mfctr r10 4038729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 404c7c5cbb4SNicholas Piggin mfcr r9 405fc589ee4SNicholas Piggin std r11,IAREA+EX_R11(r13) 406fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 407c7c5cbb4SNicholas Piggin 408c7c5cbb4SNicholas Piggin /* 409c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 410c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 411c7c5cbb4SNicholas Piggin * not recoverable if they are live. 412c7c5cbb4SNicholas Piggin */ 413c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 414fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 415a3cd35beSNicholas Piggin .if IDAR && !IISIDE 416fc589ee4SNicholas Piggin .if IHSRR 417c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 418c7c5cbb4SNicholas Piggin .else 419c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 420c7c5cbb4SNicholas Piggin .endif 421fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 422c7c5cbb4SNicholas Piggin .endif 423a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 424fc589ee4SNicholas Piggin .if IHSRR 425c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 426c7c5cbb4SNicholas Piggin .else 427c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 428c7c5cbb4SNicholas Piggin .endif 429fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 430c7c5cbb4SNicholas Piggin .endif 431c7c5cbb4SNicholas Piggin 4323f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4338729c26eSNicholas Piggin BEGIN_FTR_SECTION 4348729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 4358729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 4368729c26eSNicholas Piggin FTR_SECTION_ELSE 4378729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 4388729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 4398729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4408729c26eSNicholas Piggin .elseif IHSRR 4418729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 4428729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 4438729c26eSNicholas Piggin .else 4448729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 4458729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 446c7c5cbb4SNicholas Piggin .endif 447d73a10cbSNicholas Piggin 448d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 4498729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 4508729c26eSNicholas Piggin .endif 4518729c26eSNicholas Piggin 452c7c5cbb4SNicholas Piggin .if \ool 453c7c5cbb4SNicholas Piggin .popsection 454c7c5cbb4SNicholas Piggin .endif 455c7c5cbb4SNicholas Piggin.endm 456c7c5cbb4SNicholas Piggin 457d064151fSNicholas Piggin/* 4588729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 4599600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 4609600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 4619600f261SNicholas Piggin * 4628729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 463d064151fSNicholas Piggin */ 4648729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 4658729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 4668729c26eSNicholas Piggin\name\()_common_real: 4679600f261SNicholas Piggin .if IKVM_REAL 4689d598f93SNicholas Piggin KVMTEST \name 4699600f261SNicholas Piggin .endif 4709600f261SNicholas Piggin 4718729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 4728729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 4738729c26eSNicholas Piggin .if IHSRR == EXC_HV_OR_STD 4748729c26eSNicholas Piggin BEGIN_FTR_SECTION 4758729c26eSNicholas Piggin xori r10,r10,MSR_RI 4768729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 4778729c26eSNicholas Piggin .elseif ! IHSRR 4788729c26eSNicholas Piggin xori r10,r10,MSR_RI 4798729c26eSNicholas Piggin .endif 4808729c26eSNicholas Piggin mtmsrd r10 4818729c26eSNicholas Piggin 4828729c26eSNicholas Piggin .if IVIRT 4839600f261SNicholas Piggin .if IKVM_VIRT 4849600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 4859600f261SNicholas Piggin .endif 4869600f261SNicholas Piggin 4878729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 4888729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_virt) 4898729c26eSNicholas Piggin\name\()_common_virt: 4909600f261SNicholas Piggin .if IKVM_VIRT 4919d598f93SNicholas Piggin KVMTEST \name 4929600f261SNicholas Piggin1: 4939600f261SNicholas Piggin .endif 4948729c26eSNicholas Piggin .endif /* IVIRT */ 4958729c26eSNicholas Piggin.endm 4968729c26eSNicholas Piggin 4979600f261SNicholas Piggin/* 4989600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4999600f261SNicholas Piggin * want to run in real mode. 5009600f261SNicholas Piggin */ 5019600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 5029600f261SNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 5039600f261SNicholas Piggin\name\()_common_real: 5049600f261SNicholas Piggin .if IKVM_REAL 5059d598f93SNicholas Piggin KVMTEST \name 5069600f261SNicholas Piggin .endif 5079600f261SNicholas Piggin.endm 5089600f261SNicholas Piggin 5098729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 5100eddf327SNicholas Piggin .if IMASK 5110eddf327SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5120eddf327SNicholas Piggin andi. r10,r10,IMASK 5130eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 5140eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 5150eddf327SNicholas Piggin li r10,PACA_IRQ_EE 5160eddf327SNicholas Piggin .elseif IVEC == 0x900 5170eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 5180eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 5190eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 5200eddf327SNicholas Piggin .elseif IVEC == 0xe60 5210eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 5220eddf327SNicholas Piggin .elseif IVEC == 0xf00 5230eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 5240eddf327SNicholas Piggin .else 5250eddf327SNicholas Piggin .abort "Bad maskable vector" 5260eddf327SNicholas Piggin .endif 5270eddf327SNicholas Piggin 5283f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 5290eddf327SNicholas Piggin BEGIN_FTR_SECTION 5300eddf327SNicholas Piggin bne masked_Hinterrupt 5310eddf327SNicholas Piggin FTR_SECTION_ELSE 5320eddf327SNicholas Piggin bne masked_interrupt 5330eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 5340eddf327SNicholas Piggin .elseif IHSRR 5350eddf327SNicholas Piggin bne masked_Hinterrupt 5360eddf327SNicholas Piggin .else 5370eddf327SNicholas Piggin bne masked_interrupt 5380eddf327SNicholas Piggin .endif 5390eddf327SNicholas Piggin .endif 5400eddf327SNicholas Piggin 5416d71759aSNicholas Piggin .if ISTACK 5425d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 5435d5e0edfSNicholas Piggin mr r10,r1 /* Save r1 */ 5445d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 5451b359982SNicholas Piggin beq- 100f 5465d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 5471b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 5481b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 5495d5e0edfSNicholas Piggin .endif 5508c9fb5d4SNicholas Piggin 5518c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 5528c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 5538c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 5548c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 5558c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 5568c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 5575d5e0edfSNicholas Piggin 5588729c26eSNicholas Piggin .if ISET_RI 5598729c26eSNicholas Piggin li r10,MSR_RI 5608729c26eSNicholas Piggin mtmsrd r10,1 /* Set MSR_RI */ 5618729c26eSNicholas Piggin .endif 5628729c26eSNicholas Piggin 5636d71759aSNicholas Piggin .if ISTACK 5646d71759aSNicholas Piggin .if IKUAP 5655d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 5665d5e0edfSNicholas Piggin .endif 5671b359982SNicholas Piggin beq 101f /* if from kernel mode */ 5685d5e0edfSNicholas Piggin ACCOUNT_CPU_USER_ENTRY(r13, r9, r10) 569931dc86bSNicholas PigginBEGIN_FTR_SECTION 570931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 571931dc86bSNicholas Piggin std r9,_PPR(r1) 572931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5731b359982SNicholas Piggin101: 5745d5e0edfSNicholas Piggin .else 5756d71759aSNicholas Piggin .if IKUAP 576bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 577bcbceed4SNicholas Piggin .endif 5785d5e0edfSNicholas Piggin .endif 5795d5e0edfSNicholas Piggin 5808c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5816d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5826d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5838c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5848c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5856d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5866d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5876d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5888c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5898c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5908c9fb5d4SNicholas Piggin std r11,GPR13(r1) 591a3cd35beSNicholas Piggin 5926cc0c16dSNicholas Piggin SAVE_NVGPRS(r1) 5936cc0c16dSNicholas Piggin 5946d71759aSNicholas Piggin .if IDAR 595a3cd35beSNicholas Piggin .if IISIDE 596d1a84718SNicholas Piggin ld r10,_NIP(r1) 597d1a84718SNicholas Piggin .else 5986d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 599d1a84718SNicholas Piggin .endif 600d1a84718SNicholas Piggin std r10,_DAR(r1) 601d1a84718SNicholas Piggin .endif 602a3cd35beSNicholas Piggin 6036d71759aSNicholas Piggin .if IDSISR 604a3cd35beSNicholas Piggin .if IISIDE 605d1a84718SNicholas Piggin ld r10,_MSR(r1) 606d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 607d1a84718SNicholas Piggin and r10,r10,r11 608d1a84718SNicholas Piggin .else 6096d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 610d1a84718SNicholas Piggin .endif 611d1a84718SNicholas Piggin std r10,_DSISR(r1) 612d1a84718SNicholas Piggin .endif 613a3cd35beSNicholas Piggin 614931dc86bSNicholas PigginBEGIN_FTR_SECTION 6156d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 6168c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 617931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 6188729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 6198c9fb5d4SNicholas Piggin std r10,_CTR(r1) 6208c9fb5d4SNicholas Piggin std r2,GPR2(r1) /* save r2 in stackframe */ 6218c9fb5d4SNicholas Piggin SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ 6228c9fb5d4SNicholas Piggin SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ 6238c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 6248c9fb5d4SNicholas Piggin ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 6258c9fb5d4SNicholas Piggin std r9,_LINK(r1) 6268c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 6278c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 6288c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 6298c9fb5d4SNicholas Piggin std r11,_XER(r1) 6306cc0c16dSNicholas Piggin li r9,IVEC 6318c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 6328c9fb5d4SNicholas Piggin li r10,0 6338c9fb5d4SNicholas Piggin ld r11,exception_marker@toc(r2) 6348c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 6358c9fb5d4SNicholas Piggin std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 6365d5e0edfSNicholas Piggin 6376d71759aSNicholas Piggin .if ISTACK 6385d5e0edfSNicholas Piggin ACCOUNT_STOLEN_TIME 639bcbceed4SNicholas Piggin .endif 640d1a84718SNicholas Piggin 6416d71759aSNicholas Piggin .if IRECONCILE 642d1a84718SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 643d1a84718SNicholas Piggin .endif 644bcbceed4SNicholas Piggin.endm 645bcbceed4SNicholas Piggin 646391e941bSNicholas Piggin/* 6478729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 6488729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 6498729c26eSNicholas Piggin * SRR1, and relocation is on. 6508729c26eSNicholas Piggin * 6518729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 6528729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 6538729c26eSNicholas Piggin */ 6548729c26eSNicholas Piggin.macro GEN_COMMON name 6558729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 6568729c26eSNicholas Piggin __GEN_COMMON_BODY \name 6578729c26eSNicholas Piggin.endm 6588729c26eSNicholas Piggin 6598729c26eSNicholas Piggin/* 660391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 661391e941bSNicholas Piggin * standard exception. 662391e941bSNicholas Piggin */ 6633f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 664391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 665391e941bSNicholas Piggin ld r9,_MSR(r1) 666391e941bSNicholas Piggin .if \hsrr 667391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 668391e941bSNicholas Piggin .else 669391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 670391e941bSNicholas Piggin .endif 671391e941bSNicholas Piggin ld r9,_NIP(r1) 672391e941bSNicholas Piggin .if \hsrr 673391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 674391e941bSNicholas Piggin .else 675391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 676391e941bSNicholas Piggin .endif 677391e941bSNicholas Piggin ld r9,_CTR(r1) 678391e941bSNicholas Piggin mtctr r9 679391e941bSNicholas Piggin ld r9,_XER(r1) 680391e941bSNicholas Piggin mtxer r9 681391e941bSNicholas Piggin ld r9,_LINK(r1) 682391e941bSNicholas Piggin mtlr r9 683391e941bSNicholas Piggin ld r9,_CCR(r1) 684391e941bSNicholas Piggin mtcr r9 685391e941bSNicholas Piggin REST_8GPRS(2, r1) 686391e941bSNicholas Piggin REST_4GPRS(10, r1) 687391e941bSNicholas Piggin REST_GPR(0, r1) 688391e941bSNicholas Piggin /* restore original r1. */ 689391e941bSNicholas Piggin ld r1,GPR1(r1) 690391e941bSNicholas Piggin.endm 691d064151fSNicholas Piggin 69212a04809SNicholas Piggin#define RUNLATCH_ON \ 69312a04809SNicholas PigginBEGIN_FTR_SECTION \ 69412a04809SNicholas Piggin ld r3, PACA_THREAD_INFO(r13); \ 69512a04809SNicholas Piggin ld r4,TI_LOCAL_FLAGS(r3); \ 69612a04809SNicholas Piggin andi. r0,r4,_TLF_RUNLATCH; \ 69712a04809SNicholas Piggin beql ppc64_runlatch_on_trampoline; \ 69812a04809SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CTRL) 69912a04809SNicholas Piggin 70012a04809SNicholas Piggin/* 70112a04809SNicholas Piggin * When the idle code in power4_idle puts the CPU into NAP mode, 70212a04809SNicholas Piggin * it has to do so in a loop, and relies on the external interrupt 70312a04809SNicholas Piggin * and decrementer interrupt entry code to get it out of the loop. 70412a04809SNicholas Piggin * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags 70512a04809SNicholas Piggin * to signal that it is in the loop and needs help to get out. 70612a04809SNicholas Piggin */ 70712a04809SNicholas Piggin#ifdef CONFIG_PPC_970_NAP 70812a04809SNicholas Piggin#define FINISH_NAP \ 70912a04809SNicholas PigginBEGIN_FTR_SECTION \ 71012a04809SNicholas Piggin ld r11, PACA_THREAD_INFO(r13); \ 71112a04809SNicholas Piggin ld r9,TI_LOCAL_FLAGS(r11); \ 71212a04809SNicholas Piggin andi. r10,r9,_TLF_NAPPING; \ 71312a04809SNicholas Piggin bnel power4_fixup_nap; \ 71412a04809SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 71512a04809SNicholas Piggin#else 71612a04809SNicholas Piggin#define FINISH_NAP 71712a04809SNicholas Piggin#endif 71812a04809SNicholas Piggin 71912a04809SNicholas Piggin/* 72057f26649SNicholas Piggin * There are a few constraints to be concerned with. 72157f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 72257f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 72357f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 72457f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 72557f26649SNicholas Piggin * must be used. 72657f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 72757f26649SNicholas Piggin * virtual 0xc00... 72857f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 72957f26649SNicholas Piggin * 73057f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 73157f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 73257f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 73357f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 73457f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 73557f26649SNicholas Piggin * 73657f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 73757f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 73857f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 73957f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 74057f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 74157f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 74257f26649SNicholas Piggin * 74357f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 74457f26649SNicholas Piggin * 74557f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 74657f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 74757f26649SNicholas Piggin * 74857f26649SNicholas Piggin * 7490ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 7500ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 75157f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 75257f26649SNicholas Piggin * 0x1900 - 0x3fff : Real mode trampolines 75357f26649SNicholas Piggin * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 75457f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 7550ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 75657f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 75757f26649SNicholas Piggin * setup code, rest of kernel. 758e0319829SNicholas Piggin * 759e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 760e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 761e0319829SNicholas Piggin * vectors there. 7620ebc4cdaSBenjamin Herrenschmidt */ 76357f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 76457f26649SNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000) 76557f26649SNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900) 76657f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 767ccd47702SNicholas Piggin 768ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 769bd3524feSNicholas Piggin .globl start_real_trampolines 770bd3524feSNicholas Piggin .globl end_real_trampolines 771bd3524feSNicholas Piggin .globl start_virt_trampolines 772bd3524feSNicholas Piggin .globl end_virt_trampolines 773ccd47702SNicholas Piggin#endif 774ccd47702SNicholas Piggin 77557f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 77657f26649SNicholas Piggin/* 77757f26649SNicholas Piggin * Data area reserved for FWNMI option. 77857f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 77957f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 78057f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 78157f26649SNicholas Piggin */ 78257f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 78357f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 78457f26649SNicholas Piggin#else 78557f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 78657f26649SNicholas Piggin#endif 78757f26649SNicholas Piggin 78857f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 78957f26649SNicholas Piggin 7900ebc4cdaSBenjamin Herrenschmidt/* 7910ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 7920ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 7930ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 7940ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 7950ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 7960ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 7970ebc4cdaSBenjamin Herrenschmidt */ 7980ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 7990ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 8000ebc4cdaSBenjamin Herrenschmidt 801e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 8021a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 803e0319829SNicholas Piggin 804fb479e44SNicholas Piggin 80594325357SNicholas Piggin/** 80694325357SNicholas Piggin * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 80794325357SNicholas Piggin * This is a non-maskable, asynchronous interrupt always taken in real-mode. 80894325357SNicholas Piggin * It is caused by: 80994325357SNicholas Piggin * - Wake from power-saving state, on powernv. 81094325357SNicholas Piggin * - An NMI from another CPU, triggered by firmware or hypercall. 81194325357SNicholas Piggin * - As crash/debug signal injected from BMC, firmware or hypervisor. 81294325357SNicholas Piggin * 81394325357SNicholas Piggin * Handling: 81494325357SNicholas Piggin * Power-save wakeup is the only performance critical path, so this is 81594325357SNicholas Piggin * determined quickly as possible first. In this case volatile registers 81694325357SNicholas Piggin * can be discarded and SPRs like CFAR don't need to be read. 81794325357SNicholas Piggin * 81894325357SNicholas Piggin * If not a powersave wakeup, then it's run as a regular interrupt, however 81994325357SNicholas Piggin * it uses its own stack and PACA save area to preserve the regular kernel 82094325357SNicholas Piggin * environment for debugging. 82194325357SNicholas Piggin * 82294325357SNicholas Piggin * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 82394325357SNicholas Piggin * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 82494325357SNicholas Piggin * correct to switch to virtual mode to run the regular interrupt handler 82594325357SNicholas Piggin * because it might be interrupted when the MMU is in a bad state (e.g., SLB 82694325357SNicholas Piggin * is clear). 82794325357SNicholas Piggin * 82894325357SNicholas Piggin * FWNMI: 82994325357SNicholas Piggin * PAPR specifies a "fwnmi" facility which sends the sreset to a different 83094325357SNicholas Piggin * entry point with a different register set up. Some hypervisors will 83194325357SNicholas Piggin * send the sreset to 0x100 in the guest if it is not fwnmi capable. 83294325357SNicholas Piggin * 83394325357SNicholas Piggin * KVM: 83494325357SNicholas Piggin * Unlike most SRR interrupts, this may be taken by the host while executing 83594325357SNicholas Piggin * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 83694325357SNicholas Piggin * mode and then raise the sreset. 83794325357SNicholas Piggin */ 8384f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 8394f50541fSNicholas Piggin IVEC=0x100 8404f50541fSNicholas Piggin IAREA=PACA_EXNMI 8418729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 8424f50541fSNicholas Piggin /* 8434f50541fSNicholas Piggin * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 8444f50541fSNicholas Piggin * being used, so a nested NMI exception would corrupt it. 8454f50541fSNicholas Piggin */ 8464f50541fSNicholas Piggin ISET_RI=0 8474f50541fSNicholas Piggin ISTACK=0 8484f50541fSNicholas Piggin IRECONCILE=0 8494f50541fSNicholas Piggin IKVM_REAL=1 8504f50541fSNicholas PigginINT_DEFINE_END(system_reset) 8514f50541fSNicholas Piggin 852a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 853fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 854fb479e44SNicholas Piggin /* 855fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 856ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 857ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 858ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 859ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 8600e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 8610e10be2bSNicholas Piggin * 8620e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 8630e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 8640e10be2bSNicholas Piggin * 8650e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 8660e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 8670e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 8680e10be2bSNicholas Piggin * common exception macros to handle it. 869fb479e44SNicholas Piggin */ 870a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 8710e10be2bSNicholas Piggin SET_SCRATCH0(r13) 8720e10be2bSNicholas Piggin GET_PACA(r13) 8730e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 8740e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 8750e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 876a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 8770e10be2bSNicholas Piggin mfocrf r4,0x80 8780e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 8790e10be2bSNicholas Piggin bne+ system_reset_idle_wake 8800e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 8810e10be2bSNicholas Piggin mtocrf 0x80,r4 8820e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 8830e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 8840e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 8850e10be2bSNicholas Piggin GET_SCRATCH0(r13) 886a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 887fb479e44SNicholas Piggin#endif 888fb479e44SNicholas Piggin 8894f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 890c4f3b52cSNicholas Piggin /* 8910e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 8920e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 8930e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 8940e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 8950e10be2bSNicholas Piggin * be dangerous anyway. 896c4f3b52cSNicholas Piggin */ 8971a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 8981a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 899fb479e44SNicholas Piggin 900fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 9010e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 9020e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 9030e10be2bSNicholas Piggin cmpwi cr1,r5,2 9040e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 9050e10be2bSNicholas Piggin BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) 906371fefd6SPaul Mackerras#endif 907371fefd6SPaul Mackerras 908acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 909acc8da44SNicholas Piggin/* 910acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 911acc8da44SNicholas Piggin */ 912acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 91394325357SNicholas Piggin /* XXX: fwnmi guest could run a nested/PR guest, so why no test? */ 9144f50541fSNicholas Piggin __IKVM_REAL(system_reset)=0 9154f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 916acc8da44SNicholas Piggin 917acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 918acc8da44SNicholas Piggin 919a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 9208729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 921c4f3b52cSNicholas Piggin /* 922c4f3b52cSNicholas Piggin * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 923c4f3b52cSNicholas Piggin * to recover, but nested NMI will notice in_nmi and not recover 924c4f3b52cSNicholas Piggin * because of the use of the NMI stack. in_nmi reentrancy is tested in 925c4f3b52cSNicholas Piggin * system_reset_exception. 926c4f3b52cSNicholas Piggin */ 927c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 928c4f3b52cSNicholas Piggin addi r10,r10,1 929c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 930c4f3b52cSNicholas Piggin li r10,MSR_RI 931c4f3b52cSNicholas Piggin mtmsrd r10,1 932aca79d2bSVaidyanathan Srinivasan 933b1ee8a3dSNicholas Piggin mr r10,r1 934b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 935b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 9368729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 93747169fbaSNicholas Piggin /* 938b44fc96dSNicholas Piggin * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does 93947169fbaSNicholas Piggin * the right thing. We do not want to reconcile because that goes 94047169fbaSNicholas Piggin * through irq tracing which we don't want in NMI. 94147169fbaSNicholas Piggin * 942b44fc96dSNicholas Piggin * Save PACAIRQHAPPENED to _DAR (otherwise unused), and set HARD_DIS 943b44fc96dSNicholas Piggin * as we are running with MSR[EE]=0. 94447169fbaSNicholas Piggin */ 94547169fbaSNicholas Piggin li r10,IRQS_ALL_DISABLED 94647169fbaSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 94747169fbaSNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 94847169fbaSNicholas Piggin std r10,_DAR(r1) 949b44fc96dSNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 950b44fc96dSNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 95147169fbaSNicholas Piggin 952c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 953c06075f3SNicholas Piggin bl system_reset_exception 95415b4dd79SNicholas Piggin 95515b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 956fbc50063SNicholas Piggin li r9,0 95715b4dd79SNicholas Piggin mtmsrd r9,1 958c4f3b52cSNicholas Piggin 959c4f3b52cSNicholas Piggin /* 96015b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 961c4f3b52cSNicholas Piggin */ 962c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 963c4f3b52cSNicholas Piggin subi r10,r10,1 964c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 965c4f3b52cSNicholas Piggin 96615b4dd79SNicholas Piggin /* 96715b4dd79SNicholas Piggin * Restore soft mask settings. 96815b4dd79SNicholas Piggin */ 96915b4dd79SNicholas Piggin ld r10,_DAR(r1) 97015b4dd79SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 97115b4dd79SNicholas Piggin ld r10,SOFTE(r1) 97215b4dd79SNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 97315b4dd79SNicholas Piggin 9743f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 97515b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 976582baf44SNicholas Piggin 9779600f261SNicholas Piggin GEN_KVM system_reset 9789600f261SNicholas Piggin 9790ebc4cdaSBenjamin Herrenschmidt 98094325357SNicholas Piggin/** 98194325357SNicholas Piggin * Interrupt 0x200 - Machine Check Interrupt (MCE). 98294325357SNicholas Piggin * This is a non-maskable interrupt always taken in real-mode. It can be 98394325357SNicholas Piggin * synchronous or asynchronous, caused by hardware or software, and it may be 98494325357SNicholas Piggin * taken in a power-saving state. 98594325357SNicholas Piggin * 98694325357SNicholas Piggin * Handling: 98794325357SNicholas Piggin * Similarly to system reset, this uses its own stack and PACA save area, 98894325357SNicholas Piggin * the difference is re-entrancy is allowed on the machine check stack. 98994325357SNicholas Piggin * 99094325357SNicholas Piggin * machine_check_early is run in real mode, and carefully decodes the 99194325357SNicholas Piggin * machine check and tries to handle it (e.g., flush the SLB if there was an 99294325357SNicholas Piggin * error detected there), determines if it was recoverable and logs the 99394325357SNicholas Piggin * event. 99494325357SNicholas Piggin * 995b44fc96dSNicholas Piggin * This early code does not "reconcile" irq soft-mask state like SRESET or 996b44fc96dSNicholas Piggin * regular interrupts do, so irqs_disabled() among other things may not work 997b44fc96dSNicholas Piggin * properly (irq disable/enable already doesn't work because irq tracing can 998b44fc96dSNicholas Piggin * not work in real mode). 999b44fc96dSNicholas Piggin * 100094325357SNicholas Piggin * Then, depending on the execution context when the interrupt is taken, there 100194325357SNicholas Piggin * are 3 main actions: 100294325357SNicholas Piggin * - Executing in kernel mode. The event is queued with irq_work, which means 100394325357SNicholas Piggin * it is handled when it is next safe to do so (i.e., the kernel has enabled 100494325357SNicholas Piggin * interrupts), which could be immediately when the interrupt returns. This 100594325357SNicholas Piggin * avoids nasty issues like switching to virtual mode when the MMU is in a 100694325357SNicholas Piggin * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 100794325357SNicholas Piggin * but it has different priorities). Check to see if the CPU was in power 100894325357SNicholas Piggin * save, and return via the wake up code if it was. 100994325357SNicholas Piggin * 101094325357SNicholas Piggin * - Executing in user mode. machine_check_exception is run like a normal 101194325357SNicholas Piggin * interrupt handler, which processes the data generated by the early handler. 101294325357SNicholas Piggin * 101394325357SNicholas Piggin * - Executing in guest mode. The interrupt is run with its KVM test, and 101494325357SNicholas Piggin * branches to KVM to deal with. KVM may queue the event for the host 101594325357SNicholas Piggin * to report later. 101694325357SNicholas Piggin * 101794325357SNicholas Piggin * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 101894325357SNicholas Piggin * or SCRATCH0 is in use, it may cause a crash. 101994325357SNicholas Piggin * 102094325357SNicholas Piggin * KVM: 102194325357SNicholas Piggin * See SRESET. 102294325357SNicholas Piggin */ 10234f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 10244f50541fSNicholas Piggin IVEC=0x200 10254f50541fSNicholas Piggin IAREA=PACA_EXMC 10268729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 1027d73a10cbSNicholas Piggin IREALMODE_COMMON=1 1028c8eb54dbSNicholas Piggin /* 1029c8eb54dbSNicholas Piggin * MSR_RI is not enabled, because PACA_EXMC is being used, so a 1030c8eb54dbSNicholas Piggin * nested machine check corrupts it. machine_check_common enables 1031c8eb54dbSNicholas Piggin * MSR_RI. 1032c8eb54dbSNicholas Piggin */ 10334f50541fSNicholas Piggin ISET_RI=0 10344f50541fSNicholas Piggin ISTACK=0 10354f50541fSNicholas Piggin IDAR=1 10364f50541fSNicholas Piggin IDSISR=1 10374f50541fSNicholas Piggin IRECONCILE=0 10384f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 10394f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 10404f50541fSNicholas Piggin 10414f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 10424f50541fSNicholas Piggin IVEC=0x200 10434f50541fSNicholas Piggin IAREA=PACA_EXMC 10448729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 10454f50541fSNicholas Piggin ISET_RI=0 10464f50541fSNicholas Piggin IDAR=1 10474f50541fSNicholas Piggin IDSISR=1 10484f50541fSNicholas Piggin IKVM_SKIP=1 10494f50541fSNicholas Piggin IKVM_REAL=1 10504f50541fSNicholas PigginINT_DEFINE_END(machine_check) 10514f50541fSNicholas Piggin 10524f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 10534f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 10541a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 10551a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 1056c8eb54dbSNicholas Piggin 1057abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1058abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 1059abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 10604f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 1061abd1f4caSNicholas Piggin#endif 1062abd1f4caSNicholas Piggin 1063fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 1064fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1065fce16d48SNicholas Piggin li r9,0; \ 1066fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 1067fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 1068fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 1069fce16d48SNicholas Piggin subi r12,r12,1; \ 1070fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 10713f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 1072fce16d48SNicholas Piggin 1073c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 10749600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 10759600f261SNicholas Piggin 1076afcf0095SNicholas Piggin /* 1077afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 1078afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 1079afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1080afcf0095SNicholas Piggin * 1081afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 1082afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 1083afcf0095SNicholas Piggin * machine checks. 1084afcf0095SNicholas Piggin * 1085afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 1086afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 1087afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 1088afcf0095SNicholas Piggin * 1089afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1090afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 1091afcf0095SNicholas Piggin * rfid with MSR_ME=1. 10921945bc45SNicholas Piggin * 10931945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 10941945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 10952bf1071aSNicholas Piggin * to restore state. 1096afcf0095SNicholas Piggin */ 1097afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 1098afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 1099c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1100afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 1101afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 1102c8eb54dbSNicholas Piggin 1103c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 1104c8eb54dbSNicholas Piggin bne 1f 1105c8eb54dbSNicholas Piggin /* First machine check entry */ 1106c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1107b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1108b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1109b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1110c8eb54dbSNicholas Piggin 11118729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1112c8eb54dbSNicholas Piggin 1113db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1114296e753fSNicholas Piggin bl enable_machine_check 1115db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1116296e753fSNicholas Piggin li r10,MSR_RI 1117296e753fSNicholas Piggin mtmsrd r10,1 1118296e753fSNicholas Piggin 1119afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1120afcf0095SNicholas Piggin bl machine_check_early 1121afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1122afcf0095SNicholas Piggin ld r12,_MSR(r1) 11231945bc45SNicholas Piggin 1124afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1125afcf0095SNicholas Piggin /* 1126afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1127afcf0095SNicholas Piggin * of the following is true: 1128afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1129afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1130afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1131afcf0095SNicholas Piggin * 1132afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1133afcf0095SNicholas Piggin */ 11341945bc45SNicholas PigginBEGIN_FTR_SECTION 11351945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 11366102c005SNicholas Piggin bne machine_check_idle_common 11371945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1138afcf0095SNicholas Piggin#endif 11391945bc45SNicholas Piggin 1140afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1141afcf0095SNicholas Piggin /* 114219dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 114305f97d94SNicholas Piggin * exception handler which will take the 114405f97d94SNicholas Piggin * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event 114505f97d94SNicholas Piggin * to guest. 1146afcf0095SNicholas Piggin */ 1147afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1148afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1149b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1150afcf0095SNicholas Piggin#endif 115119dbe673SNicholas Piggin 1152afcf0095SNicholas Piggin /* 115319dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 115419dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 115519dbe673SNicholas Piggin */ 115619dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1157b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 115819dbe673SNicholas Piggin 115919dbe673SNicholas Piggin /* 116019dbe673SNicholas Piggin * At this point we are coming from kernel context. 1161afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1162afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1163afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1164afcf0095SNicholas Piggin */ 1165afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1166b7d9ccecSNicholas Piggin beq unrecoverable_mce 1167b7d9ccecSNicholas Piggin 1168afcf0095SNicholas Piggin /* 1169afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1170afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1171afcf0095SNicholas Piggin */ 1172afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1173afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1174b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1175272f6364SNicholas Piggin 1176afcf0095SNicholas Piggin /* 1177afcf0095SNicholas Piggin * Return from MC interrupt. 1178afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1179afcf0095SNicholas Piggin * returning from kernel or opal call. 1180afcf0095SNicholas Piggin */ 1181afcf0095SNicholas Piggin bl machine_check_queue_event 1182afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1183fe9d482bSNicholas Piggin RFI_TO_KERNEL 1184272f6364SNicholas Piggin 1185b3fe3526SNicholas Pigginmce_deliver: 1186b3fe3526SNicholas Piggin /* 1187b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1188b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1189b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1190b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1191b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1192b3fe3526SNicholas Piggin */ 11930b66370cSNicholas PigginBEGIN_FTR_SECTION 11940b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 11950b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 11960b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1197afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 11984f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1199afcf0095SNicholas Piggin 1200fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1201fce16d48SNicholas Piggin /* 1202fce16d48SNicholas Piggin * Machine check is different because we use a different 1203fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1204fce16d48SNicholas Piggin */ 12054f50541fSNicholas Piggin GEN_COMMON machine_check 12064f50541fSNicholas Piggin 1207fce16d48SNicholas Piggin FINISH_NAP 1208fce16d48SNicholas Piggin /* Enable MSR_RI when finished with PACA_EXMC */ 1209fce16d48SNicholas Piggin li r10,MSR_RI 1210fce16d48SNicholas Piggin mtmsrd r10,1 1211fce16d48SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1212fce16d48SNicholas Piggin bl machine_check_exception 12136cc0c16dSNicholas Piggin b interrupt_return 1214fce16d48SNicholas Piggin 12159600f261SNicholas Piggin GEN_KVM machine_check 12169600f261SNicholas Piggin 12179600f261SNicholas Piggin 1218fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1219fce16d48SNicholas Piggin/* 1220fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1221fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1222fce16d48SNicholas Piggin */ 1223fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 1224fce16d48SNicholas Piggin bl machine_check_queue_event 1225fce16d48SNicholas Piggin 1226fce16d48SNicholas Piggin /* 1227*8a5054d8SNicholas Piggin * GPR-loss wakeups are relatively straightforward, because the 1228*8a5054d8SNicholas Piggin * idle sleep code has saved all non-volatile registers on its 1229*8a5054d8SNicholas Piggin * own stack, and r1 in PACAR1. 1230fce16d48SNicholas Piggin * 1231*8a5054d8SNicholas Piggin * For no-loss wakeups the r1 and lr registers used by the 1232*8a5054d8SNicholas Piggin * early machine check handler have to be restored first. r2 is 1233*8a5054d8SNicholas Piggin * the kernel TOC, so no need to restore it. 1234fce16d48SNicholas Piggin * 1235fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1236fce16d48SNicholas Piggin */ 1237fce16d48SNicholas Piggin ld r3,_MSR(r1) 1238fce16d48SNicholas Piggin ld r4,_LINK(r1) 1239*8a5054d8SNicholas Piggin ld r1,GPR1(r1) 1240fce16d48SNicholas Piggin 1241fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1242fce16d48SNicholas Piggin subi r11,r11,1 1243fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1244fce16d48SNicholas Piggin 1245fce16d48SNicholas Piggin mtlr r4 1246fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1247fce16d48SNicholas Piggin cmpwi cr1,r10,2 1248*8a5054d8SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1249fce16d48SNicholas Piggin b idle_return_gpr_loss 1250fce16d48SNicholas Piggin#endif 1251fce16d48SNicholas Piggin 1252b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1253b7d9ccecSNicholas Piggin /* 1254b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1255b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1256b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1257b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1258b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1259b7d9ccecSNicholas Piggin */ 1260b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1261b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1262b7d9ccecSNicholas Piggin mtmsrd r10,1 1263b7d9ccecSNicholas Piggin bl disable_machine_check 1264b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1265b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1266b7d9ccecSNicholas Piggin li r3,MSR_ME 1267b7d9ccecSNicholas Piggin andc r10,r10,r3 1268b7d9ccecSNicholas Piggin mtmsrd r10 1269b7d9ccecSNicholas Piggin 1270afcf0095SNicholas Piggin /* Invoke machine_check_exception to print MCE event and panic. */ 1271afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1272afcf0095SNicholas Piggin bl machine_check_exception 1273b7d9ccecSNicholas Piggin 1274afcf0095SNicholas Piggin /* 1275b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1276b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1277afcf0095SNicholas Piggin */ 1278b7d9ccecSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1279afcf0095SNicholas Piggin bl unrecoverable_exception 1280b7d9ccecSNicholas Piggin b . 1281afcf0095SNicholas Piggin 12824f50541fSNicholas Piggin 12834f50541fSNicholas Piggin/** 128494325357SNicholas Piggin * Interrupt 0x300 - Data Storage Interrupt (DSI). 128594325357SNicholas Piggin * This is a synchronous interrupt generated due to a data access exception, 128694325357SNicholas Piggin * e.g., a load orstore which does not have a valid page table entry with 128794325357SNicholas Piggin * permissions. DAWR matches also fault here, as do RC updates, and minor misc 128894325357SNicholas Piggin * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 12894f50541fSNicholas Piggin * 129094325357SNicholas Piggin * Handling: 129194325357SNicholas Piggin * - Hash MMU 129294325357SNicholas Piggin * Go to do_hash_page first to see if the HPT can be filled from an entry in 129394325357SNicholas Piggin * the Linux page table. Hash faults can hit in kernel mode in a fairly 129494325357SNicholas Piggin * arbitrary state (e.g., interrupts disabled, locks held) when accessing 129594325357SNicholas Piggin * "non-bolted" regions, e.g., vmalloc space. However these should always be 129694325357SNicholas Piggin * backed by Linux page tables. 12974f50541fSNicholas Piggin * 129894325357SNicholas Piggin * If none is found, do a Linux page fault. Linux page faults can happen in 129994325357SNicholas Piggin * kernel mode due to user copy operations of course. 13004f50541fSNicholas Piggin * 130194325357SNicholas Piggin * - Radix MMU 130294325357SNicholas Piggin * The hardware loads from the Linux page table directly, so a fault goes 130394325357SNicholas Piggin * immediately to Linux page fault. 130494325357SNicholas Piggin * 130594325357SNicholas Piggin * Conditions like DAWR match are handled on the way in to Linux page fault. 13064f50541fSNicholas Piggin */ 1307a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1308a42a239dSNicholas Piggin IVEC=0x300 1309a42a239dSNicholas Piggin IDAR=1 1310a42a239dSNicholas Piggin IDSISR=1 13112284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1312d52fd3d3SNicholas Piggin IKVM_SKIP=1 1313a42a239dSNicholas Piggin IKVM_REAL=1 13142284ffeaSNicholas Piggin#endif 1315a42a239dSNicholas PigginINT_DEFINE_END(data_access) 13160ebc4cdaSBenjamin Herrenschmidt 1317e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1318689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1319e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1320e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1321a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1322e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 132380795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 13247cb3a1a0SNicholas Piggin GEN_COMMON data_access 13259b123d1eSNicholas Piggin ld r4,_DAR(r1) 13269b123d1eSNicholas Piggin ld r5,_DSISR(r1) 132780795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 13289b123d1eSNicholas Piggin ld r6,_MSR(r1) 13299b123d1eSNicholas Piggin li r3,0x300 133080795e6cSNicholas Piggin b do_hash_page /* Try to handle as hpte fault */ 133180795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 133280795e6cSNicholas Piggin b handle_page_fault 133380795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 133480795e6cSNicholas Piggin 13359600f261SNicholas Piggin GEN_KVM data_access 13369600f261SNicholas Piggin 13370ebc4cdaSBenjamin Herrenschmidt 133894325357SNicholas Piggin/** 133994325357SNicholas Piggin * Interrupt 0x380 - Data Segment Interrupt (DSLB). 134094325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault missing SLB 134194325357SNicholas Piggin * entry for HPT, or an address outside RPT translation range. 134294325357SNicholas Piggin * 134394325357SNicholas Piggin * Handling: 134494325357SNicholas Piggin * - HPT: 134594325357SNicholas Piggin * This refills the SLB, or reports an access fault similarly to a bad page 134694325357SNicholas Piggin * fault. When coming from user-mode, the SLB handler may access any kernel 134794325357SNicholas Piggin * data, though it may itself take a DSLB. When coming from kernel mode, 134894325357SNicholas Piggin * recursive faults must be avoided so access is restricted to the kernel 134994325357SNicholas Piggin * image text/data, kernel stack, and any data allocated below 135094325357SNicholas Piggin * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 135194325357SNicholas Piggin * on user-handler data structures. 135294325357SNicholas Piggin * 135394325357SNicholas Piggin * A dedicated save area EXSLB is used (XXX: but it actually need not be 135494325357SNicholas Piggin * these days, we could use EXGEN). 135594325357SNicholas Piggin */ 13564f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 13574f50541fSNicholas Piggin IVEC=0x380 13584f50541fSNicholas Piggin IAREA=PACA_EXSLB 13594f50541fSNicholas Piggin IRECONCILE=0 13604f50541fSNicholas Piggin IDAR=1 13612284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 13624f50541fSNicholas Piggin IKVM_SKIP=1 13634f50541fSNicholas Piggin IKVM_REAL=1 13642284ffeaSNicholas Piggin#endif 13654f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 13664f50541fSNicholas Piggin 13671a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1368689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 13691a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 13701a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 13714f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 13721a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 137348e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 13744f50541fSNicholas Piggin GEN_COMMON data_access_slb 1375d1a84718SNicholas Piggin ld r4,_DAR(r1) 137648e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 13777100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 13787100e870SNicholas Piggin /* HPT case, do SLB fault */ 137948e7b769SNicholas Piggin bl do_slb_fault 138048e7b769SNicholas Piggin cmpdi r3,0 138148e7b769SNicholas Piggin bne- 1f 13826cc0c16dSNicholas Piggin b fast_interrupt_return 138348e7b769SNicholas Piggin1: /* Error case */ 13847100e870SNicholas PigginMMU_FTR_SECTION_ELSE 13857100e870SNicholas Piggin /* Radix case, access is outside page table range */ 13867100e870SNicholas Piggin li r3,-EFAULT 13877100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 138848e7b769SNicholas Piggin std r3,RESULT(r1) 138948e7b769SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 139048e7b769SNicholas Piggin ld r4,_DAR(r1) 139148e7b769SNicholas Piggin ld r5,RESULT(r1) 139248e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 139348e7b769SNicholas Piggin bl do_bad_slb_fault 13946cc0c16dSNicholas Piggin b interrupt_return 139548e7b769SNicholas Piggin 13969600f261SNicholas Piggin GEN_KVM data_access_slb 13979600f261SNicholas Piggin 13982b9af6e4SNicholas Piggin 139994325357SNicholas Piggin/** 140094325357SNicholas Piggin * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 140194325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 140294325357SNicholas Piggin * instruction fetch. 140394325357SNicholas Piggin * 140494325357SNicholas Piggin * Handling: 140594325357SNicholas Piggin * Similar to DSI, though in response to fetch. The faulting address is found 140694325357SNicholas Piggin * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 140794325357SNicholas Piggin */ 14084f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 14094f50541fSNicholas Piggin IVEC=0x400 1410a3cd35beSNicholas Piggin IISIDE=1 1411a3cd35beSNicholas Piggin IDAR=1 1412a3cd35beSNicholas Piggin IDSISR=1 14132284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14144f50541fSNicholas Piggin IKVM_REAL=1 14152284ffeaSNicholas Piggin#endif 14164f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 14174f50541fSNicholas Piggin 14187299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 14194f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 14207299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 14217299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 14224f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 14237299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 142427ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 14254f50541fSNicholas Piggin GEN_COMMON instruction_access 14269b123d1eSNicholas Piggin ld r4,_DAR(r1) 14279b123d1eSNicholas Piggin ld r5,_DSISR(r1) 142827ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 14299b123d1eSNicholas Piggin ld r6,_MSR(r1) 14309b123d1eSNicholas Piggin li r3,0x400 143127ce77dfSNicholas Piggin b do_hash_page /* Try to handle as hpte fault */ 143227ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 143327ce77dfSNicholas Piggin b handle_page_fault 143427ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 143527ce77dfSNicholas Piggin 14369600f261SNicholas Piggin GEN_KVM instruction_access 14379600f261SNicholas Piggin 14380ebc4cdaSBenjamin Herrenschmidt 143994325357SNicholas Piggin/** 144094325357SNicholas Piggin * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 144194325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 144294325357SNicholas Piggin * instruction fetch. 144394325357SNicholas Piggin * 144494325357SNicholas Piggin * Handling: 144594325357SNicholas Piggin * Similar to DSLB, though in response to fetch. The faulting address is found 144694325357SNicholas Piggin * in SRR0 (rather than DAR). 144794325357SNicholas Piggin */ 14484f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 14494f50541fSNicholas Piggin IVEC=0x480 14504f50541fSNicholas Piggin IAREA=PACA_EXSLB 14514f50541fSNicholas Piggin IRECONCILE=0 1452a3cd35beSNicholas Piggin IISIDE=1 1453a3cd35beSNicholas Piggin IDAR=1 14542284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14554f50541fSNicholas Piggin IKVM_REAL=1 14562284ffeaSNicholas Piggin#endif 14574f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 14584f50541fSNicholas Piggin 14597299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 14604f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 14617299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 14627299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 14634f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 14647299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 146548e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 14664f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 1467d1a84718SNicholas Piggin ld r4,_DAR(r1) 146854be0b9cSMichael Ellerman addi r3,r1,STACK_FRAME_OVERHEAD 14697100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14707100e870SNicholas Piggin /* HPT case, do SLB fault */ 147148e7b769SNicholas Piggin bl do_slb_fault 147248e7b769SNicholas Piggin cmpdi r3,0 147348e7b769SNicholas Piggin bne- 1f 14746cc0c16dSNicholas Piggin b fast_interrupt_return 147548e7b769SNicholas Piggin1: /* Error case */ 14767100e870SNicholas PigginMMU_FTR_SECTION_ELSE 14777100e870SNicholas Piggin /* Radix case, access is outside page table range */ 14787100e870SNicholas Piggin li r3,-EFAULT 14797100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 148048e7b769SNicholas Piggin std r3,RESULT(r1) 148148e7b769SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1482d1a84718SNicholas Piggin ld r4,_DAR(r1) 148348e7b769SNicholas Piggin ld r5,RESULT(r1) 148448e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 148548e7b769SNicholas Piggin bl do_bad_slb_fault 14866cc0c16dSNicholas Piggin b interrupt_return 14875e46e29eSNicholas Piggin 14889600f261SNicholas Piggin GEN_KVM instruction_access_slb 14899600f261SNicholas Piggin 14909600f261SNicholas Piggin 149194325357SNicholas Piggin/** 149294325357SNicholas Piggin * Interrupt 0x500 - External Interrupt. 149394325357SNicholas Piggin * This is an asynchronous maskable interrupt in response to an "external 149494325357SNicholas Piggin * exception" from the interrupt controller or hypervisor (e.g., device 149594325357SNicholas Piggin * interrupt). It is maskable in hardware by clearing MSR[EE], and 149694325357SNicholas Piggin * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 149794325357SNicholas Piggin * 149894325357SNicholas Piggin * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 149994325357SNicholas Piggin * interrupts are delivered with HSRR registers, guests use SRRs, which 150094325357SNicholas Piggin * reqiures IHSRR_IF_HVMODE. 150194325357SNicholas Piggin * 150294325357SNicholas Piggin * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 150394325357SNicholas Piggin * external interrupts are delivered as Hypervisor Virtualization Interrupts 150494325357SNicholas Piggin * rather than External Interrupts. 150594325357SNicholas Piggin * 150694325357SNicholas Piggin * Handling: 150794325357SNicholas Piggin * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 150894325357SNicholas Piggin * because registers at the time of the interrupt are not so important as it is 150994325357SNicholas Piggin * asynchronous. 151094325357SNicholas Piggin * 151194325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 151294325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 151394325357SNicholas Piggin */ 15144f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 15154f50541fSNicholas Piggin IVEC=0x500 15163f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 15174f50541fSNicholas Piggin IMASK=IRQS_DISABLED 15184f50541fSNicholas Piggin IKVM_REAL=1 15194f50541fSNicholas Piggin IKVM_VIRT=1 15204f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 15214f50541fSNicholas Piggin 15221a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 15234f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 15241a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 15251a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 15264f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 15271a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1528eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 15294f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1530eb204d86SNicholas Piggin FINISH_NAP 1531eb204d86SNicholas Piggin RUNLATCH_ON 1532eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1533eb204d86SNicholas Piggin bl do_IRQ 1534702f0980SNicholas Piggin b interrupt_return 1535c138e588SNicholas Piggin 15369600f261SNicholas Piggin GEN_KVM hardware_interrupt 15379600f261SNicholas Piggin 1538c138e588SNicholas Piggin 153994325357SNicholas Piggin/** 154094325357SNicholas Piggin * Interrupt 0x600 - Alignment Interrupt 154194325357SNicholas Piggin * This is a synchronous interrupt in response to data alignment fault. 154294325357SNicholas Piggin */ 15434f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 15444f50541fSNicholas Piggin IVEC=0x600 15454f50541fSNicholas Piggin IDAR=1 15464f50541fSNicholas Piggin IDSISR=1 15472284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15484f50541fSNicholas Piggin IKVM_REAL=1 15492284ffeaSNicholas Piggin#endif 15504f50541fSNicholas PigginINT_DEFINE_END(alignment) 15514f50541fSNicholas Piggin 1552e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 15534f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1554e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1555e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 15564f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1557e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1558f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 15594f50541fSNicholas Piggin GEN_COMMON alignment 1560f9aa6714SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1561f9aa6714SNicholas Piggin bl alignment_exception 1562702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 15636cc0c16dSNicholas Piggin b interrupt_return 1564f9aa6714SNicholas Piggin 15659600f261SNicholas Piggin GEN_KVM alignment 15669600f261SNicholas Piggin 1567b01c8b54SPaul Mackerras 156894325357SNicholas Piggin/** 156994325357SNicholas Piggin * Interrupt 0x700 - Program Interrupt (program check). 157094325357SNicholas Piggin * This is a synchronous interrupt in response to various instruction faults: 157194325357SNicholas Piggin * traps, privilege errors, TM errors, floating point exceptions. 157294325357SNicholas Piggin * 157394325357SNicholas Piggin * Handling: 157494325357SNicholas Piggin * This interrupt may use the "emergency stack" in some cases when being taken 157594325357SNicholas Piggin * from kernel context, which complicates handling. 157694325357SNicholas Piggin */ 15774f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 15784f50541fSNicholas Piggin IVEC=0x700 15792284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15804f50541fSNicholas Piggin IKVM_REAL=1 15812284ffeaSNicholas Piggin#endif 15824f50541fSNicholas PigginINT_DEFINE_END(program_check) 15834f50541fSNicholas Piggin 15847299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 15854f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 15867299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 15877299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 15884f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 15897299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 159011e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 15918729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 15928729c26eSNicholas Piggin 1593265e60a1SCyril Bur /* 1594265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1595265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1596265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1597265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1598265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1599265e60a1SCyril Bur * the kernel. 1600265e60a1SCyril Bur */ 1601265e60a1SCyril Bur 16020a882e28SNicholas Piggin andi. r10,r12,MSR_PR 16030a882e28SNicholas Piggin bne 2f /* If userspace, go normal path */ 16040a882e28SNicholas Piggin 16050a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 16060a882e28SNicholas Piggin bne 1f /* If TM, emergency */ 16070a882e28SNicholas Piggin 16080a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 16090a882e28SNicholas Piggin blt 2f /* normal path if not */ 16100a882e28SNicholas Piggin 16110a882e28SNicholas Piggin /* Use the emergency stack */ 16120a882e28SNicholas Piggin1: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1613265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1614265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1615265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1616265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 16174f50541fSNicholas Piggin __ISTACK(program_check)=0 16188729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 16191b359982SNicholas Piggin b 3f 16200a882e28SNicholas Piggin2: 16214f50541fSNicholas Piggin __ISTACK(program_check)=1 16228729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 16231b359982SNicholas Piggin3: 162411e87346SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 162511e87346SNicholas Piggin bl program_check_exception 1626702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 16276cc0c16dSNicholas Piggin b interrupt_return 162811e87346SNicholas Piggin 16299600f261SNicholas Piggin GEN_KVM program_check 16309600f261SNicholas Piggin 1631a485c709SPaul Mackerras 163294325357SNicholas Piggin/* 163394325357SNicholas Piggin * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 163494325357SNicholas Piggin * This is a synchronous interrupt in response to executing an fp instruction 163594325357SNicholas Piggin * with MSR[FP]=0. 163694325357SNicholas Piggin * 163794325357SNicholas Piggin * Handling: 163894325357SNicholas Piggin * This will load FP registers and enable the FP bit if coming from userspace, 163994325357SNicholas Piggin * otherwise report a bad kernel use of FP. 164094325357SNicholas Piggin */ 16414f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 16424f50541fSNicholas Piggin IVEC=0x800 16434f50541fSNicholas Piggin IRECONCILE=0 16442284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16454f50541fSNicholas Piggin IKVM_REAL=1 16462284ffeaSNicholas Piggin#endif 16474f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 16484f50541fSNicholas Piggin 16497299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 16504f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 16517299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 16527299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 16534f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 16547299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1655c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 16564f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1657c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1658c78d9b97SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1659c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1660c78d9b97SNicholas Piggin bl kernel_fp_unavailable_exception 166163ce271bSChristophe Leroy0: trap 166263ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1663c78d9b97SNicholas Piggin1: 1664c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1665c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1666c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1667c78d9b97SNicholas Piggin * transaction), go do TM stuff 1668c78d9b97SNicholas Piggin */ 1669c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1670c78d9b97SNicholas Piggin bne- 2f 1671c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1672c78d9b97SNicholas Piggin#endif 1673c78d9b97SNicholas Piggin bl load_up_fpu 16746cc0c16dSNicholas Piggin b fast_interrupt_return 1675c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1676c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1677c78d9b97SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1678c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1679c78d9b97SNicholas Piggin bl fp_unavailable_tm 16806cc0c16dSNicholas Piggin b interrupt_return 1681c78d9b97SNicholas Piggin#endif 1682c78d9b97SNicholas Piggin 16839600f261SNicholas Piggin GEN_KVM fp_unavailable 16849600f261SNicholas Piggin 1685b01c8b54SPaul Mackerras 168694325357SNicholas Piggin/** 168794325357SNicholas Piggin * Interrupt 0x900 - Decrementer Interrupt. 168894325357SNicholas Piggin * This is an asynchronous interrupt in response to a decrementer exception 168994325357SNicholas Piggin * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 169094325357SNicholas Piggin * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 169194325357SNicholas Piggin * local_irq_disable()). 169294325357SNicholas Piggin * 169394325357SNicholas Piggin * Handling: 169494325357SNicholas Piggin * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 169594325357SNicholas Piggin * 169694325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 169794325357SNicholas Piggin * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 169894325357SNicholas Piggin * in the interrupted context. 169994325357SNicholas Piggin * If PPC_WATCHDOG is configured, the soft masked handler will actually set 170094325357SNicholas Piggin * things back up to run soft_nmi_interrupt as a regular interrupt handler 170194325357SNicholas Piggin * on the emergency stack. 170294325357SNicholas Piggin */ 17034f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 17044f50541fSNicholas Piggin IVEC=0x900 17054f50541fSNicholas Piggin IMASK=IRQS_DISABLED 17062284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17074f50541fSNicholas Piggin IKVM_REAL=1 17082284ffeaSNicholas Piggin#endif 17094f50541fSNicholas PigginINT_DEFINE_END(decrementer) 17104f50541fSNicholas Piggin 17117299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1712689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 17137299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 17147299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 17154f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 17167299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1717eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 17184f50541fSNicholas Piggin GEN_COMMON decrementer 1719eb204d86SNicholas Piggin FINISH_NAP 1720eb204d86SNicholas Piggin RUNLATCH_ON 1721eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1722eb204d86SNicholas Piggin bl timer_interrupt 1723702f0980SNicholas Piggin b interrupt_return 172439c0da57SNicholas Piggin 17259600f261SNicholas Piggin GEN_KVM decrementer 17269600f261SNicholas Piggin 17270ebc4cdaSBenjamin Herrenschmidt 172894325357SNicholas Piggin/** 172994325357SNicholas Piggin * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 173094325357SNicholas Piggin * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 173194325357SNicholas Piggin * register. 173294325357SNicholas Piggin * 173394325357SNicholas Piggin * Handling: 173494325357SNicholas Piggin * Linux does not use this outside KVM where it's used to keep a host timer 173594325357SNicholas Piggin * while the guest is given control of DEC. It should normally be caught by 173694325357SNicholas Piggin * the KVM test and routed there. 173794325357SNicholas Piggin */ 17384f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 17394f50541fSNicholas Piggin IVEC=0x980 17403f7fbd97SNicholas Piggin IHSRR=1 17412babd6eaSNicholas Piggin ISTACK=0 17422babd6eaSNicholas Piggin IRECONCILE=0 17434f50541fSNicholas Piggin IKVM_REAL=1 17444f50541fSNicholas Piggin IKVM_VIRT=1 17454f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 17464f50541fSNicholas Piggin 17477299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 17484f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 17497299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 17507299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 17514f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 17527299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1753eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 17542babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 17552babd6eaSNicholas Piggin /* 17562babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 17572babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 17582babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 17592babd6eaSNicholas Piggin * edge-triggered. 17602babd6eaSNicholas Piggin * 17612babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 17622babd6eaSNicholas Piggin */ 17632babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 17642babd6eaSNicholas Piggin mtctr r10 17652babd6eaSNicholas Piggin mtcrf 0x80,r9 17662babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 17672babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 17682babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 17692babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 17702babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 17712babd6eaSNicholas Piggin HRFI_TO_KERNEL 1772facc6d74SNicholas Piggin 17739600f261SNicholas Piggin GEN_KVM hdecrementer 17749600f261SNicholas Piggin 1775da2bc464SMichael Ellerman 177694325357SNicholas Piggin/** 177794325357SNicholas Piggin * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 177894325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsndp doorbell. 177994325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 178094325357SNicholas Piggin * IRQS_DISABLED mask (i.e., local_irq_disable()). 178194325357SNicholas Piggin * 178294325357SNicholas Piggin * Handling: 178394325357SNicholas Piggin * Guests may use this for IPIs between threads in a core if the 178494325357SNicholas Piggin * hypervisor supports it. NVGPRS are not saved (see 0x500). 178594325357SNicholas Piggin * 178694325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 178794325357SNicholas Piggin * replay, leaving MSR[EE] enabled in the interrupted context because the 178894325357SNicholas Piggin * doorbells are edge triggered. 178994325357SNicholas Piggin */ 17904f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 17914f50541fSNicholas Piggin IVEC=0xa00 17924f50541fSNicholas Piggin IMASK=IRQS_DISABLED 17932284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17944f50541fSNicholas Piggin IKVM_REAL=1 17952284ffeaSNicholas Piggin#endif 17964f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 17974f50541fSNicholas Piggin 17987299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 17994f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 18007299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 18017299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 18024f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 18037299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1804eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 18054f50541fSNicholas Piggin GEN_COMMON doorbell_super 1806eb204d86SNicholas Piggin FINISH_NAP 1807eb204d86SNicholas Piggin RUNLATCH_ON 1808eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1809ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1810eb204d86SNicholas Piggin bl doorbell_exception 1811ca243163SNicholas Piggin#else 1812eb204d86SNicholas Piggin bl unknown_exception 1813ca243163SNicholas Piggin#endif 1814702f0980SNicholas Piggin b interrupt_return 1815ca243163SNicholas Piggin 18169600f261SNicholas Piggin GEN_KVM doorbell_super 18179600f261SNicholas Piggin 1818da2bc464SMichael Ellerman 18195ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 18205ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1821341215dcSNicholas Piggin 182294325357SNicholas Piggin/** 182394325357SNicholas Piggin * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 182494325357SNicholas Piggin * This is a synchronous interrupt invoked with the "sc" instruction. The 182594325357SNicholas Piggin * system call is invoked with "sc 0" and does not alter the HV bit, so it 182694325357SNicholas Piggin * is directed to the currently running OS. The hypercall is invoked with 182794325357SNicholas Piggin * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1828acd7d8ceSNicholas Piggin * 1829acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1830acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1831acd7d8ceSNicholas Piggin * 183294325357SNicholas Piggin * Handling: 183394325357SNicholas Piggin * If the KVM test fires then it was due to a hypercall and is accordingly 183494325357SNicholas Piggin * routed to KVM. Otherwise this executes a normal Linux system call. 183594325357SNicholas Piggin * 1836acd7d8ceSNicholas Piggin * Call convention: 1837acd7d8ceSNicholas Piggin * 183858b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 183958b278f5SVaibhav Jain * Documentation/powerpc/syscall64-abi.rst and 184058b278f5SVaibhav Jain * Documentation/powerpc/papr_hcalls.rst respectively. 1841acd7d8ceSNicholas Piggin * 1842acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 184376fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 184476fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 184576fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1846acd7d8ceSNicholas Piggin */ 1847b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1848b177ae2fSNicholas Piggin IVEC=0xc00 1849b177ae2fSNicholas Piggin IKVM_REAL=1 1850b177ae2fSNicholas Piggin IKVM_VIRT=1 1851b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1852b177ae2fSNicholas Piggin 18531b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1854bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1855bc355125SPaul Mackerras /* 1856acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 185776fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 185876fc0cfcSNicholas Piggin * register. 1859acd7d8ceSNicholas Piggin * 1860acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1861acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1862bc355125SPaul Mackerras */ 18631b4d4a79SNicholas Piggin mtctr r13 18641b4d4a79SNicholas Piggin GET_PACA(r13) 18651b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 18661b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 18679d598f93SNicholas Piggin KVMTEST system_call /* uses r10, branch to system_call_kvm */ 18681b4d4a79SNicholas Piggin mfctr r9 1869bc355125SPaul Mackerras#else 18701b4d4a79SNicholas Piggin mr r9,r13 18711b4d4a79SNicholas Piggin GET_PACA(r13) 18721b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1873bc355125SPaul Mackerras#endif 1874bc355125SPaul Mackerras 1875727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 18761b4d4a79SNicholas PigginBEGIN_FTR_SECTION 18771b4d4a79SNicholas Piggin cmpdi r0,0x1ebe 18781b4d4a79SNicholas Piggin beq- 1f 18791b4d4a79SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 18801b4d4a79SNicholas Piggin#endif 18815c2511bfSMichael Ellerman 1882b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 18831b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 18841b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 1885b0b2a93dSNicholas Piggin 1886b0b2a93dSNicholas Piggin HMT_MEDIUM 1887b0b2a93dSNicholas Piggin 1888b0b2a93dSNicholas Piggin .if ! \virt 18891b4d4a79SNicholas Piggin __LOAD_HANDLER(r10, system_call_common) 18901b4d4a79SNicholas Piggin mtspr SPRN_SRR0,r10 18911b4d4a79SNicholas Piggin ld r10,PACAKMSR(r13) 18921b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r10 18931b4d4a79SNicholas Piggin RFI_TO_KERNEL 18941b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 18951b4d4a79SNicholas Piggin .else 18961b4d4a79SNicholas Piggin li r10,MSR_RI 18971b4d4a79SNicholas Piggin mtmsrd r10,1 /* Set RI (EE=0) */ 1898b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 1899b0b2a93dSNicholas Piggin __LOAD_HANDLER(r10, system_call_common) 1900b0b2a93dSNicholas Piggin mtctr r10 1901b0b2a93dSNicholas Piggin bctr 1902b0b2a93dSNicholas Piggin#else 19031b4d4a79SNicholas Piggin b system_call_common 1904d807ad37SNicholas Piggin#endif 19051b4d4a79SNicholas Piggin .endif 19061b4d4a79SNicholas Piggin 19071b4d4a79SNicholas Piggin#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 19081b4d4a79SNicholas Piggin /* Fast LE/BE switch system call */ 19091b4d4a79SNicholas Piggin1: mfspr r12,SPRN_SRR1 19101b4d4a79SNicholas Piggin xori r12,r12,MSR_LE 19111b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r12 19121b4d4a79SNicholas Piggin mr r13,r9 19131b4d4a79SNicholas Piggin RFI_TO_USER /* return to userspace */ 19141b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 19151b4d4a79SNicholas Piggin#endif 19161b4d4a79SNicholas Piggin.endm 1917d807ad37SNicholas Piggin 19181a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 19191b4d4a79SNicholas Piggin SYSTEM_CALL 0 19201a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 19211a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 19221b4d4a79SNicholas Piggin SYSTEM_CALL 1 19231a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 1924d807ad37SNicholas Piggin 1925acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 19269600f261SNicholas PigginTRAMP_REAL_BEGIN(system_call_kvm) 1927acd7d8ceSNicholas Piggin /* 1928acd7d8ceSNicholas Piggin * This is a hcall, so register convention is as above, with these 1929acd7d8ceSNicholas Piggin * differences: 1930acd7d8ceSNicholas Piggin * r13 = PACA 193176fc0cfcSNicholas Piggin * ctr = orig r13 193276fc0cfcSNicholas Piggin * orig r10 saved in PACA 1933acd7d8ceSNicholas Piggin */ 1934acd7d8ceSNicholas Piggin /* 1935acd7d8ceSNicholas Piggin * Save the PPR (on systems that support it) before changing to 1936acd7d8ceSNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 1937acd7d8ceSNicholas Piggin * guest state (it is the guest's PPR value). 1938acd7d8ceSNicholas Piggin */ 1939931dc86bSNicholas PigginBEGIN_FTR_SECTION 19409600f261SNicholas Piggin mfspr r10,SPRN_PPR 19419600f261SNicholas Piggin std r10,HSTATE_PPR(r13) 1942931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1943acd7d8ceSNicholas Piggin HMT_MEDIUM 1944acd7d8ceSNicholas Piggin mfctr r10 194576fc0cfcSNicholas Piggin SET_SCRATCH0(r10) 19469600f261SNicholas Piggin mfcr r10 19479600f261SNicholas Piggin std r12,HSTATE_SCRATCH0(r13) 19489600f261SNicholas Piggin sldi r12,r10,32 19499600f261SNicholas Piggin ori r12,r12,0xc00 19509600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 19519600f261SNicholas Piggin /* 19529600f261SNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives 19539600f261SNicholas Piggin * outside the head section. 19549600f261SNicholas Piggin */ 19559600f261SNicholas Piggin __LOAD_FAR_HANDLER(r10, kvmppc_interrupt) 19569600f261SNicholas Piggin mtctr r10 19579600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 19589600f261SNicholas Piggin bctr 19599600f261SNicholas Piggin#else 19609600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 19619600f261SNicholas Piggin b kvmppc_interrupt 19629600f261SNicholas Piggin#endif 1963acd7d8ceSNicholas Piggin#endif 1964da2bc464SMichael Ellerman 1965d807ad37SNicholas Piggin 196694325357SNicholas Piggin/** 196794325357SNicholas Piggin * Interrupt 0xd00 - Trace Interrupt. 196894325357SNicholas Piggin * This is a synchronous interrupt in response to instruction step or 196994325357SNicholas Piggin * breakpoint faults. 197094325357SNicholas Piggin */ 19714f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 19724f50541fSNicholas Piggin IVEC=0xd00 19732284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 19744f50541fSNicholas Piggin IKVM_REAL=1 19752284ffeaSNicholas Piggin#endif 19764f50541fSNicholas PigginINT_DEFINE_END(single_step) 19774f50541fSNicholas Piggin 19787299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 19794f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 19807299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 19817299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 19824f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 19837299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 1984eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 19854f50541fSNicholas Piggin GEN_COMMON single_step 1986eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1987eb204d86SNicholas Piggin bl single_step_exception 19886cc0c16dSNicholas Piggin b interrupt_return 1989da2bc464SMichael Ellerman 19909600f261SNicholas Piggin GEN_KVM single_step 19919600f261SNicholas Piggin 19927299417cSNicholas Piggin 199394325357SNicholas Piggin/** 199494325357SNicholas Piggin * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 199594325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 199694325357SNicholas Piggin * guest data access. 199794325357SNicholas Piggin * 199894325357SNicholas Piggin * Handling: 199994325357SNicholas Piggin * This should always get routed to KVM. In radix MMU mode, this is caused 200094325357SNicholas Piggin * by a guest nested radix access that can't be performed due to the 200194325357SNicholas Piggin * partition scope page table. In hash mode, this can be caused by guests 200294325357SNicholas Piggin * running with translation disabled (virtual real mode) or with VPM enabled. 200394325357SNicholas Piggin * KVM will update the page table structures or disallow the access. 200494325357SNicholas Piggin */ 20054f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 20064f50541fSNicholas Piggin IVEC=0xe00 20073f7fbd97SNicholas Piggin IHSRR=1 20084f50541fSNicholas Piggin IDAR=1 20094f50541fSNicholas Piggin IDSISR=1 20104f50541fSNicholas Piggin IKVM_SKIP=1 20114f50541fSNicholas Piggin IKVM_REAL=1 20124f50541fSNicholas Piggin IKVM_VIRT=1 20134f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 20144f50541fSNicholas Piggin 20157299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 20164f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 20177299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 20187299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 20194f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 20207299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2021f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 20224f50541fSNicholas Piggin GEN_COMMON h_data_storage 2023f5c32c1dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2024d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 2025d1a84718SNicholas Piggin ld r4,_DAR(r1) 2026d7b45615SSuraj Jitindar Singh li r5,SIGSEGV 2027d7b45615SSuraj Jitindar Singh bl bad_page_fault 2028d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 2029f5c32c1dSNicholas Piggin bl unknown_exception 2030d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 20316cc0c16dSNicholas Piggin b interrupt_return 2032f5c32c1dSNicholas Piggin 20339600f261SNicholas Piggin GEN_KVM h_data_storage 20349600f261SNicholas Piggin 20351707dd16SPaul Mackerras 203694325357SNicholas Piggin/** 203794325357SNicholas Piggin * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 203894325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 203994325357SNicholas Piggin * guest instruction fetch, similar to HDSI. 204094325357SNicholas Piggin */ 20414f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 20424f50541fSNicholas Piggin IVEC=0xe20 20433f7fbd97SNicholas Piggin IHSRR=1 20444f50541fSNicholas Piggin IKVM_REAL=1 20454f50541fSNicholas Piggin IKVM_VIRT=1 20464f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 20474f50541fSNicholas Piggin 20487299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 20494f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 20507299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 20517299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 20524f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 20537299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2054eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 20554f50541fSNicholas Piggin GEN_COMMON h_instr_storage 2056eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2057eb204d86SNicholas Piggin bl unknown_exception 20586cc0c16dSNicholas Piggin b interrupt_return 205982517cabSNicholas Piggin 20609600f261SNicholas Piggin GEN_KVM h_instr_storage 20619600f261SNicholas Piggin 20621707dd16SPaul Mackerras 206394325357SNicholas Piggin/** 206494325357SNicholas Piggin * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 206594325357SNicholas Piggin */ 20664f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 20674f50541fSNicholas Piggin IVEC=0xe40 20683f7fbd97SNicholas Piggin IHSRR=1 20694f50541fSNicholas Piggin IKVM_REAL=1 20704f50541fSNicholas Piggin IKVM_VIRT=1 20714f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 20724f50541fSNicholas Piggin 20737299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 20744f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 20757299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 20767299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 20774f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 20787299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2079eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 20804f50541fSNicholas Piggin GEN_COMMON emulation_assist 2081eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2082eb204d86SNicholas Piggin bl emulation_assist_interrupt 2083702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 20846cc0c16dSNicholas Piggin b interrupt_return 2085031b4026SNicholas Piggin 20869600f261SNicholas Piggin GEN_KVM emulation_assist 20879600f261SNicholas Piggin 20881707dd16SPaul Mackerras 208994325357SNicholas Piggin/** 209094325357SNicholas Piggin * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 209194325357SNicholas Piggin * This is an asynchronous interrupt caused by a Hypervisor Maintenance 209294325357SNicholas Piggin * Exception. It is always taken in real mode but uses HSRR registers 209394325357SNicholas Piggin * unlike SRESET and MCE. 209494325357SNicholas Piggin * 209594325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 209694325357SNicholas Piggin * with IRQS_DISABLED mask (i.e., local_irq_disable()). 209794325357SNicholas Piggin * 209894325357SNicholas Piggin * Handling: 209994325357SNicholas Piggin * This is a special case, this is handled similarly to machine checks, with an 210094325357SNicholas Piggin * initial real mode handler that is not soft-masked, which attempts to fix the 210194325357SNicholas Piggin * problem. Then a regular handler which is soft-maskable and reports the 210294325357SNicholas Piggin * problem. 210394325357SNicholas Piggin * 210494325357SNicholas Piggin * The emergency stack is used for the early real mode handler. 210594325357SNicholas Piggin * 210694325357SNicholas Piggin * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 210794325357SNicholas Piggin * either use soft-masking for the MCE, or use irq_work for the HMI. 210894325357SNicholas Piggin * 210994325357SNicholas Piggin * KVM: 211094325357SNicholas Piggin * Unlike MCE, this calls into KVM without calling the real mode handler 211194325357SNicholas Piggin * first. 2112e0319829SNicholas Piggin */ 21134f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 21144f50541fSNicholas Piggin IVEC=0xe60 21153f7fbd97SNicholas Piggin IHSRR=1 2116d73a10cbSNicholas Piggin IREALMODE_COMMON=1 21174f50541fSNicholas Piggin ISTACK=0 21184f50541fSNicholas Piggin IRECONCILE=0 21194f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 21204f50541fSNicholas Piggin IKVM_REAL=1 21214f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 21224f50541fSNicholas Piggin 21234f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 21244f50541fSNicholas Piggin IVEC=0xe60 21253f7fbd97SNicholas Piggin IHSRR=1 21264f50541fSNicholas Piggin IMASK=IRQS_DISABLED 21274f50541fSNicholas Piggin IKVM_REAL=1 21284f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 21294f50541fSNicholas Piggin 2130f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 21314f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2132f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 21331a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 21344f50541fSNicholas Piggin 2135293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 21369600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 21379600f261SNicholas Piggin 213862f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 2139a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 214062f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2141bcbceed4SNicholas Piggin 21428729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 2143bcbceed4SNicholas Piggin 214462f9b03bSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2145293c2e27SNicholas Piggin bl hmi_exception_realmode 21465080332cSMichael Neuling cmpdi cr0,r3,0 214767d4160aSNicholas Piggin bne 1f 21485080332cSMichael Neuling 21493f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 2150222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 21515080332cSMichael Neuling 215267d4160aSNicholas Piggin1: 215362f9b03bSNicholas Piggin /* 215462f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 215562f9b03bSNicholas Piggin * firmware. 215662f9b03bSNicholas Piggin */ 21573f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 21584f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 215962f9b03bSNicholas Piggin 21609600f261SNicholas Piggin GEN_KVM hmi_exception_early 21619600f261SNicholas Piggin 21625080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 21634f50541fSNicholas Piggin GEN_COMMON hmi_exception 216447169fbaSNicholas Piggin FINISH_NAP 216547169fbaSNicholas Piggin RUNLATCH_ON 2166c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2167c06075f3SNicholas Piggin bl handle_hmi_exception 21686cc0c16dSNicholas Piggin b interrupt_return 21691707dd16SPaul Mackerras 21709600f261SNicholas Piggin GEN_KVM hmi_exception 21719600f261SNicholas Piggin 21727299417cSNicholas Piggin 217394325357SNicholas Piggin/** 217494325357SNicholas Piggin * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 217594325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsnd doorbell. 217694325357SNicholas Piggin * Similar to the 0xa00 doorbell but for host rather than guest. 217794325357SNicholas Piggin */ 21784f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 21794f50541fSNicholas Piggin IVEC=0xe80 21803f7fbd97SNicholas Piggin IHSRR=1 21814f50541fSNicholas Piggin IMASK=IRQS_DISABLED 21824f50541fSNicholas Piggin IKVM_REAL=1 21834f50541fSNicholas Piggin IKVM_VIRT=1 21844f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 21854f50541fSNicholas Piggin 21867299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 21874f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 21887299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 21897299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 21904f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 21917299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2192eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 21934f50541fSNicholas Piggin GEN_COMMON h_doorbell 2194eb204d86SNicholas Piggin FINISH_NAP 2195eb204d86SNicholas Piggin RUNLATCH_ON 2196eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 21979bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 2198eb204d86SNicholas Piggin bl doorbell_exception 21999bcb81bfSNicholas Piggin#else 2200eb204d86SNicholas Piggin bl unknown_exception 22019bcb81bfSNicholas Piggin#endif 2202702f0980SNicholas Piggin b interrupt_return 22039bcb81bfSNicholas Piggin 22049600f261SNicholas Piggin GEN_KVM h_doorbell 22059600f261SNicholas Piggin 22060ebc4cdaSBenjamin Herrenschmidt 220794325357SNicholas Piggin/** 220894325357SNicholas Piggin * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 220994325357SNicholas Piggin * This is an asynchronous interrupt in response to an "external exception". 221094325357SNicholas Piggin * Similar to 0x500 but for host only. 221194325357SNicholas Piggin */ 22124f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 22134f50541fSNicholas Piggin IVEC=0xea0 22143f7fbd97SNicholas Piggin IHSRR=1 22154f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22164f50541fSNicholas Piggin IKVM_REAL=1 22174f50541fSNicholas Piggin IKVM_VIRT=1 22184f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 22194f50541fSNicholas Piggin 22207299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 22214f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 22227299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 22237299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 22244f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 22257299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2226eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 22274f50541fSNicholas Piggin GEN_COMMON h_virt_irq 2228eb204d86SNicholas Piggin FINISH_NAP 2229eb204d86SNicholas Piggin RUNLATCH_ON 2230eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2231eb204d86SNicholas Piggin bl do_IRQ 2232702f0980SNicholas Piggin b interrupt_return 223374408776SNicholas Piggin 22349600f261SNicholas Piggin GEN_KVM h_virt_irq 22359600f261SNicholas Piggin 22369baaef0aSBenjamin Herrenschmidt 22371a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 22381a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 22391a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 22401a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 2241bda7fea2SNicholas Piggin 22420ebc4cdaSBenjamin Herrenschmidt 224394325357SNicholas Piggin/* 224494325357SNicholas Piggin * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 224594325357SNicholas Piggin * This is an asynchronous interrupt in response to a PMU exception. 224694325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 224794325357SNicholas Piggin * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 224894325357SNicholas Piggin * 224994325357SNicholas Piggin * Handling: 225094325357SNicholas Piggin * This calls into the perf subsystem. 225194325357SNicholas Piggin * 225294325357SNicholas Piggin * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 225394325357SNicholas Piggin * runs under local_irq_disable. However it may be soft-masked in 225494325357SNicholas Piggin * powerpc-specific code. 225594325357SNicholas Piggin * 225694325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 225794325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 225894325357SNicholas Piggin */ 22594f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 22604f50541fSNicholas Piggin IVEC=0xf00 22614f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 22622284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 22634f50541fSNicholas Piggin IKVM_REAL=1 22642284ffeaSNicholas Piggin#endif 22654f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 22664f50541fSNicholas Piggin 22677299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 22684f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 22697299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 22707299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 22714f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 22727299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2273eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 22744f50541fSNicholas Piggin GEN_COMMON performance_monitor 2275eb204d86SNicholas Piggin FINISH_NAP 2276eb204d86SNicholas Piggin RUNLATCH_ON 2277eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2278eb204d86SNicholas Piggin bl performance_monitor_exception 2279702f0980SNicholas Piggin b interrupt_return 2280b1c7f150SNicholas Piggin 22819600f261SNicholas Piggin GEN_KVM performance_monitor 22829600f261SNicholas Piggin 22830ebc4cdaSBenjamin Herrenschmidt 228494325357SNicholas Piggin/** 228594325357SNicholas Piggin * Interrupt 0xf20 - Vector Unavailable Interrupt. 228694325357SNicholas Piggin * This is a synchronous interrupt in response to 228794325357SNicholas Piggin * executing a vector (or altivec) instruction with MSR[VEC]=0. 228894325357SNicholas Piggin * Similar to FP unavailable. 228994325357SNicholas Piggin */ 22904f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 22914f50541fSNicholas Piggin IVEC=0xf20 22924f50541fSNicholas Piggin IRECONCILE=0 22932284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 22944f50541fSNicholas Piggin IKVM_REAL=1 22952284ffeaSNicholas Piggin#endif 22964f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 22974f50541fSNicholas Piggin 22987299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 22994f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 23007299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 23017299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 23024f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 23037299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2304d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 23054f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 2306d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 2307d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 2308d1a0ca9cSNicholas Piggin beq 1f 2309d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2310d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2311d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2312d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 2313d1a0ca9cSNicholas Piggin */ 2314d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2315d1a0ca9cSNicholas Piggin bne- 2f 2316d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2317d1a0ca9cSNicholas Piggin#endif 2318d1a0ca9cSNicholas Piggin bl load_up_altivec 23196cc0c16dSNicholas Piggin b fast_interrupt_return 2320d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2321d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 2322d1a0ca9cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2323d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2324d1a0ca9cSNicholas Piggin bl altivec_unavailable_tm 23256cc0c16dSNicholas Piggin b interrupt_return 2326d1a0ca9cSNicholas Piggin#endif 2327d1a0ca9cSNicholas Piggin1: 2328d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2329d1a0ca9cSNicholas Piggin#endif 2330d1a0ca9cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2331d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2332d1a0ca9cSNicholas Piggin bl altivec_unavailable_exception 23336cc0c16dSNicholas Piggin b interrupt_return 2334d1a0ca9cSNicholas Piggin 23359600f261SNicholas Piggin GEN_KVM altivec_unavailable 23369600f261SNicholas Piggin 23370ebc4cdaSBenjamin Herrenschmidt 233894325357SNicholas Piggin/** 233994325357SNicholas Piggin * Interrupt 0xf40 - VSX Unavailable Interrupt. 234094325357SNicholas Piggin * This is a synchronous interrupt in response to 234194325357SNicholas Piggin * executing a VSX instruction with MSR[VSX]=0. 234294325357SNicholas Piggin * Similar to FP unavailable. 234394325357SNicholas Piggin */ 23444f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 23454f50541fSNicholas Piggin IVEC=0xf40 23464f50541fSNicholas Piggin IRECONCILE=0 23472284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23484f50541fSNicholas Piggin IKVM_REAL=1 23492284ffeaSNicholas Piggin#endif 23504f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 23514f50541fSNicholas Piggin 23527299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 23534f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 23547299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 23557299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 23564f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 23577299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2358792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 23594f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2360792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2361792cbdddSNicholas PigginBEGIN_FTR_SECTION 2362792cbdddSNicholas Piggin beq 1f 2363792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2364792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2365792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2366792cbdddSNicholas Piggin * transaction), go do TM stuff 2367792cbdddSNicholas Piggin */ 2368792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2369792cbdddSNicholas Piggin bne- 2f 2370792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2371792cbdddSNicholas Piggin#endif 2372792cbdddSNicholas Piggin b load_up_vsx 2373792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2374792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2375792cbdddSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2376792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2377792cbdddSNicholas Piggin bl vsx_unavailable_tm 23786cc0c16dSNicholas Piggin b interrupt_return 2379792cbdddSNicholas Piggin#endif 2380792cbdddSNicholas Piggin1: 2381792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2382792cbdddSNicholas Piggin#endif 2383792cbdddSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2384792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2385792cbdddSNicholas Piggin bl vsx_unavailable_exception 23866cc0c16dSNicholas Piggin b interrupt_return 2387792cbdddSNicholas Piggin 23889600f261SNicholas Piggin GEN_KVM vsx_unavailable 23899600f261SNicholas Piggin 2390d0c0c9a1SMichael Neuling 239194325357SNicholas Piggin/** 239294325357SNicholas Piggin * Interrupt 0xf60 - Facility Unavailable Interrupt. 239394325357SNicholas Piggin * This is a synchronous interrupt in response to 239494325357SNicholas Piggin * executing an instruction without access to the facility that can be 239594325357SNicholas Piggin * resolved by the OS (e.g., FSCR, MSR). 239694325357SNicholas Piggin * Similar to FP unavailable. 239794325357SNicholas Piggin */ 23984f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 23994f50541fSNicholas Piggin IVEC=0xf60 24002284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24014f50541fSNicholas Piggin IKVM_REAL=1 24022284ffeaSNicholas Piggin#endif 24034f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 24044f50541fSNicholas Piggin 24057299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 24064f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 24077299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 24087299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 24094f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 24107299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2411eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 24124f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2413eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2414eb204d86SNicholas Piggin bl facility_unavailable_exception 24156cc0c16dSNicholas Piggin b interrupt_return 24161134713cSNicholas Piggin 24179600f261SNicholas Piggin GEN_KVM facility_unavailable 24189600f261SNicholas Piggin 2419da2bc464SMichael Ellerman 242094325357SNicholas Piggin/** 242194325357SNicholas Piggin * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 242294325357SNicholas Piggin * This is a synchronous interrupt in response to 242394325357SNicholas Piggin * executing an instruction without access to the facility that can only 242494325357SNicholas Piggin * be resolved in HV mode (e.g., HFSCR). 242594325357SNicholas Piggin * Similar to FP unavailable. 242694325357SNicholas Piggin */ 24274f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 24284f50541fSNicholas Piggin IVEC=0xf80 24293f7fbd97SNicholas Piggin IHSRR=1 24304f50541fSNicholas Piggin IKVM_REAL=1 24314f50541fSNicholas Piggin IKVM_VIRT=1 24324f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 24334f50541fSNicholas Piggin 24347299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 24354f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 24367299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 24377299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 24384f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 24397299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2440eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 24414f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2442eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2443eb204d86SNicholas Piggin bl facility_unavailable_exception 24446cc0c16dSNicholas Piggin b interrupt_return 244514b0072cSNicholas Piggin 24469600f261SNicholas Piggin GEN_KVM h_facility_unavailable 24479600f261SNicholas Piggin 2448da2bc464SMichael Ellerman 24491a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 24501a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 24511a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 24521a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 24531a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 24541a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 24551a6822d1SNicholas Piggin 24561a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 24571a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 24581a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 24591a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2460da2bc464SMichael Ellerman 24610ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 24624f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 24634f50541fSNicholas Piggin IVEC=0x1200 24643f7fbd97SNicholas Piggin IHSRR=1 24654f50541fSNicholas Piggin IKVM_SKIP=1 24664f50541fSNicholas Piggin IKVM_REAL=1 24674f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 24684f50541fSNicholas Piggin 24697299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 24704f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 24717299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 24721a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2473eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 24744f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2475eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2476eb204d86SNicholas Piggin bl cbe_system_error_exception 24776cc0c16dSNicholas Piggin b interrupt_return 24789600f261SNicholas Piggin 24799600f261SNicholas Piggin GEN_KVM cbe_system_error 24809600f261SNicholas Piggin 2481da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 24821a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 24831a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2484da2bc464SMichael Ellerman#endif 2485da2bc464SMichael Ellerman 2486ff1b3206SNicholas Piggin 24874f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 24884f50541fSNicholas Piggin IVEC=0x1300 24892284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24904f50541fSNicholas Piggin IKVM_SKIP=1 24914f50541fSNicholas Piggin IKVM_REAL=1 24922284ffeaSNicholas Piggin#endif 24934f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 24944f50541fSNicholas Piggin 24957299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 24964f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 24977299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 24987299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 24994f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 25007299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2501eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 25024f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2503eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2504eb204d86SNicholas Piggin bl instruction_breakpoint_exception 25056cc0c16dSNicholas Piggin b interrupt_return 25064e96dbbfSNicholas Piggin 25079600f261SNicholas Piggin GEN_KVM instruction_breakpoint 25089600f261SNicholas Piggin 25097299417cSNicholas Piggin 25101a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 25111a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2512da2bc464SMichael Ellerman 251394325357SNicholas Piggin/** 251494325357SNicholas Piggin * Interrupt 0x1500 - Soft Patch Interrupt 251594325357SNicholas Piggin * 251694325357SNicholas Piggin * Handling: 251794325357SNicholas Piggin * This is an implementation specific interrupt which can be used for a 251894325357SNicholas Piggin * range of exceptions. 251994325357SNicholas Piggin * 252094325357SNicholas Piggin * This interrupt handler is unique in that it runs the denormal assist 252194325357SNicholas Piggin * code even for guests (and even in guest context) without going to KVM, 252294325357SNicholas Piggin * for speed. POWER9 does not raise denorm exceptions, so this special case 252394325357SNicholas Piggin * could be phased out in future to reduce special cases. 252494325357SNicholas Piggin */ 25254f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 25264f50541fSNicholas Piggin IVEC=0x1500 25273f7fbd97SNicholas Piggin IHSRR=1 25283f7fbd97SNicholas Piggin IBRANCH_COMMON=0 25299600f261SNicholas Piggin IKVM_REAL=1 25304f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 25314f50541fSNicholas Piggin 25324f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 25334f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2534b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2535d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2536b92a66a6SMichael Neuling bne+ denorm_assist 2537b92a66a6SMichael Neuling#endif 25388729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 25394f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2540d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 25411a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 25424f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2543d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 254452b98923SNicholas Piggin bne+ denorm_assist 25458729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 25461a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2547d7e89849SNicholas Piggin#else 25481a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2549d7e89849SNicholas Piggin#endif 2550b92a66a6SMichael Neuling 2551b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2552da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2553b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2554b92a66a6SMichael Neuling/* 2555b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2556b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2557b92a66a6SMichael Neuling */ 2558b92a66a6SMichael Neuling mfmsr r10 2559b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2560b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2561b92a66a6SMichael Neuling mtmsrd r10 2562b92a66a6SMichael Neuling sync 2563d7c67fb1SMichael Neuling 2564f3c8b6c6SNicholas Piggin .Lreg=0 2565f3c8b6c6SNicholas Piggin .rept 32 2566f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2567f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2568f3c8b6c6SNicholas Piggin .endr 2569d7c67fb1SMichael Neuling 2570b92a66a6SMichael NeulingFTR_SECTION_ELSE 2571b92a66a6SMichael Neuling/* 2572b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2573b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2574b92a66a6SMichael Neuling */ 2575b92a66a6SMichael Neuling mfmsr r10 2576b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2577b92a66a6SMichael Neuling mtmsrd r10 2578b92a66a6SMichael Neuling sync 2579d7c67fb1SMichael Neuling 2580f3c8b6c6SNicholas Piggin .Lreg=0 2581f3c8b6c6SNicholas Piggin .rept 32 2582f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2583f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2584f3c8b6c6SNicholas Piggin .endr 2585d7c67fb1SMichael Neuling 2586b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2587fb0fce3eSMichael Neuling 2588fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2589fb0fce3eSMichael Neuling b denorm_done 2590fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2591fb0fce3eSMichael Neuling/* 2592fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2593fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2594fb0fce3eSMichael Neuling */ 2595f3c8b6c6SNicholas Piggin .Lreg=32 2596f3c8b6c6SNicholas Piggin .rept 32 2597f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2598f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2599f3c8b6c6SNicholas Piggin .endr 2600f3c8b6c6SNicholas Piggin 2601fb0fce3eSMichael Neulingdenorm_done: 2602f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2603f14040bcSMichael Neuling subi r11,r11,4 2604b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2605b92a66a6SMichael Neuling mtcrf 0x80,r9 2606b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2607931dc86bSNicholas PigginBEGIN_FTR_SECTION 2608931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2609931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2610931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2611630573c1SPaul MackerrasBEGIN_FTR_SECTION 2612630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2613630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2614630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 2615b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2616b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2617b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2618b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2619222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2620b92a66a6SMichael Neuling b . 2621b92a66a6SMichael Neuling#endif 2622b92a66a6SMichael Neuling 26234f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 26244f50541fSNicholas Piggin GEN_COMMON denorm_exception 2625eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2626eb204d86SNicholas Piggin bl unknown_exception 26276cc0c16dSNicholas Piggin b interrupt_return 2628d7e89849SNicholas Piggin 26299600f261SNicholas Piggin GEN_KVM denorm_exception 26309600f261SNicholas Piggin 2631d7e89849SNicholas Piggin 2632d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 26334f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 26344f50541fSNicholas Piggin IVEC=0x1600 26353f7fbd97SNicholas Piggin IHSRR=1 26364f50541fSNicholas Piggin IKVM_SKIP=1 26374f50541fSNicholas Piggin IKVM_REAL=1 26384f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 26394f50541fSNicholas Piggin 26407299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 26414f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 26427299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 26431a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2644eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 26454f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2646eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2647eb204d86SNicholas Piggin bl cbe_maintenance_exception 26486cc0c16dSNicholas Piggin b interrupt_return 26499600f261SNicholas Piggin 26509600f261SNicholas Piggin GEN_KVM cbe_maintenance 26519600f261SNicholas Piggin 2652d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 26531a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 26541a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2655d7e89849SNicholas Piggin#endif 2656d7e89849SNicholas Piggin 265769a79344SNicholas Piggin 26584f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 26594f50541fSNicholas Piggin IVEC=0x1700 26602284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 26614f50541fSNicholas Piggin IKVM_REAL=1 26622284ffeaSNicholas Piggin#endif 26634f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 26644f50541fSNicholas Piggin 26657299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 26664f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 26677299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 26687299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 26694f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 26707299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2671eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 26724f50541fSNicholas Piggin GEN_COMMON altivec_assist 2673eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2674b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 2675eb204d86SNicholas Piggin bl altivec_assist_exception 2676702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2677b51c079eSNicholas Piggin#else 2678eb204d86SNicholas Piggin bl unknown_exception 2679b51c079eSNicholas Piggin#endif 26806cc0c16dSNicholas Piggin b interrupt_return 2681b51c079eSNicholas Piggin 26829600f261SNicholas Piggin GEN_KVM altivec_assist 26839600f261SNicholas Piggin 2684d7e89849SNicholas Piggin 2685d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 26864f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 26874f50541fSNicholas Piggin IVEC=0x1800 26883f7fbd97SNicholas Piggin IHSRR=1 26894f50541fSNicholas Piggin IKVM_SKIP=1 26904f50541fSNicholas Piggin IKVM_REAL=1 26914f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 26924f50541fSNicholas Piggin 26937299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 26944f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 26957299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 26961a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2697eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 26984f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2699eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2700eb204d86SNicholas Piggin bl cbe_thermal_exception 27016cc0c16dSNicholas Piggin b interrupt_return 27029600f261SNicholas Piggin 27039600f261SNicholas Piggin GEN_KVM cbe_thermal 27049600f261SNicholas Piggin 2705d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 27061a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 27071a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2708d7e89849SNicholas Piggin#endif 2709d7e89849SNicholas Piggin 27107299417cSNicholas Piggin 271175eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 27122104180aSNicholas Piggin 27130eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 27140eddf327SNicholas Piggin IVEC=0x900 27150eddf327SNicholas Piggin ISTACK=0 271671c3b05aSNicholas Piggin IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */ 27170eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 27182104180aSNicholas Piggin 2719cc491f1dSNicholas Piggin/* 2720cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2721cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2722cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2723cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2724cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2725cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2726cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2727cc491f1dSNicholas Piggin */ 27282104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 27290eddf327SNicholas Piggin mfspr r11,SPRN_SRR0 27302104180aSNicholas Piggin mr r10,r1 27312104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 27322104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 27330eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 273471c3b05aSNicholas Piggin 273571c3b05aSNicholas Piggin /* 273671c3b05aSNicholas Piggin * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see 273771c3b05aSNicholas Piggin * system_reset_common) 273871c3b05aSNicholas Piggin */ 273971c3b05aSNicholas Piggin li r10,IRQS_ALL_DISABLED 274071c3b05aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 274171c3b05aSNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 274271c3b05aSNicholas Piggin std r10,_DAR(r1) 274371c3b05aSNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 274471c3b05aSNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 274571c3b05aSNicholas Piggin 2746c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2747c06075f3SNicholas Piggin bl soft_nmi_interrupt 274871c3b05aSNicholas Piggin 274971c3b05aSNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 275071c3b05aSNicholas Piggin li r9,0 275171c3b05aSNicholas Piggin mtmsrd r9,1 275271c3b05aSNicholas Piggin 275371c3b05aSNicholas Piggin /* 275471c3b05aSNicholas Piggin * Restore soft mask settings. 275571c3b05aSNicholas Piggin */ 275671c3b05aSNicholas Piggin ld r10,_DAR(r1) 275771c3b05aSNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 275871c3b05aSNicholas Piggin ld r10,SOFTE(r1) 275971c3b05aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 276071c3b05aSNicholas Piggin 276171c3b05aSNicholas Piggin kuap_restore_amr r10 276271c3b05aSNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 276371c3b05aSNicholas Piggin RFI_TO_KERNEL 27642104180aSNicholas Piggin 276575eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2766d7e89849SNicholas Piggin 27670ebc4cdaSBenjamin Herrenschmidt/* 2768fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2769fe9e1d54SIan Munsie * - If it was a decrementer interrupt, we bump the dec to max and and return. 2770fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2771fe9e1d54SIan Munsie * triggered and won't automatically refire. 27720869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 27730869b6fdSMahesh Salgaonkar * and it won't refire. 27746cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2775fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 27760ebc4cdaSBenjamin Herrenschmidt */ 27773f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 27784508a74aSNicholas Piggin .if \hsrr 27794508a74aSNicholas Pigginmasked_Hinterrupt: 27804508a74aSNicholas Piggin .else 27814508a74aSNicholas Pigginmasked_interrupt: 27824508a74aSNicholas Piggin .endif 27834508a74aSNicholas Piggin lbz r11,PACAIRQHAPPENED(r13) 27844508a74aSNicholas Piggin or r11,r11,r10 27854508a74aSNicholas Piggin stb r11,PACAIRQHAPPENED(r13) 27864508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 27874508a74aSNicholas Piggin bne 1f 27884508a74aSNicholas Piggin lis r10,0x7fff 27894508a74aSNicholas Piggin ori r10,r10,0xffff 27904508a74aSNicholas Piggin mtspr SPRN_DEC,r10 27910eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 27920eddf327SNicholas Piggin b soft_nmi_common 27930eddf327SNicholas Piggin#else 27940eddf327SNicholas Piggin b 2f 27950eddf327SNicholas Piggin#endif 27964508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 27974508a74aSNicholas Piggin beq 2f 27980eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 27994508a74aSNicholas Piggin .if \hsrr 28000eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 28014508a74aSNicholas Piggin .else 28020eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 28034508a74aSNicholas Piggin .endif 28044508a74aSNicholas Piggin ori r11,r11,PACA_IRQ_HARD_DIS 28054508a74aSNicholas Piggin stb r11,PACAIRQHAPPENED(r13) 28064508a74aSNicholas Piggin2: /* done */ 28070eddf327SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 28080eddf327SNicholas Piggin mtctr r10 28094508a74aSNicholas Piggin mtcrf 0x80,r9 28104508a74aSNicholas Piggin std r1,PACAR1(r13) 28114508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 28124508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 28134508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 28140eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 28154508a74aSNicholas Piggin /* returns to kernel where r13 must be set up, so don't restore it */ 28164508a74aSNicholas Piggin .if \hsrr 28174508a74aSNicholas Piggin HRFI_TO_KERNEL 28184508a74aSNicholas Piggin .else 28194508a74aSNicholas Piggin RFI_TO_KERNEL 28204508a74aSNicholas Piggin .endif 28214508a74aSNicholas Piggin b . 28224508a74aSNicholas Piggin.endm 28230ebc4cdaSBenjamin Herrenschmidt 2824a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2825a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2826a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2827a048a07dSNicholas Piggin sync 2828a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2829a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2830a048a07dSNicholas Piggin ori 31,31,0 2831a048a07dSNicholas Piggin .rept 14 2832a048a07dSNicholas Piggin b 1f 2833a048a07dSNicholas Piggin1: 2834a048a07dSNicholas Piggin .endr 2835a048a07dSNicholas Piggin blr 2836a048a07dSNicholas Piggin 2837aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 2838aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2839aa8a5e00SMichael Ellerman GET_PACA(r13); 284078ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 284178ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2842aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2843aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2844aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2845aa8a5e00SMichael Ellerman mfctr r9 2846aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2847bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2848bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2849aa8a5e00SMichael Ellerman mtctr r11 285015a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2851aa8a5e00SMichael Ellerman 2852aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2853aa8a5e00SMichael Ellerman sync 2854bdcb1aefSNicholas Piggin 2855bdcb1aefSNicholas Piggin /* 2856bdcb1aefSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 2857bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2858bdcb1aefSNicholas Piggin * hurt). 2859bdcb1aefSNicholas Piggin */ 2860bdcb1aefSNicholas Piggin1: 2861bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2862bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2863bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2864bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2865bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2866bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2867bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2868bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2869bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2870aa8a5e00SMichael Ellerman bdnz 1b 2871aa8a5e00SMichael Ellerman 2872aa8a5e00SMichael Ellerman mtctr r9 2873aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2874aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2875aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 287678ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2877aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2878aa8a5e00SMichael Ellerman rfid 2879aa8a5e00SMichael Ellerman 2880aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 2881aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2882aa8a5e00SMichael Ellerman GET_PACA(r13); 288378ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 288478ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2885aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2886aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2887aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2888aa8a5e00SMichael Ellerman mfctr r9 2889aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2890bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2891bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2892aa8a5e00SMichael Ellerman mtctr r11 289315a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2894aa8a5e00SMichael Ellerman 2895aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2896aa8a5e00SMichael Ellerman sync 2897bdcb1aefSNicholas Piggin 2898bdcb1aefSNicholas Piggin /* 2899bdcb1aefSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 2900bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2901bdcb1aefSNicholas Piggin * hurt). 2902bdcb1aefSNicholas Piggin */ 2903bdcb1aefSNicholas Piggin1: 2904bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2905bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2906bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2907bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2908bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2909bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2910bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2911bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2912bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2913aa8a5e00SMichael Ellerman bdnz 1b 2914aa8a5e00SMichael Ellerman 2915aa8a5e00SMichael Ellerman mtctr r9 2916aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2917aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2918aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 291978ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2920aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2921aa8a5e00SMichael Ellerman hrfid 2922aa8a5e00SMichael Ellerman 29230eddf327SNicholas PigginUSE_TEXT_SECTION() 29243f7fbd97SNicholas Piggin MASKED_INTERRUPT 29253f7fbd97SNicholas Piggin MASKED_INTERRUPT hsrr=1 29267230c564SBenjamin Herrenschmidt 29274f6c11dbSPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 29289600f261SNicholas Pigginkvmppc_skip_interrupt: 29294f6c11dbSPaul Mackerras /* 29304f6c11dbSPaul Mackerras * Here all GPRs are unchanged from when the interrupt happened 29314f6c11dbSPaul Mackerras * except for r13, which is saved in SPRG_SCRATCH0. 29324f6c11dbSPaul Mackerras */ 29334f6c11dbSPaul Mackerras mfspr r13, SPRN_SRR0 29344f6c11dbSPaul Mackerras addi r13, r13, 4 29354f6c11dbSPaul Mackerras mtspr SPRN_SRR0, r13 29364f6c11dbSPaul Mackerras GET_SCRATCH0(r13) 2937222f20f1SNicholas Piggin RFI_TO_KERNEL 29384f6c11dbSPaul Mackerras b . 29394f6c11dbSPaul Mackerras 29409600f261SNicholas Pigginkvmppc_skip_Hinterrupt: 29414f6c11dbSPaul Mackerras /* 29424f6c11dbSPaul Mackerras * Here all GPRs are unchanged from when the interrupt happened 29434f6c11dbSPaul Mackerras * except for r13, which is saved in SPRG_SCRATCH0. 29444f6c11dbSPaul Mackerras */ 29454f6c11dbSPaul Mackerras mfspr r13, SPRN_HSRR0 29464f6c11dbSPaul Mackerras addi r13, r13, 4 29474f6c11dbSPaul Mackerras mtspr SPRN_HSRR0, r13 29484f6c11dbSPaul Mackerras GET_SCRATCH0(r13) 2949222f20f1SNicholas Piggin HRFI_TO_KERNEL 29504f6c11dbSPaul Mackerras b . 29514f6c11dbSPaul Mackerras#endif 29524f6c11dbSPaul Mackerras 29530ebc4cdaSBenjamin Herrenschmidt /* 2954c1fb6816SMichael Neuling * Relocation-on interrupts: A subset of the interrupts can be delivered 2955c1fb6816SMichael Neuling * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 2956c1fb6816SMichael Neuling * it. Addresses are the same as the original interrupt addresses, but 2957c1fb6816SMichael Neuling * offset by 0xc000000000004000. 2958c1fb6816SMichael Neuling * It's impossible to receive interrupts below 0x300 via this mechanism. 2959c1fb6816SMichael Neuling * KVM: None of these traps are from the guest ; anything that escalated 2960c1fb6816SMichael Neuling * to HV=1 from HV=0 is delivered via real mode handlers. 2961c1fb6816SMichael Neuling */ 2962c1fb6816SMichael Neuling 2963c1fb6816SMichael Neuling /* 2964c1fb6816SMichael Neuling * This uses the standard macro, since the original 0x300 vector 2965c1fb6816SMichael Neuling * only has extra guff for STAB-based processors -- which never 2966c1fb6816SMichael Neuling * come here. 2967c1fb6816SMichael Neuling */ 2968da2bc464SMichael Ellerman 296957f26649SNicholas PigginEXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) 2970b1576fecSAnton Blanchard b __ppc64_runlatch_on 2971fe1952fcSBenjamin Herrenschmidt 297257f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 29738ed8ab40SHari Bathini /* 29748ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 29758ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 29768ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 29778ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 29788ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 29798ed8ab40SHari Bathini */ 29808ed8ab40SHari Bathini .align 7 29818ed8ab40SHari Bathini .globl __end_interrupts 29828ed8ab40SHari Bathini__end_interrupts: 298357f26649SNicholas PigginDEFINE_FIXED_SYMBOL(__end_interrupts) 298461383407SBenjamin Herrenschmidt 2985087aa036SChen Gang#ifdef CONFIG_PPC_970_NAP 2986ed0bc98fSNicholas Piggin /* 2987ed0bc98fSNicholas Piggin * Called by exception entry code if _TLF_NAPPING was set, this clears 2988ed0bc98fSNicholas Piggin * the NAPPING flag, and redirects the exception exit to 2989ed0bc98fSNicholas Piggin * power4_fixup_nap_return. 2990ed0bc98fSNicholas Piggin */ 2991ed0bc98fSNicholas Piggin .globl power4_fixup_nap 29927c8cb4b5SNicholas PigginEXC_COMMON_BEGIN(power4_fixup_nap) 2993087aa036SChen Gang andc r9,r9,r10 2994087aa036SChen Gang std r9,TI_LOCAL_FLAGS(r11) 2995ed0bc98fSNicholas Piggin LOAD_REG_ADDR(r10, power4_idle_nap_return) 2996ed0bc98fSNicholas Piggin std r10,_NIP(r1) 2997ed0bc98fSNicholas Piggin blr 2998ed0bc98fSNicholas Piggin 2999ed0bc98fSNicholas Pigginpower4_idle_nap_return: 3000087aa036SChen Gang blr 3001087aa036SChen Gang#endif 3002087aa036SChen Gang 300357f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 300457f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 300557f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 300657f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 300757f26649SNicholas Piggin 300857f26649SNicholas PigginUSE_TEXT_SECTION() 300957f26649SNicholas Piggin 3010296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 3011296e753fSNicholas Pigginenable_machine_check: 3012296e753fSNicholas Piggin mflr r0 3013296e753fSNicholas Piggin bcl 20,31,$+4 3014296e753fSNicholas Piggin0: mflr r3 3015296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 3016296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 3017296e753fSNicholas Piggin mfmsr r3 3018296e753fSNicholas Piggin ori r3,r3,MSR_ME 3019296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 3020296e753fSNicholas Piggin RFI_TO_KERNEL 3021296e753fSNicholas Piggin1: mtlr r0 3022296e753fSNicholas Piggin blr 3023296e753fSNicholas Piggin 3024b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 3025b7d9ccecSNicholas Piggindisable_machine_check: 3026b7d9ccecSNicholas Piggin mflr r0 3027b7d9ccecSNicholas Piggin bcl 20,31,$+4 3028b7d9ccecSNicholas Piggin0: mflr r3 3029b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 3030b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 3031b7d9ccecSNicholas Piggin mfmsr r3 3032b7d9ccecSNicholas Piggin li r4,MSR_ME 3033b7d9ccecSNicholas Piggin andc r3,r3,r4 3034b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 3035b7d9ccecSNicholas Piggin RFI_TO_KERNEL 3036b7d9ccecSNicholas Piggin1: mtlr r0 3037b7d9ccecSNicholas Piggin blr 3038b7d9ccecSNicholas Piggin 3039087aa036SChen Gang/* 30400ebc4cdaSBenjamin Herrenschmidt * Hash table stuff 30410ebc4cdaSBenjamin Herrenschmidt */ 3042f4329f2eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 30436a3bab90SAnton Blancharddo_hash_page: 30444e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 3045e6c2a479SRam Pai lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h 3046398a719dSBenjamin Herrenschmidt ori r0,r0,DSISR_BAD_FAULT_64S@l 30479b123d1eSNicholas Piggin and. r0,r5,r0 /* weird error? */ 30480ebc4cdaSBenjamin Herrenschmidt bne- handle_page_fault /* if not, try to insert a HPTE */ 3049c911d2e1SChristophe Leroy ld r11, PACA_THREAD_INFO(r13) 30509c1e1052SPaul Mackerras lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */ 30519c1e1052SPaul Mackerras andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */ 30529c1e1052SPaul Mackerras bne 77f /* then don't call hash_page now */ 30530ebc4cdaSBenjamin Herrenschmidt 30540ebc4cdaSBenjamin Herrenschmidt /* 30559b123d1eSNicholas Piggin * r3 contains the trap number 30569b123d1eSNicholas Piggin * r4 contains the faulting address 30579b123d1eSNicholas Piggin * r5 contains dsisr 30589b123d1eSNicholas Piggin * r6 msr 30590ebc4cdaSBenjamin Herrenschmidt * 30607230c564SBenjamin Herrenschmidt * at return r3 = 0 for success, 1 for page fault, negative for error 30610ebc4cdaSBenjamin Herrenschmidt */ 3062106713a1SAneesh Kumar K.V bl __hash_page /* build HPTE if possible */ 3063106713a1SAneesh Kumar K.V cmpdi r3,0 /* see if __hash_page succeeded */ 30640ebc4cdaSBenjamin Herrenschmidt 30657230c564SBenjamin Herrenschmidt /* Success */ 3066702f0980SNicholas Piggin beq interrupt_return /* Return from exception on success */ 30670ebc4cdaSBenjamin Herrenschmidt 30687230c564SBenjamin Herrenschmidt /* Error */ 30697230c564SBenjamin Herrenschmidt blt- 13f 3070d89ba535SNaveen N. Rao 30719b123d1eSNicholas Piggin /* Reload DAR/DSISR into r4/r5 for the DABR check below */ 30729b123d1eSNicholas Piggin ld r4,_DAR(r1) 30739b123d1eSNicholas Piggin ld r5,_DSISR(r1) 30744e003747SMichael Ellerman#endif /* CONFIG_PPC_BOOK3S_64 */ 30750ebc4cdaSBenjamin Herrenschmidt 3076a546498fSBenjamin Herrenschmidt/* Here we have a page fault that hash_page can't handle. */ 3077a546498fSBenjamin Herrenschmidthandle_page_fault: 30789b123d1eSNicholas Piggin11: andis. r0,r5,DSISR_DABRMATCH@h 3079d89ba535SNaveen N. Rao bne- handle_dabr_fault 3080a546498fSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 3081b1576fecSAnton Blanchard bl do_page_fault 3082a546498fSBenjamin Herrenschmidt cmpdi r3,0 3083702f0980SNicholas Piggin beq+ interrupt_return 3084a546498fSBenjamin Herrenschmidt mr r5,r3 3085a546498fSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 3086c31f7134SNicholas Piggin ld r4,_DAR(r1) 3087b1576fecSAnton Blanchard bl bad_page_fault 30886cc0c16dSNicholas Piggin b interrupt_return 30890ebc4cdaSBenjamin Herrenschmidt 30909c7cc234SK.Prasad/* We have a data breakpoint exception - handle it */ 30919c7cc234SK.Prasadhandle_dabr_fault: 30929c7cc234SK.Prasad ld r4,_DAR(r1) 30939c7cc234SK.Prasad ld r5,_DSISR(r1) 30949c7cc234SK.Prasad addi r3,r1,STACK_FRAME_OVERHEAD 3095b1576fecSAnton Blanchard bl do_break 3096f474c28fSRavi Bangoria /* 3097f474c28fSRavi Bangoria * do_break() may have changed the NV GPRS while handling a breakpoint. 3098702f0980SNicholas Piggin * If so, we need to restore them with their updated values. 3099f474c28fSRavi Bangoria */ 3100702f0980SNicholas Piggin REST_NVGPRS(r1) 31016cc0c16dSNicholas Piggin b interrupt_return 31029c7cc234SK.Prasad 31030ebc4cdaSBenjamin Herrenschmidt 31044e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 31050ebc4cdaSBenjamin Herrenschmidt/* We have a page fault that hash_page could handle but HV refused 31060ebc4cdaSBenjamin Herrenschmidt * the PTE insertion 31070ebc4cdaSBenjamin Herrenschmidt */ 31086cc0c16dSNicholas Piggin13: mr r5,r3 31090ebc4cdaSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 31100ebc4cdaSBenjamin Herrenschmidt ld r4,_DAR(r1) 3111b1576fecSAnton Blanchard bl low_hash_fault 31126cc0c16dSNicholas Piggin b interrupt_return 3113caca285eSAneesh Kumar K.V#endif 31140ebc4cdaSBenjamin Herrenschmidt 31159c1e1052SPaul Mackerras/* 31169c1e1052SPaul Mackerras * We come here as a result of a DSI at a point where we don't want 31179c1e1052SPaul Mackerras * to call hash_page, such as when we are accessing memory (possibly 31189c1e1052SPaul Mackerras * user memory) inside a PMU interrupt that occurred while interrupts 31199c1e1052SPaul Mackerras * were soft-disabled. We want to invoke the exception handler for 31209c1e1052SPaul Mackerras * the access, or panic if there isn't a handler. 31219c1e1052SPaul Mackerras */ 31226cc0c16dSNicholas Piggin77: addi r3,r1,STACK_FRAME_OVERHEAD 31239c1e1052SPaul Mackerras li r5,SIGSEGV 3124b1576fecSAnton Blanchard bl bad_page_fault 31256cc0c16dSNicholas Piggin b interrupt_return 3126