1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 167230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 178aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1846f52210SStephen Rothwell#include <asm/ptrace.h> 197cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 20da2bc464SMichael Ellerman#include <asm/head-64.h> 212c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 22890274c2SMichael Ellerman#include <asm/kup.h> 238aa34ab8SBenjamin Herrenschmidt 247299417cSNicholas Piggin/* 257299417cSNicholas Piggin * Following are fixed section helper macros. 267299417cSNicholas Piggin * 277299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 287299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 297299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 307299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 317299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 327299417cSNicholas Piggin */ 337299417cSNicholas Piggin 34a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 35a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 36a2432811SNicholas Piggin 37a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 38a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 39a2432811SNicholas Piggin 40a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 41a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 42a2432811SNicholas Piggin 43a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 44a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 45a2432811SNicholas Piggin 46a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 47a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 48a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 49a2432811SNicholas Piggin .global name; \ 50a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 51a2432811SNicholas Piggin DEFINE_FIXED_SYMBOL(name); \ 52a2432811SNicholas Pigginname: 53a2432811SNicholas Piggin 54a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 55a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 56a2432811SNicholas Piggin 57a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 58a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 59a2432811SNicholas Piggin 60a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 61a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 62a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 63a2432811SNicholas Piggin 64a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 65a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 66a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 67a2432811SNicholas Piggin 680ebc4cdaSBenjamin Herrenschmidt/* 6912a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 7012a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 7112a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 7212a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 7312a04809SNicholas Piggin * that kernelbase be 64K aligned. 7412a04809SNicholas Piggin */ 7512a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 7612a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 7712a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 7812a04809SNicholas Piggin 7912a04809SNicholas Piggin#define __LOAD_HANDLER(reg, label) \ 8012a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 8112a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l 8212a04809SNicholas Piggin 8312a04809SNicholas Piggin/* 8412a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 8512a04809SNicholas Piggin * head-y require >64K offsets. 8612a04809SNicholas Piggin */ 8712a04809SNicholas Piggin#define __LOAD_FAR_HANDLER(reg, label) \ 8812a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 8912a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l; \ 9012a04809SNicholas Piggin addis reg,reg,(ABS_ADDR(label))@h 9112a04809SNicholas Piggin 9212a04809SNicholas Piggin/* 9312a04809SNicholas Piggin * Branch to label using its 0xC000 address. This results in instruction 9412a04809SNicholas Piggin * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned 9512a04809SNicholas Piggin * on using mtmsr rather than rfid. 9612a04809SNicholas Piggin * 9712a04809SNicholas Piggin * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than 9812a04809SNicholas Piggin * load KBASE for a slight optimisation. 9912a04809SNicholas Piggin */ 10012a04809SNicholas Piggin#define BRANCH_TO_C000(reg, label) \ 1010e10be2bSNicholas Piggin __LOAD_FAR_HANDLER(reg, label); \ 10212a04809SNicholas Piggin mtctr reg; \ 10312a04809SNicholas Piggin bctr 10412a04809SNicholas Piggin 105a42a239dSNicholas Piggin/* 106a42a239dSNicholas Piggin * Interrupt code generation macros 107a42a239dSNicholas Piggin */ 10894325357SNicholas Piggin#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 10994325357SNicholas Piggin#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 11094325357SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 11194325357SNicholas Piggin#define IAREA .L_IAREA_\name\() /* PACA save area */ 11294325357SNicholas Piggin#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 11394325357SNicholas Piggin#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 11494325357SNicholas Piggin#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 11594325357SNicholas Piggin#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 11694325357SNicholas Piggin#define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */ 11794325357SNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 11894325357SNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 11994325357SNicholas Piggin#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 12094325357SNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 1214f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 12294325357SNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 12394325357SNicholas Piggin#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 1244f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 12594325357SNicholas Piggin#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 126a42a239dSNicholas Piggin 127a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 128a42a239dSNicholas Piggin.macro int_define_ ## n name 129a42a239dSNicholas Piggin 130a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 131a42a239dSNicholas Piggin.endm ; \ 132a42a239dSNicholas Pigginint_define_ ## n n ; \ 133a42a239dSNicholas Piggindo_define_int n 134a42a239dSNicholas Piggin 135a42a239dSNicholas Piggin.macro do_define_int name 136a42a239dSNicholas Piggin .ifndef IVEC 137a42a239dSNicholas Piggin .error "IVEC not defined" 138a42a239dSNicholas Piggin .endif 139a42a239dSNicholas Piggin .ifndef IHSRR 1403f7fbd97SNicholas Piggin IHSRR=0 1413f7fbd97SNicholas Piggin .endif 1423f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 1433f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 144a42a239dSNicholas Piggin .endif 145a42a239dSNicholas Piggin .ifndef IAREA 146a42a239dSNicholas Piggin IAREA=PACA_EXGEN 147a42a239dSNicholas Piggin .endif 1488729c26eSNicholas Piggin .ifndef IVIRT 1498729c26eSNicholas Piggin IVIRT=1 1508729c26eSNicholas Piggin .endif 151a3cd35beSNicholas Piggin .ifndef IISIDE 152a3cd35beSNicholas Piggin IISIDE=0 153a3cd35beSNicholas Piggin .endif 154a42a239dSNicholas Piggin .ifndef IDAR 155a42a239dSNicholas Piggin IDAR=0 156a42a239dSNicholas Piggin .endif 157a42a239dSNicholas Piggin .ifndef IDSISR 158a42a239dSNicholas Piggin IDSISR=0 159a42a239dSNicholas Piggin .endif 160a42a239dSNicholas Piggin .ifndef ISET_RI 161a42a239dSNicholas Piggin ISET_RI=1 162a42a239dSNicholas Piggin .endif 163d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 164d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 165d73a10cbSNicholas Piggin .endif 166d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 167d73a10cbSNicholas Piggin IREALMODE_COMMON=0 168d73a10cbSNicholas Piggin .else 169d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 170d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 171d73a10cbSNicholas Piggin .endif 172a42a239dSNicholas Piggin .endif 173a42a239dSNicholas Piggin .ifndef IMASK 174a42a239dSNicholas Piggin IMASK=0 175a42a239dSNicholas Piggin .endif 176a42a239dSNicholas Piggin .ifndef IKVM_REAL 177a42a239dSNicholas Piggin IKVM_REAL=0 178a42a239dSNicholas Piggin .endif 179a42a239dSNicholas Piggin .ifndef IKVM_VIRT 180a42a239dSNicholas Piggin IKVM_VIRT=0 181a42a239dSNicholas Piggin .endif 1827cb3a1a0SNicholas Piggin .ifndef ISTACK 1837cb3a1a0SNicholas Piggin ISTACK=1 1847cb3a1a0SNicholas Piggin .endif 1857cb3a1a0SNicholas Piggin .ifndef IKUAP 1867cb3a1a0SNicholas Piggin IKUAP=1 1877cb3a1a0SNicholas Piggin .endif 188a42a239dSNicholas Piggin.endm 189a42a239dSNicholas Piggin 19012a04809SNicholas Piggin/* 1912284ffeaSNicholas Piggin * All interrupts which set HSRR registers, as well as SRESET and MCE and 1922284ffeaSNicholas Piggin * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 1932284ffeaSNicholas Piggin * so they all generally need to test whether they were taken in guest context. 1942284ffeaSNicholas Piggin * 1952284ffeaSNicholas Piggin * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 1962284ffeaSNicholas Piggin * taken with MSR[HV]=0. 1972284ffeaSNicholas Piggin * 1982284ffeaSNicholas Piggin * Interrupts which set SRR registers (with the above exceptions) do not 1992284ffeaSNicholas Piggin * elevate to MSR[HV]=1 mode, though most can be taken when running with 2002284ffeaSNicholas Piggin * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 2012284ffeaSNicholas Piggin * not need to test whether a guest is running because they get delivered to 2022284ffeaSNicholas Piggin * the guest directly, including nested HV KVM guests. 2032284ffeaSNicholas Piggin * 2042284ffeaSNicholas Piggin * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 2052284ffeaSNicholas Piggin * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 2062284ffeaSNicholas Piggin * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 2072284ffeaSNicholas Piggin * delivered to the real-mode entry point, therefore such interrupts only test 2082284ffeaSNicholas Piggin * KVM in their real mode handlers, and only when PR KVM is possible. 2092284ffeaSNicholas Piggin * 2102284ffeaSNicholas Piggin * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 2112284ffeaSNicholas Piggin * delivered in real-mode when the MMU is in hash mode because the MMU 2122284ffeaSNicholas Piggin * registers are not set appropriately to translate host addresses. In nested 2132284ffeaSNicholas Piggin * radix mode these can be delivered in virt-mode as the host translations are 2142284ffeaSNicholas Piggin * used implicitly (see: effective LPID, effective PID). 2152284ffeaSNicholas Piggin */ 2162284ffeaSNicholas Piggin 2172284ffeaSNicholas Piggin/* 2182284ffeaSNicholas Piggin * If an interrupt is taken while a guest is running, it is immediately routed 219f3601156SNicholas Piggin * to KVM to handle. 22012a04809SNicholas Piggin */ 22112a04809SNicholas Piggin 22269fdd674SNicholas Piggin.macro KVMTEST name handler 22369fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 22412a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 22512a04809SNicholas Piggin cmpwi r10,0 22612a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 2273f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 228def0db4fSNicholas Piggin BEGIN_FTR_SECTION 22969fdd674SNicholas Piggin li r10,(IVEC + 0x2) 230def0db4fSNicholas Piggin FTR_SECTION_ELSE 23169fdd674SNicholas Piggin li r10,(IVEC) 232def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 233b177ae2fSNicholas Piggin .elseif IHSRR 23469fdd674SNicholas Piggin li r10,(IVEC + 0x2) 23512a04809SNicholas Piggin .else 23669fdd674SNicholas Piggin li r10,(IVEC) 23712a04809SNicholas Piggin .endif 23869fdd674SNicholas Piggin bne \handler 23912a04809SNicholas Piggin#endif 24069fdd674SNicholas Piggin.endm 24112a04809SNicholas Piggin 242c7c5cbb4SNicholas Piggin/* 243c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 244c7c5cbb4SNicholas Piggin * 245c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 246c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 247c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 248c7c5cbb4SNicholas Piggin * the kernel. 249c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 250c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 251c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 252c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 253c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 254c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 255c7c5cbb4SNicholas Piggin * irq_soft_mask set. 256c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 257c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 258c7c5cbb4SNicholas Piggin * mode. 259c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 260c7c5cbb4SNicholas Piggin * This is done if early=2. 261c7c5cbb4SNicholas Piggin */ 2628729c26eSNicholas Piggin 2638729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 264d73a10cbSNicholas Piggin .if IREALMODE_COMMON 265d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 266d73a10cbSNicholas Piggin mtctr r10 267d73a10cbSNicholas Piggin bctr 268d73a10cbSNicholas Piggin .else 2698729c26eSNicholas Piggin .if \virt 2708729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 2718729c26eSNicholas Piggin b \name\()_common_virt 2728729c26eSNicholas Piggin#else 2738729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 2748729c26eSNicholas Piggin mtctr r10 2758729c26eSNicholas Piggin bctr 2768729c26eSNicholas Piggin#endif 2778729c26eSNicholas Piggin .else 2788729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 2798729c26eSNicholas Piggin mtctr r10 2808729c26eSNicholas Piggin bctr 2818729c26eSNicholas Piggin .endif 282d73a10cbSNicholas Piggin .endif 2838729c26eSNicholas Piggin.endm 2848729c26eSNicholas Piggin 285fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 286c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 287c7c5cbb4SNicholas Piggin GET_PACA(r13) 288fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 289931dc86bSNicholas PigginBEGIN_FTR_SECTION 290931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 291931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 292c7c5cbb4SNicholas Piggin HMT_MEDIUM 293fc589ee4SNicholas Piggin std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 294931dc86bSNicholas PigginBEGIN_FTR_SECTION 295931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 296931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 297c7c5cbb4SNicholas Piggin .if \ool 298c7c5cbb4SNicholas Piggin .if !\virt 299c7c5cbb4SNicholas Piggin b tramp_real_\name 300c7c5cbb4SNicholas Piggin .pushsection .text 301c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 302c7c5cbb4SNicholas Piggin .else 303c7c5cbb4SNicholas Piggin b tramp_virt_\name 304c7c5cbb4SNicholas Piggin .pushsection .text 305c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 306c7c5cbb4SNicholas Piggin .endif 307c7c5cbb4SNicholas Piggin .endif 308c7c5cbb4SNicholas Piggin 309931dc86bSNicholas PigginBEGIN_FTR_SECTION 310931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 311931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 312931dc86bSNicholas PigginBEGIN_FTR_SECTION 313931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 314931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 315c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 3168729c26eSNicholas Piggin mfctr r10 3178729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 318c7c5cbb4SNicholas Piggin mfcr r9 319fc589ee4SNicholas Piggin std r11,IAREA+EX_R11(r13) 320fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 321c7c5cbb4SNicholas Piggin 322c7c5cbb4SNicholas Piggin /* 323c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 324c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 325c7c5cbb4SNicholas Piggin * not recoverable if they are live. 326c7c5cbb4SNicholas Piggin */ 327c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 328fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 329a3cd35beSNicholas Piggin .if IDAR && !IISIDE 330fc589ee4SNicholas Piggin .if IHSRR 331c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 332c7c5cbb4SNicholas Piggin .else 333c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 334c7c5cbb4SNicholas Piggin .endif 335fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 336c7c5cbb4SNicholas Piggin .endif 337a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 338fc589ee4SNicholas Piggin .if IHSRR 339c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 340c7c5cbb4SNicholas Piggin .else 341c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 342c7c5cbb4SNicholas Piggin .endif 343fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 344c7c5cbb4SNicholas Piggin .endif 345c7c5cbb4SNicholas Piggin 3463f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 3478729c26eSNicholas Piggin BEGIN_FTR_SECTION 3488729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3498729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3508729c26eSNicholas Piggin FTR_SECTION_ELSE 3518729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3528729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 3538729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 3548729c26eSNicholas Piggin .elseif IHSRR 3558729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3568729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3578729c26eSNicholas Piggin .else 3588729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3598729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 360c7c5cbb4SNicholas Piggin .endif 361d73a10cbSNicholas Piggin 362d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 3638729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 3648729c26eSNicholas Piggin .endif 3658729c26eSNicholas Piggin 366c7c5cbb4SNicholas Piggin .if \ool 367c7c5cbb4SNicholas Piggin .popsection 368c7c5cbb4SNicholas Piggin .endif 369c7c5cbb4SNicholas Piggin.endm 370c7c5cbb4SNicholas Piggin 371d064151fSNicholas Piggin/* 3728729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 3739600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 3749600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 3759600f261SNicholas Piggin * 3768729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 377d064151fSNicholas Piggin */ 3788729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 3798729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 3808729c26eSNicholas Piggin\name\()_common_real: 3819600f261SNicholas Piggin .if IKVM_REAL 38269fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 3839600f261SNicholas Piggin .endif 3849600f261SNicholas Piggin 3858729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 3868729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 387c080a173SDaniel Axtens .if IHSRR_IF_HVMODE 3888729c26eSNicholas Piggin BEGIN_FTR_SECTION 3898729c26eSNicholas Piggin xori r10,r10,MSR_RI 3908729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 3918729c26eSNicholas Piggin .elseif ! IHSRR 3928729c26eSNicholas Piggin xori r10,r10,MSR_RI 3938729c26eSNicholas Piggin .endif 3948729c26eSNicholas Piggin mtmsrd r10 3958729c26eSNicholas Piggin 3968729c26eSNicholas Piggin .if IVIRT 3979600f261SNicholas Piggin .if IKVM_VIRT 3989600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 3999600f261SNicholas Piggin .endif 4009600f261SNicholas Piggin 4018729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 4028729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_virt) 4038729c26eSNicholas Piggin\name\()_common_virt: 4049600f261SNicholas Piggin .if IKVM_VIRT 40569fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4069600f261SNicholas Piggin1: 4079600f261SNicholas Piggin .endif 4088729c26eSNicholas Piggin .endif /* IVIRT */ 4098729c26eSNicholas Piggin.endm 4108729c26eSNicholas Piggin 4119600f261SNicholas Piggin/* 4129600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4139600f261SNicholas Piggin * want to run in real mode. 4149600f261SNicholas Piggin */ 4159600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 4169600f261SNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 4179600f261SNicholas Piggin\name\()_common_real: 4189600f261SNicholas Piggin .if IKVM_REAL 41969fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4209600f261SNicholas Piggin .endif 4219600f261SNicholas Piggin.endm 4229600f261SNicholas Piggin 4238729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 4240eddf327SNicholas Piggin .if IMASK 425b2dc2977SNicholas Piggin .if ! ISTACK 426b2dc2977SNicholas Piggin .error "No support for masked interrupt to use custom stack" 427b2dc2977SNicholas Piggin .endif 428b2dc2977SNicholas Piggin 429b2dc2977SNicholas Piggin /* If coming from user, skip soft-mask tests. */ 430b2dc2977SNicholas Piggin andi. r10,r12,MSR_PR 431b2dc2977SNicholas Piggin bne 2f 432b2dc2977SNicholas Piggin 433b2dc2977SNicholas Piggin /* Kernel code running below __end_interrupts is implicitly 434b2dc2977SNicholas Piggin * soft-masked */ 435b2dc2977SNicholas Piggin LOAD_HANDLER(r10, __end_interrupts) 436b2dc2977SNicholas Piggin cmpld r11,r10 437b2dc2977SNicholas Piggin li r10,IMASK 438b2dc2977SNicholas Piggin blt- 1f 439b2dc2977SNicholas Piggin 440b2dc2977SNicholas Piggin /* Test the soft mask state against our interrupt's bit */ 4410eddf327SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 442b2dc2977SNicholas Piggin1: andi. r10,r10,IMASK 4430eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 4440eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 4450eddf327SNicholas Piggin li r10,PACA_IRQ_EE 4460eddf327SNicholas Piggin .elseif IVEC == 0x900 4470eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 4480eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 4490eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 4500eddf327SNicholas Piggin .elseif IVEC == 0xe60 4510eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 4520eddf327SNicholas Piggin .elseif IVEC == 0xf00 4530eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 4540eddf327SNicholas Piggin .else 4550eddf327SNicholas Piggin .abort "Bad maskable vector" 4560eddf327SNicholas Piggin .endif 4570eddf327SNicholas Piggin 4583f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4590eddf327SNicholas Piggin BEGIN_FTR_SECTION 4600eddf327SNicholas Piggin bne masked_Hinterrupt 4610eddf327SNicholas Piggin FTR_SECTION_ELSE 4620eddf327SNicholas Piggin bne masked_interrupt 4630eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4640eddf327SNicholas Piggin .elseif IHSRR 4650eddf327SNicholas Piggin bne masked_Hinterrupt 4660eddf327SNicholas Piggin .else 4670eddf327SNicholas Piggin bne masked_interrupt 4680eddf327SNicholas Piggin .endif 4690eddf327SNicholas Piggin .endif 4700eddf327SNicholas Piggin 4716d71759aSNicholas Piggin .if ISTACK 4725d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 473b2dc2977SNicholas Piggin2: mr r10,r1 /* Save r1 */ 4745d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 4751b359982SNicholas Piggin beq- 100f 4765d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 4771b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 4781b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 4795d5e0edfSNicholas Piggin .endif 4808c9fb5d4SNicholas Piggin 4818c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 4828c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 4838c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 4848c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 4858c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 4868c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 4875d5e0edfSNicholas Piggin 4888729c26eSNicholas Piggin .if ISET_RI 4898729c26eSNicholas Piggin li r10,MSR_RI 4908729c26eSNicholas Piggin mtmsrd r10,1 /* Set MSR_RI */ 4918729c26eSNicholas Piggin .endif 4928729c26eSNicholas Piggin 4936d71759aSNicholas Piggin .if ISTACK 4946d71759aSNicholas Piggin .if IKUAP 4955d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 4965d5e0edfSNicholas Piggin .endif 4971b359982SNicholas Piggin beq 101f /* if from kernel mode */ 498931dc86bSNicholas PigginBEGIN_FTR_SECTION 499931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 500931dc86bSNicholas Piggin std r9,_PPR(r1) 501931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5021b359982SNicholas Piggin101: 5035d5e0edfSNicholas Piggin .else 5046d71759aSNicholas Piggin .if IKUAP 505bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 506bcbceed4SNicholas Piggin .endif 5075d5e0edfSNicholas Piggin .endif 5085d5e0edfSNicholas Piggin 5098c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5106d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5116d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5128c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5138c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5146d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5156d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5166d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5178c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5188c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5198c9fb5d4SNicholas Piggin std r11,GPR13(r1) 520a3cd35beSNicholas Piggin 5216cc0c16dSNicholas Piggin SAVE_NVGPRS(r1) 5226cc0c16dSNicholas Piggin 5236d71759aSNicholas Piggin .if IDAR 524a3cd35beSNicholas Piggin .if IISIDE 525d1a84718SNicholas Piggin ld r10,_NIP(r1) 526d1a84718SNicholas Piggin .else 5276d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 528d1a84718SNicholas Piggin .endif 529d1a84718SNicholas Piggin std r10,_DAR(r1) 530d1a84718SNicholas Piggin .endif 531a3cd35beSNicholas Piggin 5326d71759aSNicholas Piggin .if IDSISR 533a3cd35beSNicholas Piggin .if IISIDE 534d1a84718SNicholas Piggin ld r10,_MSR(r1) 535d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 536d1a84718SNicholas Piggin and r10,r10,r11 537d1a84718SNicholas Piggin .else 5386d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 539d1a84718SNicholas Piggin .endif 540d1a84718SNicholas Piggin std r10,_DSISR(r1) 541d1a84718SNicholas Piggin .endif 542a3cd35beSNicholas Piggin 543931dc86bSNicholas PigginBEGIN_FTR_SECTION 5446d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 5458c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 546931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 5478729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 5488c9fb5d4SNicholas Piggin std r10,_CTR(r1) 5498c9fb5d4SNicholas Piggin std r2,GPR2(r1) /* save r2 in stackframe */ 5508c9fb5d4SNicholas Piggin SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ 5518c9fb5d4SNicholas Piggin SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ 5528c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 5538c9fb5d4SNicholas Piggin ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 5548c9fb5d4SNicholas Piggin std r9,_LINK(r1) 5558c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5568c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 5578c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 5588c9fb5d4SNicholas Piggin std r11,_XER(r1) 5596cc0c16dSNicholas Piggin li r9,IVEC 5608c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 5618c9fb5d4SNicholas Piggin li r10,0 5628c9fb5d4SNicholas Piggin ld r11,exception_marker@toc(r2) 5638c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 5648c9fb5d4SNicholas Piggin std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 565bcbceed4SNicholas Piggin.endm 566bcbceed4SNicholas Piggin 567391e941bSNicholas Piggin/* 5688729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 5698729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 5708729c26eSNicholas Piggin * SRR1, and relocation is on. 5718729c26eSNicholas Piggin * 5728729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 5738729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 5748729c26eSNicholas Piggin */ 5758729c26eSNicholas Piggin.macro GEN_COMMON name 5768729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 5778729c26eSNicholas Piggin __GEN_COMMON_BODY \name 5788729c26eSNicholas Piggin.endm 5798729c26eSNicholas Piggin 5808729c26eSNicholas Piggin/* 581391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 582391e941bSNicholas Piggin * standard exception. 583391e941bSNicholas Piggin */ 5843f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 585391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 586391e941bSNicholas Piggin ld r9,_MSR(r1) 587391e941bSNicholas Piggin .if \hsrr 588391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 589391e941bSNicholas Piggin .else 590391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 591391e941bSNicholas Piggin .endif 592391e941bSNicholas Piggin ld r9,_NIP(r1) 593391e941bSNicholas Piggin .if \hsrr 594391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 595391e941bSNicholas Piggin .else 596391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 597391e941bSNicholas Piggin .endif 598391e941bSNicholas Piggin ld r9,_CTR(r1) 599391e941bSNicholas Piggin mtctr r9 600391e941bSNicholas Piggin ld r9,_XER(r1) 601391e941bSNicholas Piggin mtxer r9 602391e941bSNicholas Piggin ld r9,_LINK(r1) 603391e941bSNicholas Piggin mtlr r9 604391e941bSNicholas Piggin ld r9,_CCR(r1) 605391e941bSNicholas Piggin mtcr r9 606391e941bSNicholas Piggin REST_8GPRS(2, r1) 607391e941bSNicholas Piggin REST_4GPRS(10, r1) 608391e941bSNicholas Piggin REST_GPR(0, r1) 609391e941bSNicholas Piggin /* restore original r1. */ 610391e941bSNicholas Piggin ld r1,GPR1(r1) 611391e941bSNicholas Piggin.endm 612d064151fSNicholas Piggin 61312a04809SNicholas Piggin/* 61457f26649SNicholas Piggin * There are a few constraints to be concerned with. 61557f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 61657f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 61757f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 61857f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 61957f26649SNicholas Piggin * must be used. 62057f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 62157f26649SNicholas Piggin * virtual 0xc00... 62257f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 62357f26649SNicholas Piggin * 62457f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 62557f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 62657f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 62757f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 62857f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 62957f26649SNicholas Piggin * 63057f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 63157f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 63257f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 63357f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 63457f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 63557f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 63657f26649SNicholas Piggin * 6377fa95f9aSNicholas Piggin * The scv instructions are a special case. They get a 0x3000 offset applied. 6387fa95f9aSNicholas Piggin * scv exceptions have unique reentrancy properties, see below. 6397fa95f9aSNicholas Piggin * 64057f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 64157f26649SNicholas Piggin * 64257f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 64357f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 64457f26649SNicholas Piggin * 64557f26649SNicholas Piggin * 6460ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 6470ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 64857f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 6497fa95f9aSNicholas Piggin * 0x1900 - 0x2fff : Real mode trampolines 6507fa95f9aSNicholas Piggin * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 65157f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 6520ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 65357f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 65457f26649SNicholas Piggin * setup code, rest of kernel. 655e0319829SNicholas Piggin * 656e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 657e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 658e0319829SNicholas Piggin * vectors there. 6590ebc4cdaSBenjamin Herrenschmidt */ 66057f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 6617fa95f9aSNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000) 6627fa95f9aSNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900) 66357f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 664ccd47702SNicholas Piggin 665ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 666bd3524feSNicholas Piggin .globl start_real_trampolines 667bd3524feSNicholas Piggin .globl end_real_trampolines 668bd3524feSNicholas Piggin .globl start_virt_trampolines 669bd3524feSNicholas Piggin .globl end_virt_trampolines 670ccd47702SNicholas Piggin#endif 671ccd47702SNicholas Piggin 67257f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 67357f26649SNicholas Piggin/* 67457f26649SNicholas Piggin * Data area reserved for FWNMI option. 67557f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 67657f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 67757f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 67857f26649SNicholas Piggin */ 67957f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 68057f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 68157f26649SNicholas Piggin#else 68257f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 68357f26649SNicholas Piggin#endif 68457f26649SNicholas Piggin 68557f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 68657f26649SNicholas Piggin 6870ebc4cdaSBenjamin Herrenschmidt/* 6880ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 6890ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 6900ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 6910ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 6920ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 6930ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 6940ebc4cdaSBenjamin Herrenschmidt */ 6950ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 6960ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 6970ebc4cdaSBenjamin Herrenschmidt 6987fa95f9aSNicholas Piggin/** 6997fa95f9aSNicholas Piggin * Interrupt 0x3000 - System Call Vectored Interrupt (syscall). 7007fa95f9aSNicholas Piggin * This is a synchronous interrupt invoked with the "scv" instruction. The 7017fa95f9aSNicholas Piggin * system call does not alter the HV bit, so it is directed to the OS. 7027fa95f9aSNicholas Piggin * 7037fa95f9aSNicholas Piggin * Handling: 7047fa95f9aSNicholas Piggin * scv instructions enter the kernel without changing EE, RI, ME, or HV. 7057fa95f9aSNicholas Piggin * In particular, this means we can take a maskable interrupt at any point 7067fa95f9aSNicholas Piggin * in the scv handler, which is unlike any other interrupt. This is solved 7077fa95f9aSNicholas Piggin * by treating the instruction addresses below __end_interrupts as being 7087fa95f9aSNicholas Piggin * soft-masked. 7097fa95f9aSNicholas Piggin * 7107fa95f9aSNicholas Piggin * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and 7117fa95f9aSNicholas Piggin * ensure scv is never executed with relocation off, which means AIL-0 7127fa95f9aSNicholas Piggin * should never happen. 7137fa95f9aSNicholas Piggin * 7147fa95f9aSNicholas Piggin * Before leaving the below __end_interrupts text, at least of the following 7157fa95f9aSNicholas Piggin * must be true: 7167fa95f9aSNicholas Piggin * - MSR[PR]=1 (i.e., return to userspace) 7177fa95f9aSNicholas Piggin * - MSR_EE|MSR_RI is set (no reentrant exceptions) 7187fa95f9aSNicholas Piggin * - Standard kernel environment is set up (stack, paca, etc) 7197fa95f9aSNicholas Piggin * 7207fa95f9aSNicholas Piggin * Call convention: 7217fa95f9aSNicholas Piggin * 7227fa95f9aSNicholas Piggin * syscall register convention is in Documentation/powerpc/syscall64-abi.rst 7237fa95f9aSNicholas Piggin */ 7247fa95f9aSNicholas PigginEXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000) 7257fa95f9aSNicholas Piggin /* SCV 0 */ 7267fa95f9aSNicholas Piggin mr r9,r13 7277fa95f9aSNicholas Piggin GET_PACA(r13) 7287fa95f9aSNicholas Piggin mflr r11 7297fa95f9aSNicholas Piggin mfctr r12 7307fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 7317fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 7327fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 7337fa95f9aSNicholas Piggin b system_call_vectored_tramp 7347fa95f9aSNicholas Piggin#else 7357fa95f9aSNicholas Piggin b system_call_vectored_common 7367fa95f9aSNicholas Piggin#endif 7377fa95f9aSNicholas Piggin nop 7387fa95f9aSNicholas Piggin 7397fa95f9aSNicholas Piggin /* SCV 1 - 127 */ 7407fa95f9aSNicholas Piggin .rept 127 7417fa95f9aSNicholas Piggin mr r9,r13 7427fa95f9aSNicholas Piggin GET_PACA(r13) 7437fa95f9aSNicholas Piggin mflr r11 7447fa95f9aSNicholas Piggin mfctr r12 7457fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 7467fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 7477fa95f9aSNicholas Piggin li r0,-1 /* cause failure */ 7487fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 7497fa95f9aSNicholas Piggin b system_call_vectored_sigill_tramp 7507fa95f9aSNicholas Piggin#else 7517fa95f9aSNicholas Piggin b system_call_vectored_sigill 7527fa95f9aSNicholas Piggin#endif 7537fa95f9aSNicholas Piggin .endr 7547fa95f9aSNicholas PigginEXC_VIRT_END(system_call_vectored, 0x3000, 0x1000) 7557fa95f9aSNicholas Piggin 7567fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 7577fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_tramp) 7587fa95f9aSNicholas Piggin __LOAD_HANDLER(r10, system_call_vectored_common) 7597fa95f9aSNicholas Piggin mtctr r10 7607fa95f9aSNicholas Piggin bctr 7617fa95f9aSNicholas Piggin 7627fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) 7637fa95f9aSNicholas Piggin __LOAD_HANDLER(r10, system_call_vectored_sigill) 7647fa95f9aSNicholas Piggin mtctr r10 7657fa95f9aSNicholas Piggin bctr 7667fa95f9aSNicholas Piggin#endif 7677fa95f9aSNicholas Piggin 7687fa95f9aSNicholas Piggin 769e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 7701a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 771e0319829SNicholas Piggin 772fb479e44SNicholas Piggin 77394325357SNicholas Piggin/** 77494325357SNicholas Piggin * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 77594325357SNicholas Piggin * This is a non-maskable, asynchronous interrupt always taken in real-mode. 77694325357SNicholas Piggin * It is caused by: 77794325357SNicholas Piggin * - Wake from power-saving state, on powernv. 77894325357SNicholas Piggin * - An NMI from another CPU, triggered by firmware or hypercall. 77994325357SNicholas Piggin * - As crash/debug signal injected from BMC, firmware or hypervisor. 78094325357SNicholas Piggin * 78194325357SNicholas Piggin * Handling: 78294325357SNicholas Piggin * Power-save wakeup is the only performance critical path, so this is 78394325357SNicholas Piggin * determined quickly as possible first. In this case volatile registers 78494325357SNicholas Piggin * can be discarded and SPRs like CFAR don't need to be read. 78594325357SNicholas Piggin * 78694325357SNicholas Piggin * If not a powersave wakeup, then it's run as a regular interrupt, however 78794325357SNicholas Piggin * it uses its own stack and PACA save area to preserve the regular kernel 78894325357SNicholas Piggin * environment for debugging. 78994325357SNicholas Piggin * 79094325357SNicholas Piggin * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 79194325357SNicholas Piggin * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 79294325357SNicholas Piggin * correct to switch to virtual mode to run the regular interrupt handler 79394325357SNicholas Piggin * because it might be interrupted when the MMU is in a bad state (e.g., SLB 79494325357SNicholas Piggin * is clear). 79594325357SNicholas Piggin * 79694325357SNicholas Piggin * FWNMI: 79794325357SNicholas Piggin * PAPR specifies a "fwnmi" facility which sends the sreset to a different 79894325357SNicholas Piggin * entry point with a different register set up. Some hypervisors will 79994325357SNicholas Piggin * send the sreset to 0x100 in the guest if it is not fwnmi capable. 80094325357SNicholas Piggin * 80194325357SNicholas Piggin * KVM: 80294325357SNicholas Piggin * Unlike most SRR interrupts, this may be taken by the host while executing 80394325357SNicholas Piggin * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 80494325357SNicholas Piggin * mode and then raise the sreset. 80594325357SNicholas Piggin */ 8064f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 8074f50541fSNicholas Piggin IVEC=0x100 8084f50541fSNicholas Piggin IAREA=PACA_EXNMI 8098729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 8104f50541fSNicholas Piggin /* 8114f50541fSNicholas Piggin * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 8124f50541fSNicholas Piggin * being used, so a nested NMI exception would corrupt it. 8134f50541fSNicholas Piggin */ 8144f50541fSNicholas Piggin ISET_RI=0 8154f50541fSNicholas Piggin ISTACK=0 8164f50541fSNicholas Piggin IKVM_REAL=1 8174f50541fSNicholas PigginINT_DEFINE_END(system_reset) 8184f50541fSNicholas Piggin 819a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 820fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 821fb479e44SNicholas Piggin /* 822fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 823ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 824ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 825ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 826ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 8270e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 8280e10be2bSNicholas Piggin * 8290e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 8300e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 8310e10be2bSNicholas Piggin * 8320e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 8330e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 8340e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 8350e10be2bSNicholas Piggin * common exception macros to handle it. 836fb479e44SNicholas Piggin */ 837a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 8380e10be2bSNicholas Piggin SET_SCRATCH0(r13) 8390e10be2bSNicholas Piggin GET_PACA(r13) 8400e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 8410e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 8420e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 843a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 8440e10be2bSNicholas Piggin mfocrf r4,0x80 8450e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 8460e10be2bSNicholas Piggin bne+ system_reset_idle_wake 8470e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 8480e10be2bSNicholas Piggin mtocrf 0x80,r4 8490e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 8500e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 8510e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 8520e10be2bSNicholas Piggin GET_SCRATCH0(r13) 853a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 854fb479e44SNicholas Piggin#endif 855fb479e44SNicholas Piggin 8564f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 857c4f3b52cSNicholas Piggin /* 8580e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 8590e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 8600e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 8610e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 8620e10be2bSNicholas Piggin * be dangerous anyway. 863c4f3b52cSNicholas Piggin */ 8641a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 8651a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 866fb479e44SNicholas Piggin 867fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 8680e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 8690e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 8700e10be2bSNicholas Piggin cmpwi cr1,r5,2 8710e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 8720e10be2bSNicholas Piggin BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) 873371fefd6SPaul Mackerras#endif 874371fefd6SPaul Mackerras 875acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 876acc8da44SNicholas Piggin/* 877acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 878acc8da44SNicholas Piggin */ 879acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 8804f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 881acc8da44SNicholas Piggin 882acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 883acc8da44SNicholas Piggin 884a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 8858729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 886c4f3b52cSNicholas Piggin /* 887c4f3b52cSNicholas Piggin * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 888c4f3b52cSNicholas Piggin * to recover, but nested NMI will notice in_nmi and not recover 889c4f3b52cSNicholas Piggin * because of the use of the NMI stack. in_nmi reentrancy is tested in 890c4f3b52cSNicholas Piggin * system_reset_exception. 891c4f3b52cSNicholas Piggin */ 892c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 893c4f3b52cSNicholas Piggin addi r10,r10,1 894c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 895c4f3b52cSNicholas Piggin li r10,MSR_RI 896c4f3b52cSNicholas Piggin mtmsrd r10,1 897aca79d2bSVaidyanathan Srinivasan 898b1ee8a3dSNicholas Piggin mr r10,r1 899b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 900b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 9018729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 90247169fbaSNicholas Piggin 903c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 904c06075f3SNicholas Piggin bl system_reset_exception 90515b4dd79SNicholas Piggin 90615b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 907fbc50063SNicholas Piggin li r9,0 90815b4dd79SNicholas Piggin mtmsrd r9,1 909c4f3b52cSNicholas Piggin 910c4f3b52cSNicholas Piggin /* 91115b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 912c4f3b52cSNicholas Piggin */ 913c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 914c4f3b52cSNicholas Piggin subi r10,r10,1 915c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 916c4f3b52cSNicholas Piggin 9178e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 9183f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 91915b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 920582baf44SNicholas Piggin 9210ebc4cdaSBenjamin Herrenschmidt 92294325357SNicholas Piggin/** 92394325357SNicholas Piggin * Interrupt 0x200 - Machine Check Interrupt (MCE). 92494325357SNicholas Piggin * This is a non-maskable interrupt always taken in real-mode. It can be 92594325357SNicholas Piggin * synchronous or asynchronous, caused by hardware or software, and it may be 92694325357SNicholas Piggin * taken in a power-saving state. 92794325357SNicholas Piggin * 92894325357SNicholas Piggin * Handling: 92994325357SNicholas Piggin * Similarly to system reset, this uses its own stack and PACA save area, 93094325357SNicholas Piggin * the difference is re-entrancy is allowed on the machine check stack. 93194325357SNicholas Piggin * 93294325357SNicholas Piggin * machine_check_early is run in real mode, and carefully decodes the 93394325357SNicholas Piggin * machine check and tries to handle it (e.g., flush the SLB if there was an 93494325357SNicholas Piggin * error detected there), determines if it was recoverable and logs the 93594325357SNicholas Piggin * event. 93694325357SNicholas Piggin * 937b44fc96dSNicholas Piggin * This early code does not "reconcile" irq soft-mask state like SRESET or 938b44fc96dSNicholas Piggin * regular interrupts do, so irqs_disabled() among other things may not work 939b44fc96dSNicholas Piggin * properly (irq disable/enable already doesn't work because irq tracing can 940b44fc96dSNicholas Piggin * not work in real mode). 941b44fc96dSNicholas Piggin * 94294325357SNicholas Piggin * Then, depending on the execution context when the interrupt is taken, there 94394325357SNicholas Piggin * are 3 main actions: 94494325357SNicholas Piggin * - Executing in kernel mode. The event is queued with irq_work, which means 94594325357SNicholas Piggin * it is handled when it is next safe to do so (i.e., the kernel has enabled 94694325357SNicholas Piggin * interrupts), which could be immediately when the interrupt returns. This 94794325357SNicholas Piggin * avoids nasty issues like switching to virtual mode when the MMU is in a 94894325357SNicholas Piggin * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 94994325357SNicholas Piggin * but it has different priorities). Check to see if the CPU was in power 95094325357SNicholas Piggin * save, and return via the wake up code if it was. 95194325357SNicholas Piggin * 95294325357SNicholas Piggin * - Executing in user mode. machine_check_exception is run like a normal 95394325357SNicholas Piggin * interrupt handler, which processes the data generated by the early handler. 95494325357SNicholas Piggin * 95594325357SNicholas Piggin * - Executing in guest mode. The interrupt is run with its KVM test, and 95694325357SNicholas Piggin * branches to KVM to deal with. KVM may queue the event for the host 95794325357SNicholas Piggin * to report later. 95894325357SNicholas Piggin * 95994325357SNicholas Piggin * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 96094325357SNicholas Piggin * or SCRATCH0 is in use, it may cause a crash. 96194325357SNicholas Piggin * 96294325357SNicholas Piggin * KVM: 96394325357SNicholas Piggin * See SRESET. 96494325357SNicholas Piggin */ 9654f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 9664f50541fSNicholas Piggin IVEC=0x200 9674f50541fSNicholas Piggin IAREA=PACA_EXMC 9688729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 969d73a10cbSNicholas Piggin IREALMODE_COMMON=1 970c8eb54dbSNicholas Piggin /* 971c8eb54dbSNicholas Piggin * MSR_RI is not enabled, because PACA_EXMC is being used, so a 972c8eb54dbSNicholas Piggin * nested machine check corrupts it. machine_check_common enables 973c8eb54dbSNicholas Piggin * MSR_RI. 974c8eb54dbSNicholas Piggin */ 9754f50541fSNicholas Piggin ISET_RI=0 9764f50541fSNicholas Piggin ISTACK=0 9774f50541fSNicholas Piggin IDAR=1 9784f50541fSNicholas Piggin IDSISR=1 9794f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 9804f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 9814f50541fSNicholas Piggin 9824f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 9834f50541fSNicholas Piggin IVEC=0x200 9844f50541fSNicholas Piggin IAREA=PACA_EXMC 9858729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 9864f50541fSNicholas Piggin ISET_RI=0 9874f50541fSNicholas Piggin IDAR=1 9884f50541fSNicholas Piggin IDSISR=1 9894f50541fSNicholas Piggin IKVM_REAL=1 9904f50541fSNicholas PigginINT_DEFINE_END(machine_check) 9914f50541fSNicholas Piggin 9924f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 9934f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 9941a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 9951a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 996c8eb54dbSNicholas Piggin 997abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 998abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 999abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 10004f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 1001abd1f4caSNicholas Piggin#endif 1002abd1f4caSNicholas Piggin 1003fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 1004fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1005fce16d48SNicholas Piggin li r9,0; \ 1006fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 1007fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 1008fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 1009fce16d48SNicholas Piggin subi r12,r12,1; \ 1010fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 10113f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 1012fce16d48SNicholas Piggin 1013c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 10149600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 10159600f261SNicholas Piggin 1016afcf0095SNicholas Piggin /* 1017afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 1018afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 1019afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1020afcf0095SNicholas Piggin * 1021afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 1022afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 1023afcf0095SNicholas Piggin * machine checks. 1024afcf0095SNicholas Piggin * 1025afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 1026afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 1027afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 1028afcf0095SNicholas Piggin * 1029afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1030afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 1031afcf0095SNicholas Piggin * rfid with MSR_ME=1. 10321945bc45SNicholas Piggin * 10331945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 10341945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 10352bf1071aSNicholas Piggin * to restore state. 1036afcf0095SNicholas Piggin */ 1037afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 1038afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 1039c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1040afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 1041afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 1042c8eb54dbSNicholas Piggin 1043c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 1044c8eb54dbSNicholas Piggin bne 1f 1045c8eb54dbSNicholas Piggin /* First machine check entry */ 1046c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1047b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1048b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1049b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1050c8eb54dbSNicholas Piggin 10518729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1052c8eb54dbSNicholas Piggin 1053db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1054296e753fSNicholas Piggin bl enable_machine_check 1055db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1056296e753fSNicholas Piggin li r10,MSR_RI 1057296e753fSNicholas Piggin mtmsrd r10,1 1058296e753fSNicholas Piggin 1059afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1060afcf0095SNicholas Piggin bl machine_check_early 1061afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1062afcf0095SNicholas Piggin ld r12,_MSR(r1) 10631945bc45SNicholas Piggin 1064afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1065afcf0095SNicholas Piggin /* 1066afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1067afcf0095SNicholas Piggin * of the following is true: 1068afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1069afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1070afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1071afcf0095SNicholas Piggin * 1072afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1073afcf0095SNicholas Piggin */ 10741945bc45SNicholas PigginBEGIN_FTR_SECTION 10751945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 10766102c005SNicholas Piggin bne machine_check_idle_common 10771945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1078afcf0095SNicholas Piggin#endif 10791945bc45SNicholas Piggin 1080afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1081afcf0095SNicholas Piggin /* 108219dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 108305f97d94SNicholas Piggin * exception handler which will take the 108469fdd674SNicholas Piggin * machine_check_kvm->kvm_interrupt branch to deliver the MC event 108505f97d94SNicholas Piggin * to guest. 1086afcf0095SNicholas Piggin */ 1087afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1088afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1089b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1090afcf0095SNicholas Piggin#endif 109119dbe673SNicholas Piggin 1092afcf0095SNicholas Piggin /* 109319dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 109419dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 109519dbe673SNicholas Piggin */ 109619dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1097b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 109819dbe673SNicholas Piggin 109919dbe673SNicholas Piggin /* 110019dbe673SNicholas Piggin * At this point we are coming from kernel context. 1101afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1102afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1103afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1104afcf0095SNicholas Piggin */ 1105afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1106b7d9ccecSNicholas Piggin beq unrecoverable_mce 1107b7d9ccecSNicholas Piggin 1108afcf0095SNicholas Piggin /* 1109afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1110afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1111afcf0095SNicholas Piggin */ 1112afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1113afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1114b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1115272f6364SNicholas Piggin 1116afcf0095SNicholas Piggin /* 1117afcf0095SNicholas Piggin * Return from MC interrupt. 1118afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1119afcf0095SNicholas Piggin * returning from kernel or opal call. 1120afcf0095SNicholas Piggin */ 1121afcf0095SNicholas Piggin bl machine_check_queue_event 1122afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1123fe9d482bSNicholas Piggin RFI_TO_KERNEL 1124272f6364SNicholas Piggin 1125b3fe3526SNicholas Pigginmce_deliver: 1126b3fe3526SNicholas Piggin /* 1127b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1128b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1129b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1130b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1131b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1132b3fe3526SNicholas Piggin */ 11330b66370cSNicholas PigginBEGIN_FTR_SECTION 11340b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 11350b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 11360b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1137afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 11384f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1139afcf0095SNicholas Piggin 1140fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1141fce16d48SNicholas Piggin /* 1142fce16d48SNicholas Piggin * Machine check is different because we use a different 1143fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1144fce16d48SNicholas Piggin */ 11454f50541fSNicholas Piggin GEN_COMMON machine_check 11464f50541fSNicholas Piggin 1147fce16d48SNicholas Piggin /* Enable MSR_RI when finished with PACA_EXMC */ 1148fce16d48SNicholas Piggin li r10,MSR_RI 1149fce16d48SNicholas Piggin mtmsrd r10,1 1150fce16d48SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1151fce16d48SNicholas Piggin bl machine_check_exception 11526cc0c16dSNicholas Piggin b interrupt_return 1153fce16d48SNicholas Piggin 11549600f261SNicholas Piggin 1155fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1156fce16d48SNicholas Piggin/* 1157fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1158fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1159fce16d48SNicholas Piggin */ 1160fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 1161fce16d48SNicholas Piggin bl machine_check_queue_event 1162fce16d48SNicholas Piggin 1163fce16d48SNicholas Piggin /* 11648a5054d8SNicholas Piggin * GPR-loss wakeups are relatively straightforward, because the 11658a5054d8SNicholas Piggin * idle sleep code has saved all non-volatile registers on its 11668a5054d8SNicholas Piggin * own stack, and r1 in PACAR1. 1167fce16d48SNicholas Piggin * 11688a5054d8SNicholas Piggin * For no-loss wakeups the r1 and lr registers used by the 11698a5054d8SNicholas Piggin * early machine check handler have to be restored first. r2 is 11708a5054d8SNicholas Piggin * the kernel TOC, so no need to restore it. 1171fce16d48SNicholas Piggin * 1172fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1173fce16d48SNicholas Piggin */ 1174fce16d48SNicholas Piggin ld r3,_MSR(r1) 1175fce16d48SNicholas Piggin ld r4,_LINK(r1) 11768a5054d8SNicholas Piggin ld r1,GPR1(r1) 1177fce16d48SNicholas Piggin 1178fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1179fce16d48SNicholas Piggin subi r11,r11,1 1180fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1181fce16d48SNicholas Piggin 1182fce16d48SNicholas Piggin mtlr r4 1183fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1184fce16d48SNicholas Piggin cmpwi cr1,r10,2 11858a5054d8SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1186fce16d48SNicholas Piggin b idle_return_gpr_loss 1187fce16d48SNicholas Piggin#endif 1188fce16d48SNicholas Piggin 1189b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1190b7d9ccecSNicholas Piggin /* 1191b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1192b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1193b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1194b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1195b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1196b7d9ccecSNicholas Piggin */ 1197b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1198b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1199b7d9ccecSNicholas Piggin mtmsrd r10,1 1200b7d9ccecSNicholas Piggin bl disable_machine_check 1201b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1202b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1203b7d9ccecSNicholas Piggin li r3,MSR_ME 1204b7d9ccecSNicholas Piggin andc r10,r10,r3 1205b7d9ccecSNicholas Piggin mtmsrd r10 1206b7d9ccecSNicholas Piggin 1207ac2a2a14SNicholas Piggin lhz r12,PACA_IN_MCE(r13) 1208ac2a2a14SNicholas Piggin subi r12,r12,1 1209ac2a2a14SNicholas Piggin sth r12,PACA_IN_MCE(r13) 1210ac2a2a14SNicholas Piggin 1211afcf0095SNicholas Piggin /* Invoke machine_check_exception to print MCE event and panic. */ 1212afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1213afcf0095SNicholas Piggin bl machine_check_exception 1214b7d9ccecSNicholas Piggin 1215afcf0095SNicholas Piggin /* 1216b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1217b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1218afcf0095SNicholas Piggin */ 1219b7d9ccecSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1220afcf0095SNicholas Piggin bl unrecoverable_exception 1221b7d9ccecSNicholas Piggin b . 1222afcf0095SNicholas Piggin 12234f50541fSNicholas Piggin 12244f50541fSNicholas Piggin/** 122594325357SNicholas Piggin * Interrupt 0x300 - Data Storage Interrupt (DSI). 122694325357SNicholas Piggin * This is a synchronous interrupt generated due to a data access exception, 122794325357SNicholas Piggin * e.g., a load orstore which does not have a valid page table entry with 122894325357SNicholas Piggin * permissions. DAWR matches also fault here, as do RC updates, and minor misc 122994325357SNicholas Piggin * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 12304f50541fSNicholas Piggin * 123194325357SNicholas Piggin * Handling: 123294325357SNicholas Piggin * - Hash MMU 1233a4922f54SNicholas Piggin * Go to do_hash_fault, which attempts to fill the HPT from an entry in the 1234a4922f54SNicholas Piggin * Linux page table. Hash faults can hit in kernel mode in a fairly 123594325357SNicholas Piggin * arbitrary state (e.g., interrupts disabled, locks held) when accessing 123694325357SNicholas Piggin * "non-bolted" regions, e.g., vmalloc space. However these should always be 1237a4922f54SNicholas Piggin * backed by Linux page table entries. 12384f50541fSNicholas Piggin * 1239a4922f54SNicholas Piggin * If no entry is found the Linux page fault handler is invoked (by 1240a4922f54SNicholas Piggin * do_hash_fault). Linux page faults can happen in kernel mode due to user 1241a4922f54SNicholas Piggin * copy operations of course. 12424f50541fSNicholas Piggin * 1243cd81acc6SNicholas Piggin * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest 1244cd81acc6SNicholas Piggin * MMU context, which may cause a DSI in the host, which must go to the 1245cd81acc6SNicholas Piggin * KVM handler. MSR[IR] is not enabled, so the real-mode handler will 1246cd81acc6SNicholas Piggin * always be used regardless of AIL setting. 1247cd81acc6SNicholas Piggin * 124894325357SNicholas Piggin * - Radix MMU 124994325357SNicholas Piggin * The hardware loads from the Linux page table directly, so a fault goes 125094325357SNicholas Piggin * immediately to Linux page fault. 125194325357SNicholas Piggin * 125294325357SNicholas Piggin * Conditions like DAWR match are handled on the way in to Linux page fault. 12534f50541fSNicholas Piggin */ 1254a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1255a42a239dSNicholas Piggin IVEC=0x300 1256a42a239dSNicholas Piggin IDAR=1 1257a42a239dSNicholas Piggin IDSISR=1 1258a42a239dSNicholas Piggin IKVM_REAL=1 1259a42a239dSNicholas PigginINT_DEFINE_END(data_access) 12600ebc4cdaSBenjamin Herrenschmidt 1261e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1262689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1263e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1264e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1265a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1266e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 126780795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 12687cb3a1a0SNicholas Piggin GEN_COMMON data_access 1269a01a3f2dSNicholas Piggin ld r4,_DSISR(r1) 1270a4922f54SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1271a01a3f2dSNicholas Piggin andis. r0,r4,DSISR_DABRMATCH@h 127236f01141SNicholas Piggin bne- 1f 127380795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 1274a4922f54SNicholas Piggin bl do_hash_fault 127580795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 1276a4922f54SNicholas Piggin bl do_page_fault 127780795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1278a4922f54SNicholas Piggin b interrupt_return 1279a4922f54SNicholas Piggin 1280a4922f54SNicholas Piggin1: bl do_break 128136f01141SNicholas Piggin /* 128236f01141SNicholas Piggin * do_break() may have changed the NV GPRS while handling a breakpoint. 128336f01141SNicholas Piggin * If so, we need to restore them with their updated values. 128436f01141SNicholas Piggin */ 128536f01141SNicholas Piggin REST_NVGPRS(r1) 128636f01141SNicholas Piggin b interrupt_return 128780795e6cSNicholas Piggin 12880ebc4cdaSBenjamin Herrenschmidt 128994325357SNicholas Piggin/** 129094325357SNicholas Piggin * Interrupt 0x380 - Data Segment Interrupt (DSLB). 129194325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault missing SLB 129294325357SNicholas Piggin * entry for HPT, or an address outside RPT translation range. 129394325357SNicholas Piggin * 129494325357SNicholas Piggin * Handling: 129594325357SNicholas Piggin * - HPT: 129694325357SNicholas Piggin * This refills the SLB, or reports an access fault similarly to a bad page 129794325357SNicholas Piggin * fault. When coming from user-mode, the SLB handler may access any kernel 129894325357SNicholas Piggin * data, though it may itself take a DSLB. When coming from kernel mode, 129994325357SNicholas Piggin * recursive faults must be avoided so access is restricted to the kernel 130094325357SNicholas Piggin * image text/data, kernel stack, and any data allocated below 130194325357SNicholas Piggin * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 130294325357SNicholas Piggin * on user-handler data structures. 130394325357SNicholas Piggin * 1304cd81acc6SNicholas Piggin * KVM: Same as 0x300, DSLB must test for KVM guest. 130594325357SNicholas Piggin */ 13064f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 13074f50541fSNicholas Piggin IVEC=0x380 13084f50541fSNicholas Piggin IDAR=1 13094f50541fSNicholas Piggin IKVM_REAL=1 13104f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 13114f50541fSNicholas Piggin 13121a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1313689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 13141a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 13151a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 13164f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 13171a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 131848e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 13194f50541fSNicholas Piggin GEN_COMMON data_access_slb 13207100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 13217100e870SNicholas Piggin /* HPT case, do SLB fault */ 1322a01a3f2dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 132348e7b769SNicholas Piggin bl do_slb_fault 132448e7b769SNicholas Piggin cmpdi r3,0 132548e7b769SNicholas Piggin bne- 1f 13266cc0c16dSNicholas Piggin b fast_interrupt_return 132748e7b769SNicholas Piggin1: /* Error case */ 13287100e870SNicholas PigginMMU_FTR_SECTION_ELSE 13297100e870SNicholas Piggin /* Radix case, access is outside page table range */ 13307100e870SNicholas Piggin li r3,-EFAULT 13317100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 133248e7b769SNicholas Piggin std r3,RESULT(r1) 133348e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 133448e7b769SNicholas Piggin bl do_bad_slb_fault 13356cc0c16dSNicholas Piggin b interrupt_return 133648e7b769SNicholas Piggin 13372b9af6e4SNicholas Piggin 133894325357SNicholas Piggin/** 133994325357SNicholas Piggin * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 134094325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 134194325357SNicholas Piggin * instruction fetch. 134294325357SNicholas Piggin * 134394325357SNicholas Piggin * Handling: 134494325357SNicholas Piggin * Similar to DSI, though in response to fetch. The faulting address is found 134594325357SNicholas Piggin * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 134694325357SNicholas Piggin */ 13474f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 13484f50541fSNicholas Piggin IVEC=0x400 1349a3cd35beSNicholas Piggin IISIDE=1 1350a3cd35beSNicholas Piggin IDAR=1 1351a3cd35beSNicholas Piggin IDSISR=1 13522284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 13534f50541fSNicholas Piggin IKVM_REAL=1 13542284ffeaSNicholas Piggin#endif 13554f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 13564f50541fSNicholas Piggin 13577299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 13584f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 13597299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 13607299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 13614f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 13627299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 136327ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 13644f50541fSNicholas Piggin GEN_COMMON instruction_access 1365a4922f54SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 136627ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 1367a4922f54SNicholas Piggin bl do_hash_fault 136827ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 1369a4922f54SNicholas Piggin bl do_page_fault 137027ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1371a4922f54SNicholas Piggin b interrupt_return 137227ce77dfSNicholas Piggin 13730ebc4cdaSBenjamin Herrenschmidt 137494325357SNicholas Piggin/** 137594325357SNicholas Piggin * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 137694325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 137794325357SNicholas Piggin * instruction fetch. 137894325357SNicholas Piggin * 137994325357SNicholas Piggin * Handling: 138094325357SNicholas Piggin * Similar to DSLB, though in response to fetch. The faulting address is found 138194325357SNicholas Piggin * in SRR0 (rather than DAR). 138294325357SNicholas Piggin */ 13834f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 13844f50541fSNicholas Piggin IVEC=0x480 1385a3cd35beSNicholas Piggin IISIDE=1 1386a3cd35beSNicholas Piggin IDAR=1 13872284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 13884f50541fSNicholas Piggin IKVM_REAL=1 13892284ffeaSNicholas Piggin#endif 13904f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 13914f50541fSNicholas Piggin 13927299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 13934f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 13947299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 13957299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 13964f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 13977299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 139848e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 13994f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 14007100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14017100e870SNicholas Piggin /* HPT case, do SLB fault */ 1402a01a3f2dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 140348e7b769SNicholas Piggin bl do_slb_fault 140448e7b769SNicholas Piggin cmpdi r3,0 140548e7b769SNicholas Piggin bne- 1f 14066cc0c16dSNicholas Piggin b fast_interrupt_return 140748e7b769SNicholas Piggin1: /* Error case */ 14087100e870SNicholas PigginMMU_FTR_SECTION_ELSE 14097100e870SNicholas Piggin /* Radix case, access is outside page table range */ 14107100e870SNicholas Piggin li r3,-EFAULT 14117100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 141248e7b769SNicholas Piggin std r3,RESULT(r1) 141348e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 141448e7b769SNicholas Piggin bl do_bad_slb_fault 14156cc0c16dSNicholas Piggin b interrupt_return 14165e46e29eSNicholas Piggin 14179600f261SNicholas Piggin 141894325357SNicholas Piggin/** 141994325357SNicholas Piggin * Interrupt 0x500 - External Interrupt. 142094325357SNicholas Piggin * This is an asynchronous maskable interrupt in response to an "external 142194325357SNicholas Piggin * exception" from the interrupt controller or hypervisor (e.g., device 142294325357SNicholas Piggin * interrupt). It is maskable in hardware by clearing MSR[EE], and 142394325357SNicholas Piggin * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 142494325357SNicholas Piggin * 142594325357SNicholas Piggin * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 142694325357SNicholas Piggin * interrupts are delivered with HSRR registers, guests use SRRs, which 142794325357SNicholas Piggin * reqiures IHSRR_IF_HVMODE. 142894325357SNicholas Piggin * 142994325357SNicholas Piggin * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 143094325357SNicholas Piggin * external interrupts are delivered as Hypervisor Virtualization Interrupts 143194325357SNicholas Piggin * rather than External Interrupts. 143294325357SNicholas Piggin * 143394325357SNicholas Piggin * Handling: 143494325357SNicholas Piggin * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 143594325357SNicholas Piggin * because registers at the time of the interrupt are not so important as it is 143694325357SNicholas Piggin * asynchronous. 143794325357SNicholas Piggin * 143894325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 143994325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 144094325357SNicholas Piggin */ 14414f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 14424f50541fSNicholas Piggin IVEC=0x500 14433f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 14444f50541fSNicholas Piggin IMASK=IRQS_DISABLED 14454f50541fSNicholas Piggin IKVM_REAL=1 14464f50541fSNicholas Piggin IKVM_VIRT=1 14474f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 14484f50541fSNicholas Piggin 14491a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 14504f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 14511a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 14521a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 14534f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 14541a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1455eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 14564f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1457eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1458eb204d86SNicholas Piggin bl do_IRQ 1459702f0980SNicholas Piggin b interrupt_return 1460c138e588SNicholas Piggin 1461c138e588SNicholas Piggin 146294325357SNicholas Piggin/** 146394325357SNicholas Piggin * Interrupt 0x600 - Alignment Interrupt 146494325357SNicholas Piggin * This is a synchronous interrupt in response to data alignment fault. 146594325357SNicholas Piggin */ 14664f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 14674f50541fSNicholas Piggin IVEC=0x600 14684f50541fSNicholas Piggin IDAR=1 14694f50541fSNicholas Piggin IDSISR=1 14702284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14714f50541fSNicholas Piggin IKVM_REAL=1 14722284ffeaSNicholas Piggin#endif 14734f50541fSNicholas PigginINT_DEFINE_END(alignment) 14744f50541fSNicholas Piggin 1475e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 14764f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1477e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1478e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 14794f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1480e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1481f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 14824f50541fSNicholas Piggin GEN_COMMON alignment 1483f9aa6714SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1484f9aa6714SNicholas Piggin bl alignment_exception 1485702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 14866cc0c16dSNicholas Piggin b interrupt_return 1487f9aa6714SNicholas Piggin 1488b01c8b54SPaul Mackerras 148994325357SNicholas Piggin/** 149094325357SNicholas Piggin * Interrupt 0x700 - Program Interrupt (program check). 149194325357SNicholas Piggin * This is a synchronous interrupt in response to various instruction faults: 149294325357SNicholas Piggin * traps, privilege errors, TM errors, floating point exceptions. 149394325357SNicholas Piggin * 149494325357SNicholas Piggin * Handling: 149594325357SNicholas Piggin * This interrupt may use the "emergency stack" in some cases when being taken 149694325357SNicholas Piggin * from kernel context, which complicates handling. 149794325357SNicholas Piggin */ 14984f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 14994f50541fSNicholas Piggin IVEC=0x700 15002284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15014f50541fSNicholas Piggin IKVM_REAL=1 15022284ffeaSNicholas Piggin#endif 15034f50541fSNicholas PigginINT_DEFINE_END(program_check) 15044f50541fSNicholas Piggin 15057299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 1506e7eb9190SMichael Ellerman 1507e7eb9190SMichael Ellerman#ifdef CONFIG_CPU_LITTLE_ENDIAN 1508e7eb9190SMichael Ellerman /* 1509e7eb9190SMichael Ellerman * There's a short window during boot where although the kernel is 1510e7eb9190SMichael Ellerman * running little endian, any exceptions will cause the CPU to switch 1511e7eb9190SMichael Ellerman * back to big endian. For example a WARN() boils down to a trap 1512e7eb9190SMichael Ellerman * instruction, which will cause a program check, and we end up here but 1513e7eb9190SMichael Ellerman * with the CPU in big endian mode. The first instruction of the program 1514e7eb9190SMichael Ellerman * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when 1515e7eb9190SMichael Ellerman * executed in the wrong endian is an lhzu with a ~3GB displacement from 1516e7eb9190SMichael Ellerman * r3. The content of r3 is random, so that is a load from some random 1517e7eb9190SMichael Ellerman * location, and depending on the system can easily lead to a checkstop, 1518e7eb9190SMichael Ellerman * or an infinitely recursive page fault. 1519e7eb9190SMichael Ellerman * 1520e7eb9190SMichael Ellerman * So to handle that case we have a trampoline here that can detect we 1521e7eb9190SMichael Ellerman * are in the wrong endian and flip us back to the correct endian. We 1522e7eb9190SMichael Ellerman * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires 1523e7eb9190SMichael Ellerman * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as 1524e7eb9190SMichael Ellerman * SPRG1 is already used for the paca. SPRG3 is user readable, but this 1525e7eb9190SMichael Ellerman * trampoline is only active very early in boot, and SPRG3 will be 1526e7eb9190SMichael Ellerman * reinitialised in vdso_getcpu_init() before userspace starts. 1527e7eb9190SMichael Ellerman */ 1528e7eb9190SMichael EllermanBEGIN_FTR_SECTION 1529e7eb9190SMichael Ellerman tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 1530e7eb9190SMichael Ellerman b 1f // Skip trampoline if endian is correct 1531e7eb9190SMichael Ellerman .long 0xa643707d // mtsprg 0, r11 Backup r11 1532e7eb9190SMichael Ellerman .long 0xa6027a7d // mfsrr0 r11 1533e7eb9190SMichael Ellerman .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 1534e7eb9190SMichael Ellerman .long 0xa6027b7d // mfsrr1 r11 1535e7eb9190SMichael Ellerman .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 1536e7eb9190SMichael Ellerman .long 0xa600607d // mfmsr r11 1537e7eb9190SMichael Ellerman .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] 1538e7eb9190SMichael Ellerman .long 0xa6037b7d // mtsrr1 r11 1539e7eb9190SMichael Ellerman .long 0x34076039 // li r11, 0x734 1540e7eb9190SMichael Ellerman .long 0xa6037a7d // mtsrr0 r11 1541e7eb9190SMichael Ellerman .long 0x2400004c // rfid 1542e7eb9190SMichael Ellerman mfsprg r11, 3 1543e7eb9190SMichael Ellerman mtsrr1 r11 // Restore SRR1 1544e7eb9190SMichael Ellerman mfsprg r11, 2 1545e7eb9190SMichael Ellerman mtsrr0 r11 // Restore SRR0 1546e7eb9190SMichael Ellerman mfsprg r11, 0 // Restore r11 1547e7eb9190SMichael Ellerman1: 1548e7eb9190SMichael EllermanEND_FTR_SECTION(0, 1) // nop out after boot 1549e7eb9190SMichael Ellerman#endif /* CONFIG_CPU_LITTLE_ENDIAN */ 1550e7eb9190SMichael Ellerman 15514f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 15527299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 15537299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 15544f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 15557299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 155611e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 15578729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 15588729c26eSNicholas Piggin 1559265e60a1SCyril Bur /* 1560265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1561265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1562265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1563265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1564265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1565265e60a1SCyril Bur * the kernel. 1566265e60a1SCyril Bur */ 1567265e60a1SCyril Bur 15680a882e28SNicholas Piggin andi. r10,r12,MSR_PR 15690a882e28SNicholas Piggin bne 2f /* If userspace, go normal path */ 15700a882e28SNicholas Piggin 15710a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 15720a882e28SNicholas Piggin bne 1f /* If TM, emergency */ 15730a882e28SNicholas Piggin 15740a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 15750a882e28SNicholas Piggin blt 2f /* normal path if not */ 15760a882e28SNicholas Piggin 15770a882e28SNicholas Piggin /* Use the emergency stack */ 15780a882e28SNicholas Piggin1: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1579265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1580265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1581265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1582265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 15834f50541fSNicholas Piggin __ISTACK(program_check)=0 15848729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 15851b359982SNicholas Piggin b 3f 15860a882e28SNicholas Piggin2: 15874f50541fSNicholas Piggin __ISTACK(program_check)=1 15888729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 15891b359982SNicholas Piggin3: 159011e87346SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 159111e87346SNicholas Piggin bl program_check_exception 1592702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 15936cc0c16dSNicholas Piggin b interrupt_return 159411e87346SNicholas Piggin 1595a485c709SPaul Mackerras 159694325357SNicholas Piggin/* 159794325357SNicholas Piggin * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 159894325357SNicholas Piggin * This is a synchronous interrupt in response to executing an fp instruction 159994325357SNicholas Piggin * with MSR[FP]=0. 160094325357SNicholas Piggin * 160194325357SNicholas Piggin * Handling: 160294325357SNicholas Piggin * This will load FP registers and enable the FP bit if coming from userspace, 160394325357SNicholas Piggin * otherwise report a bad kernel use of FP. 160494325357SNicholas Piggin */ 16054f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 16064f50541fSNicholas Piggin IVEC=0x800 16072284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16084f50541fSNicholas Piggin IKVM_REAL=1 16092284ffeaSNicholas Piggin#endif 16104f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 16114f50541fSNicholas Piggin 16127299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 16134f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 16147299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 16157299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 16164f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 16177299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1618c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 16194f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1620c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1621c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1622c78d9b97SNicholas Piggin bl kernel_fp_unavailable_exception 162363ce271bSChristophe Leroy0: trap 162463ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1625c78d9b97SNicholas Piggin1: 1626c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1627c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1628c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1629c78d9b97SNicholas Piggin * transaction), go do TM stuff 1630c78d9b97SNicholas Piggin */ 1631c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1632c78d9b97SNicholas Piggin bne- 2f 1633c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1634c78d9b97SNicholas Piggin#endif 1635c78d9b97SNicholas Piggin bl load_up_fpu 16366cc0c16dSNicholas Piggin b fast_interrupt_return 1637c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1638c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1639c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1640c78d9b97SNicholas Piggin bl fp_unavailable_tm 16416cc0c16dSNicholas Piggin b interrupt_return 1642c78d9b97SNicholas Piggin#endif 1643c78d9b97SNicholas Piggin 1644b01c8b54SPaul Mackerras 164594325357SNicholas Piggin/** 164694325357SNicholas Piggin * Interrupt 0x900 - Decrementer Interrupt. 164794325357SNicholas Piggin * This is an asynchronous interrupt in response to a decrementer exception 164894325357SNicholas Piggin * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 164994325357SNicholas Piggin * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 165094325357SNicholas Piggin * local_irq_disable()). 165194325357SNicholas Piggin * 165294325357SNicholas Piggin * Handling: 165394325357SNicholas Piggin * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 165494325357SNicholas Piggin * 165594325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 165694325357SNicholas Piggin * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 165794325357SNicholas Piggin * in the interrupted context. 165894325357SNicholas Piggin * If PPC_WATCHDOG is configured, the soft masked handler will actually set 165994325357SNicholas Piggin * things back up to run soft_nmi_interrupt as a regular interrupt handler 166094325357SNicholas Piggin * on the emergency stack. 166194325357SNicholas Piggin */ 16624f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 16634f50541fSNicholas Piggin IVEC=0x900 16644f50541fSNicholas Piggin IMASK=IRQS_DISABLED 16652284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16664f50541fSNicholas Piggin IKVM_REAL=1 16672284ffeaSNicholas Piggin#endif 16684f50541fSNicholas PigginINT_DEFINE_END(decrementer) 16694f50541fSNicholas Piggin 16707299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1671689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 16727299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 16737299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 16744f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 16757299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1676eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 16774f50541fSNicholas Piggin GEN_COMMON decrementer 1678eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1679eb204d86SNicholas Piggin bl timer_interrupt 1680702f0980SNicholas Piggin b interrupt_return 168139c0da57SNicholas Piggin 16820ebc4cdaSBenjamin Herrenschmidt 168394325357SNicholas Piggin/** 168494325357SNicholas Piggin * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 168594325357SNicholas Piggin * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 168694325357SNicholas Piggin * register. 168794325357SNicholas Piggin * 168894325357SNicholas Piggin * Handling: 168994325357SNicholas Piggin * Linux does not use this outside KVM where it's used to keep a host timer 169094325357SNicholas Piggin * while the guest is given control of DEC. It should normally be caught by 169194325357SNicholas Piggin * the KVM test and routed there. 169294325357SNicholas Piggin */ 16934f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 16944f50541fSNicholas Piggin IVEC=0x980 16953f7fbd97SNicholas Piggin IHSRR=1 16962babd6eaSNicholas Piggin ISTACK=0 16974f50541fSNicholas Piggin IKVM_REAL=1 16984f50541fSNicholas Piggin IKVM_VIRT=1 16994f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 17004f50541fSNicholas Piggin 17017299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 17024f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 17037299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 17047299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 17054f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 17067299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1707eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 17082babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 17092babd6eaSNicholas Piggin /* 17102babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 17112babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 17122babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 17132babd6eaSNicholas Piggin * edge-triggered. 17142babd6eaSNicholas Piggin * 17152babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 17162babd6eaSNicholas Piggin */ 17172babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 17182babd6eaSNicholas Piggin mtctr r10 17192babd6eaSNicholas Piggin mtcrf 0x80,r9 17202babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 17212babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 17222babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 17232babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 17242babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 17252babd6eaSNicholas Piggin HRFI_TO_KERNEL 1726facc6d74SNicholas Piggin 1727da2bc464SMichael Ellerman 172894325357SNicholas Piggin/** 172994325357SNicholas Piggin * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 173094325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsndp doorbell. 173194325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 173294325357SNicholas Piggin * IRQS_DISABLED mask (i.e., local_irq_disable()). 173394325357SNicholas Piggin * 173494325357SNicholas Piggin * Handling: 173594325357SNicholas Piggin * Guests may use this for IPIs between threads in a core if the 173694325357SNicholas Piggin * hypervisor supports it. NVGPRS are not saved (see 0x500). 173794325357SNicholas Piggin * 173894325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 173994325357SNicholas Piggin * replay, leaving MSR[EE] enabled in the interrupted context because the 174094325357SNicholas Piggin * doorbells are edge triggered. 174194325357SNicholas Piggin */ 17424f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 17434f50541fSNicholas Piggin IVEC=0xa00 17444f50541fSNicholas Piggin IMASK=IRQS_DISABLED 17452284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17464f50541fSNicholas Piggin IKVM_REAL=1 17472284ffeaSNicholas Piggin#endif 17484f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 17494f50541fSNicholas Piggin 17507299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 17514f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 17527299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 17537299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 17544f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 17557299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1756eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 17574f50541fSNicholas Piggin GEN_COMMON doorbell_super 1758eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1759ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1760eb204d86SNicholas Piggin bl doorbell_exception 1761ca243163SNicholas Piggin#else 17626c6aee00SNicholas Piggin bl unknown_async_exception 1763ca243163SNicholas Piggin#endif 1764702f0980SNicholas Piggin b interrupt_return 1765ca243163SNicholas Piggin 1766da2bc464SMichael Ellerman 17675ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 17685ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1769341215dcSNicholas Piggin 177094325357SNicholas Piggin/** 177194325357SNicholas Piggin * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 177294325357SNicholas Piggin * This is a synchronous interrupt invoked with the "sc" instruction. The 177394325357SNicholas Piggin * system call is invoked with "sc 0" and does not alter the HV bit, so it 177494325357SNicholas Piggin * is directed to the currently running OS. The hypercall is invoked with 177594325357SNicholas Piggin * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1776acd7d8ceSNicholas Piggin * 1777acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1778acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1779acd7d8ceSNicholas Piggin * 178094325357SNicholas Piggin * Handling: 178194325357SNicholas Piggin * If the KVM test fires then it was due to a hypercall and is accordingly 178294325357SNicholas Piggin * routed to KVM. Otherwise this executes a normal Linux system call. 178394325357SNicholas Piggin * 1784acd7d8ceSNicholas Piggin * Call convention: 1785acd7d8ceSNicholas Piggin * 178658b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 178758b278f5SVaibhav Jain * Documentation/powerpc/syscall64-abi.rst and 178858b278f5SVaibhav Jain * Documentation/powerpc/papr_hcalls.rst respectively. 1789acd7d8ceSNicholas Piggin * 1790acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 179176fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 179276fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 179376fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1794acd7d8ceSNicholas Piggin */ 1795b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1796b177ae2fSNicholas Piggin IVEC=0xc00 1797b177ae2fSNicholas Piggin IKVM_REAL=1 1798b177ae2fSNicholas Piggin IKVM_VIRT=1 1799b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1800b177ae2fSNicholas Piggin 18011b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1802bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1803bc355125SPaul Mackerras /* 1804acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 180576fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 180676fc0cfcSNicholas Piggin * register. 1807acd7d8ceSNicholas Piggin * 1808acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1809acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1810bc355125SPaul Mackerras */ 18111b4d4a79SNicholas Piggin mtctr r13 18121b4d4a79SNicholas Piggin GET_PACA(r13) 18131b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 18141b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 181569fdd674SNicholas Piggin KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */ 18161b4d4a79SNicholas Piggin mfctr r9 1817bc355125SPaul Mackerras#else 18181b4d4a79SNicholas Piggin mr r9,r13 18191b4d4a79SNicholas Piggin GET_PACA(r13) 18201b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1821bc355125SPaul Mackerras#endif 1822bc355125SPaul Mackerras 1823727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 18241b4d4a79SNicholas PigginBEGIN_FTR_SECTION 18251b4d4a79SNicholas Piggin cmpdi r0,0x1ebe 18261b4d4a79SNicholas Piggin beq- 1f 18271b4d4a79SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 18281b4d4a79SNicholas Piggin#endif 18295c2511bfSMichael Ellerman 1830b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 18311b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 18321b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 1833b0b2a93dSNicholas Piggin 1834b0b2a93dSNicholas Piggin HMT_MEDIUM 1835b0b2a93dSNicholas Piggin 1836b0b2a93dSNicholas Piggin .if ! \virt 183714ad0e7dSNicholas Piggin __LOAD_HANDLER(r10, system_call_common_real) 183814ad0e7dSNicholas Piggin mtctr r10 183914ad0e7dSNicholas Piggin bctr 18401b4d4a79SNicholas Piggin .else 18411b4d4a79SNicholas Piggin li r10,MSR_RI 18421b4d4a79SNicholas Piggin mtmsrd r10,1 /* Set RI (EE=0) */ 1843b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 1844b0b2a93dSNicholas Piggin __LOAD_HANDLER(r10, system_call_common) 1845b0b2a93dSNicholas Piggin mtctr r10 1846b0b2a93dSNicholas Piggin bctr 1847b0b2a93dSNicholas Piggin#else 18481b4d4a79SNicholas Piggin b system_call_common 1849d807ad37SNicholas Piggin#endif 18501b4d4a79SNicholas Piggin .endif 18511b4d4a79SNicholas Piggin 18521b4d4a79SNicholas Piggin#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 18531b4d4a79SNicholas Piggin /* Fast LE/BE switch system call */ 18541b4d4a79SNicholas Piggin1: mfspr r12,SPRN_SRR1 18551b4d4a79SNicholas Piggin xori r12,r12,MSR_LE 18561b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r12 18571b4d4a79SNicholas Piggin mr r13,r9 18581b4d4a79SNicholas Piggin RFI_TO_USER /* return to userspace */ 18591b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 18601b4d4a79SNicholas Piggin#endif 18611b4d4a79SNicholas Piggin.endm 1862d807ad37SNicholas Piggin 18631a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 18641b4d4a79SNicholas Piggin SYSTEM_CALL 0 18651a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 18661a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 18671b4d4a79SNicholas Piggin SYSTEM_CALL 1 18681a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 1869d807ad37SNicholas Piggin 1870acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 187169fdd674SNicholas PigginTRAMP_REAL_BEGIN(kvm_hcall) 1872*e2762743SNicholas Piggin std r9,PACA_EXGEN+EX_R9(r13) 1873*e2762743SNicholas Piggin std r11,PACA_EXGEN+EX_R11(r13) 1874*e2762743SNicholas Piggin std r12,PACA_EXGEN+EX_R12(r13) 1875*e2762743SNicholas Piggin mfcr r9 1876acd7d8ceSNicholas Piggin mfctr r10 1877*e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_R13(r13) 1878*e2762743SNicholas Piggin li r10,0 1879*e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CFAR(r13) 1880*e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CTR(r13) 1881*e2762743SNicholas Piggin /* 1882*e2762743SNicholas Piggin * Save the PPR (on systems that support it) before changing to 1883*e2762743SNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 1884*e2762743SNicholas Piggin * guest state (it is the guest's PPR value). 1885*e2762743SNicholas Piggin */ 1886*e2762743SNicholas PigginBEGIN_FTR_SECTION 1887*e2762743SNicholas Piggin mfspr r10,SPRN_PPR 1888*e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_PPR(r13) 1889*e2762743SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1890*e2762743SNicholas Piggin 1891*e2762743SNicholas Piggin HMT_MEDIUM 1892*e2762743SNicholas Piggin 18939600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 18949600f261SNicholas Piggin /* 189531c67cfeSNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives 18969600f261SNicholas Piggin * outside the head section. 18979600f261SNicholas Piggin */ 189831c67cfeSNicholas Piggin __LOAD_FAR_HANDLER(r10, kvmppc_hcall) 18999600f261SNicholas Piggin mtctr r10 19009600f261SNicholas Piggin bctr 19019600f261SNicholas Piggin#else 190231c67cfeSNicholas Piggin b kvmppc_hcall 19039600f261SNicholas Piggin#endif 1904acd7d8ceSNicholas Piggin#endif 1905da2bc464SMichael Ellerman 190694325357SNicholas Piggin/** 190794325357SNicholas Piggin * Interrupt 0xd00 - Trace Interrupt. 190894325357SNicholas Piggin * This is a synchronous interrupt in response to instruction step or 190994325357SNicholas Piggin * breakpoint faults. 191094325357SNicholas Piggin */ 19114f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 19124f50541fSNicholas Piggin IVEC=0xd00 19132284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 19144f50541fSNicholas Piggin IKVM_REAL=1 19152284ffeaSNicholas Piggin#endif 19164f50541fSNicholas PigginINT_DEFINE_END(single_step) 19174f50541fSNicholas Piggin 19187299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 19194f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 19207299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 19217299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 19224f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 19237299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 1924eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 19254f50541fSNicholas Piggin GEN_COMMON single_step 1926eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1927eb204d86SNicholas Piggin bl single_step_exception 19286cc0c16dSNicholas Piggin b interrupt_return 1929da2bc464SMichael Ellerman 19307299417cSNicholas Piggin 193194325357SNicholas Piggin/** 193294325357SNicholas Piggin * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 193394325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 193494325357SNicholas Piggin * guest data access. 193594325357SNicholas Piggin * 193694325357SNicholas Piggin * Handling: 193794325357SNicholas Piggin * This should always get routed to KVM. In radix MMU mode, this is caused 193894325357SNicholas Piggin * by a guest nested radix access that can't be performed due to the 193994325357SNicholas Piggin * partition scope page table. In hash mode, this can be caused by guests 194094325357SNicholas Piggin * running with translation disabled (virtual real mode) or with VPM enabled. 194194325357SNicholas Piggin * KVM will update the page table structures or disallow the access. 194294325357SNicholas Piggin */ 19434f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 19444f50541fSNicholas Piggin IVEC=0xe00 19453f7fbd97SNicholas Piggin IHSRR=1 19464f50541fSNicholas Piggin IDAR=1 19474f50541fSNicholas Piggin IDSISR=1 19484f50541fSNicholas Piggin IKVM_REAL=1 19494f50541fSNicholas Piggin IKVM_VIRT=1 19504f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 19514f50541fSNicholas Piggin 19527299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 19534f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 19547299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 19557299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 19564f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 19577299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 1958f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 19594f50541fSNicholas Piggin GEN_COMMON h_data_storage 1960f5c32c1dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1961d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 196271f47976SNicholas Piggin bl do_bad_page_fault_segv 1963d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 1964f5c32c1dSNicholas Piggin bl unknown_exception 1965d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 19666cc0c16dSNicholas Piggin b interrupt_return 1967f5c32c1dSNicholas Piggin 19681707dd16SPaul Mackerras 196994325357SNicholas Piggin/** 197094325357SNicholas Piggin * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 197194325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 197294325357SNicholas Piggin * guest instruction fetch, similar to HDSI. 197394325357SNicholas Piggin */ 19744f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 19754f50541fSNicholas Piggin IVEC=0xe20 19763f7fbd97SNicholas Piggin IHSRR=1 19774f50541fSNicholas Piggin IKVM_REAL=1 19784f50541fSNicholas Piggin IKVM_VIRT=1 19794f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 19804f50541fSNicholas Piggin 19817299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 19824f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 19837299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 19847299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 19854f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 19867299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 1987eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 19884f50541fSNicholas Piggin GEN_COMMON h_instr_storage 1989eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1990eb204d86SNicholas Piggin bl unknown_exception 19916cc0c16dSNicholas Piggin b interrupt_return 199282517cabSNicholas Piggin 19931707dd16SPaul Mackerras 199494325357SNicholas Piggin/** 199594325357SNicholas Piggin * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 199694325357SNicholas Piggin */ 19974f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 19984f50541fSNicholas Piggin IVEC=0xe40 19993f7fbd97SNicholas Piggin IHSRR=1 20004f50541fSNicholas Piggin IKVM_REAL=1 20014f50541fSNicholas Piggin IKVM_VIRT=1 20024f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 20034f50541fSNicholas Piggin 20047299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 20054f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 20067299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 20077299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 20084f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 20097299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2010eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 20114f50541fSNicholas Piggin GEN_COMMON emulation_assist 2012eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2013eb204d86SNicholas Piggin bl emulation_assist_interrupt 2014702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 20156cc0c16dSNicholas Piggin b interrupt_return 2016031b4026SNicholas Piggin 20171707dd16SPaul Mackerras 201894325357SNicholas Piggin/** 201994325357SNicholas Piggin * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 202094325357SNicholas Piggin * This is an asynchronous interrupt caused by a Hypervisor Maintenance 202194325357SNicholas Piggin * Exception. It is always taken in real mode but uses HSRR registers 202294325357SNicholas Piggin * unlike SRESET and MCE. 202394325357SNicholas Piggin * 202494325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 202594325357SNicholas Piggin * with IRQS_DISABLED mask (i.e., local_irq_disable()). 202694325357SNicholas Piggin * 202794325357SNicholas Piggin * Handling: 202894325357SNicholas Piggin * This is a special case, this is handled similarly to machine checks, with an 202994325357SNicholas Piggin * initial real mode handler that is not soft-masked, which attempts to fix the 203094325357SNicholas Piggin * problem. Then a regular handler which is soft-maskable and reports the 203194325357SNicholas Piggin * problem. 203294325357SNicholas Piggin * 203394325357SNicholas Piggin * The emergency stack is used for the early real mode handler. 203494325357SNicholas Piggin * 203594325357SNicholas Piggin * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 203694325357SNicholas Piggin * either use soft-masking for the MCE, or use irq_work for the HMI. 203794325357SNicholas Piggin * 203894325357SNicholas Piggin * KVM: 203994325357SNicholas Piggin * Unlike MCE, this calls into KVM without calling the real mode handler 204094325357SNicholas Piggin * first. 2041e0319829SNicholas Piggin */ 20424f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 20434f50541fSNicholas Piggin IVEC=0xe60 20443f7fbd97SNicholas Piggin IHSRR=1 2045d73a10cbSNicholas Piggin IREALMODE_COMMON=1 20464f50541fSNicholas Piggin ISTACK=0 20474f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 20484f50541fSNicholas Piggin IKVM_REAL=1 20494f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 20504f50541fSNicholas Piggin 20514f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 20524f50541fSNicholas Piggin IVEC=0xe60 20533f7fbd97SNicholas Piggin IHSRR=1 20544f50541fSNicholas Piggin IMASK=IRQS_DISABLED 20554f50541fSNicholas Piggin IKVM_REAL=1 20564f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 20574f50541fSNicholas Piggin 2058f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 20594f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2060f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 20611a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 20624f50541fSNicholas Piggin 2063293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 20649600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 20659600f261SNicholas Piggin 206662f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 2067a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 206862f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2069bcbceed4SNicholas Piggin 20708729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 2071bcbceed4SNicholas Piggin 207262f9b03bSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2073293c2e27SNicholas Piggin bl hmi_exception_realmode 20745080332cSMichael Neuling cmpdi cr0,r3,0 207567d4160aSNicholas Piggin bne 1f 20765080332cSMichael Neuling 20773f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 2078222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 20795080332cSMichael Neuling 208067d4160aSNicholas Piggin1: 208162f9b03bSNicholas Piggin /* 208262f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 208362f9b03bSNicholas Piggin * firmware. 208462f9b03bSNicholas Piggin */ 20853f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 20864f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 208762f9b03bSNicholas Piggin 20885080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 20894f50541fSNicholas Piggin GEN_COMMON hmi_exception 2090c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2091c06075f3SNicholas Piggin bl handle_hmi_exception 20926cc0c16dSNicholas Piggin b interrupt_return 20931707dd16SPaul Mackerras 20947299417cSNicholas Piggin 209594325357SNicholas Piggin/** 209694325357SNicholas Piggin * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 209794325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsnd doorbell. 209894325357SNicholas Piggin * Similar to the 0xa00 doorbell but for host rather than guest. 209994325357SNicholas Piggin */ 21004f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 21014f50541fSNicholas Piggin IVEC=0xe80 21023f7fbd97SNicholas Piggin IHSRR=1 21034f50541fSNicholas Piggin IMASK=IRQS_DISABLED 21044f50541fSNicholas Piggin IKVM_REAL=1 21054f50541fSNicholas Piggin IKVM_VIRT=1 21064f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 21074f50541fSNicholas Piggin 21087299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 21094f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 21107299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 21117299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 21124f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 21137299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2114eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 21154f50541fSNicholas Piggin GEN_COMMON h_doorbell 2116eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 21179bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 2118eb204d86SNicholas Piggin bl doorbell_exception 21199bcb81bfSNicholas Piggin#else 21206c6aee00SNicholas Piggin bl unknown_async_exception 21219bcb81bfSNicholas Piggin#endif 2122702f0980SNicholas Piggin b interrupt_return 21239bcb81bfSNicholas Piggin 21240ebc4cdaSBenjamin Herrenschmidt 212594325357SNicholas Piggin/** 212694325357SNicholas Piggin * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 212794325357SNicholas Piggin * This is an asynchronous interrupt in response to an "external exception". 212894325357SNicholas Piggin * Similar to 0x500 but for host only. 212994325357SNicholas Piggin */ 21304f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 21314f50541fSNicholas Piggin IVEC=0xea0 21323f7fbd97SNicholas Piggin IHSRR=1 21334f50541fSNicholas Piggin IMASK=IRQS_DISABLED 21344f50541fSNicholas Piggin IKVM_REAL=1 21354f50541fSNicholas Piggin IKVM_VIRT=1 21364f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 21374f50541fSNicholas Piggin 21387299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 21394f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 21407299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 21417299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 21424f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 21437299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2144eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 21454f50541fSNicholas Piggin GEN_COMMON h_virt_irq 2146eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2147eb204d86SNicholas Piggin bl do_IRQ 2148702f0980SNicholas Piggin b interrupt_return 214974408776SNicholas Piggin 21509baaef0aSBenjamin Herrenschmidt 21511a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 21521a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 21531a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 21541a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 2155bda7fea2SNicholas Piggin 21560ebc4cdaSBenjamin Herrenschmidt 215794325357SNicholas Piggin/* 215894325357SNicholas Piggin * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 215994325357SNicholas Piggin * This is an asynchronous interrupt in response to a PMU exception. 216094325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 216194325357SNicholas Piggin * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 216294325357SNicholas Piggin * 216394325357SNicholas Piggin * Handling: 216494325357SNicholas Piggin * This calls into the perf subsystem. 216594325357SNicholas Piggin * 216694325357SNicholas Piggin * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 216794325357SNicholas Piggin * runs under local_irq_disable. However it may be soft-masked in 216894325357SNicholas Piggin * powerpc-specific code. 216994325357SNicholas Piggin * 217094325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 217194325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 217294325357SNicholas Piggin */ 21734f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 21744f50541fSNicholas Piggin IVEC=0xf00 21754f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 21762284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 21774f50541fSNicholas Piggin IKVM_REAL=1 21782284ffeaSNicholas Piggin#endif 21794f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 21804f50541fSNicholas Piggin 21817299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 21824f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 21837299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 21847299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 21854f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 21867299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2187eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 21884f50541fSNicholas Piggin GEN_COMMON performance_monitor 2189eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2190eb204d86SNicholas Piggin bl performance_monitor_exception 2191702f0980SNicholas Piggin b interrupt_return 2192b1c7f150SNicholas Piggin 21930ebc4cdaSBenjamin Herrenschmidt 219494325357SNicholas Piggin/** 219594325357SNicholas Piggin * Interrupt 0xf20 - Vector Unavailable Interrupt. 219694325357SNicholas Piggin * This is a synchronous interrupt in response to 219794325357SNicholas Piggin * executing a vector (or altivec) instruction with MSR[VEC]=0. 219894325357SNicholas Piggin * Similar to FP unavailable. 219994325357SNicholas Piggin */ 22004f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 22014f50541fSNicholas Piggin IVEC=0xf20 22022284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 22034f50541fSNicholas Piggin IKVM_REAL=1 22042284ffeaSNicholas Piggin#endif 22054f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 22064f50541fSNicholas Piggin 22077299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 22084f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 22097299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 22107299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 22114f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 22127299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2213d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 22144f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 2215d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 2216d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 2217d1a0ca9cSNicholas Piggin beq 1f 2218d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2219d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2220d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2221d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 2222d1a0ca9cSNicholas Piggin */ 2223d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2224d1a0ca9cSNicholas Piggin bne- 2f 2225d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2226d1a0ca9cSNicholas Piggin#endif 2227d1a0ca9cSNicholas Piggin bl load_up_altivec 22286cc0c16dSNicholas Piggin b fast_interrupt_return 2229d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2230d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 2231d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2232d1a0ca9cSNicholas Piggin bl altivec_unavailable_tm 22336cc0c16dSNicholas Piggin b interrupt_return 2234d1a0ca9cSNicholas Piggin#endif 2235d1a0ca9cSNicholas Piggin1: 2236d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2237d1a0ca9cSNicholas Piggin#endif 2238d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2239d1a0ca9cSNicholas Piggin bl altivec_unavailable_exception 22406cc0c16dSNicholas Piggin b interrupt_return 2241d1a0ca9cSNicholas Piggin 22420ebc4cdaSBenjamin Herrenschmidt 224394325357SNicholas Piggin/** 224494325357SNicholas Piggin * Interrupt 0xf40 - VSX Unavailable Interrupt. 224594325357SNicholas Piggin * This is a synchronous interrupt in response to 224694325357SNicholas Piggin * executing a VSX instruction with MSR[VSX]=0. 224794325357SNicholas Piggin * Similar to FP unavailable. 224894325357SNicholas Piggin */ 22494f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 22504f50541fSNicholas Piggin IVEC=0xf40 22512284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 22524f50541fSNicholas Piggin IKVM_REAL=1 22532284ffeaSNicholas Piggin#endif 22544f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 22554f50541fSNicholas Piggin 22567299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 22574f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 22587299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 22597299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 22604f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 22617299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2262792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 22634f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2264792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2265792cbdddSNicholas PigginBEGIN_FTR_SECTION 2266792cbdddSNicholas Piggin beq 1f 2267792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2268792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2269792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2270792cbdddSNicholas Piggin * transaction), go do TM stuff 2271792cbdddSNicholas Piggin */ 2272792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2273792cbdddSNicholas Piggin bne- 2f 2274792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2275792cbdddSNicholas Piggin#endif 2276792cbdddSNicholas Piggin b load_up_vsx 2277792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2278792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2279792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2280792cbdddSNicholas Piggin bl vsx_unavailable_tm 22816cc0c16dSNicholas Piggin b interrupt_return 2282792cbdddSNicholas Piggin#endif 2283792cbdddSNicholas Piggin1: 2284792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2285792cbdddSNicholas Piggin#endif 2286792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2287792cbdddSNicholas Piggin bl vsx_unavailable_exception 22886cc0c16dSNicholas Piggin b interrupt_return 2289792cbdddSNicholas Piggin 2290d0c0c9a1SMichael Neuling 229194325357SNicholas Piggin/** 229294325357SNicholas Piggin * Interrupt 0xf60 - Facility Unavailable Interrupt. 229394325357SNicholas Piggin * This is a synchronous interrupt in response to 229494325357SNicholas Piggin * executing an instruction without access to the facility that can be 229594325357SNicholas Piggin * resolved by the OS (e.g., FSCR, MSR). 229694325357SNicholas Piggin * Similar to FP unavailable. 229794325357SNicholas Piggin */ 22984f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 22994f50541fSNicholas Piggin IVEC=0xf60 23002284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23014f50541fSNicholas Piggin IKVM_REAL=1 23022284ffeaSNicholas Piggin#endif 23034f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 23044f50541fSNicholas Piggin 23057299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 23064f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 23077299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 23087299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 23094f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 23107299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2311eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 23124f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2313eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2314eb204d86SNicholas Piggin bl facility_unavailable_exception 2315595d153dSMichael Ellerman REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 23166cc0c16dSNicholas Piggin b interrupt_return 23171134713cSNicholas Piggin 2318da2bc464SMichael Ellerman 231994325357SNicholas Piggin/** 232094325357SNicholas Piggin * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 232194325357SNicholas Piggin * This is a synchronous interrupt in response to 232294325357SNicholas Piggin * executing an instruction without access to the facility that can only 232394325357SNicholas Piggin * be resolved in HV mode (e.g., HFSCR). 232494325357SNicholas Piggin * Similar to FP unavailable. 232594325357SNicholas Piggin */ 23264f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 23274f50541fSNicholas Piggin IVEC=0xf80 23283f7fbd97SNicholas Piggin IHSRR=1 23294f50541fSNicholas Piggin IKVM_REAL=1 23304f50541fSNicholas Piggin IKVM_VIRT=1 23314f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 23324f50541fSNicholas Piggin 23337299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 23344f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 23357299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 23367299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 23374f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 23387299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2339eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 23404f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2341eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2342eb204d86SNicholas Piggin bl facility_unavailable_exception 2343595d153dSMichael Ellerman REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */ 23446cc0c16dSNicholas Piggin b interrupt_return 234514b0072cSNicholas Piggin 2346da2bc464SMichael Ellerman 23471a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 23481a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 23491a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 23501a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 23511a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 23521a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 23531a6822d1SNicholas Piggin 23541a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 23551a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 23561a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 23571a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2358da2bc464SMichael Ellerman 23590ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 23604f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 23614f50541fSNicholas Piggin IVEC=0x1200 23623f7fbd97SNicholas Piggin IHSRR=1 23634f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 23644f50541fSNicholas Piggin 23657299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 23664f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 23677299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 23681a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2369eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 23704f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2371eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2372eb204d86SNicholas Piggin bl cbe_system_error_exception 23736cc0c16dSNicholas Piggin b interrupt_return 23749600f261SNicholas Piggin 2375da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 23761a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 23771a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2378da2bc464SMichael Ellerman#endif 2379da2bc464SMichael Ellerman 2380da487a5dSNicholas Piggin/** 2381da487a5dSNicholas Piggin * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. 2382da487a5dSNicholas Piggin * This has been removed from the ISA before 2.01, which is the earliest 2383da487a5dSNicholas Piggin * 64-bit BookS ISA supported, however the G5 / 970 implements this 2384da487a5dSNicholas Piggin * interrupt with a non-architected feature available through the support 2385da487a5dSNicholas Piggin * processor interface. 2386da487a5dSNicholas Piggin */ 23874f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 23884f50541fSNicholas Piggin IVEC=0x1300 23892284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23904f50541fSNicholas Piggin IKVM_REAL=1 23912284ffeaSNicholas Piggin#endif 23924f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 23934f50541fSNicholas Piggin 23947299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 23954f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 23967299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 23977299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 23984f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 23997299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2400eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 24014f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2402eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2403eb204d86SNicholas Piggin bl instruction_breakpoint_exception 24046cc0c16dSNicholas Piggin b interrupt_return 24054e96dbbfSNicholas Piggin 24067299417cSNicholas Piggin 24071a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 24081a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2409da2bc464SMichael Ellerman 241094325357SNicholas Piggin/** 241194325357SNicholas Piggin * Interrupt 0x1500 - Soft Patch Interrupt 241294325357SNicholas Piggin * 241394325357SNicholas Piggin * Handling: 241494325357SNicholas Piggin * This is an implementation specific interrupt which can be used for a 241594325357SNicholas Piggin * range of exceptions. 241694325357SNicholas Piggin * 241794325357SNicholas Piggin * This interrupt handler is unique in that it runs the denormal assist 241894325357SNicholas Piggin * code even for guests (and even in guest context) without going to KVM, 241994325357SNicholas Piggin * for speed. POWER9 does not raise denorm exceptions, so this special case 242094325357SNicholas Piggin * could be phased out in future to reduce special cases. 242194325357SNicholas Piggin */ 24224f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 24234f50541fSNicholas Piggin IVEC=0x1500 24243f7fbd97SNicholas Piggin IHSRR=1 24254557ac6bSNicholas Piggin IBRANCH_TO_COMMON=0 24269600f261SNicholas Piggin IKVM_REAL=1 24274f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 24284f50541fSNicholas Piggin 24294f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 24304f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2431b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2432d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2433b92a66a6SMichael Neuling bne+ denorm_assist 2434b92a66a6SMichael Neuling#endif 24358729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 24364f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2437d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 24381a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 24394f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2440d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 244152b98923SNicholas Piggin bne+ denorm_assist 24428729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 24431a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2444d7e89849SNicholas Piggin#else 24451a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2446d7e89849SNicholas Piggin#endif 2447b92a66a6SMichael Neuling 2448b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2449da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2450b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2451b92a66a6SMichael Neuling/* 2452b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2453b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2454b92a66a6SMichael Neuling */ 2455b92a66a6SMichael Neuling mfmsr r10 2456b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2457b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2458b92a66a6SMichael Neuling mtmsrd r10 2459b92a66a6SMichael Neuling sync 2460d7c67fb1SMichael Neuling 2461f3c8b6c6SNicholas Piggin .Lreg=0 2462f3c8b6c6SNicholas Piggin .rept 32 2463f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2464f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2465f3c8b6c6SNicholas Piggin .endr 2466d7c67fb1SMichael Neuling 2467b92a66a6SMichael NeulingFTR_SECTION_ELSE 2468b92a66a6SMichael Neuling/* 2469b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2470b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2471b92a66a6SMichael Neuling */ 2472b92a66a6SMichael Neuling mfmsr r10 2473b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2474b92a66a6SMichael Neuling mtmsrd r10 2475b92a66a6SMichael Neuling sync 2476d7c67fb1SMichael Neuling 2477f3c8b6c6SNicholas Piggin .Lreg=0 2478f3c8b6c6SNicholas Piggin .rept 32 2479f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2480f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2481f3c8b6c6SNicholas Piggin .endr 2482d7c67fb1SMichael Neuling 2483b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2484fb0fce3eSMichael Neuling 2485fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2486fb0fce3eSMichael Neuling b denorm_done 2487fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2488fb0fce3eSMichael Neuling/* 2489fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2490fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2491fb0fce3eSMichael Neuling */ 2492f3c8b6c6SNicholas Piggin .Lreg=32 2493f3c8b6c6SNicholas Piggin .rept 32 2494f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2495f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2496f3c8b6c6SNicholas Piggin .endr 2497f3c8b6c6SNicholas Piggin 2498fb0fce3eSMichael Neulingdenorm_done: 2499f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2500f14040bcSMichael Neuling subi r11,r11,4 2501b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2502b92a66a6SMichael Neuling mtcrf 0x80,r9 2503b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2504931dc86bSNicholas PigginBEGIN_FTR_SECTION 2505931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2506931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2507931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2508630573c1SPaul MackerrasBEGIN_FTR_SECTION 2509630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2510630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2511630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 2512b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2513b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2514b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2515b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2516222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2517b92a66a6SMichael Neuling b . 2518b92a66a6SMichael Neuling#endif 2519b92a66a6SMichael Neuling 25204f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 25214f50541fSNicholas Piggin GEN_COMMON denorm_exception 2522eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2523eb204d86SNicholas Piggin bl unknown_exception 25246cc0c16dSNicholas Piggin b interrupt_return 2525d7e89849SNicholas Piggin 2526d7e89849SNicholas Piggin 2527d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 25284f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 25294f50541fSNicholas Piggin IVEC=0x1600 25303f7fbd97SNicholas Piggin IHSRR=1 25314f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 25324f50541fSNicholas Piggin 25337299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 25344f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 25357299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 25361a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2537eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 25384f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2539eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2540eb204d86SNicholas Piggin bl cbe_maintenance_exception 25416cc0c16dSNicholas Piggin b interrupt_return 25429600f261SNicholas Piggin 2543d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 25441a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 25451a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2546d7e89849SNicholas Piggin#endif 2547d7e89849SNicholas Piggin 254869a79344SNicholas Piggin 25494f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 25504f50541fSNicholas Piggin IVEC=0x1700 25512284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 25524f50541fSNicholas Piggin IKVM_REAL=1 25532284ffeaSNicholas Piggin#endif 25544f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 25554f50541fSNicholas Piggin 25567299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 25574f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 25587299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 25597299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 25604f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 25617299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2562eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 25634f50541fSNicholas Piggin GEN_COMMON altivec_assist 2564eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2565b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 2566eb204d86SNicholas Piggin bl altivec_assist_exception 2567702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2568b51c079eSNicholas Piggin#else 2569eb204d86SNicholas Piggin bl unknown_exception 2570b51c079eSNicholas Piggin#endif 25716cc0c16dSNicholas Piggin b interrupt_return 2572b51c079eSNicholas Piggin 2573d7e89849SNicholas Piggin 2574d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 25754f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 25764f50541fSNicholas Piggin IVEC=0x1800 25773f7fbd97SNicholas Piggin IHSRR=1 25784f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 25794f50541fSNicholas Piggin 25807299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 25814f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 25827299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 25831a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2584eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 25854f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2586eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2587eb204d86SNicholas Piggin bl cbe_thermal_exception 25886cc0c16dSNicholas Piggin b interrupt_return 25899600f261SNicholas Piggin 2590d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 25911a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 25921a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2593d7e89849SNicholas Piggin#endif 2594d7e89849SNicholas Piggin 25957299417cSNicholas Piggin 259675eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 25972104180aSNicholas Piggin 25980eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 25990eddf327SNicholas Piggin IVEC=0x900 26000eddf327SNicholas Piggin ISTACK=0 26010eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 26022104180aSNicholas Piggin 2603cc491f1dSNicholas Piggin/* 2604cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2605cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2606cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2607cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2608cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2609cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2610cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2611cc491f1dSNicholas Piggin */ 26122104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 26130eddf327SNicholas Piggin mfspr r11,SPRN_SRR0 26142104180aSNicholas Piggin mr r10,r1 26152104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 26162104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 26170eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 261871c3b05aSNicholas Piggin 2619c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2620c06075f3SNicholas Piggin bl soft_nmi_interrupt 262171c3b05aSNicholas Piggin 262271c3b05aSNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 262371c3b05aSNicholas Piggin li r9,0 262471c3b05aSNicholas Piggin mtmsrd r9,1 262571c3b05aSNicholas Piggin 26268e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 262771c3b05aSNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 262871c3b05aSNicholas Piggin RFI_TO_KERNEL 26292104180aSNicholas Piggin 263075eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2631d7e89849SNicholas Piggin 26320ebc4cdaSBenjamin Herrenschmidt/* 2633fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2634fe9e1d54SIan Munsie * - If it was a decrementer interrupt, we bump the dec to max and and return. 2635fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2636fe9e1d54SIan Munsie * triggered and won't automatically refire. 26370869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 26380869b6fdSMahesh Salgaonkar * and it won't refire. 26396cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2640fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 26410ebc4cdaSBenjamin Herrenschmidt */ 26423f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 26434508a74aSNicholas Piggin .if \hsrr 26444508a74aSNicholas Pigginmasked_Hinterrupt: 26454508a74aSNicholas Piggin .else 26464508a74aSNicholas Pigginmasked_interrupt: 26474508a74aSNicholas Piggin .endif 26484508a74aSNicholas Piggin lbz r11,PACAIRQHAPPENED(r13) 26494508a74aSNicholas Piggin or r11,r11,r10 26504508a74aSNicholas Piggin stb r11,PACAIRQHAPPENED(r13) 26514508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 26524508a74aSNicholas Piggin bne 1f 26534508a74aSNicholas Piggin lis r10,0x7fff 26544508a74aSNicholas Piggin ori r10,r10,0xffff 26554508a74aSNicholas Piggin mtspr SPRN_DEC,r10 26560eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 26570eddf327SNicholas Piggin b soft_nmi_common 26580eddf327SNicholas Piggin#else 26590eddf327SNicholas Piggin b 2f 26600eddf327SNicholas Piggin#endif 26614508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 26624508a74aSNicholas Piggin beq 2f 26630eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 26644508a74aSNicholas Piggin .if \hsrr 26650eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 26664508a74aSNicholas Piggin .else 26670eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 26684508a74aSNicholas Piggin .endif 26694508a74aSNicholas Piggin ori r11,r11,PACA_IRQ_HARD_DIS 26704508a74aSNicholas Piggin stb r11,PACAIRQHAPPENED(r13) 26714508a74aSNicholas Piggin2: /* done */ 26720eddf327SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 26730eddf327SNicholas Piggin mtctr r10 26744508a74aSNicholas Piggin mtcrf 0x80,r9 26754508a74aSNicholas Piggin std r1,PACAR1(r13) 26764508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 26774508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 26784508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 26790eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 2680b2dc2977SNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 2681b2dc2977SNicholas Piggin /* May return to masked low address where r13 is not set up */ 26824508a74aSNicholas Piggin .if \hsrr 26834508a74aSNicholas Piggin HRFI_TO_KERNEL 26844508a74aSNicholas Piggin .else 26854508a74aSNicholas Piggin RFI_TO_KERNEL 26864508a74aSNicholas Piggin .endif 26874508a74aSNicholas Piggin b . 26884508a74aSNicholas Piggin.endm 26890ebc4cdaSBenjamin Herrenschmidt 2690a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2691a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2692a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2693a048a07dSNicholas Piggin sync 2694a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2695a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2696a048a07dSNicholas Piggin ori 31,31,0 2697a048a07dSNicholas Piggin .rept 14 2698a048a07dSNicholas Piggin b 1f 2699a048a07dSNicholas Piggin1: 2700a048a07dSNicholas Piggin .endr 2701a048a07dSNicholas Piggin blr 2702a048a07dSNicholas Piggin 27039a32a7e7SNicholas Piggin/* Clobbers r10, r11, ctr */ 27049a32a7e7SNicholas Piggin.macro L1D_DISPLACEMENT_FLUSH 2705aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2706bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2707bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2708aa8a5e00SMichael Ellerman mtctr r11 270915a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2710aa8a5e00SMichael Ellerman 2711aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2712aa8a5e00SMichael Ellerman sync 2713bdcb1aefSNicholas Piggin 2714bdcb1aefSNicholas Piggin /* 2715f7964378SNicholas Piggin * The load addresses are at staggered offsets within cachelines, 2716bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2717bdcb1aefSNicholas Piggin * hurt). 2718bdcb1aefSNicholas Piggin */ 2719bdcb1aefSNicholas Piggin1: 2720bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2721bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2722bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2723bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2724bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2725bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2726bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2727bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2728bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2729aa8a5e00SMichael Ellerman bdnz 1b 27309a32a7e7SNicholas Piggin.endm 2731aa8a5e00SMichael Ellerman 27329a32a7e7SNicholas PigginTRAMP_REAL_BEGIN(entry_flush_fallback) 27339a32a7e7SNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 27349a32a7e7SNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 27359a32a7e7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 27369a32a7e7SNicholas Piggin mfctr r9 27379a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2738f7964378SNicholas Piggin mtctr r9 2739f7964378SNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2740f7964378SNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2741f7964378SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 2742f7964378SNicholas Piggin blr 2743f7964378SNicholas Piggin 274408685be7SNicholas Piggin/* 274508685be7SNicholas Piggin * The SCV entry flush happens with interrupts enabled, so it must disable 274608685be7SNicholas Piggin * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10 274708685be7SNicholas Piggin * (containing LR) does not need to be preserved here because scv entry 274808685be7SNicholas Piggin * puts 0 in the pt_regs, CTR can be clobbered for the same reason. 274908685be7SNicholas Piggin */ 275008685be7SNicholas PigginTRAMP_REAL_BEGIN(scv_entry_flush_fallback) 275108685be7SNicholas Piggin li r10,0 275208685be7SNicholas Piggin mtmsrd r10,1 275308685be7SNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 275408685be7SNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 275508685be7SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 275608685be7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 275708685be7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 275808685be7SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 275908685be7SNicholas Piggin li r10,MSR_RI 276008685be7SNicholas Piggin mtmsrd r10,1 276108685be7SNicholas Piggin blr 276208685be7SNicholas Piggin 2763aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 2764aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2765aa8a5e00SMichael Ellerman GET_PACA(r13); 2766aa8a5e00SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 2767aa8a5e00SMichael Ellerman ld r1,PACAKSAVE(r13) 2768aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2769aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2770aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2771aa8a5e00SMichael Ellerman mfctr r9 27729a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2773aa8a5e00SMichael Ellerman mtctr r9 2774aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2775aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2776aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 277778ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2778aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2779aa8a5e00SMichael Ellerman rfid 2780aa8a5e00SMichael Ellerman 2781aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 2782aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2783aa8a5e00SMichael Ellerman GET_PACA(r13); 278478ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 278578ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2786aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2787aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2788aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2789aa8a5e00SMichael Ellerman mfctr r9 27909a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2791aa8a5e00SMichael Ellerman mtctr r9 2792aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2793aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2794aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 279578ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2796aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2797aa8a5e00SMichael Ellerman hrfid 2798aa8a5e00SMichael Ellerman 27997fa95f9aSNicholas PigginTRAMP_REAL_BEGIN(rfscv_flush_fallback) 28007fa95f9aSNicholas Piggin /* system call volatile */ 28017fa95f9aSNicholas Piggin mr r7,r13 28027fa95f9aSNicholas Piggin GET_PACA(r13); 28037fa95f9aSNicholas Piggin mr r8,r1 28047fa95f9aSNicholas Piggin ld r1,PACAKSAVE(r13) 28057fa95f9aSNicholas Piggin mfctr r9 28067fa95f9aSNicholas Piggin ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 28077fa95f9aSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 28087fa95f9aSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 28097fa95f9aSNicholas Piggin mtctr r11 28107fa95f9aSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 28117fa95f9aSNicholas Piggin 28127fa95f9aSNicholas Piggin /* order ld/st prior to dcbt stop all streams with flushing */ 28137fa95f9aSNicholas Piggin sync 28147fa95f9aSNicholas Piggin 28157fa95f9aSNicholas Piggin /* 28167fa95f9aSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 28177fa95f9aSNicholas Piggin * which suits some pipelines better (on others it should not 28187fa95f9aSNicholas Piggin * hurt). 28197fa95f9aSNicholas Piggin */ 28207fa95f9aSNicholas Piggin1: 28217fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 28227fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 28237fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 28247fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 28257fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 28267fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 28277fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 28287fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 28297fa95f9aSNicholas Piggin addi r10,r10,0x80*8 28307fa95f9aSNicholas Piggin bdnz 1b 28317fa95f9aSNicholas Piggin 28327fa95f9aSNicholas Piggin mtctr r9 28337fa95f9aSNicholas Piggin li r9,0 28347fa95f9aSNicholas Piggin li r10,0 28357fa95f9aSNicholas Piggin li r11,0 28367fa95f9aSNicholas Piggin mr r1,r8 28377fa95f9aSNicholas Piggin mr r13,r7 28387fa95f9aSNicholas Piggin RFSCV 28397fa95f9aSNicholas Piggin 28400eddf327SNicholas PigginUSE_TEXT_SECTION() 28419a32a7e7SNicholas Piggin 284269fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 284369fdd674SNicholas Pigginkvm_interrupt: 284469fdd674SNicholas Piggin /* 284569fdd674SNicholas Piggin * The conditional branch in KVMTEST can't reach all the way, 284669fdd674SNicholas Piggin * make a stub. 284769fdd674SNicholas Piggin */ 284869fdd674SNicholas Piggin b kvmppc_interrupt 284969fdd674SNicholas Piggin#endif 285069fdd674SNicholas Piggin 28519a32a7e7SNicholas Piggin_GLOBAL(do_uaccess_flush) 28529a32a7e7SNicholas Piggin UACCESS_FLUSH_FIXUP_SECTION 28539a32a7e7SNicholas Piggin nop 28549a32a7e7SNicholas Piggin nop 28559a32a7e7SNicholas Piggin nop 28569a32a7e7SNicholas Piggin blr 28579a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 28589a32a7e7SNicholas Piggin blr 28599a32a7e7SNicholas Piggin_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) 28609a32a7e7SNicholas PigginEXPORT_SYMBOL(do_uaccess_flush) 28619a32a7e7SNicholas Piggin 28629a32a7e7SNicholas Piggin 28633f7fbd97SNicholas PigginMASKED_INTERRUPT 28643f7fbd97SNicholas PigginMASKED_INTERRUPT hsrr=1 28657230c564SBenjamin Herrenschmidt 28660ebc4cdaSBenjamin Herrenschmidt /* 2867c1fb6816SMichael Neuling * Relocation-on interrupts: A subset of the interrupts can be delivered 2868c1fb6816SMichael Neuling * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 2869c1fb6816SMichael Neuling * it. Addresses are the same as the original interrupt addresses, but 2870c1fb6816SMichael Neuling * offset by 0xc000000000004000. 2871c1fb6816SMichael Neuling * It's impossible to receive interrupts below 0x300 via this mechanism. 2872c1fb6816SMichael Neuling * KVM: None of these traps are from the guest ; anything that escalated 2873c1fb6816SMichael Neuling * to HV=1 from HV=0 is delivered via real mode handlers. 2874c1fb6816SMichael Neuling */ 2875c1fb6816SMichael Neuling 2876c1fb6816SMichael Neuling /* 2877c1fb6816SMichael Neuling * This uses the standard macro, since the original 0x300 vector 2878c1fb6816SMichael Neuling * only has extra guff for STAB-based processors -- which never 2879c1fb6816SMichael Neuling * come here. 2880c1fb6816SMichael Neuling */ 2881da2bc464SMichael Ellerman 288257f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 28838ed8ab40SHari Bathini /* 2884b2dc2977SNicholas Piggin * All code below __end_interrupts is treated as soft-masked. If 2885b2dc2977SNicholas Piggin * any code runs here with MSR[EE]=1, it must then cope with pending 2886b2dc2977SNicholas Piggin * soft interrupt being raised (i.e., by ensuring it is replayed). 2887b2dc2977SNicholas Piggin * 28888ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 28898ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 28908ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 28918ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 28928ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 28938ed8ab40SHari Bathini */ 28948ed8ab40SHari Bathini .align 7 28958ed8ab40SHari Bathini .globl __end_interrupts 28968ed8ab40SHari Bathini__end_interrupts: 289757f26649SNicholas PigginDEFINE_FIXED_SYMBOL(__end_interrupts) 289861383407SBenjamin Herrenschmidt 289957f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 290057f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 290157f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 290257f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 290357f26649SNicholas Piggin 290457f26649SNicholas PigginUSE_TEXT_SECTION() 290557f26649SNicholas Piggin 2906296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 2907296e753fSNicholas Pigginenable_machine_check: 2908296e753fSNicholas Piggin mflr r0 2909296e753fSNicholas Piggin bcl 20,31,$+4 2910296e753fSNicholas Piggin0: mflr r3 2911296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 2912296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 2913296e753fSNicholas Piggin mfmsr r3 2914296e753fSNicholas Piggin ori r3,r3,MSR_ME 2915296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 2916296e753fSNicholas Piggin RFI_TO_KERNEL 2917296e753fSNicholas Piggin1: mtlr r0 2918296e753fSNicholas Piggin blr 2919296e753fSNicholas Piggin 2920b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 2921b7d9ccecSNicholas Piggindisable_machine_check: 2922b7d9ccecSNicholas Piggin mflr r0 2923b7d9ccecSNicholas Piggin bcl 20,31,$+4 2924b7d9ccecSNicholas Piggin0: mflr r3 2925b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 2926b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 2927b7d9ccecSNicholas Piggin mfmsr r3 2928b7d9ccecSNicholas Piggin li r4,MSR_ME 2929b7d9ccecSNicholas Piggin andc r3,r3,r4 2930b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 2931b7d9ccecSNicholas Piggin RFI_TO_KERNEL 2932b7d9ccecSNicholas Piggin1: mtlr r0 2933b7d9ccecSNicholas Piggin blr 2934