1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 167230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 178aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1846f52210SStephen Rothwell#include <asm/ptrace.h> 197cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 20da2bc464SMichael Ellerman#include <asm/head-64.h> 212c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 22890274c2SMichael Ellerman#include <asm/kup.h> 238aa34ab8SBenjamin Herrenschmidt 247299417cSNicholas Piggin/* 257299417cSNicholas Piggin * Following are fixed section helper macros. 267299417cSNicholas Piggin * 277299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 287299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 297299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 307299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 317299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 327299417cSNicholas Piggin */ 337299417cSNicholas Piggin 34a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 35a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 36a2432811SNicholas Piggin 37a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 38a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 39a2432811SNicholas Piggin 40a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 41a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 42a2432811SNicholas Piggin 43a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 44a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 45a2432811SNicholas Piggin 46a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 47a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 48a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 49a2432811SNicholas Piggin .global name; \ 50a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 51*d72c4a36SDaniel Axtens DEFINE_FIXED_SYMBOL(name, text); \ 52a2432811SNicholas Pigginname: 53a2432811SNicholas Piggin 54a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 55a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 56a2432811SNicholas Piggin 57a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 58a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 59a2432811SNicholas Piggin 60a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 61a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 62a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 63a2432811SNicholas Piggin 64a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 65a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 66a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 67a2432811SNicholas Piggin 680ebc4cdaSBenjamin Herrenschmidt/* 6912a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 7012a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 7112a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 7212a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 7312a04809SNicholas Piggin * that kernelbase be 64K aligned. 7412a04809SNicholas Piggin */ 7512a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 7612a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 7712a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 7812a04809SNicholas Piggin 79*d72c4a36SDaniel Axtens#define __LOAD_HANDLER(reg, label, section) \ 8012a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 81*d72c4a36SDaniel Axtens ori reg,reg,(ABS_ADDR(label, section))@l 8212a04809SNicholas Piggin 8312a04809SNicholas Piggin/* 8412a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 8512a04809SNicholas Piggin * head-y require >64K offsets. 8612a04809SNicholas Piggin */ 87*d72c4a36SDaniel Axtens#define __LOAD_FAR_HANDLER(reg, label, section) \ 8812a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 89*d72c4a36SDaniel Axtens ori reg,reg,(ABS_ADDR(label, section))@l; \ 90*d72c4a36SDaniel Axtens addis reg,reg,(ABS_ADDR(label, section))@h 9112a04809SNicholas Piggin 9212a04809SNicholas Piggin/* 93a42a239dSNicholas Piggin * Interrupt code generation macros 94a42a239dSNicholas Piggin */ 9594325357SNicholas Piggin#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 9694325357SNicholas Piggin#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 9794325357SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 9894325357SNicholas Piggin#define IAREA .L_IAREA_\name\() /* PACA save area */ 9994325357SNicholas Piggin#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 10094325357SNicholas Piggin#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 101af47d79bSNicholas Piggin#define ICFAR .L_ICFAR_\name\() /* Uses CFAR */ 102af47d79bSNicholas Piggin#define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */ 10394325357SNicholas Piggin#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 10494325357SNicholas Piggin#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 10594325357SNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 10694325357SNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 10794325357SNicholas Piggin#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 10894325357SNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 1094f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 11094325357SNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 11194325357SNicholas Piggin#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 1124f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 11394325357SNicholas Piggin#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 114a42a239dSNicholas Piggin 115a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 116a42a239dSNicholas Piggin.macro int_define_ ## n name 117a42a239dSNicholas Piggin 118a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 119a42a239dSNicholas Piggin.endm ; \ 120a42a239dSNicholas Pigginint_define_ ## n n ; \ 121a42a239dSNicholas Piggindo_define_int n 122a42a239dSNicholas Piggin 123a42a239dSNicholas Piggin.macro do_define_int name 124a42a239dSNicholas Piggin .ifndef IVEC 125a42a239dSNicholas Piggin .error "IVEC not defined" 126a42a239dSNicholas Piggin .endif 127a42a239dSNicholas Piggin .ifndef IHSRR 1283f7fbd97SNicholas Piggin IHSRR=0 1293f7fbd97SNicholas Piggin .endif 1303f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 1313f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 132a42a239dSNicholas Piggin .endif 133a42a239dSNicholas Piggin .ifndef IAREA 134a42a239dSNicholas Piggin IAREA=PACA_EXGEN 135a42a239dSNicholas Piggin .endif 1368729c26eSNicholas Piggin .ifndef IVIRT 1378729c26eSNicholas Piggin IVIRT=1 1388729c26eSNicholas Piggin .endif 139a3cd35beSNicholas Piggin .ifndef IISIDE 140a3cd35beSNicholas Piggin IISIDE=0 141a3cd35beSNicholas Piggin .endif 142af47d79bSNicholas Piggin .ifndef ICFAR 143af47d79bSNicholas Piggin ICFAR=1 144af47d79bSNicholas Piggin .endif 145af47d79bSNicholas Piggin .ifndef ICFAR_IF_HVMODE 146af47d79bSNicholas Piggin ICFAR_IF_HVMODE=0 147af47d79bSNicholas Piggin .endif 148a42a239dSNicholas Piggin .ifndef IDAR 149a42a239dSNicholas Piggin IDAR=0 150a42a239dSNicholas Piggin .endif 151a42a239dSNicholas Piggin .ifndef IDSISR 152a42a239dSNicholas Piggin IDSISR=0 153a42a239dSNicholas Piggin .endif 154d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 155d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 156d73a10cbSNicholas Piggin .endif 157d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 158d73a10cbSNicholas Piggin IREALMODE_COMMON=0 159d73a10cbSNicholas Piggin .else 160d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 161d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 162d73a10cbSNicholas Piggin .endif 163a42a239dSNicholas Piggin .endif 164a42a239dSNicholas Piggin .ifndef IMASK 165a42a239dSNicholas Piggin IMASK=0 166a42a239dSNicholas Piggin .endif 167a42a239dSNicholas Piggin .ifndef IKVM_REAL 168a42a239dSNicholas Piggin IKVM_REAL=0 169a42a239dSNicholas Piggin .endif 170a42a239dSNicholas Piggin .ifndef IKVM_VIRT 171a42a239dSNicholas Piggin IKVM_VIRT=0 172a42a239dSNicholas Piggin .endif 1737cb3a1a0SNicholas Piggin .ifndef ISTACK 1747cb3a1a0SNicholas Piggin ISTACK=1 1757cb3a1a0SNicholas Piggin .endif 1767cb3a1a0SNicholas Piggin .ifndef IKUAP 1777cb3a1a0SNicholas Piggin IKUAP=1 1787cb3a1a0SNicholas Piggin .endif 179a42a239dSNicholas Piggin.endm 180a42a239dSNicholas Piggin 18112a04809SNicholas Piggin/* 1822284ffeaSNicholas Piggin * All interrupts which set HSRR registers, as well as SRESET and MCE and 1832284ffeaSNicholas Piggin * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 1842284ffeaSNicholas Piggin * so they all generally need to test whether they were taken in guest context. 1852284ffeaSNicholas Piggin * 1862284ffeaSNicholas Piggin * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 1872284ffeaSNicholas Piggin * taken with MSR[HV]=0. 1882284ffeaSNicholas Piggin * 1892284ffeaSNicholas Piggin * Interrupts which set SRR registers (with the above exceptions) do not 1902284ffeaSNicholas Piggin * elevate to MSR[HV]=1 mode, though most can be taken when running with 1912284ffeaSNicholas Piggin * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 1922284ffeaSNicholas Piggin * not need to test whether a guest is running because they get delivered to 1932284ffeaSNicholas Piggin * the guest directly, including nested HV KVM guests. 1942284ffeaSNicholas Piggin * 1952284ffeaSNicholas Piggin * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 1962284ffeaSNicholas Piggin * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 1972284ffeaSNicholas Piggin * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 1982284ffeaSNicholas Piggin * delivered to the real-mode entry point, therefore such interrupts only test 1992284ffeaSNicholas Piggin * KVM in their real mode handlers, and only when PR KVM is possible. 2002284ffeaSNicholas Piggin * 2012284ffeaSNicholas Piggin * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 2022284ffeaSNicholas Piggin * delivered in real-mode when the MMU is in hash mode because the MMU 2032284ffeaSNicholas Piggin * registers are not set appropriately to translate host addresses. In nested 2042284ffeaSNicholas Piggin * radix mode these can be delivered in virt-mode as the host translations are 2052284ffeaSNicholas Piggin * used implicitly (see: effective LPID, effective PID). 2062284ffeaSNicholas Piggin */ 2072284ffeaSNicholas Piggin 2082284ffeaSNicholas Piggin/* 2092284ffeaSNicholas Piggin * If an interrupt is taken while a guest is running, it is immediately routed 210f3601156SNicholas Piggin * to KVM to handle. 21112a04809SNicholas Piggin */ 21212a04809SNicholas Piggin 21369fdd674SNicholas Piggin.macro KVMTEST name handler 21469fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 21512a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 21612a04809SNicholas Piggin cmpwi r10,0 21712a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 2183f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 219def0db4fSNicholas Piggin BEGIN_FTR_SECTION 22069fdd674SNicholas Piggin li r10,(IVEC + 0x2) 221def0db4fSNicholas Piggin FTR_SECTION_ELSE 22269fdd674SNicholas Piggin li r10,(IVEC) 223def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 224b177ae2fSNicholas Piggin .elseif IHSRR 22569fdd674SNicholas Piggin li r10,(IVEC + 0x2) 22612a04809SNicholas Piggin .else 22769fdd674SNicholas Piggin li r10,(IVEC) 22812a04809SNicholas Piggin .endif 22969fdd674SNicholas Piggin bne \handler 23012a04809SNicholas Piggin#endif 23169fdd674SNicholas Piggin.endm 23212a04809SNicholas Piggin 233c7c5cbb4SNicholas Piggin/* 234c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 235c7c5cbb4SNicholas Piggin * 236c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 237c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 238c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 239c7c5cbb4SNicholas Piggin * the kernel. 240c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 241c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 242c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 243c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 244c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 245c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 246c7c5cbb4SNicholas Piggin * irq_soft_mask set. 247c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 248c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 249c7c5cbb4SNicholas Piggin * mode. 250c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 251c7c5cbb4SNicholas Piggin * This is done if early=2. 252c7c5cbb4SNicholas Piggin */ 2538729c26eSNicholas Piggin 2548729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 255d73a10cbSNicholas Piggin .if IREALMODE_COMMON 256d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 257d73a10cbSNicholas Piggin mtctr r10 258d73a10cbSNicholas Piggin bctr 259d73a10cbSNicholas Piggin .else 2608729c26eSNicholas Piggin .if \virt 2618729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 2628729c26eSNicholas Piggin b \name\()_common_virt 2638729c26eSNicholas Piggin#else 2648729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 2658729c26eSNicholas Piggin mtctr r10 2668729c26eSNicholas Piggin bctr 2678729c26eSNicholas Piggin#endif 2688729c26eSNicholas Piggin .else 2698729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 2708729c26eSNicholas Piggin mtctr r10 2718729c26eSNicholas Piggin bctr 2728729c26eSNicholas Piggin .endif 273d73a10cbSNicholas Piggin .endif 2748729c26eSNicholas Piggin.endm 2758729c26eSNicholas Piggin 276fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 277c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 278c7c5cbb4SNicholas Piggin GET_PACA(r13) 279fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 280931dc86bSNicholas PigginBEGIN_FTR_SECTION 281931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 282931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 283c7c5cbb4SNicholas Piggin HMT_MEDIUM 284fc589ee4SNicholas Piggin std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 285af47d79bSNicholas Piggin .if ICFAR 286931dc86bSNicholas PigginBEGIN_FTR_SECTION 287931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 288931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 289af47d79bSNicholas Piggin .elseif ICFAR_IF_HVMODE 290af47d79bSNicholas PigginBEGIN_FTR_SECTION 291af47d79bSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 292af47d79bSNicholas Piggin mfspr r10,SPRN_CFAR 293af47d79bSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) 294af47d79bSNicholas PigginFTR_SECTION_ELSE 295af47d79bSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 296af47d79bSNicholas Piggin li r10,0 297af47d79bSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) 298af47d79bSNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 299af47d79bSNicholas Piggin .endif 300c7c5cbb4SNicholas Piggin .if \ool 301c7c5cbb4SNicholas Piggin .if !\virt 302c7c5cbb4SNicholas Piggin b tramp_real_\name 303c7c5cbb4SNicholas Piggin .pushsection .text 304c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 305c7c5cbb4SNicholas Piggin .else 306c7c5cbb4SNicholas Piggin b tramp_virt_\name 307c7c5cbb4SNicholas Piggin .pushsection .text 308c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 309c7c5cbb4SNicholas Piggin .endif 310c7c5cbb4SNicholas Piggin .endif 311c7c5cbb4SNicholas Piggin 312931dc86bSNicholas PigginBEGIN_FTR_SECTION 313931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 314931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 315af47d79bSNicholas Piggin .if ICFAR || ICFAR_IF_HVMODE 316931dc86bSNicholas PigginBEGIN_FTR_SECTION 317931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 318931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 319af47d79bSNicholas Piggin .endif 320c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 3218729c26eSNicholas Piggin mfctr r10 3228729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 323c7c5cbb4SNicholas Piggin mfcr r9 324fc589ee4SNicholas Piggin std r11,IAREA+EX_R11(r13) 325fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 326c7c5cbb4SNicholas Piggin 327c7c5cbb4SNicholas Piggin /* 328c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 329c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 330c7c5cbb4SNicholas Piggin * not recoverable if they are live. 331c7c5cbb4SNicholas Piggin */ 332c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 333fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 334a3cd35beSNicholas Piggin .if IDAR && !IISIDE 335fc589ee4SNicholas Piggin .if IHSRR 336c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 337c7c5cbb4SNicholas Piggin .else 338c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 339c7c5cbb4SNicholas Piggin .endif 340fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 341c7c5cbb4SNicholas Piggin .endif 342a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 343fc589ee4SNicholas Piggin .if IHSRR 344c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 345c7c5cbb4SNicholas Piggin .else 346c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 347c7c5cbb4SNicholas Piggin .endif 348fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 349c7c5cbb4SNicholas Piggin .endif 350c7c5cbb4SNicholas Piggin 3513f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 3528729c26eSNicholas Piggin BEGIN_FTR_SECTION 3538729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3548729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3558729c26eSNicholas Piggin FTR_SECTION_ELSE 3568729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3578729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 3588729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 3598729c26eSNicholas Piggin .elseif IHSRR 3608729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3618729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3628729c26eSNicholas Piggin .else 3638729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3648729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 365c7c5cbb4SNicholas Piggin .endif 366d73a10cbSNicholas Piggin 367d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 3688729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 3698729c26eSNicholas Piggin .endif 3708729c26eSNicholas Piggin 371c7c5cbb4SNicholas Piggin .if \ool 372c7c5cbb4SNicholas Piggin .popsection 373c7c5cbb4SNicholas Piggin .endif 374c7c5cbb4SNicholas Piggin.endm 375c7c5cbb4SNicholas Piggin 376d064151fSNicholas Piggin/* 3778729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 3789600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 3799600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 3809600f261SNicholas Piggin * 3818729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 382d064151fSNicholas Piggin */ 3838729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 384*d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_real, text) 3858729c26eSNicholas Piggin\name\()_common_real: 3869600f261SNicholas Piggin .if IKVM_REAL 38769fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 3889600f261SNicholas Piggin .endif 3899600f261SNicholas Piggin 3908729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 3918729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 392c080a173SDaniel Axtens .if IHSRR_IF_HVMODE 3938729c26eSNicholas Piggin BEGIN_FTR_SECTION 3948729c26eSNicholas Piggin xori r10,r10,MSR_RI 3958729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 3968729c26eSNicholas Piggin .elseif ! IHSRR 3978729c26eSNicholas Piggin xori r10,r10,MSR_RI 3988729c26eSNicholas Piggin .endif 3998729c26eSNicholas Piggin mtmsrd r10 4008729c26eSNicholas Piggin 4018729c26eSNicholas Piggin .if IVIRT 4029600f261SNicholas Piggin .if IKVM_VIRT 4039600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 4049600f261SNicholas Piggin .endif 4059600f261SNicholas Piggin 4068729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 407*d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_virt, text) 4088729c26eSNicholas Piggin\name\()_common_virt: 4099600f261SNicholas Piggin .if IKVM_VIRT 41069fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4119600f261SNicholas Piggin1: 4129600f261SNicholas Piggin .endif 4138729c26eSNicholas Piggin .endif /* IVIRT */ 4148729c26eSNicholas Piggin.endm 4158729c26eSNicholas Piggin 4169600f261SNicholas Piggin/* 4179600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4189600f261SNicholas Piggin * want to run in real mode. 4199600f261SNicholas Piggin */ 4209600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 421*d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_real, text) 4229600f261SNicholas Piggin\name\()_common_real: 4239600f261SNicholas Piggin .if IKVM_REAL 42469fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4259600f261SNicholas Piggin .endif 4269600f261SNicholas Piggin.endm 4279600f261SNicholas Piggin 4288729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 4290eddf327SNicholas Piggin .if IMASK 430b2dc2977SNicholas Piggin .if ! ISTACK 431b2dc2977SNicholas Piggin .error "No support for masked interrupt to use custom stack" 432b2dc2977SNicholas Piggin .endif 433b2dc2977SNicholas Piggin 434b2dc2977SNicholas Piggin /* If coming from user, skip soft-mask tests. */ 435b2dc2977SNicholas Piggin andi. r10,r12,MSR_PR 436325678fdSNicholas Piggin bne 3f 437b2dc2977SNicholas Piggin 4389d1988caSNicholas Piggin /* 439325678fdSNicholas Piggin * Kernel code running below __end_soft_masked may be 440325678fdSNicholas Piggin * implicitly soft-masked if it is within the regions 441325678fdSNicholas Piggin * in the soft mask table. 4429d1988caSNicholas Piggin */ 4439d1988caSNicholas Piggin LOAD_HANDLER(r10, __end_soft_masked) 444b2dc2977SNicholas Piggin cmpld r11,r10 445325678fdSNicholas Piggin bge+ 1f 4469d1988caSNicholas Piggin 447325678fdSNicholas Piggin /* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */ 448325678fdSNicholas Piggin mtctr r12 449325678fdSNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 450325678fdSNicholas Piggin SEARCH_SOFT_MASK_TABLE 451325678fdSNicholas Piggin cmpdi r12,0 452325678fdSNicholas Piggin mfctr r12 /* Restore r12 to SRR1 */ 453325678fdSNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 454325678fdSNicholas Piggin beq 1f /* Not in soft-mask table */ 455b2dc2977SNicholas Piggin li r10,IMASK 456325678fdSNicholas Piggin b 2f /* In soft-mask table, always mask */ 457b2dc2977SNicholas Piggin 458b2dc2977SNicholas Piggin /* Test the soft mask state against our interrupt's bit */ 459325678fdSNicholas Piggin1: lbz r10,PACAIRQSOFTMASK(r13) 460325678fdSNicholas Piggin2: andi. r10,r10,IMASK 4610eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 4620eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 4630eddf327SNicholas Piggin li r10,PACA_IRQ_EE 4640eddf327SNicholas Piggin .elseif IVEC == 0x900 4650eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 4660eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 4670eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 4680eddf327SNicholas Piggin .elseif IVEC == 0xe60 4690eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 4700eddf327SNicholas Piggin .elseif IVEC == 0xf00 4710eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 4720eddf327SNicholas Piggin .else 4730eddf327SNicholas Piggin .abort "Bad maskable vector" 4740eddf327SNicholas Piggin .endif 4750eddf327SNicholas Piggin 4763f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4770eddf327SNicholas Piggin BEGIN_FTR_SECTION 4780eddf327SNicholas Piggin bne masked_Hinterrupt 4790eddf327SNicholas Piggin FTR_SECTION_ELSE 4800eddf327SNicholas Piggin bne masked_interrupt 4810eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4820eddf327SNicholas Piggin .elseif IHSRR 4830eddf327SNicholas Piggin bne masked_Hinterrupt 4840eddf327SNicholas Piggin .else 4850eddf327SNicholas Piggin bne masked_interrupt 4860eddf327SNicholas Piggin .endif 4870eddf327SNicholas Piggin .endif 4880eddf327SNicholas Piggin 4896d71759aSNicholas Piggin .if ISTACK 4905d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 491325678fdSNicholas Piggin3: mr r10,r1 /* Save r1 */ 4925d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 4931b359982SNicholas Piggin beq- 100f 4945d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 4951b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 4961b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 4975d5e0edfSNicholas Piggin .endif 4988c9fb5d4SNicholas Piggin 4998c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 5008c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 5018c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 5028c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 5038c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 5048c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 5055d5e0edfSNicholas Piggin 50659dc5bfcSNicholas Piggin /* Mark our [H]SRRs valid for return */ 50759dc5bfcSNicholas Piggin li r10,1 50859dc5bfcSNicholas Piggin .if IHSRR_IF_HVMODE 50959dc5bfcSNicholas Piggin BEGIN_FTR_SECTION 51059dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 51159dc5bfcSNicholas Piggin FTR_SECTION_ELSE 51259dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 51359dc5bfcSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 51459dc5bfcSNicholas Piggin .elseif IHSRR 51559dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 51659dc5bfcSNicholas Piggin .else 51759dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 51859dc5bfcSNicholas Piggin .endif 51959dc5bfcSNicholas Piggin 5206d71759aSNicholas Piggin .if ISTACK 5216d71759aSNicholas Piggin .if IKUAP 5225d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 5235d5e0edfSNicholas Piggin .endif 5241b359982SNicholas Piggin beq 101f /* if from kernel mode */ 525931dc86bSNicholas PigginBEGIN_FTR_SECTION 526931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 527931dc86bSNicholas Piggin std r9,_PPR(r1) 528931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5291b359982SNicholas Piggin101: 5305d5e0edfSNicholas Piggin .else 5316d71759aSNicholas Piggin .if IKUAP 532bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 533bcbceed4SNicholas Piggin .endif 5345d5e0edfSNicholas Piggin .endif 5355d5e0edfSNicholas Piggin 5368c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5376d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5386d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5398c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5408c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5416d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5426d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5436d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5448c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5458c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5468c9fb5d4SNicholas Piggin std r11,GPR13(r1) 547a3cd35beSNicholas Piggin 5486cc0c16dSNicholas Piggin SAVE_NVGPRS(r1) 5496cc0c16dSNicholas Piggin 5506d71759aSNicholas Piggin .if IDAR 551a3cd35beSNicholas Piggin .if IISIDE 552d1a84718SNicholas Piggin ld r10,_NIP(r1) 553d1a84718SNicholas Piggin .else 5546d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 555d1a84718SNicholas Piggin .endif 556d1a84718SNicholas Piggin std r10,_DAR(r1) 557d1a84718SNicholas Piggin .endif 558a3cd35beSNicholas Piggin 5596d71759aSNicholas Piggin .if IDSISR 560a3cd35beSNicholas Piggin .if IISIDE 561d1a84718SNicholas Piggin ld r10,_MSR(r1) 562d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 563d1a84718SNicholas Piggin and r10,r10,r11 564d1a84718SNicholas Piggin .else 5656d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 566d1a84718SNicholas Piggin .endif 567d1a84718SNicholas Piggin std r10,_DSISR(r1) 568d1a84718SNicholas Piggin .endif 569a3cd35beSNicholas Piggin 570931dc86bSNicholas PigginBEGIN_FTR_SECTION 571af47d79bSNicholas Piggin .if ICFAR || ICFAR_IF_HVMODE 5726d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 573af47d79bSNicholas Piggin .else 574af47d79bSNicholas Piggin li r10,0 575af47d79bSNicholas Piggin .endif 5768c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 577931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 5788729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 5798c9fb5d4SNicholas Piggin std r10,_CTR(r1) 5808c9fb5d4SNicholas Piggin std r2,GPR2(r1) /* save r2 in stackframe */ 581aebd1fb4SNicholas Piggin SAVE_GPRS(3, 8, r1) /* save r3 - r8 in stackframe */ 5828c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 5838c9fb5d4SNicholas Piggin ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 5848c9fb5d4SNicholas Piggin std r9,_LINK(r1) 5858c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5868c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 5878c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 5888c9fb5d4SNicholas Piggin std r11,_XER(r1) 5896cc0c16dSNicholas Piggin li r9,IVEC 5908c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 5918c9fb5d4SNicholas Piggin li r10,0 5928c9fb5d4SNicholas Piggin ld r11,exception_marker@toc(r2) 5938c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 5948c9fb5d4SNicholas Piggin std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 595bcbceed4SNicholas Piggin.endm 596bcbceed4SNicholas Piggin 597391e941bSNicholas Piggin/* 5988729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 5998729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 6008729c26eSNicholas Piggin * SRR1, and relocation is on. 6018729c26eSNicholas Piggin * 6028729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 6038729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 6048729c26eSNicholas Piggin */ 6058729c26eSNicholas Piggin.macro GEN_COMMON name 6068729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 6078729c26eSNicholas Piggin __GEN_COMMON_BODY \name 6088729c26eSNicholas Piggin.endm 6098729c26eSNicholas Piggin 610f23699c9SNicholas Piggin.macro SEARCH_RESTART_TABLE 611f23699c9SNicholas Piggin#ifdef CONFIG_RELOCATABLE 612f23699c9SNicholas Piggin mr r12,r2 613f23699c9SNicholas Piggin ld r2,PACATOC(r13) 614f23699c9SNicholas Piggin LOAD_REG_ADDR(r9, __start___restart_table) 615f23699c9SNicholas Piggin LOAD_REG_ADDR(r10, __stop___restart_table) 616f23699c9SNicholas Piggin mr r2,r12 617f23699c9SNicholas Piggin#else 618f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___restart_table) 619f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___restart_table) 620f23699c9SNicholas Piggin#endif 621f23699c9SNicholas Piggin300: 622f23699c9SNicholas Piggin cmpd r9,r10 623f23699c9SNicholas Piggin beq 302f 624f23699c9SNicholas Piggin ld r12,0(r9) 625f23699c9SNicholas Piggin cmpld r11,r12 626f23699c9SNicholas Piggin blt 301f 627f23699c9SNicholas Piggin ld r12,8(r9) 628f23699c9SNicholas Piggin cmpld r11,r12 629f23699c9SNicholas Piggin bge 301f 630f23699c9SNicholas Piggin ld r12,16(r9) 631f23699c9SNicholas Piggin b 303f 632f23699c9SNicholas Piggin301: 633f23699c9SNicholas Piggin addi r9,r9,24 634f23699c9SNicholas Piggin b 300b 635f23699c9SNicholas Piggin302: 636f23699c9SNicholas Piggin li r12,0 637f23699c9SNicholas Piggin303: 638f23699c9SNicholas Piggin.endm 639f23699c9SNicholas Piggin 640325678fdSNicholas Piggin.macro SEARCH_SOFT_MASK_TABLE 641325678fdSNicholas Piggin#ifdef CONFIG_RELOCATABLE 642325678fdSNicholas Piggin mr r12,r2 643325678fdSNicholas Piggin ld r2,PACATOC(r13) 644325678fdSNicholas Piggin LOAD_REG_ADDR(r9, __start___soft_mask_table) 645325678fdSNicholas Piggin LOAD_REG_ADDR(r10, __stop___soft_mask_table) 646325678fdSNicholas Piggin mr r2,r12 647325678fdSNicholas Piggin#else 648325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___soft_mask_table) 649325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___soft_mask_table) 650325678fdSNicholas Piggin#endif 651325678fdSNicholas Piggin300: 652325678fdSNicholas Piggin cmpd r9,r10 653325678fdSNicholas Piggin beq 302f 654325678fdSNicholas Piggin ld r12,0(r9) 655325678fdSNicholas Piggin cmpld r11,r12 656325678fdSNicholas Piggin blt 301f 657325678fdSNicholas Piggin ld r12,8(r9) 658325678fdSNicholas Piggin cmpld r11,r12 659325678fdSNicholas Piggin bge 301f 660325678fdSNicholas Piggin li r12,1 661325678fdSNicholas Piggin b 303f 662325678fdSNicholas Piggin301: 663325678fdSNicholas Piggin addi r9,r9,16 664325678fdSNicholas Piggin b 300b 665325678fdSNicholas Piggin302: 666325678fdSNicholas Piggin li r12,0 667325678fdSNicholas Piggin303: 668325678fdSNicholas Piggin.endm 669325678fdSNicholas Piggin 6708729c26eSNicholas Piggin/* 671391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 672391e941bSNicholas Piggin * standard exception. 673391e941bSNicholas Piggin */ 6743f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 675391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 676391e941bSNicholas Piggin ld r9,_MSR(r1) 67759dc5bfcSNicholas Piggin li r10,0 678391e941bSNicholas Piggin .if \hsrr 679391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 68059dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 681391e941bSNicholas Piggin .else 682391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 68359dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 684391e941bSNicholas Piggin .endif 685391e941bSNicholas Piggin ld r9,_NIP(r1) 686391e941bSNicholas Piggin .if \hsrr 687391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 688391e941bSNicholas Piggin .else 689391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 690391e941bSNicholas Piggin .endif 691391e941bSNicholas Piggin ld r9,_CTR(r1) 692391e941bSNicholas Piggin mtctr r9 693391e941bSNicholas Piggin ld r9,_XER(r1) 694391e941bSNicholas Piggin mtxer r9 695391e941bSNicholas Piggin ld r9,_LINK(r1) 696391e941bSNicholas Piggin mtlr r9 697391e941bSNicholas Piggin ld r9,_CCR(r1) 698391e941bSNicholas Piggin mtcr r9 699aebd1fb4SNicholas Piggin REST_GPRS(2, 13, r1) 700391e941bSNicholas Piggin REST_GPR(0, r1) 701391e941bSNicholas Piggin /* restore original r1. */ 702391e941bSNicholas Piggin ld r1,GPR1(r1) 703391e941bSNicholas Piggin.endm 704d064151fSNicholas Piggin 70512a04809SNicholas Piggin/* 70657f26649SNicholas Piggin * There are a few constraints to be concerned with. 70757f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 70857f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 70957f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 71057f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 71157f26649SNicholas Piggin * must be used. 71257f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 71357f26649SNicholas Piggin * virtual 0xc00... 71457f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 71557f26649SNicholas Piggin * 71657f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 71757f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 71857f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 71957f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 72057f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 72157f26649SNicholas Piggin * 72257f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 72357f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 72457f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 72557f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 72657f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 72757f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 72857f26649SNicholas Piggin * 7297fa95f9aSNicholas Piggin * The scv instructions are a special case. They get a 0x3000 offset applied. 7307fa95f9aSNicholas Piggin * scv exceptions have unique reentrancy properties, see below. 7317fa95f9aSNicholas Piggin * 73257f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 73357f26649SNicholas Piggin * 73457f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 73557f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 73657f26649SNicholas Piggin * 73757f26649SNicholas Piggin * 7380ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 7390ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 74057f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 7417fa95f9aSNicholas Piggin * 0x1900 - 0x2fff : Real mode trampolines 7427fa95f9aSNicholas Piggin * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 74357f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 7440ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 74557f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 74657f26649SNicholas Piggin * setup code, rest of kernel. 747e0319829SNicholas Piggin * 748e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 749e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 750e0319829SNicholas Piggin * vectors there. 7510ebc4cdaSBenjamin Herrenschmidt */ 75257f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 7537fa95f9aSNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000) 7547fa95f9aSNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900) 75557f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 756ccd47702SNicholas Piggin 757ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 758bd3524feSNicholas Piggin .globl start_real_trampolines 759bd3524feSNicholas Piggin .globl end_real_trampolines 760bd3524feSNicholas Piggin .globl start_virt_trampolines 761bd3524feSNicholas Piggin .globl end_virt_trampolines 762ccd47702SNicholas Piggin#endif 763ccd47702SNicholas Piggin 76457f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 76557f26649SNicholas Piggin/* 76657f26649SNicholas Piggin * Data area reserved for FWNMI option. 76757f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 76857f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 76957f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 77057f26649SNicholas Piggin */ 77157f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 77257f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 77357f26649SNicholas Piggin#else 77457f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 77557f26649SNicholas Piggin#endif 77657f26649SNicholas Piggin 77757f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 77857f26649SNicholas Piggin 7790ebc4cdaSBenjamin Herrenschmidt/* 7800ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 7810ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 7820ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 7830ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 7840ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 7850ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 7860ebc4cdaSBenjamin Herrenschmidt */ 7870ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 7880ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 7890ebc4cdaSBenjamin Herrenschmidt 7907fa95f9aSNicholas Piggin/** 7917fa95f9aSNicholas Piggin * Interrupt 0x3000 - System Call Vectored Interrupt (syscall). 7927fa95f9aSNicholas Piggin * This is a synchronous interrupt invoked with the "scv" instruction. The 7937fa95f9aSNicholas Piggin * system call does not alter the HV bit, so it is directed to the OS. 7947fa95f9aSNicholas Piggin * 7957fa95f9aSNicholas Piggin * Handling: 7967fa95f9aSNicholas Piggin * scv instructions enter the kernel without changing EE, RI, ME, or HV. 7977fa95f9aSNicholas Piggin * In particular, this means we can take a maskable interrupt at any point 7987fa95f9aSNicholas Piggin * in the scv handler, which is unlike any other interrupt. This is solved 799325678fdSNicholas Piggin * by treating the instruction addresses in the handler as being soft-masked, 800325678fdSNicholas Piggin * by adding a SOFT_MASK_TABLE entry for them. 8017fa95f9aSNicholas Piggin * 8027fa95f9aSNicholas Piggin * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and 8037fa95f9aSNicholas Piggin * ensure scv is never executed with relocation off, which means AIL-0 8047fa95f9aSNicholas Piggin * should never happen. 8057fa95f9aSNicholas Piggin * 8069d1988caSNicholas Piggin * Before leaving the following inside-__end_soft_masked text, at least of the 8079d1988caSNicholas Piggin * following must be true: 8087fa95f9aSNicholas Piggin * - MSR[PR]=1 (i.e., return to userspace) 8099d1988caSNicholas Piggin * - MSR_EE|MSR_RI is clear (no reentrant exceptions) 8107fa95f9aSNicholas Piggin * - Standard kernel environment is set up (stack, paca, etc) 8117fa95f9aSNicholas Piggin * 8127fa95f9aSNicholas Piggin * Call convention: 8137fa95f9aSNicholas Piggin * 8147fa95f9aSNicholas Piggin * syscall register convention is in Documentation/powerpc/syscall64-abi.rst 8157fa95f9aSNicholas Piggin */ 8167fa95f9aSNicholas PigginEXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000) 8177fa95f9aSNicholas Piggin /* SCV 0 */ 8187fa95f9aSNicholas Piggin mr r9,r13 8197fa95f9aSNicholas Piggin GET_PACA(r13) 8207fa95f9aSNicholas Piggin mflr r11 8217fa95f9aSNicholas Piggin mfctr r12 8227fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 8237fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 8247fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 8257fa95f9aSNicholas Piggin b system_call_vectored_tramp 8267fa95f9aSNicholas Piggin#else 8277fa95f9aSNicholas Piggin b system_call_vectored_common 8287fa95f9aSNicholas Piggin#endif 8297fa95f9aSNicholas Piggin nop 8307fa95f9aSNicholas Piggin 8317fa95f9aSNicholas Piggin /* SCV 1 - 127 */ 8327fa95f9aSNicholas Piggin .rept 127 8337fa95f9aSNicholas Piggin mr r9,r13 8347fa95f9aSNicholas Piggin GET_PACA(r13) 8357fa95f9aSNicholas Piggin mflr r11 8367fa95f9aSNicholas Piggin mfctr r12 8377fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 8387fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 8397fa95f9aSNicholas Piggin li r0,-1 /* cause failure */ 8407fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 8417fa95f9aSNicholas Piggin b system_call_vectored_sigill_tramp 8427fa95f9aSNicholas Piggin#else 8437fa95f9aSNicholas Piggin b system_call_vectored_sigill 8447fa95f9aSNicholas Piggin#endif 8457fa95f9aSNicholas Piggin .endr 8467fa95f9aSNicholas PigginEXC_VIRT_END(system_call_vectored, 0x3000, 0x1000) 8477fa95f9aSNicholas Piggin 848787c70f2SNicholas Piggin// Treat scv vectors as soft-masked, see comment above. 849787c70f2SNicholas Piggin// Use absolute values rather than labels here, so they don't get relocated, 850787c70f2SNicholas Piggin// because this code runs unrelocated. 851787c70f2SNicholas PigginSOFT_MASK_TABLE(0xc000000000003000, 0xc000000000004000) 852325678fdSNicholas Piggin 8537fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 8547fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_tramp) 855*d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_vectored_common, virt_trampolines) 8567fa95f9aSNicholas Piggin mtctr r10 8577fa95f9aSNicholas Piggin bctr 8587fa95f9aSNicholas Piggin 8597fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) 860*d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_vectored_sigill, virt_trampolines) 8617fa95f9aSNicholas Piggin mtctr r10 8627fa95f9aSNicholas Piggin bctr 8637fa95f9aSNicholas Piggin#endif 8647fa95f9aSNicholas Piggin 8657fa95f9aSNicholas Piggin 866e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 8671a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 868e0319829SNicholas Piggin 869fb479e44SNicholas Piggin 87094325357SNicholas Piggin/** 87194325357SNicholas Piggin * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 87294325357SNicholas Piggin * This is a non-maskable, asynchronous interrupt always taken in real-mode. 87394325357SNicholas Piggin * It is caused by: 87494325357SNicholas Piggin * - Wake from power-saving state, on powernv. 87594325357SNicholas Piggin * - An NMI from another CPU, triggered by firmware or hypercall. 87694325357SNicholas Piggin * - As crash/debug signal injected from BMC, firmware or hypervisor. 87794325357SNicholas Piggin * 87894325357SNicholas Piggin * Handling: 87994325357SNicholas Piggin * Power-save wakeup is the only performance critical path, so this is 88094325357SNicholas Piggin * determined quickly as possible first. In this case volatile registers 88194325357SNicholas Piggin * can be discarded and SPRs like CFAR don't need to be read. 88294325357SNicholas Piggin * 88394325357SNicholas Piggin * If not a powersave wakeup, then it's run as a regular interrupt, however 88494325357SNicholas Piggin * it uses its own stack and PACA save area to preserve the regular kernel 88594325357SNicholas Piggin * environment for debugging. 88694325357SNicholas Piggin * 88794325357SNicholas Piggin * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 88894325357SNicholas Piggin * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 88994325357SNicholas Piggin * correct to switch to virtual mode to run the regular interrupt handler 89094325357SNicholas Piggin * because it might be interrupted when the MMU is in a bad state (e.g., SLB 89194325357SNicholas Piggin * is clear). 89294325357SNicholas Piggin * 89394325357SNicholas Piggin * FWNMI: 89494325357SNicholas Piggin * PAPR specifies a "fwnmi" facility which sends the sreset to a different 89594325357SNicholas Piggin * entry point with a different register set up. Some hypervisors will 89694325357SNicholas Piggin * send the sreset to 0x100 in the guest if it is not fwnmi capable. 89794325357SNicholas Piggin * 89894325357SNicholas Piggin * KVM: 89994325357SNicholas Piggin * Unlike most SRR interrupts, this may be taken by the host while executing 90094325357SNicholas Piggin * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 90194325357SNicholas Piggin * mode and then raise the sreset. 90294325357SNicholas Piggin */ 9034f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 9044f50541fSNicholas Piggin IVEC=0x100 9054f50541fSNicholas Piggin IAREA=PACA_EXNMI 9068729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 9074f50541fSNicholas Piggin ISTACK=0 9084f50541fSNicholas Piggin IKVM_REAL=1 9094f50541fSNicholas PigginINT_DEFINE_END(system_reset) 9104f50541fSNicholas Piggin 911a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 912fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 913fb479e44SNicholas Piggin /* 914fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 915ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 916ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 917ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 918ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 9190e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 9200e10be2bSNicholas Piggin * 9210e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 9220e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 9230e10be2bSNicholas Piggin * 9240e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 9250e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 9260e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 9270e10be2bSNicholas Piggin * common exception macros to handle it. 928fb479e44SNicholas Piggin */ 929a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 9300e10be2bSNicholas Piggin SET_SCRATCH0(r13) 9310e10be2bSNicholas Piggin GET_PACA(r13) 9320e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 9330e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 9340e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 935a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 9360e10be2bSNicholas Piggin mfocrf r4,0x80 9370e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 9380e10be2bSNicholas Piggin bne+ system_reset_idle_wake 9390e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 9400e10be2bSNicholas Piggin mtocrf 0x80,r4 9410e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 9420e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 9430e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 9440e10be2bSNicholas Piggin GET_SCRATCH0(r13) 945a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 946fb479e44SNicholas Piggin#endif 947fb479e44SNicholas Piggin 9484f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 949c4f3b52cSNicholas Piggin /* 9500e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 9510e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 9520e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 9530e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 9540e10be2bSNicholas Piggin * be dangerous anyway. 955c4f3b52cSNicholas Piggin */ 9561a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 9571a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 958fb479e44SNicholas Piggin 959fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 9600e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 9610e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 9620e10be2bSNicholas Piggin cmpwi cr1,r5,2 9630e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 964*d72c4a36SDaniel Axtens __LOAD_FAR_HANDLER(r12, DOTSYM(idle_return_gpr_loss), real_trampolines) 965fd983957SAlexey Kardashevskiy mtctr r12 966fd983957SAlexey Kardashevskiy bctr 967371fefd6SPaul Mackerras#endif 968371fefd6SPaul Mackerras 969acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 970acc8da44SNicholas Piggin/* 971acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 972acc8da44SNicholas Piggin */ 973acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 9744f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 975acc8da44SNicholas Piggin 976acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 977acc8da44SNicholas Piggin 978a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 9798729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 980c4f3b52cSNicholas Piggin /* 981ff0b0d6eSNicholas Piggin * Increment paca->in_nmi. When the interrupt entry wrapper later 982ff0b0d6eSNicholas Piggin * enable MSR_RI, then SLB or MCE will be able to recover, but a nested 983ff0b0d6eSNicholas Piggin * NMI will notice in_nmi and not recover because of the use of the NMI 984ff0b0d6eSNicholas Piggin * stack. in_nmi reentrancy is tested in system_reset_exception. 985c4f3b52cSNicholas Piggin */ 986c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 987c4f3b52cSNicholas Piggin addi r10,r10,1 988c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 989aca79d2bSVaidyanathan Srinivasan 990b1ee8a3dSNicholas Piggin mr r10,r1 991b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 992b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 9938729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 99447169fbaSNicholas Piggin 995c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 996c06075f3SNicholas Piggin bl system_reset_exception 99715b4dd79SNicholas Piggin 99815b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 999fbc50063SNicholas Piggin li r9,0 100015b4dd79SNicholas Piggin mtmsrd r9,1 1001c4f3b52cSNicholas Piggin 1002c4f3b52cSNicholas Piggin /* 100315b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 1004c4f3b52cSNicholas Piggin */ 1005c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 1006c4f3b52cSNicholas Piggin subi r10,r10,1 1007c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 1008c4f3b52cSNicholas Piggin 10098e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 10103f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 101115b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 1012582baf44SNicholas Piggin 10130ebc4cdaSBenjamin Herrenschmidt 101494325357SNicholas Piggin/** 101594325357SNicholas Piggin * Interrupt 0x200 - Machine Check Interrupt (MCE). 101694325357SNicholas Piggin * This is a non-maskable interrupt always taken in real-mode. It can be 101794325357SNicholas Piggin * synchronous or asynchronous, caused by hardware or software, and it may be 101894325357SNicholas Piggin * taken in a power-saving state. 101994325357SNicholas Piggin * 102094325357SNicholas Piggin * Handling: 102194325357SNicholas Piggin * Similarly to system reset, this uses its own stack and PACA save area, 102294325357SNicholas Piggin * the difference is re-entrancy is allowed on the machine check stack. 102394325357SNicholas Piggin * 102494325357SNicholas Piggin * machine_check_early is run in real mode, and carefully decodes the 102594325357SNicholas Piggin * machine check and tries to handle it (e.g., flush the SLB if there was an 102694325357SNicholas Piggin * error detected there), determines if it was recoverable and logs the 102794325357SNicholas Piggin * event. 102894325357SNicholas Piggin * 1029b44fc96dSNicholas Piggin * This early code does not "reconcile" irq soft-mask state like SRESET or 1030b44fc96dSNicholas Piggin * regular interrupts do, so irqs_disabled() among other things may not work 1031b44fc96dSNicholas Piggin * properly (irq disable/enable already doesn't work because irq tracing can 1032b44fc96dSNicholas Piggin * not work in real mode). 1033b44fc96dSNicholas Piggin * 103494325357SNicholas Piggin * Then, depending on the execution context when the interrupt is taken, there 103594325357SNicholas Piggin * are 3 main actions: 103694325357SNicholas Piggin * - Executing in kernel mode. The event is queued with irq_work, which means 103794325357SNicholas Piggin * it is handled when it is next safe to do so (i.e., the kernel has enabled 103894325357SNicholas Piggin * interrupts), which could be immediately when the interrupt returns. This 103994325357SNicholas Piggin * avoids nasty issues like switching to virtual mode when the MMU is in a 104094325357SNicholas Piggin * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 104194325357SNicholas Piggin * but it has different priorities). Check to see if the CPU was in power 104294325357SNicholas Piggin * save, and return via the wake up code if it was. 104394325357SNicholas Piggin * 104494325357SNicholas Piggin * - Executing in user mode. machine_check_exception is run like a normal 104594325357SNicholas Piggin * interrupt handler, which processes the data generated by the early handler. 104694325357SNicholas Piggin * 104794325357SNicholas Piggin * - Executing in guest mode. The interrupt is run with its KVM test, and 104894325357SNicholas Piggin * branches to KVM to deal with. KVM may queue the event for the host 104994325357SNicholas Piggin * to report later. 105094325357SNicholas Piggin * 105194325357SNicholas Piggin * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 105294325357SNicholas Piggin * or SCRATCH0 is in use, it may cause a crash. 105394325357SNicholas Piggin * 105494325357SNicholas Piggin * KVM: 105594325357SNicholas Piggin * See SRESET. 105694325357SNicholas Piggin */ 10574f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 10584f50541fSNicholas Piggin IVEC=0x200 10594f50541fSNicholas Piggin IAREA=PACA_EXMC 10608729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 1061d73a10cbSNicholas Piggin IREALMODE_COMMON=1 10624f50541fSNicholas Piggin ISTACK=0 10634f50541fSNicholas Piggin IDAR=1 10644f50541fSNicholas Piggin IDSISR=1 10654f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 10664f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 10674f50541fSNicholas Piggin 10684f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 10694f50541fSNicholas Piggin IVEC=0x200 10704f50541fSNicholas Piggin IAREA=PACA_EXMC 10718729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 10724f50541fSNicholas Piggin IDAR=1 10734f50541fSNicholas Piggin IDSISR=1 10744f50541fSNicholas Piggin IKVM_REAL=1 10754f50541fSNicholas PigginINT_DEFINE_END(machine_check) 10764f50541fSNicholas Piggin 10774f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 10784f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 10791a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 10801a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 1081c8eb54dbSNicholas Piggin 1082abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1083abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 1084abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 10854f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 1086abd1f4caSNicholas Piggin#endif 1087abd1f4caSNicholas Piggin 1088fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 1089fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1090fce16d48SNicholas Piggin li r9,0; \ 1091fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 1092fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 1093fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 1094fce16d48SNicholas Piggin subi r12,r12,1; \ 1095fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 10963f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 1097fce16d48SNicholas Piggin 1098c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 10999600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 11009600f261SNicholas Piggin 1101afcf0095SNicholas Piggin /* 1102afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 1103afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 1104afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1105afcf0095SNicholas Piggin * 1106afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 1107afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 1108afcf0095SNicholas Piggin * machine checks. 1109afcf0095SNicholas Piggin * 1110afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 1111afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 1112afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 1113afcf0095SNicholas Piggin * 1114afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1115afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 1116afcf0095SNicholas Piggin * rfid with MSR_ME=1. 11171945bc45SNicholas Piggin * 11181945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 11191945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 11202bf1071aSNicholas Piggin * to restore state. 1121afcf0095SNicholas Piggin */ 1122afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 1123afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 1124c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1125afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 1126afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 1127c8eb54dbSNicholas Piggin 1128c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 1129c8eb54dbSNicholas Piggin bne 1f 1130c8eb54dbSNicholas Piggin /* First machine check entry */ 1131c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1132b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1133b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1134b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1135c8eb54dbSNicholas Piggin 11368729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1137c8eb54dbSNicholas Piggin 1138db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1139296e753fSNicholas Piggin bl enable_machine_check 1140db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1141afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1142afcf0095SNicholas Piggin bl machine_check_early 1143afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1144afcf0095SNicholas Piggin ld r12,_MSR(r1) 11451945bc45SNicholas Piggin 1146afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1147afcf0095SNicholas Piggin /* 1148afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1149afcf0095SNicholas Piggin * of the following is true: 1150afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1151afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1152afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1153afcf0095SNicholas Piggin * 1154afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1155afcf0095SNicholas Piggin */ 11561945bc45SNicholas PigginBEGIN_FTR_SECTION 11571945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 11586102c005SNicholas Piggin bne machine_check_idle_common 11591945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1160afcf0095SNicholas Piggin#endif 11611945bc45SNicholas Piggin 1162afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1163afcf0095SNicholas Piggin /* 116419dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 116505f97d94SNicholas Piggin * exception handler which will take the 116669fdd674SNicholas Piggin * machine_check_kvm->kvm_interrupt branch to deliver the MC event 116705f97d94SNicholas Piggin * to guest. 1168afcf0095SNicholas Piggin */ 1169afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1170afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1171b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1172afcf0095SNicholas Piggin#endif 117319dbe673SNicholas Piggin 1174afcf0095SNicholas Piggin /* 117519dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 117619dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 117719dbe673SNicholas Piggin */ 117819dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1179b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 118019dbe673SNicholas Piggin 118119dbe673SNicholas Piggin /* 118219dbe673SNicholas Piggin * At this point we are coming from kernel context. 1183afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1184afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1185afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1186afcf0095SNicholas Piggin */ 1187afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1188b7d9ccecSNicholas Piggin beq unrecoverable_mce 1189b7d9ccecSNicholas Piggin 1190afcf0095SNicholas Piggin /* 1191afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1192afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1193afcf0095SNicholas Piggin */ 1194afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1195afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1196b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1197272f6364SNicholas Piggin 1198afcf0095SNicholas Piggin /* 1199afcf0095SNicholas Piggin * Return from MC interrupt. 1200afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1201afcf0095SNicholas Piggin * returning from kernel or opal call. 1202afcf0095SNicholas Piggin */ 1203afcf0095SNicholas Piggin bl machine_check_queue_event 1204afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1205fe9d482bSNicholas Piggin RFI_TO_KERNEL 1206272f6364SNicholas Piggin 1207b3fe3526SNicholas Pigginmce_deliver: 1208b3fe3526SNicholas Piggin /* 1209b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1210b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1211b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1212b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1213b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1214b3fe3526SNicholas Piggin */ 12150b66370cSNicholas PigginBEGIN_FTR_SECTION 12160b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 12170b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 12180b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1219afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 12204f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1221afcf0095SNicholas Piggin 1222fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1223fce16d48SNicholas Piggin /* 1224fce16d48SNicholas Piggin * Machine check is different because we use a different 1225fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1226fce16d48SNicholas Piggin */ 12274f50541fSNicholas Piggin GEN_COMMON machine_check 1228fce16d48SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1229f08fb25bSNicholas Piggin bl machine_check_exception_async 12301df7d5e4SNicholas Piggin b interrupt_return_srr 1231fce16d48SNicholas Piggin 12329600f261SNicholas Piggin 1233fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1234fce16d48SNicholas Piggin/* 1235fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1236fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1237fce16d48SNicholas Piggin */ 1238fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 1239fce16d48SNicholas Piggin bl machine_check_queue_event 1240fce16d48SNicholas Piggin 1241fce16d48SNicholas Piggin /* 12428a5054d8SNicholas Piggin * GPR-loss wakeups are relatively straightforward, because the 12438a5054d8SNicholas Piggin * idle sleep code has saved all non-volatile registers on its 12448a5054d8SNicholas Piggin * own stack, and r1 in PACAR1. 1245fce16d48SNicholas Piggin * 12468a5054d8SNicholas Piggin * For no-loss wakeups the r1 and lr registers used by the 12478a5054d8SNicholas Piggin * early machine check handler have to be restored first. r2 is 12488a5054d8SNicholas Piggin * the kernel TOC, so no need to restore it. 1249fce16d48SNicholas Piggin * 1250fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1251fce16d48SNicholas Piggin */ 1252fce16d48SNicholas Piggin ld r3,_MSR(r1) 1253fce16d48SNicholas Piggin ld r4,_LINK(r1) 12548a5054d8SNicholas Piggin ld r1,GPR1(r1) 1255fce16d48SNicholas Piggin 1256fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1257fce16d48SNicholas Piggin subi r11,r11,1 1258fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1259fce16d48SNicholas Piggin 1260fce16d48SNicholas Piggin mtlr r4 1261fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1262fce16d48SNicholas Piggin cmpwi cr1,r10,2 12638a5054d8SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1264fce16d48SNicholas Piggin b idle_return_gpr_loss 1265fce16d48SNicholas Piggin#endif 1266fce16d48SNicholas Piggin 1267b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1268b7d9ccecSNicholas Piggin /* 1269b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1270b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1271b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1272b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1273b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1274b7d9ccecSNicholas Piggin */ 1275b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1276b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1277b7d9ccecSNicholas Piggin mtmsrd r10,1 1278b7d9ccecSNicholas Piggin bl disable_machine_check 1279b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1280b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1281b7d9ccecSNicholas Piggin li r3,MSR_ME 1282b7d9ccecSNicholas Piggin andc r10,r10,r3 1283b7d9ccecSNicholas Piggin mtmsrd r10 1284b7d9ccecSNicholas Piggin 1285ac2a2a14SNicholas Piggin lhz r12,PACA_IN_MCE(r13) 1286ac2a2a14SNicholas Piggin subi r12,r12,1 1287ac2a2a14SNicholas Piggin sth r12,PACA_IN_MCE(r13) 1288ac2a2a14SNicholas Piggin 1289f08fb25bSNicholas Piggin /* 1290f08fb25bSNicholas Piggin * Invoke machine_check_exception to print MCE event and panic. 1291f08fb25bSNicholas Piggin * This is the NMI version of the handler because we are called from 1292f08fb25bSNicholas Piggin * the early handler which is a true NMI. 1293f08fb25bSNicholas Piggin */ 1294afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1295afcf0095SNicholas Piggin bl machine_check_exception 1296b7d9ccecSNicholas Piggin 1297afcf0095SNicholas Piggin /* 1298b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1299b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1300afcf0095SNicholas Piggin */ 1301b7d9ccecSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1302afcf0095SNicholas Piggin bl unrecoverable_exception 1303b7d9ccecSNicholas Piggin b . 1304afcf0095SNicholas Piggin 13054f50541fSNicholas Piggin 13064f50541fSNicholas Piggin/** 130794325357SNicholas Piggin * Interrupt 0x300 - Data Storage Interrupt (DSI). 130894325357SNicholas Piggin * This is a synchronous interrupt generated due to a data access exception, 130994325357SNicholas Piggin * e.g., a load orstore which does not have a valid page table entry with 131094325357SNicholas Piggin * permissions. DAWR matches also fault here, as do RC updates, and minor misc 131194325357SNicholas Piggin * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 13124f50541fSNicholas Piggin * 131394325357SNicholas Piggin * Handling: 131494325357SNicholas Piggin * - Hash MMU 1315a4922f54SNicholas Piggin * Go to do_hash_fault, which attempts to fill the HPT from an entry in the 1316a4922f54SNicholas Piggin * Linux page table. Hash faults can hit in kernel mode in a fairly 131794325357SNicholas Piggin * arbitrary state (e.g., interrupts disabled, locks held) when accessing 131894325357SNicholas Piggin * "non-bolted" regions, e.g., vmalloc space. However these should always be 1319a4922f54SNicholas Piggin * backed by Linux page table entries. 13204f50541fSNicholas Piggin * 1321a4922f54SNicholas Piggin * If no entry is found the Linux page fault handler is invoked (by 1322a4922f54SNicholas Piggin * do_hash_fault). Linux page faults can happen in kernel mode due to user 1323a4922f54SNicholas Piggin * copy operations of course. 13244f50541fSNicholas Piggin * 1325cd81acc6SNicholas Piggin * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest 1326cd81acc6SNicholas Piggin * MMU context, which may cause a DSI in the host, which must go to the 1327cd81acc6SNicholas Piggin * KVM handler. MSR[IR] is not enabled, so the real-mode handler will 1328cd81acc6SNicholas Piggin * always be used regardless of AIL setting. 1329cd81acc6SNicholas Piggin * 133094325357SNicholas Piggin * - Radix MMU 133194325357SNicholas Piggin * The hardware loads from the Linux page table directly, so a fault goes 133294325357SNicholas Piggin * immediately to Linux page fault. 133394325357SNicholas Piggin * 133494325357SNicholas Piggin * Conditions like DAWR match are handled on the way in to Linux page fault. 13354f50541fSNicholas Piggin */ 1336a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1337a42a239dSNicholas Piggin IVEC=0x300 1338a42a239dSNicholas Piggin IDAR=1 1339a42a239dSNicholas Piggin IDSISR=1 1340a42a239dSNicholas Piggin IKVM_REAL=1 1341a42a239dSNicholas PigginINT_DEFINE_END(data_access) 13420ebc4cdaSBenjamin Herrenschmidt 1343e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1344689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1345e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1346e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1347a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1348e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 134980795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 13507cb3a1a0SNicholas Piggin GEN_COMMON data_access 1351a01a3f2dSNicholas Piggin ld r4,_DSISR(r1) 1352a4922f54SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1353a01a3f2dSNicholas Piggin andis. r0,r4,DSISR_DABRMATCH@h 135436f01141SNicholas Piggin bne- 1f 1355387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 135680795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 1357a4922f54SNicholas Piggin bl do_hash_fault 135880795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 1359a4922f54SNicholas Piggin bl do_page_fault 136080795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1361387e220aSNicholas Piggin#else 1362387e220aSNicholas Piggin bl do_page_fault 1363387e220aSNicholas Piggin#endif 13641df7d5e4SNicholas Piggin b interrupt_return_srr 1365a4922f54SNicholas Piggin 1366a4922f54SNicholas Piggin1: bl do_break 136736f01141SNicholas Piggin /* 136836f01141SNicholas Piggin * do_break() may have changed the NV GPRS while handling a breakpoint. 136936f01141SNicholas Piggin * If so, we need to restore them with their updated values. 137036f01141SNicholas Piggin */ 137136f01141SNicholas Piggin REST_NVGPRS(r1) 13721df7d5e4SNicholas Piggin b interrupt_return_srr 137380795e6cSNicholas Piggin 13740ebc4cdaSBenjamin Herrenschmidt 137594325357SNicholas Piggin/** 137694325357SNicholas Piggin * Interrupt 0x380 - Data Segment Interrupt (DSLB). 137794325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault missing SLB 137894325357SNicholas Piggin * entry for HPT, or an address outside RPT translation range. 137994325357SNicholas Piggin * 138094325357SNicholas Piggin * Handling: 138194325357SNicholas Piggin * - HPT: 138294325357SNicholas Piggin * This refills the SLB, or reports an access fault similarly to a bad page 138394325357SNicholas Piggin * fault. When coming from user-mode, the SLB handler may access any kernel 138494325357SNicholas Piggin * data, though it may itself take a DSLB. When coming from kernel mode, 138594325357SNicholas Piggin * recursive faults must be avoided so access is restricted to the kernel 138694325357SNicholas Piggin * image text/data, kernel stack, and any data allocated below 138794325357SNicholas Piggin * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 138894325357SNicholas Piggin * on user-handler data structures. 138994325357SNicholas Piggin * 1390cd81acc6SNicholas Piggin * KVM: Same as 0x300, DSLB must test for KVM guest. 139194325357SNicholas Piggin */ 13924f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 13934f50541fSNicholas Piggin IVEC=0x380 13944f50541fSNicholas Piggin IDAR=1 13954f50541fSNicholas Piggin IKVM_REAL=1 13964f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 13974f50541fSNicholas Piggin 13981a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1399689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 14001a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 14011a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 14024f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 14031a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 140448e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 14054f50541fSNicholas Piggin GEN_COMMON data_access_slb 1406387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 14077100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14087100e870SNicholas Piggin /* HPT case, do SLB fault */ 1409a01a3f2dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 141048e7b769SNicholas Piggin bl do_slb_fault 141148e7b769SNicholas Piggin cmpdi r3,0 141248e7b769SNicholas Piggin bne- 1f 14131df7d5e4SNicholas Piggin b fast_interrupt_return_srr 141448e7b769SNicholas Piggin1: /* Error case */ 14157100e870SNicholas PigginMMU_FTR_SECTION_ELSE 14167100e870SNicholas Piggin /* Radix case, access is outside page table range */ 14177100e870SNicholas Piggin li r3,-EFAULT 14187100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1419387e220aSNicholas Piggin#else 1420387e220aSNicholas Piggin li r3,-EFAULT 1421387e220aSNicholas Piggin#endif 142248e7b769SNicholas Piggin std r3,RESULT(r1) 142348e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1424935b534cSNicholas Piggin bl do_bad_segment_interrupt 14251df7d5e4SNicholas Piggin b interrupt_return_srr 142648e7b769SNicholas Piggin 14272b9af6e4SNicholas Piggin 142894325357SNicholas Piggin/** 142994325357SNicholas Piggin * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 143094325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 143194325357SNicholas Piggin * instruction fetch. 143294325357SNicholas Piggin * 143394325357SNicholas Piggin * Handling: 143494325357SNicholas Piggin * Similar to DSI, though in response to fetch. The faulting address is found 143594325357SNicholas Piggin * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 143694325357SNicholas Piggin */ 14374f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 14384f50541fSNicholas Piggin IVEC=0x400 1439a3cd35beSNicholas Piggin IISIDE=1 1440a3cd35beSNicholas Piggin IDAR=1 1441a3cd35beSNicholas Piggin IDSISR=1 14422284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14434f50541fSNicholas Piggin IKVM_REAL=1 14442284ffeaSNicholas Piggin#endif 14454f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 14464f50541fSNicholas Piggin 14477299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 14484f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 14497299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 14507299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 14514f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 14527299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 145327ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 14544f50541fSNicholas Piggin GEN_COMMON instruction_access 1455a4922f54SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1456387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 145727ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 1458a4922f54SNicholas Piggin bl do_hash_fault 145927ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 1460a4922f54SNicholas Piggin bl do_page_fault 146127ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1462387e220aSNicholas Piggin#else 1463387e220aSNicholas Piggin bl do_page_fault 1464387e220aSNicholas Piggin#endif 14651df7d5e4SNicholas Piggin b interrupt_return_srr 146627ce77dfSNicholas Piggin 14670ebc4cdaSBenjamin Herrenschmidt 146894325357SNicholas Piggin/** 146994325357SNicholas Piggin * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 147094325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 147194325357SNicholas Piggin * instruction fetch. 147294325357SNicholas Piggin * 147394325357SNicholas Piggin * Handling: 147494325357SNicholas Piggin * Similar to DSLB, though in response to fetch. The faulting address is found 147594325357SNicholas Piggin * in SRR0 (rather than DAR). 147694325357SNicholas Piggin */ 14774f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 14784f50541fSNicholas Piggin IVEC=0x480 1479a3cd35beSNicholas Piggin IISIDE=1 1480a3cd35beSNicholas Piggin IDAR=1 14812284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14824f50541fSNicholas Piggin IKVM_REAL=1 14832284ffeaSNicholas Piggin#endif 14844f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 14854f50541fSNicholas Piggin 14867299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 14874f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 14887299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 14897299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 14904f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 14917299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 149248e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 14934f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 1494387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 14957100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14967100e870SNicholas Piggin /* HPT case, do SLB fault */ 1497a01a3f2dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 149848e7b769SNicholas Piggin bl do_slb_fault 149948e7b769SNicholas Piggin cmpdi r3,0 150048e7b769SNicholas Piggin bne- 1f 15011df7d5e4SNicholas Piggin b fast_interrupt_return_srr 150248e7b769SNicholas Piggin1: /* Error case */ 15037100e870SNicholas PigginMMU_FTR_SECTION_ELSE 15047100e870SNicholas Piggin /* Radix case, access is outside page table range */ 15057100e870SNicholas Piggin li r3,-EFAULT 15067100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1507387e220aSNicholas Piggin#else 1508387e220aSNicholas Piggin li r3,-EFAULT 1509387e220aSNicholas Piggin#endif 151048e7b769SNicholas Piggin std r3,RESULT(r1) 151148e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1512935b534cSNicholas Piggin bl do_bad_segment_interrupt 15131df7d5e4SNicholas Piggin b interrupt_return_srr 15145e46e29eSNicholas Piggin 15159600f261SNicholas Piggin 151694325357SNicholas Piggin/** 151794325357SNicholas Piggin * Interrupt 0x500 - External Interrupt. 151894325357SNicholas Piggin * This is an asynchronous maskable interrupt in response to an "external 151994325357SNicholas Piggin * exception" from the interrupt controller or hypervisor (e.g., device 152094325357SNicholas Piggin * interrupt). It is maskable in hardware by clearing MSR[EE], and 152194325357SNicholas Piggin * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 152294325357SNicholas Piggin * 152394325357SNicholas Piggin * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 152494325357SNicholas Piggin * interrupts are delivered with HSRR registers, guests use SRRs, which 152594325357SNicholas Piggin * reqiures IHSRR_IF_HVMODE. 152694325357SNicholas Piggin * 152794325357SNicholas Piggin * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 152894325357SNicholas Piggin * external interrupts are delivered as Hypervisor Virtualization Interrupts 152994325357SNicholas Piggin * rather than External Interrupts. 153094325357SNicholas Piggin * 153194325357SNicholas Piggin * Handling: 153294325357SNicholas Piggin * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 153394325357SNicholas Piggin * because registers at the time of the interrupt are not so important as it is 153494325357SNicholas Piggin * asynchronous. 153594325357SNicholas Piggin * 153694325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 153794325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 1538af47d79bSNicholas Piggin * 1539af47d79bSNicholas Piggin * CFAR is not required because this is an asynchronous interrupt that in 1540af47d79bSNicholas Piggin * general won't have much bearing on the state of the CPU, with the possible 1541af47d79bSNicholas Piggin * exception of crash/debug IPIs, but those are generally moving to use SRESET 1542af47d79bSNicholas Piggin * IPIs. Unless this is an HV interrupt and KVM HV is possible, in which case 1543af47d79bSNicholas Piggin * it may be exiting the guest and need CFAR to be saved. 154494325357SNicholas Piggin */ 15454f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 15464f50541fSNicholas Piggin IVEC=0x500 15473f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 15484f50541fSNicholas Piggin IMASK=IRQS_DISABLED 15494f50541fSNicholas Piggin IKVM_REAL=1 15504f50541fSNicholas Piggin IKVM_VIRT=1 1551af47d79bSNicholas Piggin ICFAR=0 1552af47d79bSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1553af47d79bSNicholas Piggin ICFAR_IF_HVMODE=1 1554af47d79bSNicholas Piggin#endif 15554f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 15564f50541fSNicholas Piggin 15571a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 15584f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 15591a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 15601a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 15614f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 15621a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1563eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 15644f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1565eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1566eb204d86SNicholas Piggin bl do_IRQ 15671df7d5e4SNicholas Piggin BEGIN_FTR_SECTION 15681df7d5e4SNicholas Piggin b interrupt_return_hsrr 15691df7d5e4SNicholas Piggin FTR_SECTION_ELSE 15701df7d5e4SNicholas Piggin b interrupt_return_srr 15711df7d5e4SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1572c138e588SNicholas Piggin 1573c138e588SNicholas Piggin 157494325357SNicholas Piggin/** 157594325357SNicholas Piggin * Interrupt 0x600 - Alignment Interrupt 157694325357SNicholas Piggin * This is a synchronous interrupt in response to data alignment fault. 157794325357SNicholas Piggin */ 15784f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 15794f50541fSNicholas Piggin IVEC=0x600 15804f50541fSNicholas Piggin IDAR=1 15814f50541fSNicholas Piggin IDSISR=1 15822284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15834f50541fSNicholas Piggin IKVM_REAL=1 15842284ffeaSNicholas Piggin#endif 15854f50541fSNicholas PigginINT_DEFINE_END(alignment) 15864f50541fSNicholas Piggin 1587e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 15884f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1589e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1590e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 15914f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1592e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1593f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 15944f50541fSNicholas Piggin GEN_COMMON alignment 1595f9aa6714SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1596f9aa6714SNicholas Piggin bl alignment_exception 1597702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 15981df7d5e4SNicholas Piggin b interrupt_return_srr 1599f9aa6714SNicholas Piggin 1600b01c8b54SPaul Mackerras 160194325357SNicholas Piggin/** 160294325357SNicholas Piggin * Interrupt 0x700 - Program Interrupt (program check). 160394325357SNicholas Piggin * This is a synchronous interrupt in response to various instruction faults: 160494325357SNicholas Piggin * traps, privilege errors, TM errors, floating point exceptions. 160594325357SNicholas Piggin * 160694325357SNicholas Piggin * Handling: 160794325357SNicholas Piggin * This interrupt may use the "emergency stack" in some cases when being taken 160894325357SNicholas Piggin * from kernel context, which complicates handling. 160994325357SNicholas Piggin */ 16104f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 16114f50541fSNicholas Piggin IVEC=0x700 16122284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16134f50541fSNicholas Piggin IKVM_REAL=1 16142284ffeaSNicholas Piggin#endif 16154f50541fSNicholas PigginINT_DEFINE_END(program_check) 16164f50541fSNicholas Piggin 16177299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 1618e7eb9190SMichael Ellerman 1619e7eb9190SMichael Ellerman#ifdef CONFIG_CPU_LITTLE_ENDIAN 1620e7eb9190SMichael Ellerman /* 1621e7eb9190SMichael Ellerman * There's a short window during boot where although the kernel is 1622e7eb9190SMichael Ellerman * running little endian, any exceptions will cause the CPU to switch 1623e7eb9190SMichael Ellerman * back to big endian. For example a WARN() boils down to a trap 1624e7eb9190SMichael Ellerman * instruction, which will cause a program check, and we end up here but 1625e7eb9190SMichael Ellerman * with the CPU in big endian mode. The first instruction of the program 1626e7eb9190SMichael Ellerman * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when 1627e7eb9190SMichael Ellerman * executed in the wrong endian is an lhzu with a ~3GB displacement from 1628e7eb9190SMichael Ellerman * r3. The content of r3 is random, so that is a load from some random 1629e7eb9190SMichael Ellerman * location, and depending on the system can easily lead to a checkstop, 1630e7eb9190SMichael Ellerman * or an infinitely recursive page fault. 1631e7eb9190SMichael Ellerman * 1632e7eb9190SMichael Ellerman * So to handle that case we have a trampoline here that can detect we 1633e7eb9190SMichael Ellerman * are in the wrong endian and flip us back to the correct endian. We 1634e7eb9190SMichael Ellerman * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires 1635e7eb9190SMichael Ellerman * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as 1636e7eb9190SMichael Ellerman * SPRG1 is already used for the paca. SPRG3 is user readable, but this 1637e7eb9190SMichael Ellerman * trampoline is only active very early in boot, and SPRG3 will be 1638e7eb9190SMichael Ellerman * reinitialised in vdso_getcpu_init() before userspace starts. 1639e7eb9190SMichael Ellerman */ 1640e7eb9190SMichael EllermanBEGIN_FTR_SECTION 1641e7eb9190SMichael Ellerman tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 1642e7eb9190SMichael Ellerman b 1f // Skip trampoline if endian is correct 1643e7eb9190SMichael Ellerman .long 0xa643707d // mtsprg 0, r11 Backup r11 1644e7eb9190SMichael Ellerman .long 0xa6027a7d // mfsrr0 r11 1645e7eb9190SMichael Ellerman .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 1646e7eb9190SMichael Ellerman .long 0xa6027b7d // mfsrr1 r11 1647e7eb9190SMichael Ellerman .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 1648e7eb9190SMichael Ellerman .long 0xa600607d // mfmsr r11 1649e7eb9190SMichael Ellerman .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] 1650e7eb9190SMichael Ellerman .long 0xa6037b7d // mtsrr1 r11 1651e7eb9190SMichael Ellerman .long 0x34076039 // li r11, 0x734 1652e7eb9190SMichael Ellerman .long 0xa6037a7d // mtsrr0 r11 1653e7eb9190SMichael Ellerman .long 0x2400004c // rfid 1654e7eb9190SMichael Ellerman mfsprg r11, 3 1655e7eb9190SMichael Ellerman mtsrr1 r11 // Restore SRR1 1656e7eb9190SMichael Ellerman mfsprg r11, 2 1657e7eb9190SMichael Ellerman mtsrr0 r11 // Restore SRR0 1658e7eb9190SMichael Ellerman mfsprg r11, 0 // Restore r11 1659e7eb9190SMichael Ellerman1: 1660e7eb9190SMichael EllermanEND_FTR_SECTION(0, 1) // nop out after boot 1661e7eb9190SMichael Ellerman#endif /* CONFIG_CPU_LITTLE_ENDIAN */ 1662e7eb9190SMichael Ellerman 16634f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 16647299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 16657299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 16664f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 16677299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 166811e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 16698729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 16708729c26eSNicholas Piggin 1671265e60a1SCyril Bur /* 1672265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1673265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1674265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1675265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1676265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1677265e60a1SCyril Bur * the kernel. 1678265e60a1SCyril Bur */ 1679265e60a1SCyril Bur 16800a882e28SNicholas Piggin andi. r10,r12,MSR_PR 16813e607dc4SNicholas Piggin bne .Lnormal_stack /* If userspace, go normal path */ 16820a882e28SNicholas Piggin 16830a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 16843e607dc4SNicholas Piggin bne .Lemergency_stack /* If TM, emergency */ 16850a882e28SNicholas Piggin 16860a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 16873e607dc4SNicholas Piggin blt .Lnormal_stack /* normal path if not */ 16880a882e28SNicholas Piggin 16890a882e28SNicholas Piggin /* Use the emergency stack */ 16903e607dc4SNicholas Piggin.Lemergency_stack: 16913e607dc4SNicholas Piggin andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1692265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1693265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1694265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1695265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 16964f50541fSNicholas Piggin __ISTACK(program_check)=0 16978729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 16983e607dc4SNicholas Piggin b .Ldo_program_check 16993e607dc4SNicholas Piggin 17003e607dc4SNicholas Piggin.Lnormal_stack: 17014f50541fSNicholas Piggin __ISTACK(program_check)=1 17028729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 17033e607dc4SNicholas Piggin 17043e607dc4SNicholas Piggin.Ldo_program_check: 170511e87346SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 170611e87346SNicholas Piggin bl program_check_exception 1707702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 17081df7d5e4SNicholas Piggin b interrupt_return_srr 170911e87346SNicholas Piggin 1710a485c709SPaul Mackerras 171194325357SNicholas Piggin/* 171294325357SNicholas Piggin * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 171394325357SNicholas Piggin * This is a synchronous interrupt in response to executing an fp instruction 171494325357SNicholas Piggin * with MSR[FP]=0. 171594325357SNicholas Piggin * 171694325357SNicholas Piggin * Handling: 171794325357SNicholas Piggin * This will load FP registers and enable the FP bit if coming from userspace, 171894325357SNicholas Piggin * otherwise report a bad kernel use of FP. 171994325357SNicholas Piggin */ 17204f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 17214f50541fSNicholas Piggin IVEC=0x800 17222284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17234f50541fSNicholas Piggin IKVM_REAL=1 17242284ffeaSNicholas Piggin#endif 17254f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 17264f50541fSNicholas Piggin 17277299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 17284f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 17297299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 17307299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 17314f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 17327299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1733c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 17344f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1735c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1736c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1737c78d9b97SNicholas Piggin bl kernel_fp_unavailable_exception 173863ce271bSChristophe Leroy0: trap 173963ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1740c78d9b97SNicholas Piggin1: 1741c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1742c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1743c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1744c78d9b97SNicholas Piggin * transaction), go do TM stuff 1745c78d9b97SNicholas Piggin */ 1746c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1747c78d9b97SNicholas Piggin bne- 2f 1748c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1749c78d9b97SNicholas Piggin#endif 1750c78d9b97SNicholas Piggin bl load_up_fpu 17511df7d5e4SNicholas Piggin b fast_interrupt_return_srr 1752c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1753c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1754c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1755c78d9b97SNicholas Piggin bl fp_unavailable_tm 17561df7d5e4SNicholas Piggin b interrupt_return_srr 1757c78d9b97SNicholas Piggin#endif 1758c78d9b97SNicholas Piggin 1759b01c8b54SPaul Mackerras 176094325357SNicholas Piggin/** 176194325357SNicholas Piggin * Interrupt 0x900 - Decrementer Interrupt. 176294325357SNicholas Piggin * This is an asynchronous interrupt in response to a decrementer exception 176394325357SNicholas Piggin * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 176494325357SNicholas Piggin * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 176594325357SNicholas Piggin * local_irq_disable()). 176694325357SNicholas Piggin * 176794325357SNicholas Piggin * Handling: 176894325357SNicholas Piggin * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 176994325357SNicholas Piggin * 177094325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 177194325357SNicholas Piggin * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 177294325357SNicholas Piggin * in the interrupted context. 177394325357SNicholas Piggin * If PPC_WATCHDOG is configured, the soft masked handler will actually set 177494325357SNicholas Piggin * things back up to run soft_nmi_interrupt as a regular interrupt handler 177594325357SNicholas Piggin * on the emergency stack. 1776af47d79bSNicholas Piggin * 1777af47d79bSNicholas Piggin * CFAR is not required because this is asynchronous (see hardware_interrupt). 1778af47d79bSNicholas Piggin * A watchdog interrupt may like to have CFAR, but usually the interesting 1779af47d79bSNicholas Piggin * branch is long gone by that point (e.g., infinite loop). 178094325357SNicholas Piggin */ 17814f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 17824f50541fSNicholas Piggin IVEC=0x900 17834f50541fSNicholas Piggin IMASK=IRQS_DISABLED 17842284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17854f50541fSNicholas Piggin IKVM_REAL=1 17862284ffeaSNicholas Piggin#endif 1787af47d79bSNicholas Piggin ICFAR=0 17884f50541fSNicholas PigginINT_DEFINE_END(decrementer) 17894f50541fSNicholas Piggin 17907299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1791689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 17927299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 17937299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 17944f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 17957299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1796eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 17974f50541fSNicholas Piggin GEN_COMMON decrementer 1798eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1799eb204d86SNicholas Piggin bl timer_interrupt 18001df7d5e4SNicholas Piggin b interrupt_return_srr 180139c0da57SNicholas Piggin 18020ebc4cdaSBenjamin Herrenschmidt 180394325357SNicholas Piggin/** 180494325357SNicholas Piggin * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 180594325357SNicholas Piggin * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 180694325357SNicholas Piggin * register. 180794325357SNicholas Piggin * 180894325357SNicholas Piggin * Handling: 180994325357SNicholas Piggin * Linux does not use this outside KVM where it's used to keep a host timer 181094325357SNicholas Piggin * while the guest is given control of DEC. It should normally be caught by 181194325357SNicholas Piggin * the KVM test and routed there. 181294325357SNicholas Piggin */ 18134f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 18144f50541fSNicholas Piggin IVEC=0x980 18153f7fbd97SNicholas Piggin IHSRR=1 18162babd6eaSNicholas Piggin ISTACK=0 18174f50541fSNicholas Piggin IKVM_REAL=1 18184f50541fSNicholas Piggin IKVM_VIRT=1 18194f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 18204f50541fSNicholas Piggin 18217299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 18224f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 18237299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 18247299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 18254f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 18267299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1827eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 18282babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 18292babd6eaSNicholas Piggin /* 18302babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 18312babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 18322babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 18332babd6eaSNicholas Piggin * edge-triggered. 18342babd6eaSNicholas Piggin * 18352babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 18362babd6eaSNicholas Piggin */ 183759dc5bfcSNicholas Piggin li r10,0 183859dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 18392babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 18402babd6eaSNicholas Piggin mtctr r10 18412babd6eaSNicholas Piggin mtcrf 0x80,r9 18422babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 18432babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 18442babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 18452babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 18462babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 18472babd6eaSNicholas Piggin HRFI_TO_KERNEL 1848facc6d74SNicholas Piggin 1849da2bc464SMichael Ellerman 185094325357SNicholas Piggin/** 185194325357SNicholas Piggin * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 185294325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsndp doorbell. 185394325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 185494325357SNicholas Piggin * IRQS_DISABLED mask (i.e., local_irq_disable()). 185594325357SNicholas Piggin * 185694325357SNicholas Piggin * Handling: 185794325357SNicholas Piggin * Guests may use this for IPIs between threads in a core if the 185894325357SNicholas Piggin * hypervisor supports it. NVGPRS are not saved (see 0x500). 185994325357SNicholas Piggin * 186094325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 186194325357SNicholas Piggin * replay, leaving MSR[EE] enabled in the interrupted context because the 186294325357SNicholas Piggin * doorbells are edge triggered. 1863af47d79bSNicholas Piggin * 1864af47d79bSNicholas Piggin * CFAR is not required, similarly to hardware_interrupt. 186594325357SNicholas Piggin */ 18664f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 18674f50541fSNicholas Piggin IVEC=0xa00 18684f50541fSNicholas Piggin IMASK=IRQS_DISABLED 18692284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 18704f50541fSNicholas Piggin IKVM_REAL=1 18712284ffeaSNicholas Piggin#endif 1872af47d79bSNicholas Piggin ICFAR=0 18734f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 18744f50541fSNicholas Piggin 18757299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 18764f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 18777299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 18787299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 18794f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 18807299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1881eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 18824f50541fSNicholas Piggin GEN_COMMON doorbell_super 1883eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1884ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1885eb204d86SNicholas Piggin bl doorbell_exception 1886ca243163SNicholas Piggin#else 18876c6aee00SNicholas Piggin bl unknown_async_exception 1888ca243163SNicholas Piggin#endif 18891df7d5e4SNicholas Piggin b interrupt_return_srr 1890ca243163SNicholas Piggin 1891da2bc464SMichael Ellerman 18925ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 18935ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1894341215dcSNicholas Piggin 189594325357SNicholas Piggin/** 189694325357SNicholas Piggin * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 189794325357SNicholas Piggin * This is a synchronous interrupt invoked with the "sc" instruction. The 189894325357SNicholas Piggin * system call is invoked with "sc 0" and does not alter the HV bit, so it 189994325357SNicholas Piggin * is directed to the currently running OS. The hypercall is invoked with 190094325357SNicholas Piggin * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1901acd7d8ceSNicholas Piggin * 1902acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1903acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1904acd7d8ceSNicholas Piggin * 190594325357SNicholas Piggin * Handling: 190694325357SNicholas Piggin * If the KVM test fires then it was due to a hypercall and is accordingly 190794325357SNicholas Piggin * routed to KVM. Otherwise this executes a normal Linux system call. 190894325357SNicholas Piggin * 1909acd7d8ceSNicholas Piggin * Call convention: 1910acd7d8ceSNicholas Piggin * 191158b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 191258b278f5SVaibhav Jain * Documentation/powerpc/syscall64-abi.rst and 191358b278f5SVaibhav Jain * Documentation/powerpc/papr_hcalls.rst respectively. 1914acd7d8ceSNicholas Piggin * 1915acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 191676fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 191776fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 191876fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1919acd7d8ceSNicholas Piggin */ 1920b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1921b177ae2fSNicholas Piggin IVEC=0xc00 1922b177ae2fSNicholas Piggin IKVM_REAL=1 1923b177ae2fSNicholas Piggin IKVM_VIRT=1 1924af47d79bSNicholas Piggin ICFAR=0 1925b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1926b177ae2fSNicholas Piggin 19271b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1928bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1929bc355125SPaul Mackerras /* 1930acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 193176fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 193276fc0cfcSNicholas Piggin * register. 1933acd7d8ceSNicholas Piggin * 1934acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1935acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1936bc355125SPaul Mackerras */ 19371b4d4a79SNicholas Piggin mtctr r13 19381b4d4a79SNicholas Piggin GET_PACA(r13) 19391b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 19401b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 194169fdd674SNicholas Piggin KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */ 19421b4d4a79SNicholas Piggin mfctr r9 1943bc355125SPaul Mackerras#else 19441b4d4a79SNicholas Piggin mr r9,r13 19451b4d4a79SNicholas Piggin GET_PACA(r13) 19461b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1947bc355125SPaul Mackerras#endif 1948bc355125SPaul Mackerras 1949727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 19501b4d4a79SNicholas PigginBEGIN_FTR_SECTION 19511b4d4a79SNicholas Piggin cmpdi r0,0x1ebe 19521b4d4a79SNicholas Piggin beq- 1f 19531b4d4a79SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 19541b4d4a79SNicholas Piggin#endif 19555c2511bfSMichael Ellerman 1956b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 19571b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 19581b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 1959b0b2a93dSNicholas Piggin 1960b0b2a93dSNicholas Piggin HMT_MEDIUM 1961b0b2a93dSNicholas Piggin 1962b0b2a93dSNicholas Piggin .if ! \virt 1963*d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_common_real, real_vectors) 196414ad0e7dSNicholas Piggin mtctr r10 196514ad0e7dSNicholas Piggin bctr 19661b4d4a79SNicholas Piggin .else 1967b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 1968*d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_common, virt_vectors) 1969b0b2a93dSNicholas Piggin mtctr r10 1970b0b2a93dSNicholas Piggin bctr 1971b0b2a93dSNicholas Piggin#else 19721b4d4a79SNicholas Piggin b system_call_common 1973d807ad37SNicholas Piggin#endif 19741b4d4a79SNicholas Piggin .endif 19751b4d4a79SNicholas Piggin 19761b4d4a79SNicholas Piggin#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 19771b4d4a79SNicholas Piggin /* Fast LE/BE switch system call */ 19781b4d4a79SNicholas Piggin1: mfspr r12,SPRN_SRR1 19791b4d4a79SNicholas Piggin xori r12,r12,MSR_LE 19801b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r12 19811b4d4a79SNicholas Piggin mr r13,r9 19821b4d4a79SNicholas Piggin RFI_TO_USER /* return to userspace */ 19831b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 19841b4d4a79SNicholas Piggin#endif 19851b4d4a79SNicholas Piggin.endm 1986d807ad37SNicholas Piggin 19871a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 19881b4d4a79SNicholas Piggin SYSTEM_CALL 0 19891a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 19901a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 19911b4d4a79SNicholas Piggin SYSTEM_CALL 1 19921a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 1993d807ad37SNicholas Piggin 1994acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 199569fdd674SNicholas PigginTRAMP_REAL_BEGIN(kvm_hcall) 1996e2762743SNicholas Piggin std r9,PACA_EXGEN+EX_R9(r13) 1997e2762743SNicholas Piggin std r11,PACA_EXGEN+EX_R11(r13) 1998e2762743SNicholas Piggin std r12,PACA_EXGEN+EX_R12(r13) 1999e2762743SNicholas Piggin mfcr r9 2000acd7d8ceSNicholas Piggin mfctr r10 2001e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_R13(r13) 2002e2762743SNicholas Piggin li r10,0 2003e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CFAR(r13) 2004e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CTR(r13) 2005e2762743SNicholas Piggin /* 2006e2762743SNicholas Piggin * Save the PPR (on systems that support it) before changing to 2007e2762743SNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 2008e2762743SNicholas Piggin * guest state (it is the guest's PPR value). 2009e2762743SNicholas Piggin */ 2010e2762743SNicholas PigginBEGIN_FTR_SECTION 2011e2762743SNicholas Piggin mfspr r10,SPRN_PPR 2012e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_PPR(r13) 2013e2762743SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2014e2762743SNicholas Piggin 2015e2762743SNicholas Piggin HMT_MEDIUM 2016e2762743SNicholas Piggin 20179600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 20189600f261SNicholas Piggin /* 201931c67cfeSNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives 20209600f261SNicholas Piggin * outside the head section. 20219600f261SNicholas Piggin */ 2022*d72c4a36SDaniel Axtens __LOAD_FAR_HANDLER(r10, kvmppc_hcall, real_trampolines) 20239600f261SNicholas Piggin mtctr r10 20249600f261SNicholas Piggin bctr 20259600f261SNicholas Piggin#else 202631c67cfeSNicholas Piggin b kvmppc_hcall 20279600f261SNicholas Piggin#endif 2028acd7d8ceSNicholas Piggin#endif 2029da2bc464SMichael Ellerman 203094325357SNicholas Piggin/** 203194325357SNicholas Piggin * Interrupt 0xd00 - Trace Interrupt. 203294325357SNicholas Piggin * This is a synchronous interrupt in response to instruction step or 203394325357SNicholas Piggin * breakpoint faults. 203494325357SNicholas Piggin */ 20354f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 20364f50541fSNicholas Piggin IVEC=0xd00 20372284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 20384f50541fSNicholas Piggin IKVM_REAL=1 20392284ffeaSNicholas Piggin#endif 20404f50541fSNicholas PigginINT_DEFINE_END(single_step) 20414f50541fSNicholas Piggin 20427299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 20434f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 20447299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 20457299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 20464f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 20477299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 2048eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 20494f50541fSNicholas Piggin GEN_COMMON single_step 2050eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2051eb204d86SNicholas Piggin bl single_step_exception 20521df7d5e4SNicholas Piggin b interrupt_return_srr 2053da2bc464SMichael Ellerman 20547299417cSNicholas Piggin 205594325357SNicholas Piggin/** 205694325357SNicholas Piggin * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 205794325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 205894325357SNicholas Piggin * guest data access. 205994325357SNicholas Piggin * 206094325357SNicholas Piggin * Handling: 206194325357SNicholas Piggin * This should always get routed to KVM. In radix MMU mode, this is caused 206294325357SNicholas Piggin * by a guest nested radix access that can't be performed due to the 206394325357SNicholas Piggin * partition scope page table. In hash mode, this can be caused by guests 206494325357SNicholas Piggin * running with translation disabled (virtual real mode) or with VPM enabled. 206594325357SNicholas Piggin * KVM will update the page table structures or disallow the access. 206694325357SNicholas Piggin */ 20674f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 20684f50541fSNicholas Piggin IVEC=0xe00 20693f7fbd97SNicholas Piggin IHSRR=1 20704f50541fSNicholas Piggin IDAR=1 20714f50541fSNicholas Piggin IDSISR=1 20724f50541fSNicholas Piggin IKVM_REAL=1 20734f50541fSNicholas Piggin IKVM_VIRT=1 20744f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 20754f50541fSNicholas Piggin 20767299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 20774f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 20787299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 20797299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 20804f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 20817299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2082f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 20834f50541fSNicholas Piggin GEN_COMMON h_data_storage 2084f5c32c1dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2085d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 208671f47976SNicholas Piggin bl do_bad_page_fault_segv 2087d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 2088f5c32c1dSNicholas Piggin bl unknown_exception 2089d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 20901df7d5e4SNicholas Piggin b interrupt_return_hsrr 2091f5c32c1dSNicholas Piggin 20921707dd16SPaul Mackerras 209394325357SNicholas Piggin/** 209494325357SNicholas Piggin * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 209594325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 209694325357SNicholas Piggin * guest instruction fetch, similar to HDSI. 209794325357SNicholas Piggin */ 20984f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 20994f50541fSNicholas Piggin IVEC=0xe20 21003f7fbd97SNicholas Piggin IHSRR=1 21014f50541fSNicholas Piggin IKVM_REAL=1 21024f50541fSNicholas Piggin IKVM_VIRT=1 21034f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 21044f50541fSNicholas Piggin 21057299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 21064f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 21077299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 21087299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 21094f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 21107299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2111eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 21124f50541fSNicholas Piggin GEN_COMMON h_instr_storage 2113eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2114eb204d86SNicholas Piggin bl unknown_exception 21151df7d5e4SNicholas Piggin b interrupt_return_hsrr 211682517cabSNicholas Piggin 21171707dd16SPaul Mackerras 211894325357SNicholas Piggin/** 211994325357SNicholas Piggin * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 212094325357SNicholas Piggin */ 21214f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 21224f50541fSNicholas Piggin IVEC=0xe40 21233f7fbd97SNicholas Piggin IHSRR=1 21244f50541fSNicholas Piggin IKVM_REAL=1 21254f50541fSNicholas Piggin IKVM_VIRT=1 21264f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 21274f50541fSNicholas Piggin 21287299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 21294f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 21307299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 21317299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 21324f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 21337299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2134eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 21354f50541fSNicholas Piggin GEN_COMMON emulation_assist 2136eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2137eb204d86SNicholas Piggin bl emulation_assist_interrupt 2138702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 21391df7d5e4SNicholas Piggin b interrupt_return_hsrr 2140031b4026SNicholas Piggin 21411707dd16SPaul Mackerras 214294325357SNicholas Piggin/** 214394325357SNicholas Piggin * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 214494325357SNicholas Piggin * This is an asynchronous interrupt caused by a Hypervisor Maintenance 214594325357SNicholas Piggin * Exception. It is always taken in real mode but uses HSRR registers 214694325357SNicholas Piggin * unlike SRESET and MCE. 214794325357SNicholas Piggin * 214894325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 214994325357SNicholas Piggin * with IRQS_DISABLED mask (i.e., local_irq_disable()). 215094325357SNicholas Piggin * 215194325357SNicholas Piggin * Handling: 215294325357SNicholas Piggin * This is a special case, this is handled similarly to machine checks, with an 215394325357SNicholas Piggin * initial real mode handler that is not soft-masked, which attempts to fix the 215494325357SNicholas Piggin * problem. Then a regular handler which is soft-maskable and reports the 215594325357SNicholas Piggin * problem. 215694325357SNicholas Piggin * 215794325357SNicholas Piggin * The emergency stack is used for the early real mode handler. 215894325357SNicholas Piggin * 215994325357SNicholas Piggin * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 216094325357SNicholas Piggin * either use soft-masking for the MCE, or use irq_work for the HMI. 216194325357SNicholas Piggin * 216294325357SNicholas Piggin * KVM: 216394325357SNicholas Piggin * Unlike MCE, this calls into KVM without calling the real mode handler 216494325357SNicholas Piggin * first. 2165e0319829SNicholas Piggin */ 21664f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 21674f50541fSNicholas Piggin IVEC=0xe60 21683f7fbd97SNicholas Piggin IHSRR=1 2169d73a10cbSNicholas Piggin IREALMODE_COMMON=1 21704f50541fSNicholas Piggin ISTACK=0 21714f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 21724f50541fSNicholas Piggin IKVM_REAL=1 21734f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 21744f50541fSNicholas Piggin 21754f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 21764f50541fSNicholas Piggin IVEC=0xe60 21773f7fbd97SNicholas Piggin IHSRR=1 21784f50541fSNicholas Piggin IMASK=IRQS_DISABLED 21794f50541fSNicholas Piggin IKVM_REAL=1 21804f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 21814f50541fSNicholas Piggin 2182f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 21834f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2184f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 21851a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 21864f50541fSNicholas Piggin 2187293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 21889600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 21899600f261SNicholas Piggin 219062f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 2191a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 219262f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2193bcbceed4SNicholas Piggin 21948729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 2195bcbceed4SNicholas Piggin 219662f9b03bSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2197293c2e27SNicholas Piggin bl hmi_exception_realmode 21985080332cSMichael Neuling cmpdi cr0,r3,0 219967d4160aSNicholas Piggin bne 1f 22005080332cSMichael Neuling 22013f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 2202222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 22035080332cSMichael Neuling 220467d4160aSNicholas Piggin1: 220562f9b03bSNicholas Piggin /* 220662f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 220762f9b03bSNicholas Piggin * firmware. 220862f9b03bSNicholas Piggin */ 22093f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 22104f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 221162f9b03bSNicholas Piggin 22125080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 22134f50541fSNicholas Piggin GEN_COMMON hmi_exception 2214c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2215c06075f3SNicholas Piggin bl handle_hmi_exception 22161df7d5e4SNicholas Piggin b interrupt_return_hsrr 22171707dd16SPaul Mackerras 22187299417cSNicholas Piggin 221994325357SNicholas Piggin/** 222094325357SNicholas Piggin * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 222194325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsnd doorbell. 222294325357SNicholas Piggin * Similar to the 0xa00 doorbell but for host rather than guest. 2223af47d79bSNicholas Piggin * 2224af47d79bSNicholas Piggin * CFAR is not required (similar to doorbell_interrupt), unless KVM HV 2225af47d79bSNicholas Piggin * is enabled, in which case it may be a guest exit. Most PowerNV kernels 2226af47d79bSNicholas Piggin * include KVM support so it would be nice if this could be dynamically 2227af47d79bSNicholas Piggin * patched out if KVM was not currently running any guests. 222894325357SNicholas Piggin */ 22294f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 22304f50541fSNicholas Piggin IVEC=0xe80 22313f7fbd97SNicholas Piggin IHSRR=1 22324f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22334f50541fSNicholas Piggin IKVM_REAL=1 22344f50541fSNicholas Piggin IKVM_VIRT=1 2235af47d79bSNicholas Piggin#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2236af47d79bSNicholas Piggin ICFAR=0 2237af47d79bSNicholas Piggin#endif 22384f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 22394f50541fSNicholas Piggin 22407299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 22414f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 22427299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 22437299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 22444f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 22457299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2246eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 22474f50541fSNicholas Piggin GEN_COMMON h_doorbell 2248eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 22499bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 2250eb204d86SNicholas Piggin bl doorbell_exception 22519bcb81bfSNicholas Piggin#else 22526c6aee00SNicholas Piggin bl unknown_async_exception 22539bcb81bfSNicholas Piggin#endif 22541df7d5e4SNicholas Piggin b interrupt_return_hsrr 22559bcb81bfSNicholas Piggin 22560ebc4cdaSBenjamin Herrenschmidt 225794325357SNicholas Piggin/** 225894325357SNicholas Piggin * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 225994325357SNicholas Piggin * This is an asynchronous interrupt in response to an "external exception". 226094325357SNicholas Piggin * Similar to 0x500 but for host only. 2261af47d79bSNicholas Piggin * 2262af47d79bSNicholas Piggin * Like h_doorbell, CFAR is only required for KVM HV because this can be 2263af47d79bSNicholas Piggin * a guest exit. 226494325357SNicholas Piggin */ 22654f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 22664f50541fSNicholas Piggin IVEC=0xea0 22673f7fbd97SNicholas Piggin IHSRR=1 22684f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22694f50541fSNicholas Piggin IKVM_REAL=1 22704f50541fSNicholas Piggin IKVM_VIRT=1 2271af47d79bSNicholas Piggin#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2272af47d79bSNicholas Piggin ICFAR=0 2273af47d79bSNicholas Piggin#endif 22744f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 22754f50541fSNicholas Piggin 22767299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 22774f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 22787299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 22797299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 22804f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 22817299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2282eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 22834f50541fSNicholas Piggin GEN_COMMON h_virt_irq 2284eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2285eb204d86SNicholas Piggin bl do_IRQ 22861df7d5e4SNicholas Piggin b interrupt_return_hsrr 228774408776SNicholas Piggin 22889baaef0aSBenjamin Herrenschmidt 22891a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 22901a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 22911a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 22921a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 2293bda7fea2SNicholas Piggin 22940ebc4cdaSBenjamin Herrenschmidt 229594325357SNicholas Piggin/* 229694325357SNicholas Piggin * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 229794325357SNicholas Piggin * This is an asynchronous interrupt in response to a PMU exception. 229894325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 229994325357SNicholas Piggin * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 230094325357SNicholas Piggin * 230194325357SNicholas Piggin * Handling: 230294325357SNicholas Piggin * This calls into the perf subsystem. 230394325357SNicholas Piggin * 230494325357SNicholas Piggin * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 230594325357SNicholas Piggin * runs under local_irq_disable. However it may be soft-masked in 230694325357SNicholas Piggin * powerpc-specific code. 230794325357SNicholas Piggin * 230894325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 230994325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 2310af47d79bSNicholas Piggin * 2311af47d79bSNicholas Piggin * CFAR is not used by perf interrupts so not required. 231294325357SNicholas Piggin */ 23134f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 23144f50541fSNicholas Piggin IVEC=0xf00 23154f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 23162284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23174f50541fSNicholas Piggin IKVM_REAL=1 23182284ffeaSNicholas Piggin#endif 2319af47d79bSNicholas Piggin ICFAR=0 23204f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 23214f50541fSNicholas Piggin 23227299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 23234f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 23247299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 23257299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 23264f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 23277299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2328eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 23294f50541fSNicholas Piggin GEN_COMMON performance_monitor 2330eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2331eb204d86SNicholas Piggin bl performance_monitor_exception 23321df7d5e4SNicholas Piggin b interrupt_return_srr 2333b1c7f150SNicholas Piggin 23340ebc4cdaSBenjamin Herrenschmidt 233594325357SNicholas Piggin/** 233694325357SNicholas Piggin * Interrupt 0xf20 - Vector Unavailable Interrupt. 233794325357SNicholas Piggin * This is a synchronous interrupt in response to 233894325357SNicholas Piggin * executing a vector (or altivec) instruction with MSR[VEC]=0. 233994325357SNicholas Piggin * Similar to FP unavailable. 234094325357SNicholas Piggin */ 23414f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 23424f50541fSNicholas Piggin IVEC=0xf20 23432284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23444f50541fSNicholas Piggin IKVM_REAL=1 23452284ffeaSNicholas Piggin#endif 23464f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 23474f50541fSNicholas Piggin 23487299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 23494f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 23507299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 23517299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 23524f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 23537299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2354d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 23554f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 2356d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 2357d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 2358d1a0ca9cSNicholas Piggin beq 1f 2359d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2360d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2361d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2362d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 2363d1a0ca9cSNicholas Piggin */ 2364d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2365d1a0ca9cSNicholas Piggin bne- 2f 2366d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2367d1a0ca9cSNicholas Piggin#endif 2368d1a0ca9cSNicholas Piggin bl load_up_altivec 23691df7d5e4SNicholas Piggin b fast_interrupt_return_srr 2370d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2371d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 2372d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2373d1a0ca9cSNicholas Piggin bl altivec_unavailable_tm 23741df7d5e4SNicholas Piggin b interrupt_return_srr 2375d1a0ca9cSNicholas Piggin#endif 2376d1a0ca9cSNicholas Piggin1: 2377d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2378d1a0ca9cSNicholas Piggin#endif 2379d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2380d1a0ca9cSNicholas Piggin bl altivec_unavailable_exception 23811df7d5e4SNicholas Piggin b interrupt_return_srr 2382d1a0ca9cSNicholas Piggin 23830ebc4cdaSBenjamin Herrenschmidt 238494325357SNicholas Piggin/** 238594325357SNicholas Piggin * Interrupt 0xf40 - VSX Unavailable Interrupt. 238694325357SNicholas Piggin * This is a synchronous interrupt in response to 238794325357SNicholas Piggin * executing a VSX instruction with MSR[VSX]=0. 238894325357SNicholas Piggin * Similar to FP unavailable. 238994325357SNicholas Piggin */ 23904f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 23914f50541fSNicholas Piggin IVEC=0xf40 23922284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23934f50541fSNicholas Piggin IKVM_REAL=1 23942284ffeaSNicholas Piggin#endif 23954f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 23964f50541fSNicholas Piggin 23977299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 23984f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 23997299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 24007299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 24014f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 24027299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2403792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 24044f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2405792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2406792cbdddSNicholas PigginBEGIN_FTR_SECTION 2407792cbdddSNicholas Piggin beq 1f 2408792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2409792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2410792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2411792cbdddSNicholas Piggin * transaction), go do TM stuff 2412792cbdddSNicholas Piggin */ 2413792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2414792cbdddSNicholas Piggin bne- 2f 2415792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2416792cbdddSNicholas Piggin#endif 2417792cbdddSNicholas Piggin b load_up_vsx 2418792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2419792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2420792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2421792cbdddSNicholas Piggin bl vsx_unavailable_tm 24221df7d5e4SNicholas Piggin b interrupt_return_srr 2423792cbdddSNicholas Piggin#endif 2424792cbdddSNicholas Piggin1: 2425792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2426792cbdddSNicholas Piggin#endif 2427792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2428792cbdddSNicholas Piggin bl vsx_unavailable_exception 24291df7d5e4SNicholas Piggin b interrupt_return_srr 2430792cbdddSNicholas Piggin 2431d0c0c9a1SMichael Neuling 243294325357SNicholas Piggin/** 243394325357SNicholas Piggin * Interrupt 0xf60 - Facility Unavailable Interrupt. 243494325357SNicholas Piggin * This is a synchronous interrupt in response to 243594325357SNicholas Piggin * executing an instruction without access to the facility that can be 243694325357SNicholas Piggin * resolved by the OS (e.g., FSCR, MSR). 243794325357SNicholas Piggin * Similar to FP unavailable. 243894325357SNicholas Piggin */ 24394f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 24404f50541fSNicholas Piggin IVEC=0xf60 24412284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24424f50541fSNicholas Piggin IKVM_REAL=1 24432284ffeaSNicholas Piggin#endif 24444f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 24454f50541fSNicholas Piggin 24467299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 24474f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 24487299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 24497299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 24504f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 24517299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2452eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 24534f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2454eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2455eb204d86SNicholas Piggin bl facility_unavailable_exception 2456595d153dSMichael Ellerman REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 24571df7d5e4SNicholas Piggin b interrupt_return_srr 24581134713cSNicholas Piggin 2459da2bc464SMichael Ellerman 246094325357SNicholas Piggin/** 246194325357SNicholas Piggin * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 246294325357SNicholas Piggin * This is a synchronous interrupt in response to 246394325357SNicholas Piggin * executing an instruction without access to the facility that can only 246494325357SNicholas Piggin * be resolved in HV mode (e.g., HFSCR). 246594325357SNicholas Piggin * Similar to FP unavailable. 246694325357SNicholas Piggin */ 24674f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 24684f50541fSNicholas Piggin IVEC=0xf80 24693f7fbd97SNicholas Piggin IHSRR=1 24704f50541fSNicholas Piggin IKVM_REAL=1 24714f50541fSNicholas Piggin IKVM_VIRT=1 24724f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 24734f50541fSNicholas Piggin 24747299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 24754f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 24767299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 24777299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 24784f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 24797299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2480eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 24814f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2482eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2483eb204d86SNicholas Piggin bl facility_unavailable_exception 2484595d153dSMichael Ellerman REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */ 24851df7d5e4SNicholas Piggin b interrupt_return_hsrr 248614b0072cSNicholas Piggin 2487da2bc464SMichael Ellerman 24881a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 24891a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 24901a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 24911a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 24921a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 24931a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 24941a6822d1SNicholas Piggin 24951a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 24961a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 24971a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 24981a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2499da2bc464SMichael Ellerman 25000ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 25014f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 25024f50541fSNicholas Piggin IVEC=0x1200 25033f7fbd97SNicholas Piggin IHSRR=1 25044f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 25054f50541fSNicholas Piggin 25067299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 25074f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 25087299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 25091a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2510eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 25114f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2512eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2513eb204d86SNicholas Piggin bl cbe_system_error_exception 25141df7d5e4SNicholas Piggin b interrupt_return_hsrr 25159600f261SNicholas Piggin 2516da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 25171a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 25181a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2519da2bc464SMichael Ellerman#endif 2520da2bc464SMichael Ellerman 2521da487a5dSNicholas Piggin/** 2522da487a5dSNicholas Piggin * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. 2523da487a5dSNicholas Piggin * This has been removed from the ISA before 2.01, which is the earliest 2524da487a5dSNicholas Piggin * 64-bit BookS ISA supported, however the G5 / 970 implements this 2525da487a5dSNicholas Piggin * interrupt with a non-architected feature available through the support 2526da487a5dSNicholas Piggin * processor interface. 2527da487a5dSNicholas Piggin */ 25284f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 25294f50541fSNicholas Piggin IVEC=0x1300 25302284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 25314f50541fSNicholas Piggin IKVM_REAL=1 25322284ffeaSNicholas Piggin#endif 25334f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 25344f50541fSNicholas Piggin 25357299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 25364f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 25377299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 25387299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 25394f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 25407299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2541eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 25424f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2543eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2544eb204d86SNicholas Piggin bl instruction_breakpoint_exception 25451df7d5e4SNicholas Piggin b interrupt_return_srr 25464e96dbbfSNicholas Piggin 25477299417cSNicholas Piggin 25481a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 25491a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2550da2bc464SMichael Ellerman 255194325357SNicholas Piggin/** 255294325357SNicholas Piggin * Interrupt 0x1500 - Soft Patch Interrupt 255394325357SNicholas Piggin * 255494325357SNicholas Piggin * Handling: 255594325357SNicholas Piggin * This is an implementation specific interrupt which can be used for a 255694325357SNicholas Piggin * range of exceptions. 255794325357SNicholas Piggin * 255894325357SNicholas Piggin * This interrupt handler is unique in that it runs the denormal assist 255994325357SNicholas Piggin * code even for guests (and even in guest context) without going to KVM, 256094325357SNicholas Piggin * for speed. POWER9 does not raise denorm exceptions, so this special case 256194325357SNicholas Piggin * could be phased out in future to reduce special cases. 256294325357SNicholas Piggin */ 25634f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 25644f50541fSNicholas Piggin IVEC=0x1500 25653f7fbd97SNicholas Piggin IHSRR=1 25664557ac6bSNicholas Piggin IBRANCH_TO_COMMON=0 25679600f261SNicholas Piggin IKVM_REAL=1 25684f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 25694f50541fSNicholas Piggin 25704f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 25714f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2572b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2573d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2574b92a66a6SMichael Neuling bne+ denorm_assist 2575b92a66a6SMichael Neuling#endif 25768729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 25774f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2578d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 25791a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 25804f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2581d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 258252b98923SNicholas Piggin bne+ denorm_assist 25838729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 25841a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2585d7e89849SNicholas Piggin#else 25861a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2587d7e89849SNicholas Piggin#endif 2588b92a66a6SMichael Neuling 2589b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2590da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2591b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2592b92a66a6SMichael Neuling/* 2593b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2594b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2595b92a66a6SMichael Neuling */ 2596b92a66a6SMichael Neuling mfmsr r10 2597b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2598b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2599b92a66a6SMichael Neuling mtmsrd r10 2600b92a66a6SMichael Neuling sync 2601d7c67fb1SMichael Neuling 2602f3c8b6c6SNicholas Piggin .Lreg=0 2603f3c8b6c6SNicholas Piggin .rept 32 2604f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2605f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2606f3c8b6c6SNicholas Piggin .endr 2607d7c67fb1SMichael Neuling 2608b92a66a6SMichael NeulingFTR_SECTION_ELSE 2609b92a66a6SMichael Neuling/* 2610b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2611b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2612b92a66a6SMichael Neuling */ 2613b92a66a6SMichael Neuling mfmsr r10 2614b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2615b92a66a6SMichael Neuling mtmsrd r10 2616b92a66a6SMichael Neuling sync 2617d7c67fb1SMichael Neuling 2618f3c8b6c6SNicholas Piggin .Lreg=0 2619f3c8b6c6SNicholas Piggin .rept 32 2620f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2621f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2622f3c8b6c6SNicholas Piggin .endr 2623d7c67fb1SMichael Neuling 2624b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2625fb0fce3eSMichael Neuling 2626fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2627fb0fce3eSMichael Neuling b denorm_done 2628fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2629fb0fce3eSMichael Neuling/* 2630fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2631fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2632fb0fce3eSMichael Neuling */ 2633f3c8b6c6SNicholas Piggin .Lreg=32 2634f3c8b6c6SNicholas Piggin .rept 32 2635f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2636f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2637f3c8b6c6SNicholas Piggin .endr 2638f3c8b6c6SNicholas Piggin 2639fb0fce3eSMichael Neulingdenorm_done: 2640f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2641f14040bcSMichael Neuling subi r11,r11,4 2642b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2643b92a66a6SMichael Neuling mtcrf 0x80,r9 2644b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2645931dc86bSNicholas PigginBEGIN_FTR_SECTION 2646931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2647931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2648931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2649630573c1SPaul MackerrasBEGIN_FTR_SECTION 2650630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2651630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2652630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 265359dc5bfcSNicholas Piggin li r10,0 265459dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 2655b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2656b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2657b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2658b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2659222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2660b92a66a6SMichael Neuling b . 2661b92a66a6SMichael Neuling#endif 2662b92a66a6SMichael Neuling 26634f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 26644f50541fSNicholas Piggin GEN_COMMON denorm_exception 2665eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2666eb204d86SNicholas Piggin bl unknown_exception 26671df7d5e4SNicholas Piggin b interrupt_return_hsrr 2668d7e89849SNicholas Piggin 2669d7e89849SNicholas Piggin 2670d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 26714f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 26724f50541fSNicholas Piggin IVEC=0x1600 26733f7fbd97SNicholas Piggin IHSRR=1 26744f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 26754f50541fSNicholas Piggin 26767299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 26774f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 26787299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 26791a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2680eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 26814f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2682eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2683eb204d86SNicholas Piggin bl cbe_maintenance_exception 26841df7d5e4SNicholas Piggin b interrupt_return_hsrr 26859600f261SNicholas Piggin 2686d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 26871a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 26881a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2689d7e89849SNicholas Piggin#endif 2690d7e89849SNicholas Piggin 269169a79344SNicholas Piggin 26924f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 26934f50541fSNicholas Piggin IVEC=0x1700 26942284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 26954f50541fSNicholas Piggin IKVM_REAL=1 26962284ffeaSNicholas Piggin#endif 26974f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 26984f50541fSNicholas Piggin 26997299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 27004f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 27017299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 27027299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 27034f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 27047299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2705eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 27064f50541fSNicholas Piggin GEN_COMMON altivec_assist 2707eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2708b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 2709eb204d86SNicholas Piggin bl altivec_assist_exception 2710702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2711b51c079eSNicholas Piggin#else 2712eb204d86SNicholas Piggin bl unknown_exception 2713b51c079eSNicholas Piggin#endif 27141df7d5e4SNicholas Piggin b interrupt_return_srr 2715b51c079eSNicholas Piggin 2716d7e89849SNicholas Piggin 2717d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 27184f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 27194f50541fSNicholas Piggin IVEC=0x1800 27203f7fbd97SNicholas Piggin IHSRR=1 27214f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 27224f50541fSNicholas Piggin 27237299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 27244f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 27257299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 27261a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2727eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 27284f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2729eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2730eb204d86SNicholas Piggin bl cbe_thermal_exception 27311df7d5e4SNicholas Piggin b interrupt_return_hsrr 27329600f261SNicholas Piggin 2733d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 27341a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 27351a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2736d7e89849SNicholas Piggin#endif 2737d7e89849SNicholas Piggin 27387299417cSNicholas Piggin 273975eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 27402104180aSNicholas Piggin 27410eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 27420eddf327SNicholas Piggin IVEC=0x900 27430eddf327SNicholas Piggin ISTACK=0 2744af47d79bSNicholas Piggin ICFAR=0 27450eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 27462104180aSNicholas Piggin 2747cc491f1dSNicholas Piggin/* 2748cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2749cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2750cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2751cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2752cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2753cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2754cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2755cc491f1dSNicholas Piggin */ 27562104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 27572104180aSNicholas Piggin mr r10,r1 27582104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 27592104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 27600eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 276171c3b05aSNicholas Piggin 2762c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2763c06075f3SNicholas Piggin bl soft_nmi_interrupt 276471c3b05aSNicholas Piggin 276571c3b05aSNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 276671c3b05aSNicholas Piggin li r9,0 276771c3b05aSNicholas Piggin mtmsrd r9,1 276871c3b05aSNicholas Piggin 27698e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 2770f23699c9SNicholas Piggin 277171c3b05aSNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 277271c3b05aSNicholas Piggin RFI_TO_KERNEL 27732104180aSNicholas Piggin 277475eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2775d7e89849SNicholas Piggin 27760ebc4cdaSBenjamin Herrenschmidt/* 2777fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2778fe9e1d54SIan Munsie * - If it was a decrementer interrupt, we bump the dec to max and and return. 2779fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2780fe9e1d54SIan Munsie * triggered and won't automatically refire. 27810869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 27820869b6fdSMahesh Salgaonkar * and it won't refire. 27836cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2784fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 27850ebc4cdaSBenjamin Herrenschmidt */ 27863f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 27874508a74aSNicholas Piggin .if \hsrr 27884508a74aSNicholas Pigginmasked_Hinterrupt: 27894508a74aSNicholas Piggin .else 27904508a74aSNicholas Pigginmasked_interrupt: 27914508a74aSNicholas Piggin .endif 279263e40806SNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 279363e40806SNicholas Piggin lbz r9,PACAIRQHAPPENED(r13) 279463e40806SNicholas Piggin or r9,r9,r10 279563e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 279663e40806SNicholas Piggin 279763e40806SNicholas Piggin .if ! \hsrr 27984508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 27994508a74aSNicholas Piggin bne 1f 280063e40806SNicholas Piggin LOAD_REG_IMMEDIATE(r9, 0x7fffffff) 280163e40806SNicholas Piggin mtspr SPRN_DEC,r9 28020eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 280363e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 28040eddf327SNicholas Piggin b soft_nmi_common 28050eddf327SNicholas Piggin#else 28060eddf327SNicholas Piggin b 2f 28070eddf327SNicholas Piggin#endif 280863e40806SNicholas Piggin .endif 280963e40806SNicholas Piggin 28104508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 28114508a74aSNicholas Piggin beq 2f 28120eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 28134508a74aSNicholas Piggin .if \hsrr 28140eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 28154508a74aSNicholas Piggin .else 28160eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 28174508a74aSNicholas Piggin .endif 281863e40806SNicholas Piggin ori r9,r9,PACA_IRQ_HARD_DIS 281963e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 28204508a74aSNicholas Piggin2: /* done */ 282163e40806SNicholas Piggin li r9,0 282259dc5bfcSNicholas Piggin .if \hsrr 282363e40806SNicholas Piggin stb r9,PACAHSRR_VALID(r13) 282459dc5bfcSNicholas Piggin .else 282563e40806SNicholas Piggin stb r9,PACASRR_VALID(r13) 282659dc5bfcSNicholas Piggin .endif 282763e40806SNicholas Piggin 2828f23699c9SNicholas Piggin SEARCH_RESTART_TABLE 2829f23699c9SNicholas Piggin cmpdi r12,0 2830f23699c9SNicholas Piggin beq 3f 2831f23699c9SNicholas Piggin .if \hsrr 2832f23699c9SNicholas Piggin mtspr SPRN_HSRR0,r12 2833f23699c9SNicholas Piggin .else 2834f23699c9SNicholas Piggin mtspr SPRN_SRR0,r12 2835f23699c9SNicholas Piggin .endif 2836f23699c9SNicholas Piggin3: 2837f23699c9SNicholas Piggin 283863e40806SNicholas Piggin ld r9,PACA_EXGEN+EX_CTR(r13) 283963e40806SNicholas Piggin mtctr r9 284063e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 28414508a74aSNicholas Piggin mtcrf 0x80,r9 28424508a74aSNicholas Piggin std r1,PACAR1(r13) 28434508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 28444508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 28454508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 28460eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 2847b2dc2977SNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 2848b2dc2977SNicholas Piggin /* May return to masked low address where r13 is not set up */ 28494508a74aSNicholas Piggin .if \hsrr 28504508a74aSNicholas Piggin HRFI_TO_KERNEL 28514508a74aSNicholas Piggin .else 28524508a74aSNicholas Piggin RFI_TO_KERNEL 28534508a74aSNicholas Piggin .endif 28544508a74aSNicholas Piggin b . 28554508a74aSNicholas Piggin.endm 28560ebc4cdaSBenjamin Herrenschmidt 2857a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2858a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2859a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2860a048a07dSNicholas Piggin sync 2861a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2862a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2863a048a07dSNicholas Piggin ori 31,31,0 2864a048a07dSNicholas Piggin .rept 14 2865a048a07dSNicholas Piggin b 1f 2866a048a07dSNicholas Piggin1: 2867a048a07dSNicholas Piggin .endr 2868a048a07dSNicholas Piggin blr 2869a048a07dSNicholas Piggin 28709a32a7e7SNicholas Piggin/* Clobbers r10, r11, ctr */ 28719a32a7e7SNicholas Piggin.macro L1D_DISPLACEMENT_FLUSH 2872aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2873bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2874bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2875aa8a5e00SMichael Ellerman mtctr r11 287615a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2877aa8a5e00SMichael Ellerman 2878aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2879aa8a5e00SMichael Ellerman sync 2880bdcb1aefSNicholas Piggin 2881bdcb1aefSNicholas Piggin /* 2882f7964378SNicholas Piggin * The load addresses are at staggered offsets within cachelines, 2883bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2884bdcb1aefSNicholas Piggin * hurt). 2885bdcb1aefSNicholas Piggin */ 2886bdcb1aefSNicholas Piggin1: 2887bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2888bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2889bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2890bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2891bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2892bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2893bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2894bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2895bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2896aa8a5e00SMichael Ellerman bdnz 1b 28979a32a7e7SNicholas Piggin.endm 2898aa8a5e00SMichael Ellerman 28999a32a7e7SNicholas PigginTRAMP_REAL_BEGIN(entry_flush_fallback) 29009a32a7e7SNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 29019a32a7e7SNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 29029a32a7e7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 29039a32a7e7SNicholas Piggin mfctr r9 29049a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2905f7964378SNicholas Piggin mtctr r9 2906f7964378SNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2907f7964378SNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2908f7964378SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 2909f7964378SNicholas Piggin blr 2910f7964378SNicholas Piggin 291108685be7SNicholas Piggin/* 291208685be7SNicholas Piggin * The SCV entry flush happens with interrupts enabled, so it must disable 291308685be7SNicholas Piggin * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10 291408685be7SNicholas Piggin * (containing LR) does not need to be preserved here because scv entry 291508685be7SNicholas Piggin * puts 0 in the pt_regs, CTR can be clobbered for the same reason. 291608685be7SNicholas Piggin */ 291708685be7SNicholas PigginTRAMP_REAL_BEGIN(scv_entry_flush_fallback) 291808685be7SNicholas Piggin li r10,0 291908685be7SNicholas Piggin mtmsrd r10,1 292008685be7SNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 292108685be7SNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 292208685be7SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 292308685be7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 292408685be7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 292508685be7SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 292608685be7SNicholas Piggin li r10,MSR_RI 292708685be7SNicholas Piggin mtmsrd r10,1 292808685be7SNicholas Piggin blr 292908685be7SNicholas Piggin 2930aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 2931aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2932aa8a5e00SMichael Ellerman GET_PACA(r13); 2933aa8a5e00SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 2934aa8a5e00SMichael Ellerman ld r1,PACAKSAVE(r13) 2935aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2936aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2937aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2938aa8a5e00SMichael Ellerman mfctr r9 29399a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2940aa8a5e00SMichael Ellerman mtctr r9 2941aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2942aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2943aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 294478ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2945aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2946aa8a5e00SMichael Ellerman rfid 2947aa8a5e00SMichael Ellerman 2948aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 2949aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2950aa8a5e00SMichael Ellerman GET_PACA(r13); 295178ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 295278ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2953aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2954aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2955aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2956aa8a5e00SMichael Ellerman mfctr r9 29579a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2958aa8a5e00SMichael Ellerman mtctr r9 2959aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2960aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2961aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 296278ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2963aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2964aa8a5e00SMichael Ellerman hrfid 2965aa8a5e00SMichael Ellerman 29667fa95f9aSNicholas PigginTRAMP_REAL_BEGIN(rfscv_flush_fallback) 29677fa95f9aSNicholas Piggin /* system call volatile */ 29687fa95f9aSNicholas Piggin mr r7,r13 29697fa95f9aSNicholas Piggin GET_PACA(r13); 29707fa95f9aSNicholas Piggin mr r8,r1 29717fa95f9aSNicholas Piggin ld r1,PACAKSAVE(r13) 29727fa95f9aSNicholas Piggin mfctr r9 29737fa95f9aSNicholas Piggin ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 29747fa95f9aSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 29757fa95f9aSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 29767fa95f9aSNicholas Piggin mtctr r11 29777fa95f9aSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 29787fa95f9aSNicholas Piggin 29797fa95f9aSNicholas Piggin /* order ld/st prior to dcbt stop all streams with flushing */ 29807fa95f9aSNicholas Piggin sync 29817fa95f9aSNicholas Piggin 29827fa95f9aSNicholas Piggin /* 29837fa95f9aSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 29847fa95f9aSNicholas Piggin * which suits some pipelines better (on others it should not 29857fa95f9aSNicholas Piggin * hurt). 29867fa95f9aSNicholas Piggin */ 29877fa95f9aSNicholas Piggin1: 29887fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 29897fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 29907fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 29917fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 29927fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 29937fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 29947fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 29957fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 29967fa95f9aSNicholas Piggin addi r10,r10,0x80*8 29977fa95f9aSNicholas Piggin bdnz 1b 29987fa95f9aSNicholas Piggin 29997fa95f9aSNicholas Piggin mtctr r9 30007fa95f9aSNicholas Piggin li r9,0 30017fa95f9aSNicholas Piggin li r10,0 30027fa95f9aSNicholas Piggin li r11,0 30037fa95f9aSNicholas Piggin mr r1,r8 30047fa95f9aSNicholas Piggin mr r13,r7 30057fa95f9aSNicholas Piggin RFSCV 30067fa95f9aSNicholas Piggin 30070eddf327SNicholas PigginUSE_TEXT_SECTION() 30089a32a7e7SNicholas Piggin 300969fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 301069fdd674SNicholas Pigginkvm_interrupt: 301169fdd674SNicholas Piggin /* 301269fdd674SNicholas Piggin * The conditional branch in KVMTEST can't reach all the way, 301369fdd674SNicholas Piggin * make a stub. 301469fdd674SNicholas Piggin */ 301569fdd674SNicholas Piggin b kvmppc_interrupt 301669fdd674SNicholas Piggin#endif 301769fdd674SNicholas Piggin 30189a32a7e7SNicholas Piggin_GLOBAL(do_uaccess_flush) 30199a32a7e7SNicholas Piggin UACCESS_FLUSH_FIXUP_SECTION 30209a32a7e7SNicholas Piggin nop 30219a32a7e7SNicholas Piggin nop 30229a32a7e7SNicholas Piggin nop 30239a32a7e7SNicholas Piggin blr 30249a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 30259a32a7e7SNicholas Piggin blr 30269a32a7e7SNicholas Piggin_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) 30279a32a7e7SNicholas PigginEXPORT_SYMBOL(do_uaccess_flush) 30289a32a7e7SNicholas Piggin 30299a32a7e7SNicholas Piggin 30303f7fbd97SNicholas PigginMASKED_INTERRUPT 30313f7fbd97SNicholas PigginMASKED_INTERRUPT hsrr=1 30327230c564SBenjamin Herrenschmidt 30330ebc4cdaSBenjamin Herrenschmidt /* 3034c1fb6816SMichael Neuling * Relocation-on interrupts: A subset of the interrupts can be delivered 3035c1fb6816SMichael Neuling * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 3036c1fb6816SMichael Neuling * it. Addresses are the same as the original interrupt addresses, but 3037c1fb6816SMichael Neuling * offset by 0xc000000000004000. 3038c1fb6816SMichael Neuling * It's impossible to receive interrupts below 0x300 via this mechanism. 3039c1fb6816SMichael Neuling * KVM: None of these traps are from the guest ; anything that escalated 3040c1fb6816SMichael Neuling * to HV=1 from HV=0 is delivered via real mode handlers. 3041c1fb6816SMichael Neuling */ 3042c1fb6816SMichael Neuling 3043c1fb6816SMichael Neuling /* 3044c1fb6816SMichael Neuling * This uses the standard macro, since the original 0x300 vector 3045c1fb6816SMichael Neuling * only has extra guff for STAB-based processors -- which never 3046c1fb6816SMichael Neuling * come here. 3047c1fb6816SMichael Neuling */ 3048da2bc464SMichael Ellerman 304957f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 30508ed8ab40SHari Bathini /* 30519d1988caSNicholas Piggin * All code below __end_soft_masked is treated as soft-masked. If 3052b2dc2977SNicholas Piggin * any code runs here with MSR[EE]=1, it must then cope with pending 3053b2dc2977SNicholas Piggin * soft interrupt being raised (i.e., by ensuring it is replayed). 3054b2dc2977SNicholas Piggin * 30558ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 30568ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 30578ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 30588ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 30598ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 30608ed8ab40SHari Bathini */ 30618ed8ab40SHari Bathini .align 7 30628ed8ab40SHari Bathini .globl __end_interrupts 30638ed8ab40SHari Bathini__end_interrupts: 3064*d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(__end_interrupts, virt_trampolines) 306561383407SBenjamin Herrenschmidt 306657f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 306757f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 306857f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 306957f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 307057f26649SNicholas Piggin 307157f26649SNicholas PigginUSE_TEXT_SECTION() 307257f26649SNicholas Piggin 3073296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 3074296e753fSNicholas Pigginenable_machine_check: 3075296e753fSNicholas Piggin mflr r0 3076296e753fSNicholas Piggin bcl 20,31,$+4 3077296e753fSNicholas Piggin0: mflr r3 3078296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 3079296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 3080296e753fSNicholas Piggin mfmsr r3 3081296e753fSNicholas Piggin ori r3,r3,MSR_ME 3082296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 3083296e753fSNicholas Piggin RFI_TO_KERNEL 3084296e753fSNicholas Piggin1: mtlr r0 3085296e753fSNicholas Piggin blr 3086296e753fSNicholas Piggin 3087b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 3088b7d9ccecSNicholas Piggindisable_machine_check: 3089b7d9ccecSNicholas Piggin mflr r0 3090b7d9ccecSNicholas Piggin bcl 20,31,$+4 3091b7d9ccecSNicholas Piggin0: mflr r3 3092b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 3093b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 3094b7d9ccecSNicholas Piggin mfmsr r3 3095b7d9ccecSNicholas Piggin li r4,MSR_ME 3096b7d9ccecSNicholas Piggin andc r3,r3,r4 3097b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 3098b7d9ccecSNicholas Piggin RFI_TO_KERNEL 3099b7d9ccecSNicholas Piggin1: mtlr r0 3100b7d9ccecSNicholas Piggin blr 3101