1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 167230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 178aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1846f52210SStephen Rothwell#include <asm/ptrace.h> 197cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 20da2bc464SMichael Ellerman#include <asm/head-64.h> 212c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 22890274c2SMichael Ellerman#include <asm/kup.h> 238aa34ab8SBenjamin Herrenschmidt 247299417cSNicholas Piggin/* 257299417cSNicholas Piggin * Following are fixed section helper macros. 267299417cSNicholas Piggin * 277299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 287299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 297299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 307299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 317299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 327299417cSNicholas Piggin */ 337299417cSNicholas Piggin 34a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 35a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 36a2432811SNicholas Piggin 37a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 38a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 39a2432811SNicholas Piggin 40a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 41a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 42a2432811SNicholas Piggin 43a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 44a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 45a2432811SNicholas Piggin 46a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 47a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 48a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 49a2432811SNicholas Piggin .global name; \ 50a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 51a2432811SNicholas Piggin DEFINE_FIXED_SYMBOL(name); \ 52a2432811SNicholas Pigginname: 53a2432811SNicholas Piggin 54a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 55a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 56a2432811SNicholas Piggin 57a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 58a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 59a2432811SNicholas Piggin 60a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 61a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 62a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 63a2432811SNicholas Piggin 64a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 65a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 66a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 67a2432811SNicholas Piggin 680ebc4cdaSBenjamin Herrenschmidt/* 6912a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 7012a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 7112a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 7212a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 7312a04809SNicholas Piggin * that kernelbase be 64K aligned. 7412a04809SNicholas Piggin */ 7512a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 7612a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 7712a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 7812a04809SNicholas Piggin 7912a04809SNicholas Piggin#define __LOAD_HANDLER(reg, label) \ 8012a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 8112a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l 8212a04809SNicholas Piggin 8312a04809SNicholas Piggin/* 8412a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 8512a04809SNicholas Piggin * head-y require >64K offsets. 8612a04809SNicholas Piggin */ 8712a04809SNicholas Piggin#define __LOAD_FAR_HANDLER(reg, label) \ 8812a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 8912a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l; \ 9012a04809SNicholas Piggin addis reg,reg,(ABS_ADDR(label))@h 9112a04809SNicholas Piggin 9212a04809SNicholas Piggin/* 9312a04809SNicholas Piggin * Branch to label using its 0xC000 address. This results in instruction 9412a04809SNicholas Piggin * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned 9512a04809SNicholas Piggin * on using mtmsr rather than rfid. 9612a04809SNicholas Piggin * 9712a04809SNicholas Piggin * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than 9812a04809SNicholas Piggin * load KBASE for a slight optimisation. 9912a04809SNicholas Piggin */ 10012a04809SNicholas Piggin#define BRANCH_TO_C000(reg, label) \ 1010e10be2bSNicholas Piggin __LOAD_FAR_HANDLER(reg, label); \ 10212a04809SNicholas Piggin mtctr reg; \ 10312a04809SNicholas Piggin bctr 10412a04809SNicholas Piggin 105a42a239dSNicholas Piggin/* 106a42a239dSNicholas Piggin * Interrupt code generation macros 107a42a239dSNicholas Piggin */ 10894325357SNicholas Piggin#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 10994325357SNicholas Piggin#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 11094325357SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 11194325357SNicholas Piggin#define IAREA .L_IAREA_\name\() /* PACA save area */ 11294325357SNicholas Piggin#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 11394325357SNicholas Piggin#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 11494325357SNicholas Piggin#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 11594325357SNicholas Piggin#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 11694325357SNicholas Piggin#define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */ 11794325357SNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 11894325357SNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 11994325357SNicholas Piggin#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 12094325357SNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 1214f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 12294325357SNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 12394325357SNicholas Piggin#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 1244f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 12594325357SNicholas Piggin#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 126a42a239dSNicholas Piggin 127a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 128a42a239dSNicholas Piggin.macro int_define_ ## n name 129a42a239dSNicholas Piggin 130a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 131a42a239dSNicholas Piggin.endm ; \ 132a42a239dSNicholas Pigginint_define_ ## n n ; \ 133a42a239dSNicholas Piggindo_define_int n 134a42a239dSNicholas Piggin 135a42a239dSNicholas Piggin.macro do_define_int name 136a42a239dSNicholas Piggin .ifndef IVEC 137a42a239dSNicholas Piggin .error "IVEC not defined" 138a42a239dSNicholas Piggin .endif 139a42a239dSNicholas Piggin .ifndef IHSRR 1403f7fbd97SNicholas Piggin IHSRR=0 1413f7fbd97SNicholas Piggin .endif 1423f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 1433f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 144a42a239dSNicholas Piggin .endif 145a42a239dSNicholas Piggin .ifndef IAREA 146a42a239dSNicholas Piggin IAREA=PACA_EXGEN 147a42a239dSNicholas Piggin .endif 1488729c26eSNicholas Piggin .ifndef IVIRT 1498729c26eSNicholas Piggin IVIRT=1 1508729c26eSNicholas Piggin .endif 151a3cd35beSNicholas Piggin .ifndef IISIDE 152a3cd35beSNicholas Piggin IISIDE=0 153a3cd35beSNicholas Piggin .endif 154a42a239dSNicholas Piggin .ifndef IDAR 155a42a239dSNicholas Piggin IDAR=0 156a42a239dSNicholas Piggin .endif 157a42a239dSNicholas Piggin .ifndef IDSISR 158a42a239dSNicholas Piggin IDSISR=0 159a42a239dSNicholas Piggin .endif 160a42a239dSNicholas Piggin .ifndef ISET_RI 161a42a239dSNicholas Piggin ISET_RI=1 162a42a239dSNicholas Piggin .endif 163d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 164d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 165d73a10cbSNicholas Piggin .endif 166d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 167d73a10cbSNicholas Piggin IREALMODE_COMMON=0 168d73a10cbSNicholas Piggin .else 169d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 170d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 171d73a10cbSNicholas Piggin .endif 172a42a239dSNicholas Piggin .endif 173a42a239dSNicholas Piggin .ifndef IMASK 174a42a239dSNicholas Piggin IMASK=0 175a42a239dSNicholas Piggin .endif 176a42a239dSNicholas Piggin .ifndef IKVM_REAL 177a42a239dSNicholas Piggin IKVM_REAL=0 178a42a239dSNicholas Piggin .endif 179a42a239dSNicholas Piggin .ifndef IKVM_VIRT 180a42a239dSNicholas Piggin IKVM_VIRT=0 181a42a239dSNicholas Piggin .endif 1827cb3a1a0SNicholas Piggin .ifndef ISTACK 1837cb3a1a0SNicholas Piggin ISTACK=1 1847cb3a1a0SNicholas Piggin .endif 1857cb3a1a0SNicholas Piggin .ifndef IKUAP 1867cb3a1a0SNicholas Piggin IKUAP=1 1877cb3a1a0SNicholas Piggin .endif 188a42a239dSNicholas Piggin.endm 189a42a239dSNicholas Piggin 19012a04809SNicholas Piggin/* 1912284ffeaSNicholas Piggin * All interrupts which set HSRR registers, as well as SRESET and MCE and 1922284ffeaSNicholas Piggin * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 1932284ffeaSNicholas Piggin * so they all generally need to test whether they were taken in guest context. 1942284ffeaSNicholas Piggin * 1952284ffeaSNicholas Piggin * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 1962284ffeaSNicholas Piggin * taken with MSR[HV]=0. 1972284ffeaSNicholas Piggin * 1982284ffeaSNicholas Piggin * Interrupts which set SRR registers (with the above exceptions) do not 1992284ffeaSNicholas Piggin * elevate to MSR[HV]=1 mode, though most can be taken when running with 2002284ffeaSNicholas Piggin * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 2012284ffeaSNicholas Piggin * not need to test whether a guest is running because they get delivered to 2022284ffeaSNicholas Piggin * the guest directly, including nested HV KVM guests. 2032284ffeaSNicholas Piggin * 2042284ffeaSNicholas Piggin * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 2052284ffeaSNicholas Piggin * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 2062284ffeaSNicholas Piggin * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 2072284ffeaSNicholas Piggin * delivered to the real-mode entry point, therefore such interrupts only test 2082284ffeaSNicholas Piggin * KVM in their real mode handlers, and only when PR KVM is possible. 2092284ffeaSNicholas Piggin * 2102284ffeaSNicholas Piggin * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 2112284ffeaSNicholas Piggin * delivered in real-mode when the MMU is in hash mode because the MMU 2122284ffeaSNicholas Piggin * registers are not set appropriately to translate host addresses. In nested 2132284ffeaSNicholas Piggin * radix mode these can be delivered in virt-mode as the host translations are 2142284ffeaSNicholas Piggin * used implicitly (see: effective LPID, effective PID). 2152284ffeaSNicholas Piggin */ 2162284ffeaSNicholas Piggin 2172284ffeaSNicholas Piggin/* 2182284ffeaSNicholas Piggin * If an interrupt is taken while a guest is running, it is immediately routed 219f3601156SNicholas Piggin * to KVM to handle. 22012a04809SNicholas Piggin */ 22112a04809SNicholas Piggin 22269fdd674SNicholas Piggin.macro KVMTEST name handler 22369fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 22412a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 22512a04809SNicholas Piggin cmpwi r10,0 22612a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 2273f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 228def0db4fSNicholas Piggin BEGIN_FTR_SECTION 22969fdd674SNicholas Piggin li r10,(IVEC + 0x2) 230def0db4fSNicholas Piggin FTR_SECTION_ELSE 23169fdd674SNicholas Piggin li r10,(IVEC) 232def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 233b177ae2fSNicholas Piggin .elseif IHSRR 23469fdd674SNicholas Piggin li r10,(IVEC + 0x2) 23512a04809SNicholas Piggin .else 23669fdd674SNicholas Piggin li r10,(IVEC) 23712a04809SNicholas Piggin .endif 23869fdd674SNicholas Piggin bne \handler 23912a04809SNicholas Piggin#endif 24069fdd674SNicholas Piggin.endm 24112a04809SNicholas Piggin 242c7c5cbb4SNicholas Piggin/* 243c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 244c7c5cbb4SNicholas Piggin * 245c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 246c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 247c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 248c7c5cbb4SNicholas Piggin * the kernel. 249c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 250c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 251c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 252c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 253c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 254c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 255c7c5cbb4SNicholas Piggin * irq_soft_mask set. 256c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 257c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 258c7c5cbb4SNicholas Piggin * mode. 259c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 260c7c5cbb4SNicholas Piggin * This is done if early=2. 261c7c5cbb4SNicholas Piggin */ 2628729c26eSNicholas Piggin 2638729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 264d73a10cbSNicholas Piggin .if IREALMODE_COMMON 265d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 266d73a10cbSNicholas Piggin mtctr r10 267d73a10cbSNicholas Piggin bctr 268d73a10cbSNicholas Piggin .else 2698729c26eSNicholas Piggin .if \virt 2708729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 2718729c26eSNicholas Piggin b \name\()_common_virt 2728729c26eSNicholas Piggin#else 2738729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 2748729c26eSNicholas Piggin mtctr r10 2758729c26eSNicholas Piggin bctr 2768729c26eSNicholas Piggin#endif 2778729c26eSNicholas Piggin .else 2788729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 2798729c26eSNicholas Piggin mtctr r10 2808729c26eSNicholas Piggin bctr 2818729c26eSNicholas Piggin .endif 282d73a10cbSNicholas Piggin .endif 2838729c26eSNicholas Piggin.endm 2848729c26eSNicholas Piggin 285fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 286c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 287c7c5cbb4SNicholas Piggin GET_PACA(r13) 288fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 289931dc86bSNicholas PigginBEGIN_FTR_SECTION 290931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 291931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 292c7c5cbb4SNicholas Piggin HMT_MEDIUM 293fc589ee4SNicholas Piggin std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 294931dc86bSNicholas PigginBEGIN_FTR_SECTION 295931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 296931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 297c7c5cbb4SNicholas Piggin .if \ool 298c7c5cbb4SNicholas Piggin .if !\virt 299c7c5cbb4SNicholas Piggin b tramp_real_\name 300c7c5cbb4SNicholas Piggin .pushsection .text 301c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 302c7c5cbb4SNicholas Piggin .else 303c7c5cbb4SNicholas Piggin b tramp_virt_\name 304c7c5cbb4SNicholas Piggin .pushsection .text 305c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 306c7c5cbb4SNicholas Piggin .endif 307c7c5cbb4SNicholas Piggin .endif 308c7c5cbb4SNicholas Piggin 309931dc86bSNicholas PigginBEGIN_FTR_SECTION 310931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 311931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 312931dc86bSNicholas PigginBEGIN_FTR_SECTION 313931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 314931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 315c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 3168729c26eSNicholas Piggin mfctr r10 3178729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 318c7c5cbb4SNicholas Piggin mfcr r9 319fc589ee4SNicholas Piggin std r11,IAREA+EX_R11(r13) 320fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 321c7c5cbb4SNicholas Piggin 322c7c5cbb4SNicholas Piggin /* 323c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 324c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 325c7c5cbb4SNicholas Piggin * not recoverable if they are live. 326c7c5cbb4SNicholas Piggin */ 327c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 328fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 329a3cd35beSNicholas Piggin .if IDAR && !IISIDE 330fc589ee4SNicholas Piggin .if IHSRR 331c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 332c7c5cbb4SNicholas Piggin .else 333c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 334c7c5cbb4SNicholas Piggin .endif 335fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 336c7c5cbb4SNicholas Piggin .endif 337a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 338fc589ee4SNicholas Piggin .if IHSRR 339c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 340c7c5cbb4SNicholas Piggin .else 341c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 342c7c5cbb4SNicholas Piggin .endif 343fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 344c7c5cbb4SNicholas Piggin .endif 345c7c5cbb4SNicholas Piggin 3463f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 3478729c26eSNicholas Piggin BEGIN_FTR_SECTION 3488729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3498729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3508729c26eSNicholas Piggin FTR_SECTION_ELSE 3518729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3528729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 3538729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 3548729c26eSNicholas Piggin .elseif IHSRR 3558729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3568729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3578729c26eSNicholas Piggin .else 3588729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3598729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 360c7c5cbb4SNicholas Piggin .endif 361d73a10cbSNicholas Piggin 362d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 3638729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 3648729c26eSNicholas Piggin .endif 3658729c26eSNicholas Piggin 366c7c5cbb4SNicholas Piggin .if \ool 367c7c5cbb4SNicholas Piggin .popsection 368c7c5cbb4SNicholas Piggin .endif 369c7c5cbb4SNicholas Piggin.endm 370c7c5cbb4SNicholas Piggin 371d064151fSNicholas Piggin/* 3728729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 3739600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 3749600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 3759600f261SNicholas Piggin * 3768729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 377d064151fSNicholas Piggin */ 3788729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 3798729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 3808729c26eSNicholas Piggin\name\()_common_real: 3819600f261SNicholas Piggin .if IKVM_REAL 38269fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 3839600f261SNicholas Piggin .endif 3849600f261SNicholas Piggin 3858729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 3868729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 387c080a173SDaniel Axtens .if IHSRR_IF_HVMODE 3888729c26eSNicholas Piggin BEGIN_FTR_SECTION 3898729c26eSNicholas Piggin xori r10,r10,MSR_RI 3908729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 3918729c26eSNicholas Piggin .elseif ! IHSRR 3928729c26eSNicholas Piggin xori r10,r10,MSR_RI 3938729c26eSNicholas Piggin .endif 3948729c26eSNicholas Piggin mtmsrd r10 3958729c26eSNicholas Piggin 3968729c26eSNicholas Piggin .if IVIRT 3979600f261SNicholas Piggin .if IKVM_VIRT 3989600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 3999600f261SNicholas Piggin .endif 4009600f261SNicholas Piggin 4018729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 4028729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_virt) 4038729c26eSNicholas Piggin\name\()_common_virt: 4049600f261SNicholas Piggin .if IKVM_VIRT 40569fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4069600f261SNicholas Piggin1: 4079600f261SNicholas Piggin .endif 4088729c26eSNicholas Piggin .endif /* IVIRT */ 4098729c26eSNicholas Piggin.endm 4108729c26eSNicholas Piggin 4119600f261SNicholas Piggin/* 4129600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4139600f261SNicholas Piggin * want to run in real mode. 4149600f261SNicholas Piggin */ 4159600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 4169600f261SNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 4179600f261SNicholas Piggin\name\()_common_real: 4189600f261SNicholas Piggin .if IKVM_REAL 41969fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4209600f261SNicholas Piggin .endif 4219600f261SNicholas Piggin.endm 4229600f261SNicholas Piggin 4238729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 4240eddf327SNicholas Piggin .if IMASK 425b2dc2977SNicholas Piggin .if ! ISTACK 426b2dc2977SNicholas Piggin .error "No support for masked interrupt to use custom stack" 427b2dc2977SNicholas Piggin .endif 428b2dc2977SNicholas Piggin 429b2dc2977SNicholas Piggin /* If coming from user, skip soft-mask tests. */ 430b2dc2977SNicholas Piggin andi. r10,r12,MSR_PR 431325678fdSNicholas Piggin bne 3f 432b2dc2977SNicholas Piggin 4339d1988caSNicholas Piggin /* 434325678fdSNicholas Piggin * Kernel code running below __end_soft_masked may be 435325678fdSNicholas Piggin * implicitly soft-masked if it is within the regions 436325678fdSNicholas Piggin * in the soft mask table. 4379d1988caSNicholas Piggin */ 4389d1988caSNicholas Piggin LOAD_HANDLER(r10, __end_soft_masked) 439b2dc2977SNicholas Piggin cmpld r11,r10 440325678fdSNicholas Piggin bge+ 1f 4419d1988caSNicholas Piggin 442325678fdSNicholas Piggin /* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */ 443325678fdSNicholas Piggin mtctr r12 444325678fdSNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 445325678fdSNicholas Piggin SEARCH_SOFT_MASK_TABLE 446325678fdSNicholas Piggin cmpdi r12,0 447325678fdSNicholas Piggin mfctr r12 /* Restore r12 to SRR1 */ 448325678fdSNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 449325678fdSNicholas Piggin beq 1f /* Not in soft-mask table */ 450b2dc2977SNicholas Piggin li r10,IMASK 451325678fdSNicholas Piggin b 2f /* In soft-mask table, always mask */ 452b2dc2977SNicholas Piggin 453b2dc2977SNicholas Piggin /* Test the soft mask state against our interrupt's bit */ 454325678fdSNicholas Piggin1: lbz r10,PACAIRQSOFTMASK(r13) 455325678fdSNicholas Piggin2: andi. r10,r10,IMASK 4560eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 4570eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 4580eddf327SNicholas Piggin li r10,PACA_IRQ_EE 4590eddf327SNicholas Piggin .elseif IVEC == 0x900 4600eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 4610eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 4620eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 4630eddf327SNicholas Piggin .elseif IVEC == 0xe60 4640eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 4650eddf327SNicholas Piggin .elseif IVEC == 0xf00 4660eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 4670eddf327SNicholas Piggin .else 4680eddf327SNicholas Piggin .abort "Bad maskable vector" 4690eddf327SNicholas Piggin .endif 4700eddf327SNicholas Piggin 4713f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4720eddf327SNicholas Piggin BEGIN_FTR_SECTION 4730eddf327SNicholas Piggin bne masked_Hinterrupt 4740eddf327SNicholas Piggin FTR_SECTION_ELSE 4750eddf327SNicholas Piggin bne masked_interrupt 4760eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4770eddf327SNicholas Piggin .elseif IHSRR 4780eddf327SNicholas Piggin bne masked_Hinterrupt 4790eddf327SNicholas Piggin .else 4800eddf327SNicholas Piggin bne masked_interrupt 4810eddf327SNicholas Piggin .endif 4820eddf327SNicholas Piggin .endif 4830eddf327SNicholas Piggin 4846d71759aSNicholas Piggin .if ISTACK 4855d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 486325678fdSNicholas Piggin3: mr r10,r1 /* Save r1 */ 4875d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 4881b359982SNicholas Piggin beq- 100f 4895d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 4901b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 4911b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 4925d5e0edfSNicholas Piggin .endif 4938c9fb5d4SNicholas Piggin 4948c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 4958c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 4968c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 4978c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 4988c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 4998c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 5005d5e0edfSNicholas Piggin 50159dc5bfcSNicholas Piggin /* Mark our [H]SRRs valid for return */ 50259dc5bfcSNicholas Piggin li r10,1 50359dc5bfcSNicholas Piggin .if IHSRR_IF_HVMODE 50459dc5bfcSNicholas Piggin BEGIN_FTR_SECTION 50559dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 50659dc5bfcSNicholas Piggin FTR_SECTION_ELSE 50759dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 50859dc5bfcSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 50959dc5bfcSNicholas Piggin .elseif IHSRR 51059dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 51159dc5bfcSNicholas Piggin .else 51259dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 51359dc5bfcSNicholas Piggin .endif 51459dc5bfcSNicholas Piggin 5158729c26eSNicholas Piggin .if ISET_RI 5168729c26eSNicholas Piggin li r10,MSR_RI 5178729c26eSNicholas Piggin mtmsrd r10,1 /* Set MSR_RI */ 5188729c26eSNicholas Piggin .endif 5198729c26eSNicholas Piggin 5206d71759aSNicholas Piggin .if ISTACK 5216d71759aSNicholas Piggin .if IKUAP 5225d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 5235d5e0edfSNicholas Piggin .endif 5241b359982SNicholas Piggin beq 101f /* if from kernel mode */ 525931dc86bSNicholas PigginBEGIN_FTR_SECTION 526931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 527931dc86bSNicholas Piggin std r9,_PPR(r1) 528931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5291b359982SNicholas Piggin101: 5305d5e0edfSNicholas Piggin .else 5316d71759aSNicholas Piggin .if IKUAP 532bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 533bcbceed4SNicholas Piggin .endif 5345d5e0edfSNicholas Piggin .endif 5355d5e0edfSNicholas Piggin 5368c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5376d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5386d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5398c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5408c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5416d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5426d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5436d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5448c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5458c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5468c9fb5d4SNicholas Piggin std r11,GPR13(r1) 547a3cd35beSNicholas Piggin 5486cc0c16dSNicholas Piggin SAVE_NVGPRS(r1) 5496cc0c16dSNicholas Piggin 5506d71759aSNicholas Piggin .if IDAR 551a3cd35beSNicholas Piggin .if IISIDE 552d1a84718SNicholas Piggin ld r10,_NIP(r1) 553d1a84718SNicholas Piggin .else 5546d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 555d1a84718SNicholas Piggin .endif 556d1a84718SNicholas Piggin std r10,_DAR(r1) 557d1a84718SNicholas Piggin .endif 558a3cd35beSNicholas Piggin 5596d71759aSNicholas Piggin .if IDSISR 560a3cd35beSNicholas Piggin .if IISIDE 561d1a84718SNicholas Piggin ld r10,_MSR(r1) 562d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 563d1a84718SNicholas Piggin and r10,r10,r11 564d1a84718SNicholas Piggin .else 5656d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 566d1a84718SNicholas Piggin .endif 567d1a84718SNicholas Piggin std r10,_DSISR(r1) 568d1a84718SNicholas Piggin .endif 569a3cd35beSNicholas Piggin 570931dc86bSNicholas PigginBEGIN_FTR_SECTION 5716d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 5728c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 573931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 5748729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 5758c9fb5d4SNicholas Piggin std r10,_CTR(r1) 5768c9fb5d4SNicholas Piggin std r2,GPR2(r1) /* save r2 in stackframe */ 5778c9fb5d4SNicholas Piggin SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ 5788c9fb5d4SNicholas Piggin SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ 5798c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 5808c9fb5d4SNicholas Piggin ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 5818c9fb5d4SNicholas Piggin std r9,_LINK(r1) 5828c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5838c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 5848c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 5858c9fb5d4SNicholas Piggin std r11,_XER(r1) 5866cc0c16dSNicholas Piggin li r9,IVEC 5878c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 5888c9fb5d4SNicholas Piggin li r10,0 5898c9fb5d4SNicholas Piggin ld r11,exception_marker@toc(r2) 5908c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 5918c9fb5d4SNicholas Piggin std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 592bcbceed4SNicholas Piggin.endm 593bcbceed4SNicholas Piggin 594391e941bSNicholas Piggin/* 5958729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 5968729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 5978729c26eSNicholas Piggin * SRR1, and relocation is on. 5988729c26eSNicholas Piggin * 5998729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 6008729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 6018729c26eSNicholas Piggin */ 6028729c26eSNicholas Piggin.macro GEN_COMMON name 6038729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 6048729c26eSNicholas Piggin __GEN_COMMON_BODY \name 6058729c26eSNicholas Piggin.endm 6068729c26eSNicholas Piggin 607f23699c9SNicholas Piggin.macro SEARCH_RESTART_TABLE 608f23699c9SNicholas Piggin#ifdef CONFIG_RELOCATABLE 609f23699c9SNicholas Piggin mr r12,r2 610f23699c9SNicholas Piggin ld r2,PACATOC(r13) 611f23699c9SNicholas Piggin LOAD_REG_ADDR(r9, __start___restart_table) 612f23699c9SNicholas Piggin LOAD_REG_ADDR(r10, __stop___restart_table) 613f23699c9SNicholas Piggin mr r2,r12 614f23699c9SNicholas Piggin#else 615f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___restart_table) 616f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___restart_table) 617f23699c9SNicholas Piggin#endif 618f23699c9SNicholas Piggin300: 619f23699c9SNicholas Piggin cmpd r9,r10 620f23699c9SNicholas Piggin beq 302f 621f23699c9SNicholas Piggin ld r12,0(r9) 622f23699c9SNicholas Piggin cmpld r11,r12 623f23699c9SNicholas Piggin blt 301f 624f23699c9SNicholas Piggin ld r12,8(r9) 625f23699c9SNicholas Piggin cmpld r11,r12 626f23699c9SNicholas Piggin bge 301f 627f23699c9SNicholas Piggin ld r12,16(r9) 628f23699c9SNicholas Piggin b 303f 629f23699c9SNicholas Piggin301: 630f23699c9SNicholas Piggin addi r9,r9,24 631f23699c9SNicholas Piggin b 300b 632f23699c9SNicholas Piggin302: 633f23699c9SNicholas Piggin li r12,0 634f23699c9SNicholas Piggin303: 635f23699c9SNicholas Piggin.endm 636f23699c9SNicholas Piggin 637325678fdSNicholas Piggin.macro SEARCH_SOFT_MASK_TABLE 638325678fdSNicholas Piggin#ifdef CONFIG_RELOCATABLE 639325678fdSNicholas Piggin mr r12,r2 640325678fdSNicholas Piggin ld r2,PACATOC(r13) 641325678fdSNicholas Piggin LOAD_REG_ADDR(r9, __start___soft_mask_table) 642325678fdSNicholas Piggin LOAD_REG_ADDR(r10, __stop___soft_mask_table) 643325678fdSNicholas Piggin mr r2,r12 644325678fdSNicholas Piggin#else 645325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___soft_mask_table) 646325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___soft_mask_table) 647325678fdSNicholas Piggin#endif 648325678fdSNicholas Piggin300: 649325678fdSNicholas Piggin cmpd r9,r10 650325678fdSNicholas Piggin beq 302f 651325678fdSNicholas Piggin ld r12,0(r9) 652325678fdSNicholas Piggin cmpld r11,r12 653325678fdSNicholas Piggin blt 301f 654325678fdSNicholas Piggin ld r12,8(r9) 655325678fdSNicholas Piggin cmpld r11,r12 656325678fdSNicholas Piggin bge 301f 657325678fdSNicholas Piggin li r12,1 658325678fdSNicholas Piggin b 303f 659325678fdSNicholas Piggin301: 660325678fdSNicholas Piggin addi r9,r9,16 661325678fdSNicholas Piggin b 300b 662325678fdSNicholas Piggin302: 663325678fdSNicholas Piggin li r12,0 664325678fdSNicholas Piggin303: 665325678fdSNicholas Piggin.endm 666325678fdSNicholas Piggin 6678729c26eSNicholas Piggin/* 668391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 669391e941bSNicholas Piggin * standard exception. 670391e941bSNicholas Piggin */ 6713f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 672391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 673391e941bSNicholas Piggin ld r9,_MSR(r1) 67459dc5bfcSNicholas Piggin li r10,0 675391e941bSNicholas Piggin .if \hsrr 676391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 67759dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 678391e941bSNicholas Piggin .else 679391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 68059dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 681391e941bSNicholas Piggin .endif 682391e941bSNicholas Piggin ld r9,_NIP(r1) 683391e941bSNicholas Piggin .if \hsrr 684391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 685391e941bSNicholas Piggin .else 686391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 687391e941bSNicholas Piggin .endif 688391e941bSNicholas Piggin ld r9,_CTR(r1) 689391e941bSNicholas Piggin mtctr r9 690391e941bSNicholas Piggin ld r9,_XER(r1) 691391e941bSNicholas Piggin mtxer r9 692391e941bSNicholas Piggin ld r9,_LINK(r1) 693391e941bSNicholas Piggin mtlr r9 694391e941bSNicholas Piggin ld r9,_CCR(r1) 695391e941bSNicholas Piggin mtcr r9 696391e941bSNicholas Piggin REST_8GPRS(2, r1) 697391e941bSNicholas Piggin REST_4GPRS(10, r1) 698391e941bSNicholas Piggin REST_GPR(0, r1) 699391e941bSNicholas Piggin /* restore original r1. */ 700391e941bSNicholas Piggin ld r1,GPR1(r1) 701391e941bSNicholas Piggin.endm 702d064151fSNicholas Piggin 70312a04809SNicholas Piggin/* 70457f26649SNicholas Piggin * There are a few constraints to be concerned with. 70557f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 70657f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 70757f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 70857f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 70957f26649SNicholas Piggin * must be used. 71057f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 71157f26649SNicholas Piggin * virtual 0xc00... 71257f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 71357f26649SNicholas Piggin * 71457f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 71557f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 71657f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 71757f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 71857f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 71957f26649SNicholas Piggin * 72057f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 72157f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 72257f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 72357f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 72457f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 72557f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 72657f26649SNicholas Piggin * 7277fa95f9aSNicholas Piggin * The scv instructions are a special case. They get a 0x3000 offset applied. 7287fa95f9aSNicholas Piggin * scv exceptions have unique reentrancy properties, see below. 7297fa95f9aSNicholas Piggin * 73057f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 73157f26649SNicholas Piggin * 73257f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 73357f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 73457f26649SNicholas Piggin * 73557f26649SNicholas Piggin * 7360ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 7370ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 73857f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 7397fa95f9aSNicholas Piggin * 0x1900 - 0x2fff : Real mode trampolines 7407fa95f9aSNicholas Piggin * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 74157f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 7420ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 74357f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 74457f26649SNicholas Piggin * setup code, rest of kernel. 745e0319829SNicholas Piggin * 746e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 747e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 748e0319829SNicholas Piggin * vectors there. 7490ebc4cdaSBenjamin Herrenschmidt */ 75057f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 7517fa95f9aSNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000) 7527fa95f9aSNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900) 75357f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 754ccd47702SNicholas Piggin 755ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 756bd3524feSNicholas Piggin .globl start_real_trampolines 757bd3524feSNicholas Piggin .globl end_real_trampolines 758bd3524feSNicholas Piggin .globl start_virt_trampolines 759bd3524feSNicholas Piggin .globl end_virt_trampolines 760ccd47702SNicholas Piggin#endif 761ccd47702SNicholas Piggin 76257f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 76357f26649SNicholas Piggin/* 76457f26649SNicholas Piggin * Data area reserved for FWNMI option. 76557f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 76657f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 76757f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 76857f26649SNicholas Piggin */ 76957f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 77057f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 77157f26649SNicholas Piggin#else 77257f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 77357f26649SNicholas Piggin#endif 77457f26649SNicholas Piggin 77557f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 77657f26649SNicholas Piggin 7770ebc4cdaSBenjamin Herrenschmidt/* 7780ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 7790ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 7800ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 7810ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 7820ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 7830ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 7840ebc4cdaSBenjamin Herrenschmidt */ 7850ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 7860ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 7870ebc4cdaSBenjamin Herrenschmidt 7887fa95f9aSNicholas Piggin/** 7897fa95f9aSNicholas Piggin * Interrupt 0x3000 - System Call Vectored Interrupt (syscall). 7907fa95f9aSNicholas Piggin * This is a synchronous interrupt invoked with the "scv" instruction. The 7917fa95f9aSNicholas Piggin * system call does not alter the HV bit, so it is directed to the OS. 7927fa95f9aSNicholas Piggin * 7937fa95f9aSNicholas Piggin * Handling: 7947fa95f9aSNicholas Piggin * scv instructions enter the kernel without changing EE, RI, ME, or HV. 7957fa95f9aSNicholas Piggin * In particular, this means we can take a maskable interrupt at any point 7967fa95f9aSNicholas Piggin * in the scv handler, which is unlike any other interrupt. This is solved 797325678fdSNicholas Piggin * by treating the instruction addresses in the handler as being soft-masked, 798325678fdSNicholas Piggin * by adding a SOFT_MASK_TABLE entry for them. 7997fa95f9aSNicholas Piggin * 8007fa95f9aSNicholas Piggin * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and 8017fa95f9aSNicholas Piggin * ensure scv is never executed with relocation off, which means AIL-0 8027fa95f9aSNicholas Piggin * should never happen. 8037fa95f9aSNicholas Piggin * 8049d1988caSNicholas Piggin * Before leaving the following inside-__end_soft_masked text, at least of the 8059d1988caSNicholas Piggin * following must be true: 8067fa95f9aSNicholas Piggin * - MSR[PR]=1 (i.e., return to userspace) 8079d1988caSNicholas Piggin * - MSR_EE|MSR_RI is clear (no reentrant exceptions) 8087fa95f9aSNicholas Piggin * - Standard kernel environment is set up (stack, paca, etc) 8097fa95f9aSNicholas Piggin * 8107fa95f9aSNicholas Piggin * Call convention: 8117fa95f9aSNicholas Piggin * 8127fa95f9aSNicholas Piggin * syscall register convention is in Documentation/powerpc/syscall64-abi.rst 8137fa95f9aSNicholas Piggin */ 8147fa95f9aSNicholas PigginEXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000) 8157fa95f9aSNicholas Piggin /* SCV 0 */ 8167fa95f9aSNicholas Piggin mr r9,r13 8177fa95f9aSNicholas Piggin GET_PACA(r13) 8187fa95f9aSNicholas Piggin mflr r11 8197fa95f9aSNicholas Piggin mfctr r12 8207fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 8217fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 8227fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 8237fa95f9aSNicholas Piggin b system_call_vectored_tramp 8247fa95f9aSNicholas Piggin#else 8257fa95f9aSNicholas Piggin b system_call_vectored_common 8267fa95f9aSNicholas Piggin#endif 8277fa95f9aSNicholas Piggin nop 8287fa95f9aSNicholas Piggin 8297fa95f9aSNicholas Piggin /* SCV 1 - 127 */ 8307fa95f9aSNicholas Piggin .rept 127 8317fa95f9aSNicholas Piggin mr r9,r13 8327fa95f9aSNicholas Piggin GET_PACA(r13) 8337fa95f9aSNicholas Piggin mflr r11 8347fa95f9aSNicholas Piggin mfctr r12 8357fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 8367fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 8377fa95f9aSNicholas Piggin li r0,-1 /* cause failure */ 8387fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 8397fa95f9aSNicholas Piggin b system_call_vectored_sigill_tramp 8407fa95f9aSNicholas Piggin#else 8417fa95f9aSNicholas Piggin b system_call_vectored_sigill 8427fa95f9aSNicholas Piggin#endif 8437fa95f9aSNicholas Piggin .endr 8447fa95f9aSNicholas PigginEXC_VIRT_END(system_call_vectored, 0x3000, 0x1000) 8457fa95f9aSNicholas Piggin 846*787c70f2SNicholas Piggin// Treat scv vectors as soft-masked, see comment above. 847*787c70f2SNicholas Piggin// Use absolute values rather than labels here, so they don't get relocated, 848*787c70f2SNicholas Piggin// because this code runs unrelocated. 849*787c70f2SNicholas PigginSOFT_MASK_TABLE(0xc000000000003000, 0xc000000000004000) 850325678fdSNicholas Piggin 8517fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 8527fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_tramp) 8537fa95f9aSNicholas Piggin __LOAD_HANDLER(r10, system_call_vectored_common) 8547fa95f9aSNicholas Piggin mtctr r10 8557fa95f9aSNicholas Piggin bctr 8567fa95f9aSNicholas Piggin 8577fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) 8587fa95f9aSNicholas Piggin __LOAD_HANDLER(r10, system_call_vectored_sigill) 8597fa95f9aSNicholas Piggin mtctr r10 8607fa95f9aSNicholas Piggin bctr 8617fa95f9aSNicholas Piggin#endif 8627fa95f9aSNicholas Piggin 8637fa95f9aSNicholas Piggin 864e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 8651a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 866e0319829SNicholas Piggin 867fb479e44SNicholas Piggin 86894325357SNicholas Piggin/** 86994325357SNicholas Piggin * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 87094325357SNicholas Piggin * This is a non-maskable, asynchronous interrupt always taken in real-mode. 87194325357SNicholas Piggin * It is caused by: 87294325357SNicholas Piggin * - Wake from power-saving state, on powernv. 87394325357SNicholas Piggin * - An NMI from another CPU, triggered by firmware or hypercall. 87494325357SNicholas Piggin * - As crash/debug signal injected from BMC, firmware or hypervisor. 87594325357SNicholas Piggin * 87694325357SNicholas Piggin * Handling: 87794325357SNicholas Piggin * Power-save wakeup is the only performance critical path, so this is 87894325357SNicholas Piggin * determined quickly as possible first. In this case volatile registers 87994325357SNicholas Piggin * can be discarded and SPRs like CFAR don't need to be read. 88094325357SNicholas Piggin * 88194325357SNicholas Piggin * If not a powersave wakeup, then it's run as a regular interrupt, however 88294325357SNicholas Piggin * it uses its own stack and PACA save area to preserve the regular kernel 88394325357SNicholas Piggin * environment for debugging. 88494325357SNicholas Piggin * 88594325357SNicholas Piggin * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 88694325357SNicholas Piggin * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 88794325357SNicholas Piggin * correct to switch to virtual mode to run the regular interrupt handler 88894325357SNicholas Piggin * because it might be interrupted when the MMU is in a bad state (e.g., SLB 88994325357SNicholas Piggin * is clear). 89094325357SNicholas Piggin * 89194325357SNicholas Piggin * FWNMI: 89294325357SNicholas Piggin * PAPR specifies a "fwnmi" facility which sends the sreset to a different 89394325357SNicholas Piggin * entry point with a different register set up. Some hypervisors will 89494325357SNicholas Piggin * send the sreset to 0x100 in the guest if it is not fwnmi capable. 89594325357SNicholas Piggin * 89694325357SNicholas Piggin * KVM: 89794325357SNicholas Piggin * Unlike most SRR interrupts, this may be taken by the host while executing 89894325357SNicholas Piggin * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 89994325357SNicholas Piggin * mode and then raise the sreset. 90094325357SNicholas Piggin */ 9014f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 9024f50541fSNicholas Piggin IVEC=0x100 9034f50541fSNicholas Piggin IAREA=PACA_EXNMI 9048729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 9054f50541fSNicholas Piggin /* 9064f50541fSNicholas Piggin * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 9074f50541fSNicholas Piggin * being used, so a nested NMI exception would corrupt it. 9084f50541fSNicholas Piggin */ 9094f50541fSNicholas Piggin ISET_RI=0 9104f50541fSNicholas Piggin ISTACK=0 9114f50541fSNicholas Piggin IKVM_REAL=1 9124f50541fSNicholas PigginINT_DEFINE_END(system_reset) 9134f50541fSNicholas Piggin 914a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 915fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 916fb479e44SNicholas Piggin /* 917fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 918ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 919ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 920ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 921ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 9220e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 9230e10be2bSNicholas Piggin * 9240e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 9250e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 9260e10be2bSNicholas Piggin * 9270e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 9280e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 9290e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 9300e10be2bSNicholas Piggin * common exception macros to handle it. 931fb479e44SNicholas Piggin */ 932a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 9330e10be2bSNicholas Piggin SET_SCRATCH0(r13) 9340e10be2bSNicholas Piggin GET_PACA(r13) 9350e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 9360e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 9370e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 938a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 9390e10be2bSNicholas Piggin mfocrf r4,0x80 9400e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 9410e10be2bSNicholas Piggin bne+ system_reset_idle_wake 9420e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 9430e10be2bSNicholas Piggin mtocrf 0x80,r4 9440e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 9450e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 9460e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 9470e10be2bSNicholas Piggin GET_SCRATCH0(r13) 948a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 949fb479e44SNicholas Piggin#endif 950fb479e44SNicholas Piggin 9514f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 952c4f3b52cSNicholas Piggin /* 9530e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 9540e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 9550e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 9560e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 9570e10be2bSNicholas Piggin * be dangerous anyway. 958c4f3b52cSNicholas Piggin */ 9591a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 9601a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 961fb479e44SNicholas Piggin 962fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 9630e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 9640e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 9650e10be2bSNicholas Piggin cmpwi cr1,r5,2 9660e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 9670e10be2bSNicholas Piggin BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) 968371fefd6SPaul Mackerras#endif 969371fefd6SPaul Mackerras 970acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 971acc8da44SNicholas Piggin/* 972acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 973acc8da44SNicholas Piggin */ 974acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 9754f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 976acc8da44SNicholas Piggin 977acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 978acc8da44SNicholas Piggin 979a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 9808729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 981c4f3b52cSNicholas Piggin /* 982c4f3b52cSNicholas Piggin * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 983c4f3b52cSNicholas Piggin * to recover, but nested NMI will notice in_nmi and not recover 984c4f3b52cSNicholas Piggin * because of the use of the NMI stack. in_nmi reentrancy is tested in 985c4f3b52cSNicholas Piggin * system_reset_exception. 986c4f3b52cSNicholas Piggin */ 987c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 988c4f3b52cSNicholas Piggin addi r10,r10,1 989c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 990c4f3b52cSNicholas Piggin li r10,MSR_RI 991c4f3b52cSNicholas Piggin mtmsrd r10,1 992aca79d2bSVaidyanathan Srinivasan 993b1ee8a3dSNicholas Piggin mr r10,r1 994b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 995b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 9968729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 99747169fbaSNicholas Piggin 998c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 999c06075f3SNicholas Piggin bl system_reset_exception 100015b4dd79SNicholas Piggin 100115b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 1002fbc50063SNicholas Piggin li r9,0 100315b4dd79SNicholas Piggin mtmsrd r9,1 1004c4f3b52cSNicholas Piggin 1005c4f3b52cSNicholas Piggin /* 100615b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 1007c4f3b52cSNicholas Piggin */ 1008c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 1009c4f3b52cSNicholas Piggin subi r10,r10,1 1010c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 1011c4f3b52cSNicholas Piggin 10128e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 10133f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 101415b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 1015582baf44SNicholas Piggin 10160ebc4cdaSBenjamin Herrenschmidt 101794325357SNicholas Piggin/** 101894325357SNicholas Piggin * Interrupt 0x200 - Machine Check Interrupt (MCE). 101994325357SNicholas Piggin * This is a non-maskable interrupt always taken in real-mode. It can be 102094325357SNicholas Piggin * synchronous or asynchronous, caused by hardware or software, and it may be 102194325357SNicholas Piggin * taken in a power-saving state. 102294325357SNicholas Piggin * 102394325357SNicholas Piggin * Handling: 102494325357SNicholas Piggin * Similarly to system reset, this uses its own stack and PACA save area, 102594325357SNicholas Piggin * the difference is re-entrancy is allowed on the machine check stack. 102694325357SNicholas Piggin * 102794325357SNicholas Piggin * machine_check_early is run in real mode, and carefully decodes the 102894325357SNicholas Piggin * machine check and tries to handle it (e.g., flush the SLB if there was an 102994325357SNicholas Piggin * error detected there), determines if it was recoverable and logs the 103094325357SNicholas Piggin * event. 103194325357SNicholas Piggin * 1032b44fc96dSNicholas Piggin * This early code does not "reconcile" irq soft-mask state like SRESET or 1033b44fc96dSNicholas Piggin * regular interrupts do, so irqs_disabled() among other things may not work 1034b44fc96dSNicholas Piggin * properly (irq disable/enable already doesn't work because irq tracing can 1035b44fc96dSNicholas Piggin * not work in real mode). 1036b44fc96dSNicholas Piggin * 103794325357SNicholas Piggin * Then, depending on the execution context when the interrupt is taken, there 103894325357SNicholas Piggin * are 3 main actions: 103994325357SNicholas Piggin * - Executing in kernel mode. The event is queued with irq_work, which means 104094325357SNicholas Piggin * it is handled when it is next safe to do so (i.e., the kernel has enabled 104194325357SNicholas Piggin * interrupts), which could be immediately when the interrupt returns. This 104294325357SNicholas Piggin * avoids nasty issues like switching to virtual mode when the MMU is in a 104394325357SNicholas Piggin * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 104494325357SNicholas Piggin * but it has different priorities). Check to see if the CPU was in power 104594325357SNicholas Piggin * save, and return via the wake up code if it was. 104694325357SNicholas Piggin * 104794325357SNicholas Piggin * - Executing in user mode. machine_check_exception is run like a normal 104894325357SNicholas Piggin * interrupt handler, which processes the data generated by the early handler. 104994325357SNicholas Piggin * 105094325357SNicholas Piggin * - Executing in guest mode. The interrupt is run with its KVM test, and 105194325357SNicholas Piggin * branches to KVM to deal with. KVM may queue the event for the host 105294325357SNicholas Piggin * to report later. 105394325357SNicholas Piggin * 105494325357SNicholas Piggin * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 105594325357SNicholas Piggin * or SCRATCH0 is in use, it may cause a crash. 105694325357SNicholas Piggin * 105794325357SNicholas Piggin * KVM: 105894325357SNicholas Piggin * See SRESET. 105994325357SNicholas Piggin */ 10604f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 10614f50541fSNicholas Piggin IVEC=0x200 10624f50541fSNicholas Piggin IAREA=PACA_EXMC 10638729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 1064d73a10cbSNicholas Piggin IREALMODE_COMMON=1 1065c8eb54dbSNicholas Piggin /* 1066c8eb54dbSNicholas Piggin * MSR_RI is not enabled, because PACA_EXMC is being used, so a 1067c8eb54dbSNicholas Piggin * nested machine check corrupts it. machine_check_common enables 1068c8eb54dbSNicholas Piggin * MSR_RI. 1069c8eb54dbSNicholas Piggin */ 10704f50541fSNicholas Piggin ISET_RI=0 10714f50541fSNicholas Piggin ISTACK=0 10724f50541fSNicholas Piggin IDAR=1 10734f50541fSNicholas Piggin IDSISR=1 10744f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 10754f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 10764f50541fSNicholas Piggin 10774f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 10784f50541fSNicholas Piggin IVEC=0x200 10794f50541fSNicholas Piggin IAREA=PACA_EXMC 10808729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 10814f50541fSNicholas Piggin ISET_RI=0 10824f50541fSNicholas Piggin IDAR=1 10834f50541fSNicholas Piggin IDSISR=1 10844f50541fSNicholas Piggin IKVM_REAL=1 10854f50541fSNicholas PigginINT_DEFINE_END(machine_check) 10864f50541fSNicholas Piggin 10874f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 10884f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 10891a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 10901a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 1091c8eb54dbSNicholas Piggin 1092abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1093abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 1094abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 10954f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 1096abd1f4caSNicholas Piggin#endif 1097abd1f4caSNicholas Piggin 1098fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 1099fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1100fce16d48SNicholas Piggin li r9,0; \ 1101fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 1102fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 1103fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 1104fce16d48SNicholas Piggin subi r12,r12,1; \ 1105fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 11063f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 1107fce16d48SNicholas Piggin 1108c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 11099600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 11109600f261SNicholas Piggin 1111afcf0095SNicholas Piggin /* 1112afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 1113afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 1114afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1115afcf0095SNicholas Piggin * 1116afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 1117afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 1118afcf0095SNicholas Piggin * machine checks. 1119afcf0095SNicholas Piggin * 1120afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 1121afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 1122afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 1123afcf0095SNicholas Piggin * 1124afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1125afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 1126afcf0095SNicholas Piggin * rfid with MSR_ME=1. 11271945bc45SNicholas Piggin * 11281945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 11291945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 11302bf1071aSNicholas Piggin * to restore state. 1131afcf0095SNicholas Piggin */ 1132afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 1133afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 1134c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1135afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 1136afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 1137c8eb54dbSNicholas Piggin 1138c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 1139c8eb54dbSNicholas Piggin bne 1f 1140c8eb54dbSNicholas Piggin /* First machine check entry */ 1141c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1142b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1143b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1144b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1145c8eb54dbSNicholas Piggin 11468729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1147c8eb54dbSNicholas Piggin 1148db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1149296e753fSNicholas Piggin bl enable_machine_check 1150db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1151296e753fSNicholas Piggin li r10,MSR_RI 1152296e753fSNicholas Piggin mtmsrd r10,1 1153296e753fSNicholas Piggin 1154afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1155afcf0095SNicholas Piggin bl machine_check_early 1156afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1157afcf0095SNicholas Piggin ld r12,_MSR(r1) 11581945bc45SNicholas Piggin 1159afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1160afcf0095SNicholas Piggin /* 1161afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1162afcf0095SNicholas Piggin * of the following is true: 1163afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1164afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1165afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1166afcf0095SNicholas Piggin * 1167afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1168afcf0095SNicholas Piggin */ 11691945bc45SNicholas PigginBEGIN_FTR_SECTION 11701945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 11716102c005SNicholas Piggin bne machine_check_idle_common 11721945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1173afcf0095SNicholas Piggin#endif 11741945bc45SNicholas Piggin 1175afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1176afcf0095SNicholas Piggin /* 117719dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 117805f97d94SNicholas Piggin * exception handler which will take the 117969fdd674SNicholas Piggin * machine_check_kvm->kvm_interrupt branch to deliver the MC event 118005f97d94SNicholas Piggin * to guest. 1181afcf0095SNicholas Piggin */ 1182afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1183afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1184b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1185afcf0095SNicholas Piggin#endif 118619dbe673SNicholas Piggin 1187afcf0095SNicholas Piggin /* 118819dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 118919dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 119019dbe673SNicholas Piggin */ 119119dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1192b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 119319dbe673SNicholas Piggin 119419dbe673SNicholas Piggin /* 119519dbe673SNicholas Piggin * At this point we are coming from kernel context. 1196afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1197afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1198afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1199afcf0095SNicholas Piggin */ 1200afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1201b7d9ccecSNicholas Piggin beq unrecoverable_mce 1202b7d9ccecSNicholas Piggin 1203afcf0095SNicholas Piggin /* 1204afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1205afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1206afcf0095SNicholas Piggin */ 1207afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1208afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1209b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1210272f6364SNicholas Piggin 1211afcf0095SNicholas Piggin /* 1212afcf0095SNicholas Piggin * Return from MC interrupt. 1213afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1214afcf0095SNicholas Piggin * returning from kernel or opal call. 1215afcf0095SNicholas Piggin */ 1216afcf0095SNicholas Piggin bl machine_check_queue_event 1217afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1218fe9d482bSNicholas Piggin RFI_TO_KERNEL 1219272f6364SNicholas Piggin 1220b3fe3526SNicholas Pigginmce_deliver: 1221b3fe3526SNicholas Piggin /* 1222b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1223b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1224b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1225b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1226b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1227b3fe3526SNicholas Piggin */ 12280b66370cSNicholas PigginBEGIN_FTR_SECTION 12290b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 12300b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 12310b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1232afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 12334f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1234afcf0095SNicholas Piggin 1235fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1236fce16d48SNicholas Piggin /* 1237fce16d48SNicholas Piggin * Machine check is different because we use a different 1238fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1239fce16d48SNicholas Piggin */ 12404f50541fSNicholas Piggin GEN_COMMON machine_check 12414f50541fSNicholas Piggin 1242fce16d48SNicholas Piggin /* Enable MSR_RI when finished with PACA_EXMC */ 1243fce16d48SNicholas Piggin li r10,MSR_RI 1244fce16d48SNicholas Piggin mtmsrd r10,1 1245fce16d48SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1246fce16d48SNicholas Piggin bl machine_check_exception 12471df7d5e4SNicholas Piggin b interrupt_return_srr 1248fce16d48SNicholas Piggin 12499600f261SNicholas Piggin 1250fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1251fce16d48SNicholas Piggin/* 1252fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1253fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1254fce16d48SNicholas Piggin */ 1255fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 1256fce16d48SNicholas Piggin bl machine_check_queue_event 1257fce16d48SNicholas Piggin 1258fce16d48SNicholas Piggin /* 12598a5054d8SNicholas Piggin * GPR-loss wakeups are relatively straightforward, because the 12608a5054d8SNicholas Piggin * idle sleep code has saved all non-volatile registers on its 12618a5054d8SNicholas Piggin * own stack, and r1 in PACAR1. 1262fce16d48SNicholas Piggin * 12638a5054d8SNicholas Piggin * For no-loss wakeups the r1 and lr registers used by the 12648a5054d8SNicholas Piggin * early machine check handler have to be restored first. r2 is 12658a5054d8SNicholas Piggin * the kernel TOC, so no need to restore it. 1266fce16d48SNicholas Piggin * 1267fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1268fce16d48SNicholas Piggin */ 1269fce16d48SNicholas Piggin ld r3,_MSR(r1) 1270fce16d48SNicholas Piggin ld r4,_LINK(r1) 12718a5054d8SNicholas Piggin ld r1,GPR1(r1) 1272fce16d48SNicholas Piggin 1273fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1274fce16d48SNicholas Piggin subi r11,r11,1 1275fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1276fce16d48SNicholas Piggin 1277fce16d48SNicholas Piggin mtlr r4 1278fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1279fce16d48SNicholas Piggin cmpwi cr1,r10,2 12808a5054d8SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1281fce16d48SNicholas Piggin b idle_return_gpr_loss 1282fce16d48SNicholas Piggin#endif 1283fce16d48SNicholas Piggin 1284b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1285b7d9ccecSNicholas Piggin /* 1286b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1287b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1288b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1289b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1290b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1291b7d9ccecSNicholas Piggin */ 1292b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1293b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1294b7d9ccecSNicholas Piggin mtmsrd r10,1 1295b7d9ccecSNicholas Piggin bl disable_machine_check 1296b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1297b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1298b7d9ccecSNicholas Piggin li r3,MSR_ME 1299b7d9ccecSNicholas Piggin andc r10,r10,r3 1300b7d9ccecSNicholas Piggin mtmsrd r10 1301b7d9ccecSNicholas Piggin 1302ac2a2a14SNicholas Piggin lhz r12,PACA_IN_MCE(r13) 1303ac2a2a14SNicholas Piggin subi r12,r12,1 1304ac2a2a14SNicholas Piggin sth r12,PACA_IN_MCE(r13) 1305ac2a2a14SNicholas Piggin 1306afcf0095SNicholas Piggin /* Invoke machine_check_exception to print MCE event and panic. */ 1307afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1308afcf0095SNicholas Piggin bl machine_check_exception 1309b7d9ccecSNicholas Piggin 1310afcf0095SNicholas Piggin /* 1311b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1312b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1313afcf0095SNicholas Piggin */ 1314b7d9ccecSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1315afcf0095SNicholas Piggin bl unrecoverable_exception 1316b7d9ccecSNicholas Piggin b . 1317afcf0095SNicholas Piggin 13184f50541fSNicholas Piggin 13194f50541fSNicholas Piggin/** 132094325357SNicholas Piggin * Interrupt 0x300 - Data Storage Interrupt (DSI). 132194325357SNicholas Piggin * This is a synchronous interrupt generated due to a data access exception, 132294325357SNicholas Piggin * e.g., a load orstore which does not have a valid page table entry with 132394325357SNicholas Piggin * permissions. DAWR matches also fault here, as do RC updates, and minor misc 132494325357SNicholas Piggin * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 13254f50541fSNicholas Piggin * 132694325357SNicholas Piggin * Handling: 132794325357SNicholas Piggin * - Hash MMU 1328a4922f54SNicholas Piggin * Go to do_hash_fault, which attempts to fill the HPT from an entry in the 1329a4922f54SNicholas Piggin * Linux page table. Hash faults can hit in kernel mode in a fairly 133094325357SNicholas Piggin * arbitrary state (e.g., interrupts disabled, locks held) when accessing 133194325357SNicholas Piggin * "non-bolted" regions, e.g., vmalloc space. However these should always be 1332a4922f54SNicholas Piggin * backed by Linux page table entries. 13334f50541fSNicholas Piggin * 1334a4922f54SNicholas Piggin * If no entry is found the Linux page fault handler is invoked (by 1335a4922f54SNicholas Piggin * do_hash_fault). Linux page faults can happen in kernel mode due to user 1336a4922f54SNicholas Piggin * copy operations of course. 13374f50541fSNicholas Piggin * 1338cd81acc6SNicholas Piggin * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest 1339cd81acc6SNicholas Piggin * MMU context, which may cause a DSI in the host, which must go to the 1340cd81acc6SNicholas Piggin * KVM handler. MSR[IR] is not enabled, so the real-mode handler will 1341cd81acc6SNicholas Piggin * always be used regardless of AIL setting. 1342cd81acc6SNicholas Piggin * 134394325357SNicholas Piggin * - Radix MMU 134494325357SNicholas Piggin * The hardware loads from the Linux page table directly, so a fault goes 134594325357SNicholas Piggin * immediately to Linux page fault. 134694325357SNicholas Piggin * 134794325357SNicholas Piggin * Conditions like DAWR match are handled on the way in to Linux page fault. 13484f50541fSNicholas Piggin */ 1349a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1350a42a239dSNicholas Piggin IVEC=0x300 1351a42a239dSNicholas Piggin IDAR=1 1352a42a239dSNicholas Piggin IDSISR=1 1353a42a239dSNicholas Piggin IKVM_REAL=1 1354a42a239dSNicholas PigginINT_DEFINE_END(data_access) 13550ebc4cdaSBenjamin Herrenschmidt 1356e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1357689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1358e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1359e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1360a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1361e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 136280795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 13637cb3a1a0SNicholas Piggin GEN_COMMON data_access 1364a01a3f2dSNicholas Piggin ld r4,_DSISR(r1) 1365a4922f54SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1366a01a3f2dSNicholas Piggin andis. r0,r4,DSISR_DABRMATCH@h 136736f01141SNicholas Piggin bne- 1f 136880795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 1369a4922f54SNicholas Piggin bl do_hash_fault 137080795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 1371a4922f54SNicholas Piggin bl do_page_fault 137280795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 13731df7d5e4SNicholas Piggin b interrupt_return_srr 1374a4922f54SNicholas Piggin 1375a4922f54SNicholas Piggin1: bl do_break 137636f01141SNicholas Piggin /* 137736f01141SNicholas Piggin * do_break() may have changed the NV GPRS while handling a breakpoint. 137836f01141SNicholas Piggin * If so, we need to restore them with their updated values. 137936f01141SNicholas Piggin */ 138036f01141SNicholas Piggin REST_NVGPRS(r1) 13811df7d5e4SNicholas Piggin b interrupt_return_srr 138280795e6cSNicholas Piggin 13830ebc4cdaSBenjamin Herrenschmidt 138494325357SNicholas Piggin/** 138594325357SNicholas Piggin * Interrupt 0x380 - Data Segment Interrupt (DSLB). 138694325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault missing SLB 138794325357SNicholas Piggin * entry for HPT, or an address outside RPT translation range. 138894325357SNicholas Piggin * 138994325357SNicholas Piggin * Handling: 139094325357SNicholas Piggin * - HPT: 139194325357SNicholas Piggin * This refills the SLB, or reports an access fault similarly to a bad page 139294325357SNicholas Piggin * fault. When coming from user-mode, the SLB handler may access any kernel 139394325357SNicholas Piggin * data, though it may itself take a DSLB. When coming from kernel mode, 139494325357SNicholas Piggin * recursive faults must be avoided so access is restricted to the kernel 139594325357SNicholas Piggin * image text/data, kernel stack, and any data allocated below 139694325357SNicholas Piggin * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 139794325357SNicholas Piggin * on user-handler data structures. 139894325357SNicholas Piggin * 1399cd81acc6SNicholas Piggin * KVM: Same as 0x300, DSLB must test for KVM guest. 140094325357SNicholas Piggin */ 14014f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 14024f50541fSNicholas Piggin IVEC=0x380 14034f50541fSNicholas Piggin IDAR=1 14044f50541fSNicholas Piggin IKVM_REAL=1 14054f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 14064f50541fSNicholas Piggin 14071a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1408689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 14091a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 14101a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 14114f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 14121a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 141348e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 14144f50541fSNicholas Piggin GEN_COMMON data_access_slb 14157100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14167100e870SNicholas Piggin /* HPT case, do SLB fault */ 1417a01a3f2dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 141848e7b769SNicholas Piggin bl do_slb_fault 141948e7b769SNicholas Piggin cmpdi r3,0 142048e7b769SNicholas Piggin bne- 1f 14211df7d5e4SNicholas Piggin b fast_interrupt_return_srr 142248e7b769SNicholas Piggin1: /* Error case */ 14237100e870SNicholas PigginMMU_FTR_SECTION_ELSE 14247100e870SNicholas Piggin /* Radix case, access is outside page table range */ 14257100e870SNicholas Piggin li r3,-EFAULT 14267100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 142748e7b769SNicholas Piggin std r3,RESULT(r1) 142848e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 142948e7b769SNicholas Piggin bl do_bad_slb_fault 14301df7d5e4SNicholas Piggin b interrupt_return_srr 143148e7b769SNicholas Piggin 14322b9af6e4SNicholas Piggin 143394325357SNicholas Piggin/** 143494325357SNicholas Piggin * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 143594325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 143694325357SNicholas Piggin * instruction fetch. 143794325357SNicholas Piggin * 143894325357SNicholas Piggin * Handling: 143994325357SNicholas Piggin * Similar to DSI, though in response to fetch. The faulting address is found 144094325357SNicholas Piggin * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 144194325357SNicholas Piggin */ 14424f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 14434f50541fSNicholas Piggin IVEC=0x400 1444a3cd35beSNicholas Piggin IISIDE=1 1445a3cd35beSNicholas Piggin IDAR=1 1446a3cd35beSNicholas Piggin IDSISR=1 14472284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14484f50541fSNicholas Piggin IKVM_REAL=1 14492284ffeaSNicholas Piggin#endif 14504f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 14514f50541fSNicholas Piggin 14527299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 14534f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 14547299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 14557299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 14564f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 14577299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 145827ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 14594f50541fSNicholas Piggin GEN_COMMON instruction_access 1460a4922f54SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 146127ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 1462a4922f54SNicholas Piggin bl do_hash_fault 146327ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 1464a4922f54SNicholas Piggin bl do_page_fault 146527ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 14661df7d5e4SNicholas Piggin b interrupt_return_srr 146727ce77dfSNicholas Piggin 14680ebc4cdaSBenjamin Herrenschmidt 146994325357SNicholas Piggin/** 147094325357SNicholas Piggin * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 147194325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 147294325357SNicholas Piggin * instruction fetch. 147394325357SNicholas Piggin * 147494325357SNicholas Piggin * Handling: 147594325357SNicholas Piggin * Similar to DSLB, though in response to fetch. The faulting address is found 147694325357SNicholas Piggin * in SRR0 (rather than DAR). 147794325357SNicholas Piggin */ 14784f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 14794f50541fSNicholas Piggin IVEC=0x480 1480a3cd35beSNicholas Piggin IISIDE=1 1481a3cd35beSNicholas Piggin IDAR=1 14822284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 14834f50541fSNicholas Piggin IKVM_REAL=1 14842284ffeaSNicholas Piggin#endif 14854f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 14864f50541fSNicholas Piggin 14877299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 14884f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 14897299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 14907299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 14914f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 14927299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 149348e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 14944f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 14957100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14967100e870SNicholas Piggin /* HPT case, do SLB fault */ 1497a01a3f2dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 149848e7b769SNicholas Piggin bl do_slb_fault 149948e7b769SNicholas Piggin cmpdi r3,0 150048e7b769SNicholas Piggin bne- 1f 15011df7d5e4SNicholas Piggin b fast_interrupt_return_srr 150248e7b769SNicholas Piggin1: /* Error case */ 15037100e870SNicholas PigginMMU_FTR_SECTION_ELSE 15047100e870SNicholas Piggin /* Radix case, access is outside page table range */ 15057100e870SNicholas Piggin li r3,-EFAULT 15067100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 150748e7b769SNicholas Piggin std r3,RESULT(r1) 150848e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 150948e7b769SNicholas Piggin bl do_bad_slb_fault 15101df7d5e4SNicholas Piggin b interrupt_return_srr 15115e46e29eSNicholas Piggin 15129600f261SNicholas Piggin 151394325357SNicholas Piggin/** 151494325357SNicholas Piggin * Interrupt 0x500 - External Interrupt. 151594325357SNicholas Piggin * This is an asynchronous maskable interrupt in response to an "external 151694325357SNicholas Piggin * exception" from the interrupt controller or hypervisor (e.g., device 151794325357SNicholas Piggin * interrupt). It is maskable in hardware by clearing MSR[EE], and 151894325357SNicholas Piggin * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 151994325357SNicholas Piggin * 152094325357SNicholas Piggin * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 152194325357SNicholas Piggin * interrupts are delivered with HSRR registers, guests use SRRs, which 152294325357SNicholas Piggin * reqiures IHSRR_IF_HVMODE. 152394325357SNicholas Piggin * 152494325357SNicholas Piggin * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 152594325357SNicholas Piggin * external interrupts are delivered as Hypervisor Virtualization Interrupts 152694325357SNicholas Piggin * rather than External Interrupts. 152794325357SNicholas Piggin * 152894325357SNicholas Piggin * Handling: 152994325357SNicholas Piggin * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 153094325357SNicholas Piggin * because registers at the time of the interrupt are not so important as it is 153194325357SNicholas Piggin * asynchronous. 153294325357SNicholas Piggin * 153394325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 153494325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 153594325357SNicholas Piggin */ 15364f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 15374f50541fSNicholas Piggin IVEC=0x500 15383f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 15394f50541fSNicholas Piggin IMASK=IRQS_DISABLED 15404f50541fSNicholas Piggin IKVM_REAL=1 15414f50541fSNicholas Piggin IKVM_VIRT=1 15424f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 15434f50541fSNicholas Piggin 15441a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 15454f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 15461a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 15471a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 15484f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 15491a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1550eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 15514f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1552eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1553eb204d86SNicholas Piggin bl do_IRQ 15541df7d5e4SNicholas Piggin BEGIN_FTR_SECTION 15551df7d5e4SNicholas Piggin b interrupt_return_hsrr 15561df7d5e4SNicholas Piggin FTR_SECTION_ELSE 15571df7d5e4SNicholas Piggin b interrupt_return_srr 15581df7d5e4SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1559c138e588SNicholas Piggin 1560c138e588SNicholas Piggin 156194325357SNicholas Piggin/** 156294325357SNicholas Piggin * Interrupt 0x600 - Alignment Interrupt 156394325357SNicholas Piggin * This is a synchronous interrupt in response to data alignment fault. 156494325357SNicholas Piggin */ 15654f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 15664f50541fSNicholas Piggin IVEC=0x600 15674f50541fSNicholas Piggin IDAR=1 15684f50541fSNicholas Piggin IDSISR=1 15692284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15704f50541fSNicholas Piggin IKVM_REAL=1 15712284ffeaSNicholas Piggin#endif 15724f50541fSNicholas PigginINT_DEFINE_END(alignment) 15734f50541fSNicholas Piggin 1574e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 15754f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1576e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1577e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 15784f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1579e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1580f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 15814f50541fSNicholas Piggin GEN_COMMON alignment 1582f9aa6714SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1583f9aa6714SNicholas Piggin bl alignment_exception 1584702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 15851df7d5e4SNicholas Piggin b interrupt_return_srr 1586f9aa6714SNicholas Piggin 1587b01c8b54SPaul Mackerras 158894325357SNicholas Piggin/** 158994325357SNicholas Piggin * Interrupt 0x700 - Program Interrupt (program check). 159094325357SNicholas Piggin * This is a synchronous interrupt in response to various instruction faults: 159194325357SNicholas Piggin * traps, privilege errors, TM errors, floating point exceptions. 159294325357SNicholas Piggin * 159394325357SNicholas Piggin * Handling: 159494325357SNicholas Piggin * This interrupt may use the "emergency stack" in some cases when being taken 159594325357SNicholas Piggin * from kernel context, which complicates handling. 159694325357SNicholas Piggin */ 15974f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 15984f50541fSNicholas Piggin IVEC=0x700 15992284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16004f50541fSNicholas Piggin IKVM_REAL=1 16012284ffeaSNicholas Piggin#endif 16024f50541fSNicholas PigginINT_DEFINE_END(program_check) 16034f50541fSNicholas Piggin 16047299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 1605e7eb9190SMichael Ellerman 1606e7eb9190SMichael Ellerman#ifdef CONFIG_CPU_LITTLE_ENDIAN 1607e7eb9190SMichael Ellerman /* 1608e7eb9190SMichael Ellerman * There's a short window during boot where although the kernel is 1609e7eb9190SMichael Ellerman * running little endian, any exceptions will cause the CPU to switch 1610e7eb9190SMichael Ellerman * back to big endian. For example a WARN() boils down to a trap 1611e7eb9190SMichael Ellerman * instruction, which will cause a program check, and we end up here but 1612e7eb9190SMichael Ellerman * with the CPU in big endian mode. The first instruction of the program 1613e7eb9190SMichael Ellerman * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when 1614e7eb9190SMichael Ellerman * executed in the wrong endian is an lhzu with a ~3GB displacement from 1615e7eb9190SMichael Ellerman * r3. The content of r3 is random, so that is a load from some random 1616e7eb9190SMichael Ellerman * location, and depending on the system can easily lead to a checkstop, 1617e7eb9190SMichael Ellerman * or an infinitely recursive page fault. 1618e7eb9190SMichael Ellerman * 1619e7eb9190SMichael Ellerman * So to handle that case we have a trampoline here that can detect we 1620e7eb9190SMichael Ellerman * are in the wrong endian and flip us back to the correct endian. We 1621e7eb9190SMichael Ellerman * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires 1622e7eb9190SMichael Ellerman * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as 1623e7eb9190SMichael Ellerman * SPRG1 is already used for the paca. SPRG3 is user readable, but this 1624e7eb9190SMichael Ellerman * trampoline is only active very early in boot, and SPRG3 will be 1625e7eb9190SMichael Ellerman * reinitialised in vdso_getcpu_init() before userspace starts. 1626e7eb9190SMichael Ellerman */ 1627e7eb9190SMichael EllermanBEGIN_FTR_SECTION 1628e7eb9190SMichael Ellerman tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 1629e7eb9190SMichael Ellerman b 1f // Skip trampoline if endian is correct 1630e7eb9190SMichael Ellerman .long 0xa643707d // mtsprg 0, r11 Backup r11 1631e7eb9190SMichael Ellerman .long 0xa6027a7d // mfsrr0 r11 1632e7eb9190SMichael Ellerman .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 1633e7eb9190SMichael Ellerman .long 0xa6027b7d // mfsrr1 r11 1634e7eb9190SMichael Ellerman .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 1635e7eb9190SMichael Ellerman .long 0xa600607d // mfmsr r11 1636e7eb9190SMichael Ellerman .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] 1637e7eb9190SMichael Ellerman .long 0xa6037b7d // mtsrr1 r11 1638e7eb9190SMichael Ellerman .long 0x34076039 // li r11, 0x734 1639e7eb9190SMichael Ellerman .long 0xa6037a7d // mtsrr0 r11 1640e7eb9190SMichael Ellerman .long 0x2400004c // rfid 1641e7eb9190SMichael Ellerman mfsprg r11, 3 1642e7eb9190SMichael Ellerman mtsrr1 r11 // Restore SRR1 1643e7eb9190SMichael Ellerman mfsprg r11, 2 1644e7eb9190SMichael Ellerman mtsrr0 r11 // Restore SRR0 1645e7eb9190SMichael Ellerman mfsprg r11, 0 // Restore r11 1646e7eb9190SMichael Ellerman1: 1647e7eb9190SMichael EllermanEND_FTR_SECTION(0, 1) // nop out after boot 1648e7eb9190SMichael Ellerman#endif /* CONFIG_CPU_LITTLE_ENDIAN */ 1649e7eb9190SMichael Ellerman 16504f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 16517299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 16527299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 16534f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 16547299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 165511e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 16568729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 16578729c26eSNicholas Piggin 1658265e60a1SCyril Bur /* 1659265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1660265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1661265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1662265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1663265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1664265e60a1SCyril Bur * the kernel. 1665265e60a1SCyril Bur */ 1666265e60a1SCyril Bur 16670a882e28SNicholas Piggin andi. r10,r12,MSR_PR 16680a882e28SNicholas Piggin bne 2f /* If userspace, go normal path */ 16690a882e28SNicholas Piggin 16700a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 16710a882e28SNicholas Piggin bne 1f /* If TM, emergency */ 16720a882e28SNicholas Piggin 16730a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 16740a882e28SNicholas Piggin blt 2f /* normal path if not */ 16750a882e28SNicholas Piggin 16760a882e28SNicholas Piggin /* Use the emergency stack */ 16770a882e28SNicholas Piggin1: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1678265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1679265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1680265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1681265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 16824f50541fSNicholas Piggin __ISTACK(program_check)=0 16838729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 16841b359982SNicholas Piggin b 3f 16850a882e28SNicholas Piggin2: 16864f50541fSNicholas Piggin __ISTACK(program_check)=1 16878729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 16881b359982SNicholas Piggin3: 168911e87346SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 169011e87346SNicholas Piggin bl program_check_exception 1691702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 16921df7d5e4SNicholas Piggin b interrupt_return_srr 169311e87346SNicholas Piggin 1694a485c709SPaul Mackerras 169594325357SNicholas Piggin/* 169694325357SNicholas Piggin * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 169794325357SNicholas Piggin * This is a synchronous interrupt in response to executing an fp instruction 169894325357SNicholas Piggin * with MSR[FP]=0. 169994325357SNicholas Piggin * 170094325357SNicholas Piggin * Handling: 170194325357SNicholas Piggin * This will load FP registers and enable the FP bit if coming from userspace, 170294325357SNicholas Piggin * otherwise report a bad kernel use of FP. 170394325357SNicholas Piggin */ 17044f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 17054f50541fSNicholas Piggin IVEC=0x800 17062284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17074f50541fSNicholas Piggin IKVM_REAL=1 17082284ffeaSNicholas Piggin#endif 17094f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 17104f50541fSNicholas Piggin 17117299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 17124f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 17137299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 17147299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 17154f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 17167299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1717c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 17184f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1719c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1720c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1721c78d9b97SNicholas Piggin bl kernel_fp_unavailable_exception 172263ce271bSChristophe Leroy0: trap 172363ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1724c78d9b97SNicholas Piggin1: 1725c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1726c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1727c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1728c78d9b97SNicholas Piggin * transaction), go do TM stuff 1729c78d9b97SNicholas Piggin */ 1730c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1731c78d9b97SNicholas Piggin bne- 2f 1732c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1733c78d9b97SNicholas Piggin#endif 1734c78d9b97SNicholas Piggin bl load_up_fpu 17351df7d5e4SNicholas Piggin b fast_interrupt_return_srr 1736c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1737c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1738c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1739c78d9b97SNicholas Piggin bl fp_unavailable_tm 17401df7d5e4SNicholas Piggin b interrupt_return_srr 1741c78d9b97SNicholas Piggin#endif 1742c78d9b97SNicholas Piggin 1743b01c8b54SPaul Mackerras 174494325357SNicholas Piggin/** 174594325357SNicholas Piggin * Interrupt 0x900 - Decrementer Interrupt. 174694325357SNicholas Piggin * This is an asynchronous interrupt in response to a decrementer exception 174794325357SNicholas Piggin * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 174894325357SNicholas Piggin * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 174994325357SNicholas Piggin * local_irq_disable()). 175094325357SNicholas Piggin * 175194325357SNicholas Piggin * Handling: 175294325357SNicholas Piggin * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 175394325357SNicholas Piggin * 175494325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 175594325357SNicholas Piggin * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 175694325357SNicholas Piggin * in the interrupted context. 175794325357SNicholas Piggin * If PPC_WATCHDOG is configured, the soft masked handler will actually set 175894325357SNicholas Piggin * things back up to run soft_nmi_interrupt as a regular interrupt handler 175994325357SNicholas Piggin * on the emergency stack. 176094325357SNicholas Piggin */ 17614f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 17624f50541fSNicholas Piggin IVEC=0x900 17634f50541fSNicholas Piggin IMASK=IRQS_DISABLED 17642284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17654f50541fSNicholas Piggin IKVM_REAL=1 17662284ffeaSNicholas Piggin#endif 17674f50541fSNicholas PigginINT_DEFINE_END(decrementer) 17684f50541fSNicholas Piggin 17697299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1770689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 17717299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 17727299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 17734f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 17747299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1775eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 17764f50541fSNicholas Piggin GEN_COMMON decrementer 1777eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1778eb204d86SNicholas Piggin bl timer_interrupt 17791df7d5e4SNicholas Piggin b interrupt_return_srr 178039c0da57SNicholas Piggin 17810ebc4cdaSBenjamin Herrenschmidt 178294325357SNicholas Piggin/** 178394325357SNicholas Piggin * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 178494325357SNicholas Piggin * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 178594325357SNicholas Piggin * register. 178694325357SNicholas Piggin * 178794325357SNicholas Piggin * Handling: 178894325357SNicholas Piggin * Linux does not use this outside KVM where it's used to keep a host timer 178994325357SNicholas Piggin * while the guest is given control of DEC. It should normally be caught by 179094325357SNicholas Piggin * the KVM test and routed there. 179194325357SNicholas Piggin */ 17924f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 17934f50541fSNicholas Piggin IVEC=0x980 17943f7fbd97SNicholas Piggin IHSRR=1 17952babd6eaSNicholas Piggin ISTACK=0 17964f50541fSNicholas Piggin IKVM_REAL=1 17974f50541fSNicholas Piggin IKVM_VIRT=1 17984f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 17994f50541fSNicholas Piggin 18007299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 18014f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 18027299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 18037299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 18044f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 18057299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1806eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 18072babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 18082babd6eaSNicholas Piggin /* 18092babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 18102babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 18112babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 18122babd6eaSNicholas Piggin * edge-triggered. 18132babd6eaSNicholas Piggin * 18142babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 18152babd6eaSNicholas Piggin */ 181659dc5bfcSNicholas Piggin li r10,0 181759dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 18182babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 18192babd6eaSNicholas Piggin mtctr r10 18202babd6eaSNicholas Piggin mtcrf 0x80,r9 18212babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 18222babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 18232babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 18242babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 18252babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 18262babd6eaSNicholas Piggin HRFI_TO_KERNEL 1827facc6d74SNicholas Piggin 1828da2bc464SMichael Ellerman 182994325357SNicholas Piggin/** 183094325357SNicholas Piggin * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 183194325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsndp doorbell. 183294325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 183394325357SNicholas Piggin * IRQS_DISABLED mask (i.e., local_irq_disable()). 183494325357SNicholas Piggin * 183594325357SNicholas Piggin * Handling: 183694325357SNicholas Piggin * Guests may use this for IPIs between threads in a core if the 183794325357SNicholas Piggin * hypervisor supports it. NVGPRS are not saved (see 0x500). 183894325357SNicholas Piggin * 183994325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 184094325357SNicholas Piggin * replay, leaving MSR[EE] enabled in the interrupted context because the 184194325357SNicholas Piggin * doorbells are edge triggered. 184294325357SNicholas Piggin */ 18434f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 18444f50541fSNicholas Piggin IVEC=0xa00 18454f50541fSNicholas Piggin IMASK=IRQS_DISABLED 18462284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 18474f50541fSNicholas Piggin IKVM_REAL=1 18482284ffeaSNicholas Piggin#endif 18494f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 18504f50541fSNicholas Piggin 18517299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 18524f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 18537299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 18547299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 18554f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 18567299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1857eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 18584f50541fSNicholas Piggin GEN_COMMON doorbell_super 1859eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1860ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1861eb204d86SNicholas Piggin bl doorbell_exception 1862ca243163SNicholas Piggin#else 18636c6aee00SNicholas Piggin bl unknown_async_exception 1864ca243163SNicholas Piggin#endif 18651df7d5e4SNicholas Piggin b interrupt_return_srr 1866ca243163SNicholas Piggin 1867da2bc464SMichael Ellerman 18685ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 18695ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1870341215dcSNicholas Piggin 187194325357SNicholas Piggin/** 187294325357SNicholas Piggin * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 187394325357SNicholas Piggin * This is a synchronous interrupt invoked with the "sc" instruction. The 187494325357SNicholas Piggin * system call is invoked with "sc 0" and does not alter the HV bit, so it 187594325357SNicholas Piggin * is directed to the currently running OS. The hypercall is invoked with 187694325357SNicholas Piggin * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1877acd7d8ceSNicholas Piggin * 1878acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1879acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1880acd7d8ceSNicholas Piggin * 188194325357SNicholas Piggin * Handling: 188294325357SNicholas Piggin * If the KVM test fires then it was due to a hypercall and is accordingly 188394325357SNicholas Piggin * routed to KVM. Otherwise this executes a normal Linux system call. 188494325357SNicholas Piggin * 1885acd7d8ceSNicholas Piggin * Call convention: 1886acd7d8ceSNicholas Piggin * 188758b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 188858b278f5SVaibhav Jain * Documentation/powerpc/syscall64-abi.rst and 188958b278f5SVaibhav Jain * Documentation/powerpc/papr_hcalls.rst respectively. 1890acd7d8ceSNicholas Piggin * 1891acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 189276fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 189376fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 189476fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1895acd7d8ceSNicholas Piggin */ 1896b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1897b177ae2fSNicholas Piggin IVEC=0xc00 1898b177ae2fSNicholas Piggin IKVM_REAL=1 1899b177ae2fSNicholas Piggin IKVM_VIRT=1 1900b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1901b177ae2fSNicholas Piggin 19021b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1903bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1904bc355125SPaul Mackerras /* 1905acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 190676fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 190776fc0cfcSNicholas Piggin * register. 1908acd7d8ceSNicholas Piggin * 1909acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1910acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1911bc355125SPaul Mackerras */ 19121b4d4a79SNicholas Piggin mtctr r13 19131b4d4a79SNicholas Piggin GET_PACA(r13) 19141b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 19151b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 191669fdd674SNicholas Piggin KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */ 19171b4d4a79SNicholas Piggin mfctr r9 1918bc355125SPaul Mackerras#else 19191b4d4a79SNicholas Piggin mr r9,r13 19201b4d4a79SNicholas Piggin GET_PACA(r13) 19211b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1922bc355125SPaul Mackerras#endif 1923bc355125SPaul Mackerras 1924727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 19251b4d4a79SNicholas PigginBEGIN_FTR_SECTION 19261b4d4a79SNicholas Piggin cmpdi r0,0x1ebe 19271b4d4a79SNicholas Piggin beq- 1f 19281b4d4a79SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 19291b4d4a79SNicholas Piggin#endif 19305c2511bfSMichael Ellerman 1931b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 19321b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 19331b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 1934b0b2a93dSNicholas Piggin 1935b0b2a93dSNicholas Piggin HMT_MEDIUM 1936b0b2a93dSNicholas Piggin 1937b0b2a93dSNicholas Piggin .if ! \virt 193814ad0e7dSNicholas Piggin __LOAD_HANDLER(r10, system_call_common_real) 193914ad0e7dSNicholas Piggin mtctr r10 194014ad0e7dSNicholas Piggin bctr 19411b4d4a79SNicholas Piggin .else 1942b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 1943b0b2a93dSNicholas Piggin __LOAD_HANDLER(r10, system_call_common) 1944b0b2a93dSNicholas Piggin mtctr r10 1945b0b2a93dSNicholas Piggin bctr 1946b0b2a93dSNicholas Piggin#else 19471b4d4a79SNicholas Piggin b system_call_common 1948d807ad37SNicholas Piggin#endif 19491b4d4a79SNicholas Piggin .endif 19501b4d4a79SNicholas Piggin 19511b4d4a79SNicholas Piggin#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 19521b4d4a79SNicholas Piggin /* Fast LE/BE switch system call */ 19531b4d4a79SNicholas Piggin1: mfspr r12,SPRN_SRR1 19541b4d4a79SNicholas Piggin xori r12,r12,MSR_LE 19551b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r12 19561b4d4a79SNicholas Piggin mr r13,r9 19571b4d4a79SNicholas Piggin RFI_TO_USER /* return to userspace */ 19581b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 19591b4d4a79SNicholas Piggin#endif 19601b4d4a79SNicholas Piggin.endm 1961d807ad37SNicholas Piggin 19621a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 19631b4d4a79SNicholas Piggin SYSTEM_CALL 0 19641a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 19651a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 19661b4d4a79SNicholas Piggin SYSTEM_CALL 1 19671a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 1968d807ad37SNicholas Piggin 1969acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 197069fdd674SNicholas PigginTRAMP_REAL_BEGIN(kvm_hcall) 1971e2762743SNicholas Piggin std r9,PACA_EXGEN+EX_R9(r13) 1972e2762743SNicholas Piggin std r11,PACA_EXGEN+EX_R11(r13) 1973e2762743SNicholas Piggin std r12,PACA_EXGEN+EX_R12(r13) 1974e2762743SNicholas Piggin mfcr r9 1975acd7d8ceSNicholas Piggin mfctr r10 1976e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_R13(r13) 1977e2762743SNicholas Piggin li r10,0 1978e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CFAR(r13) 1979e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CTR(r13) 1980e2762743SNicholas Piggin /* 1981e2762743SNicholas Piggin * Save the PPR (on systems that support it) before changing to 1982e2762743SNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 1983e2762743SNicholas Piggin * guest state (it is the guest's PPR value). 1984e2762743SNicholas Piggin */ 1985e2762743SNicholas PigginBEGIN_FTR_SECTION 1986e2762743SNicholas Piggin mfspr r10,SPRN_PPR 1987e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_PPR(r13) 1988e2762743SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1989e2762743SNicholas Piggin 1990e2762743SNicholas Piggin HMT_MEDIUM 1991e2762743SNicholas Piggin 19929600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 19939600f261SNicholas Piggin /* 199431c67cfeSNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives 19959600f261SNicholas Piggin * outside the head section. 19969600f261SNicholas Piggin */ 199731c67cfeSNicholas Piggin __LOAD_FAR_HANDLER(r10, kvmppc_hcall) 19989600f261SNicholas Piggin mtctr r10 19999600f261SNicholas Piggin bctr 20009600f261SNicholas Piggin#else 200131c67cfeSNicholas Piggin b kvmppc_hcall 20029600f261SNicholas Piggin#endif 2003acd7d8ceSNicholas Piggin#endif 2004da2bc464SMichael Ellerman 200594325357SNicholas Piggin/** 200694325357SNicholas Piggin * Interrupt 0xd00 - Trace Interrupt. 200794325357SNicholas Piggin * This is a synchronous interrupt in response to instruction step or 200894325357SNicholas Piggin * breakpoint faults. 200994325357SNicholas Piggin */ 20104f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 20114f50541fSNicholas Piggin IVEC=0xd00 20122284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 20134f50541fSNicholas Piggin IKVM_REAL=1 20142284ffeaSNicholas Piggin#endif 20154f50541fSNicholas PigginINT_DEFINE_END(single_step) 20164f50541fSNicholas Piggin 20177299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 20184f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 20197299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 20207299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 20214f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 20227299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 2023eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 20244f50541fSNicholas Piggin GEN_COMMON single_step 2025eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2026eb204d86SNicholas Piggin bl single_step_exception 20271df7d5e4SNicholas Piggin b interrupt_return_srr 2028da2bc464SMichael Ellerman 20297299417cSNicholas Piggin 203094325357SNicholas Piggin/** 203194325357SNicholas Piggin * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 203294325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 203394325357SNicholas Piggin * guest data access. 203494325357SNicholas Piggin * 203594325357SNicholas Piggin * Handling: 203694325357SNicholas Piggin * This should always get routed to KVM. In radix MMU mode, this is caused 203794325357SNicholas Piggin * by a guest nested radix access that can't be performed due to the 203894325357SNicholas Piggin * partition scope page table. In hash mode, this can be caused by guests 203994325357SNicholas Piggin * running with translation disabled (virtual real mode) or with VPM enabled. 204094325357SNicholas Piggin * KVM will update the page table structures or disallow the access. 204194325357SNicholas Piggin */ 20424f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 20434f50541fSNicholas Piggin IVEC=0xe00 20443f7fbd97SNicholas Piggin IHSRR=1 20454f50541fSNicholas Piggin IDAR=1 20464f50541fSNicholas Piggin IDSISR=1 20474f50541fSNicholas Piggin IKVM_REAL=1 20484f50541fSNicholas Piggin IKVM_VIRT=1 20494f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 20504f50541fSNicholas Piggin 20517299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 20524f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 20537299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 20547299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 20554f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 20567299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2057f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 20584f50541fSNicholas Piggin GEN_COMMON h_data_storage 2059f5c32c1dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2060d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 206171f47976SNicholas Piggin bl do_bad_page_fault_segv 2062d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 2063f5c32c1dSNicholas Piggin bl unknown_exception 2064d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 20651df7d5e4SNicholas Piggin b interrupt_return_hsrr 2066f5c32c1dSNicholas Piggin 20671707dd16SPaul Mackerras 206894325357SNicholas Piggin/** 206994325357SNicholas Piggin * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 207094325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 207194325357SNicholas Piggin * guest instruction fetch, similar to HDSI. 207294325357SNicholas Piggin */ 20734f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 20744f50541fSNicholas Piggin IVEC=0xe20 20753f7fbd97SNicholas Piggin IHSRR=1 20764f50541fSNicholas Piggin IKVM_REAL=1 20774f50541fSNicholas Piggin IKVM_VIRT=1 20784f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 20794f50541fSNicholas Piggin 20807299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 20814f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 20827299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 20837299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 20844f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 20857299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2086eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 20874f50541fSNicholas Piggin GEN_COMMON h_instr_storage 2088eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2089eb204d86SNicholas Piggin bl unknown_exception 20901df7d5e4SNicholas Piggin b interrupt_return_hsrr 209182517cabSNicholas Piggin 20921707dd16SPaul Mackerras 209394325357SNicholas Piggin/** 209494325357SNicholas Piggin * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 209594325357SNicholas Piggin */ 20964f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 20974f50541fSNicholas Piggin IVEC=0xe40 20983f7fbd97SNicholas Piggin IHSRR=1 20994f50541fSNicholas Piggin IKVM_REAL=1 21004f50541fSNicholas Piggin IKVM_VIRT=1 21014f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 21024f50541fSNicholas Piggin 21037299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 21044f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 21057299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 21067299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 21074f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 21087299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2109eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 21104f50541fSNicholas Piggin GEN_COMMON emulation_assist 2111eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2112eb204d86SNicholas Piggin bl emulation_assist_interrupt 2113702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 21141df7d5e4SNicholas Piggin b interrupt_return_hsrr 2115031b4026SNicholas Piggin 21161707dd16SPaul Mackerras 211794325357SNicholas Piggin/** 211894325357SNicholas Piggin * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 211994325357SNicholas Piggin * This is an asynchronous interrupt caused by a Hypervisor Maintenance 212094325357SNicholas Piggin * Exception. It is always taken in real mode but uses HSRR registers 212194325357SNicholas Piggin * unlike SRESET and MCE. 212294325357SNicholas Piggin * 212394325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 212494325357SNicholas Piggin * with IRQS_DISABLED mask (i.e., local_irq_disable()). 212594325357SNicholas Piggin * 212694325357SNicholas Piggin * Handling: 212794325357SNicholas Piggin * This is a special case, this is handled similarly to machine checks, with an 212894325357SNicholas Piggin * initial real mode handler that is not soft-masked, which attempts to fix the 212994325357SNicholas Piggin * problem. Then a regular handler which is soft-maskable and reports the 213094325357SNicholas Piggin * problem. 213194325357SNicholas Piggin * 213294325357SNicholas Piggin * The emergency stack is used for the early real mode handler. 213394325357SNicholas Piggin * 213494325357SNicholas Piggin * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 213594325357SNicholas Piggin * either use soft-masking for the MCE, or use irq_work for the HMI. 213694325357SNicholas Piggin * 213794325357SNicholas Piggin * KVM: 213894325357SNicholas Piggin * Unlike MCE, this calls into KVM without calling the real mode handler 213994325357SNicholas Piggin * first. 2140e0319829SNicholas Piggin */ 21414f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 21424f50541fSNicholas Piggin IVEC=0xe60 21433f7fbd97SNicholas Piggin IHSRR=1 2144d73a10cbSNicholas Piggin IREALMODE_COMMON=1 21454f50541fSNicholas Piggin ISTACK=0 21464f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 21474f50541fSNicholas Piggin IKVM_REAL=1 21484f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 21494f50541fSNicholas Piggin 21504f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 21514f50541fSNicholas Piggin IVEC=0xe60 21523f7fbd97SNicholas Piggin IHSRR=1 21534f50541fSNicholas Piggin IMASK=IRQS_DISABLED 21544f50541fSNicholas Piggin IKVM_REAL=1 21554f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 21564f50541fSNicholas Piggin 2157f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 21584f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2159f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 21601a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 21614f50541fSNicholas Piggin 2162293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 21639600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 21649600f261SNicholas Piggin 216562f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 2166a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 216762f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2168bcbceed4SNicholas Piggin 21698729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 2170bcbceed4SNicholas Piggin 217162f9b03bSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2172293c2e27SNicholas Piggin bl hmi_exception_realmode 21735080332cSMichael Neuling cmpdi cr0,r3,0 217467d4160aSNicholas Piggin bne 1f 21755080332cSMichael Neuling 21763f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 2177222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 21785080332cSMichael Neuling 217967d4160aSNicholas Piggin1: 218062f9b03bSNicholas Piggin /* 218162f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 218262f9b03bSNicholas Piggin * firmware. 218362f9b03bSNicholas Piggin */ 21843f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 21854f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 218662f9b03bSNicholas Piggin 21875080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 21884f50541fSNicholas Piggin GEN_COMMON hmi_exception 2189c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2190c06075f3SNicholas Piggin bl handle_hmi_exception 21911df7d5e4SNicholas Piggin b interrupt_return_hsrr 21921707dd16SPaul Mackerras 21937299417cSNicholas Piggin 219494325357SNicholas Piggin/** 219594325357SNicholas Piggin * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 219694325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsnd doorbell. 219794325357SNicholas Piggin * Similar to the 0xa00 doorbell but for host rather than guest. 219894325357SNicholas Piggin */ 21994f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 22004f50541fSNicholas Piggin IVEC=0xe80 22013f7fbd97SNicholas Piggin IHSRR=1 22024f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22034f50541fSNicholas Piggin IKVM_REAL=1 22044f50541fSNicholas Piggin IKVM_VIRT=1 22054f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 22064f50541fSNicholas Piggin 22077299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 22084f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 22097299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 22107299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 22114f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 22127299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2213eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 22144f50541fSNicholas Piggin GEN_COMMON h_doorbell 2215eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 22169bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 2217eb204d86SNicholas Piggin bl doorbell_exception 22189bcb81bfSNicholas Piggin#else 22196c6aee00SNicholas Piggin bl unknown_async_exception 22209bcb81bfSNicholas Piggin#endif 22211df7d5e4SNicholas Piggin b interrupt_return_hsrr 22229bcb81bfSNicholas Piggin 22230ebc4cdaSBenjamin Herrenschmidt 222494325357SNicholas Piggin/** 222594325357SNicholas Piggin * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 222694325357SNicholas Piggin * This is an asynchronous interrupt in response to an "external exception". 222794325357SNicholas Piggin * Similar to 0x500 but for host only. 222894325357SNicholas Piggin */ 22294f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 22304f50541fSNicholas Piggin IVEC=0xea0 22313f7fbd97SNicholas Piggin IHSRR=1 22324f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22334f50541fSNicholas Piggin IKVM_REAL=1 22344f50541fSNicholas Piggin IKVM_VIRT=1 22354f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 22364f50541fSNicholas Piggin 22377299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 22384f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 22397299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 22407299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 22414f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 22427299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2243eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 22444f50541fSNicholas Piggin GEN_COMMON h_virt_irq 2245eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2246eb204d86SNicholas Piggin bl do_IRQ 22471df7d5e4SNicholas Piggin b interrupt_return_hsrr 224874408776SNicholas Piggin 22499baaef0aSBenjamin Herrenschmidt 22501a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 22511a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 22521a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 22531a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 2254bda7fea2SNicholas Piggin 22550ebc4cdaSBenjamin Herrenschmidt 225694325357SNicholas Piggin/* 225794325357SNicholas Piggin * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 225894325357SNicholas Piggin * This is an asynchronous interrupt in response to a PMU exception. 225994325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 226094325357SNicholas Piggin * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 226194325357SNicholas Piggin * 226294325357SNicholas Piggin * Handling: 226394325357SNicholas Piggin * This calls into the perf subsystem. 226494325357SNicholas Piggin * 226594325357SNicholas Piggin * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 226694325357SNicholas Piggin * runs under local_irq_disable. However it may be soft-masked in 226794325357SNicholas Piggin * powerpc-specific code. 226894325357SNicholas Piggin * 226994325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 227094325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 227194325357SNicholas Piggin */ 22724f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 22734f50541fSNicholas Piggin IVEC=0xf00 22744f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 22752284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 22764f50541fSNicholas Piggin IKVM_REAL=1 22772284ffeaSNicholas Piggin#endif 22784f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 22794f50541fSNicholas Piggin 22807299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 22814f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 22827299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 22837299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 22844f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 22857299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2286eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 22874f50541fSNicholas Piggin GEN_COMMON performance_monitor 2288eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2289eb204d86SNicholas Piggin bl performance_monitor_exception 22901df7d5e4SNicholas Piggin b interrupt_return_srr 2291b1c7f150SNicholas Piggin 22920ebc4cdaSBenjamin Herrenschmidt 229394325357SNicholas Piggin/** 229494325357SNicholas Piggin * Interrupt 0xf20 - Vector Unavailable Interrupt. 229594325357SNicholas Piggin * This is a synchronous interrupt in response to 229694325357SNicholas Piggin * executing a vector (or altivec) instruction with MSR[VEC]=0. 229794325357SNicholas Piggin * Similar to FP unavailable. 229894325357SNicholas Piggin */ 22994f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 23004f50541fSNicholas Piggin IVEC=0xf20 23012284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23024f50541fSNicholas Piggin IKVM_REAL=1 23032284ffeaSNicholas Piggin#endif 23044f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 23054f50541fSNicholas Piggin 23067299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 23074f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 23087299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 23097299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 23104f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 23117299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2312d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 23134f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 2314d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 2315d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 2316d1a0ca9cSNicholas Piggin beq 1f 2317d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2318d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2319d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2320d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 2321d1a0ca9cSNicholas Piggin */ 2322d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2323d1a0ca9cSNicholas Piggin bne- 2f 2324d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2325d1a0ca9cSNicholas Piggin#endif 2326d1a0ca9cSNicholas Piggin bl load_up_altivec 23271df7d5e4SNicholas Piggin b fast_interrupt_return_srr 2328d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2329d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 2330d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2331d1a0ca9cSNicholas Piggin bl altivec_unavailable_tm 23321df7d5e4SNicholas Piggin b interrupt_return_srr 2333d1a0ca9cSNicholas Piggin#endif 2334d1a0ca9cSNicholas Piggin1: 2335d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2336d1a0ca9cSNicholas Piggin#endif 2337d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2338d1a0ca9cSNicholas Piggin bl altivec_unavailable_exception 23391df7d5e4SNicholas Piggin b interrupt_return_srr 2340d1a0ca9cSNicholas Piggin 23410ebc4cdaSBenjamin Herrenschmidt 234294325357SNicholas Piggin/** 234394325357SNicholas Piggin * Interrupt 0xf40 - VSX Unavailable Interrupt. 234494325357SNicholas Piggin * This is a synchronous interrupt in response to 234594325357SNicholas Piggin * executing a VSX instruction with MSR[VSX]=0. 234694325357SNicholas Piggin * Similar to FP unavailable. 234794325357SNicholas Piggin */ 23484f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 23494f50541fSNicholas Piggin IVEC=0xf40 23502284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23514f50541fSNicholas Piggin IKVM_REAL=1 23522284ffeaSNicholas Piggin#endif 23534f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 23544f50541fSNicholas Piggin 23557299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 23564f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 23577299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 23587299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 23594f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 23607299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2361792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 23624f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2363792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2364792cbdddSNicholas PigginBEGIN_FTR_SECTION 2365792cbdddSNicholas Piggin beq 1f 2366792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2367792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2368792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2369792cbdddSNicholas Piggin * transaction), go do TM stuff 2370792cbdddSNicholas Piggin */ 2371792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2372792cbdddSNicholas Piggin bne- 2f 2373792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2374792cbdddSNicholas Piggin#endif 2375792cbdddSNicholas Piggin b load_up_vsx 2376792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2377792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2378792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2379792cbdddSNicholas Piggin bl vsx_unavailable_tm 23801df7d5e4SNicholas Piggin b interrupt_return_srr 2381792cbdddSNicholas Piggin#endif 2382792cbdddSNicholas Piggin1: 2383792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2384792cbdddSNicholas Piggin#endif 2385792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2386792cbdddSNicholas Piggin bl vsx_unavailable_exception 23871df7d5e4SNicholas Piggin b interrupt_return_srr 2388792cbdddSNicholas Piggin 2389d0c0c9a1SMichael Neuling 239094325357SNicholas Piggin/** 239194325357SNicholas Piggin * Interrupt 0xf60 - Facility Unavailable Interrupt. 239294325357SNicholas Piggin * This is a synchronous interrupt in response to 239394325357SNicholas Piggin * executing an instruction without access to the facility that can be 239494325357SNicholas Piggin * resolved by the OS (e.g., FSCR, MSR). 239594325357SNicholas Piggin * Similar to FP unavailable. 239694325357SNicholas Piggin */ 23974f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 23984f50541fSNicholas Piggin IVEC=0xf60 23992284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24004f50541fSNicholas Piggin IKVM_REAL=1 24012284ffeaSNicholas Piggin#endif 24024f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 24034f50541fSNicholas Piggin 24047299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 24054f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 24067299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 24077299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 24084f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 24097299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2410eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 24114f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2412eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2413eb204d86SNicholas Piggin bl facility_unavailable_exception 2414595d153dSMichael Ellerman REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 24151df7d5e4SNicholas Piggin b interrupt_return_srr 24161134713cSNicholas Piggin 2417da2bc464SMichael Ellerman 241894325357SNicholas Piggin/** 241994325357SNicholas Piggin * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 242094325357SNicholas Piggin * This is a synchronous interrupt in response to 242194325357SNicholas Piggin * executing an instruction without access to the facility that can only 242294325357SNicholas Piggin * be resolved in HV mode (e.g., HFSCR). 242394325357SNicholas Piggin * Similar to FP unavailable. 242494325357SNicholas Piggin */ 24254f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 24264f50541fSNicholas Piggin IVEC=0xf80 24273f7fbd97SNicholas Piggin IHSRR=1 24284f50541fSNicholas Piggin IKVM_REAL=1 24294f50541fSNicholas Piggin IKVM_VIRT=1 24304f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 24314f50541fSNicholas Piggin 24327299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 24334f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 24347299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 24357299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 24364f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 24377299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2438eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 24394f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2440eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2441eb204d86SNicholas Piggin bl facility_unavailable_exception 2442595d153dSMichael Ellerman REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */ 24431df7d5e4SNicholas Piggin b interrupt_return_hsrr 244414b0072cSNicholas Piggin 2445da2bc464SMichael Ellerman 24461a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 24471a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 24481a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 24491a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 24501a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 24511a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 24521a6822d1SNicholas Piggin 24531a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 24541a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 24551a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 24561a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2457da2bc464SMichael Ellerman 24580ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 24594f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 24604f50541fSNicholas Piggin IVEC=0x1200 24613f7fbd97SNicholas Piggin IHSRR=1 24624f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 24634f50541fSNicholas Piggin 24647299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 24654f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 24667299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 24671a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2468eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 24694f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2470eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2471eb204d86SNicholas Piggin bl cbe_system_error_exception 24721df7d5e4SNicholas Piggin b interrupt_return_hsrr 24739600f261SNicholas Piggin 2474da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 24751a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 24761a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2477da2bc464SMichael Ellerman#endif 2478da2bc464SMichael Ellerman 2479da487a5dSNicholas Piggin/** 2480da487a5dSNicholas Piggin * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. 2481da487a5dSNicholas Piggin * This has been removed from the ISA before 2.01, which is the earliest 2482da487a5dSNicholas Piggin * 64-bit BookS ISA supported, however the G5 / 970 implements this 2483da487a5dSNicholas Piggin * interrupt with a non-architected feature available through the support 2484da487a5dSNicholas Piggin * processor interface. 2485da487a5dSNicholas Piggin */ 24864f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 24874f50541fSNicholas Piggin IVEC=0x1300 24882284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24894f50541fSNicholas Piggin IKVM_REAL=1 24902284ffeaSNicholas Piggin#endif 24914f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 24924f50541fSNicholas Piggin 24937299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 24944f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 24957299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 24967299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 24974f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 24987299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2499eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 25004f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2501eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2502eb204d86SNicholas Piggin bl instruction_breakpoint_exception 25031df7d5e4SNicholas Piggin b interrupt_return_srr 25044e96dbbfSNicholas Piggin 25057299417cSNicholas Piggin 25061a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 25071a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2508da2bc464SMichael Ellerman 250994325357SNicholas Piggin/** 251094325357SNicholas Piggin * Interrupt 0x1500 - Soft Patch Interrupt 251194325357SNicholas Piggin * 251294325357SNicholas Piggin * Handling: 251394325357SNicholas Piggin * This is an implementation specific interrupt which can be used for a 251494325357SNicholas Piggin * range of exceptions. 251594325357SNicholas Piggin * 251694325357SNicholas Piggin * This interrupt handler is unique in that it runs the denormal assist 251794325357SNicholas Piggin * code even for guests (and even in guest context) without going to KVM, 251894325357SNicholas Piggin * for speed. POWER9 does not raise denorm exceptions, so this special case 251994325357SNicholas Piggin * could be phased out in future to reduce special cases. 252094325357SNicholas Piggin */ 25214f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 25224f50541fSNicholas Piggin IVEC=0x1500 25233f7fbd97SNicholas Piggin IHSRR=1 25244557ac6bSNicholas Piggin IBRANCH_TO_COMMON=0 25259600f261SNicholas Piggin IKVM_REAL=1 25264f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 25274f50541fSNicholas Piggin 25284f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 25294f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2530b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2531d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2532b92a66a6SMichael Neuling bne+ denorm_assist 2533b92a66a6SMichael Neuling#endif 25348729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 25354f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2536d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 25371a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 25384f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2539d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 254052b98923SNicholas Piggin bne+ denorm_assist 25418729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 25421a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2543d7e89849SNicholas Piggin#else 25441a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2545d7e89849SNicholas Piggin#endif 2546b92a66a6SMichael Neuling 2547b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2548da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2549b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2550b92a66a6SMichael Neuling/* 2551b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2552b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2553b92a66a6SMichael Neuling */ 2554b92a66a6SMichael Neuling mfmsr r10 2555b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2556b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2557b92a66a6SMichael Neuling mtmsrd r10 2558b92a66a6SMichael Neuling sync 2559d7c67fb1SMichael Neuling 2560f3c8b6c6SNicholas Piggin .Lreg=0 2561f3c8b6c6SNicholas Piggin .rept 32 2562f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2563f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2564f3c8b6c6SNicholas Piggin .endr 2565d7c67fb1SMichael Neuling 2566b92a66a6SMichael NeulingFTR_SECTION_ELSE 2567b92a66a6SMichael Neuling/* 2568b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2569b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2570b92a66a6SMichael Neuling */ 2571b92a66a6SMichael Neuling mfmsr r10 2572b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2573b92a66a6SMichael Neuling mtmsrd r10 2574b92a66a6SMichael Neuling sync 2575d7c67fb1SMichael Neuling 2576f3c8b6c6SNicholas Piggin .Lreg=0 2577f3c8b6c6SNicholas Piggin .rept 32 2578f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2579f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2580f3c8b6c6SNicholas Piggin .endr 2581d7c67fb1SMichael Neuling 2582b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2583fb0fce3eSMichael Neuling 2584fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2585fb0fce3eSMichael Neuling b denorm_done 2586fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2587fb0fce3eSMichael Neuling/* 2588fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2589fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2590fb0fce3eSMichael Neuling */ 2591f3c8b6c6SNicholas Piggin .Lreg=32 2592f3c8b6c6SNicholas Piggin .rept 32 2593f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2594f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2595f3c8b6c6SNicholas Piggin .endr 2596f3c8b6c6SNicholas Piggin 2597fb0fce3eSMichael Neulingdenorm_done: 2598f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2599f14040bcSMichael Neuling subi r11,r11,4 2600b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2601b92a66a6SMichael Neuling mtcrf 0x80,r9 2602b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2603931dc86bSNicholas PigginBEGIN_FTR_SECTION 2604931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2605931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2606931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2607630573c1SPaul MackerrasBEGIN_FTR_SECTION 2608630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2609630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2610630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 261159dc5bfcSNicholas Piggin li r10,0 261259dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 2613b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2614b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2615b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2616b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2617222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2618b92a66a6SMichael Neuling b . 2619b92a66a6SMichael Neuling#endif 2620b92a66a6SMichael Neuling 26214f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 26224f50541fSNicholas Piggin GEN_COMMON denorm_exception 2623eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2624eb204d86SNicholas Piggin bl unknown_exception 26251df7d5e4SNicholas Piggin b interrupt_return_hsrr 2626d7e89849SNicholas Piggin 2627d7e89849SNicholas Piggin 2628d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 26294f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 26304f50541fSNicholas Piggin IVEC=0x1600 26313f7fbd97SNicholas Piggin IHSRR=1 26324f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 26334f50541fSNicholas Piggin 26347299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 26354f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 26367299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 26371a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2638eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 26394f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2640eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2641eb204d86SNicholas Piggin bl cbe_maintenance_exception 26421df7d5e4SNicholas Piggin b interrupt_return_hsrr 26439600f261SNicholas Piggin 2644d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 26451a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 26461a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2647d7e89849SNicholas Piggin#endif 2648d7e89849SNicholas Piggin 264969a79344SNicholas Piggin 26504f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 26514f50541fSNicholas Piggin IVEC=0x1700 26522284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 26534f50541fSNicholas Piggin IKVM_REAL=1 26542284ffeaSNicholas Piggin#endif 26554f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 26564f50541fSNicholas Piggin 26577299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 26584f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 26597299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 26607299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 26614f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 26627299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2663eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 26644f50541fSNicholas Piggin GEN_COMMON altivec_assist 2665eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2666b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 2667eb204d86SNicholas Piggin bl altivec_assist_exception 2668702f0980SNicholas Piggin REST_NVGPRS(r1) /* instruction emulation may change GPRs */ 2669b51c079eSNicholas Piggin#else 2670eb204d86SNicholas Piggin bl unknown_exception 2671b51c079eSNicholas Piggin#endif 26721df7d5e4SNicholas Piggin b interrupt_return_srr 2673b51c079eSNicholas Piggin 2674d7e89849SNicholas Piggin 2675d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 26764f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 26774f50541fSNicholas Piggin IVEC=0x1800 26783f7fbd97SNicholas Piggin IHSRR=1 26794f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 26804f50541fSNicholas Piggin 26817299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 26824f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 26837299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 26841a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2685eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 26864f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2687eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2688eb204d86SNicholas Piggin bl cbe_thermal_exception 26891df7d5e4SNicholas Piggin b interrupt_return_hsrr 26909600f261SNicholas Piggin 2691d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 26921a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 26931a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2694d7e89849SNicholas Piggin#endif 2695d7e89849SNicholas Piggin 26967299417cSNicholas Piggin 269775eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 26982104180aSNicholas Piggin 26990eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 27000eddf327SNicholas Piggin IVEC=0x900 27010eddf327SNicholas Piggin ISTACK=0 27020eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 27032104180aSNicholas Piggin 2704cc491f1dSNicholas Piggin/* 2705cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2706cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2707cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2708cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2709cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2710cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2711cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2712cc491f1dSNicholas Piggin */ 27132104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 27142104180aSNicholas Piggin mr r10,r1 27152104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 27162104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 27170eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 271871c3b05aSNicholas Piggin 2719c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2720c06075f3SNicholas Piggin bl soft_nmi_interrupt 272171c3b05aSNicholas Piggin 272271c3b05aSNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 272371c3b05aSNicholas Piggin li r9,0 272471c3b05aSNicholas Piggin mtmsrd r9,1 272571c3b05aSNicholas Piggin 27268e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 2727f23699c9SNicholas Piggin 272871c3b05aSNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 272971c3b05aSNicholas Piggin RFI_TO_KERNEL 27302104180aSNicholas Piggin 273175eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2732d7e89849SNicholas Piggin 27330ebc4cdaSBenjamin Herrenschmidt/* 2734fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2735fe9e1d54SIan Munsie * - If it was a decrementer interrupt, we bump the dec to max and and return. 2736fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2737fe9e1d54SIan Munsie * triggered and won't automatically refire. 27380869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 27390869b6fdSMahesh Salgaonkar * and it won't refire. 27406cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2741fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 27420ebc4cdaSBenjamin Herrenschmidt */ 27433f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 27444508a74aSNicholas Piggin .if \hsrr 27454508a74aSNicholas Pigginmasked_Hinterrupt: 27464508a74aSNicholas Piggin .else 27474508a74aSNicholas Pigginmasked_interrupt: 27484508a74aSNicholas Piggin .endif 274963e40806SNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 275063e40806SNicholas Piggin lbz r9,PACAIRQHAPPENED(r13) 275163e40806SNicholas Piggin or r9,r9,r10 275263e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 275363e40806SNicholas Piggin 275463e40806SNicholas Piggin .if ! \hsrr 27554508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 27564508a74aSNicholas Piggin bne 1f 275763e40806SNicholas Piggin LOAD_REG_IMMEDIATE(r9, 0x7fffffff) 275863e40806SNicholas Piggin mtspr SPRN_DEC,r9 27590eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 276063e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 27610eddf327SNicholas Piggin b soft_nmi_common 27620eddf327SNicholas Piggin#else 27630eddf327SNicholas Piggin b 2f 27640eddf327SNicholas Piggin#endif 276563e40806SNicholas Piggin .endif 276663e40806SNicholas Piggin 27674508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 27684508a74aSNicholas Piggin beq 2f 27690eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 27704508a74aSNicholas Piggin .if \hsrr 27710eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 27724508a74aSNicholas Piggin .else 27730eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 27744508a74aSNicholas Piggin .endif 277563e40806SNicholas Piggin ori r9,r9,PACA_IRQ_HARD_DIS 277663e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 27774508a74aSNicholas Piggin2: /* done */ 277863e40806SNicholas Piggin li r9,0 277959dc5bfcSNicholas Piggin .if \hsrr 278063e40806SNicholas Piggin stb r9,PACAHSRR_VALID(r13) 278159dc5bfcSNicholas Piggin .else 278263e40806SNicholas Piggin stb r9,PACASRR_VALID(r13) 278359dc5bfcSNicholas Piggin .endif 278463e40806SNicholas Piggin 2785f23699c9SNicholas Piggin SEARCH_RESTART_TABLE 2786f23699c9SNicholas Piggin cmpdi r12,0 2787f23699c9SNicholas Piggin beq 3f 2788f23699c9SNicholas Piggin .if \hsrr 2789f23699c9SNicholas Piggin mtspr SPRN_HSRR0,r12 2790f23699c9SNicholas Piggin .else 2791f23699c9SNicholas Piggin mtspr SPRN_SRR0,r12 2792f23699c9SNicholas Piggin .endif 2793f23699c9SNicholas Piggin3: 2794f23699c9SNicholas Piggin 279563e40806SNicholas Piggin ld r9,PACA_EXGEN+EX_CTR(r13) 279663e40806SNicholas Piggin mtctr r9 279763e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 27984508a74aSNicholas Piggin mtcrf 0x80,r9 27994508a74aSNicholas Piggin std r1,PACAR1(r13) 28004508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 28014508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 28024508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 28030eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 2804b2dc2977SNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 2805b2dc2977SNicholas Piggin /* May return to masked low address where r13 is not set up */ 28064508a74aSNicholas Piggin .if \hsrr 28074508a74aSNicholas Piggin HRFI_TO_KERNEL 28084508a74aSNicholas Piggin .else 28094508a74aSNicholas Piggin RFI_TO_KERNEL 28104508a74aSNicholas Piggin .endif 28114508a74aSNicholas Piggin b . 28124508a74aSNicholas Piggin.endm 28130ebc4cdaSBenjamin Herrenschmidt 2814a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2815a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2816a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2817a048a07dSNicholas Piggin sync 2818a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2819a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2820a048a07dSNicholas Piggin ori 31,31,0 2821a048a07dSNicholas Piggin .rept 14 2822a048a07dSNicholas Piggin b 1f 2823a048a07dSNicholas Piggin1: 2824a048a07dSNicholas Piggin .endr 2825a048a07dSNicholas Piggin blr 2826a048a07dSNicholas Piggin 28279a32a7e7SNicholas Piggin/* Clobbers r10, r11, ctr */ 28289a32a7e7SNicholas Piggin.macro L1D_DISPLACEMENT_FLUSH 2829aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2830bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2831bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2832aa8a5e00SMichael Ellerman mtctr r11 283315a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2834aa8a5e00SMichael Ellerman 2835aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2836aa8a5e00SMichael Ellerman sync 2837bdcb1aefSNicholas Piggin 2838bdcb1aefSNicholas Piggin /* 2839f7964378SNicholas Piggin * The load addresses are at staggered offsets within cachelines, 2840bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2841bdcb1aefSNicholas Piggin * hurt). 2842bdcb1aefSNicholas Piggin */ 2843bdcb1aefSNicholas Piggin1: 2844bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2845bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2846bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2847bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2848bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2849bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2850bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2851bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2852bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2853aa8a5e00SMichael Ellerman bdnz 1b 28549a32a7e7SNicholas Piggin.endm 2855aa8a5e00SMichael Ellerman 28569a32a7e7SNicholas PigginTRAMP_REAL_BEGIN(entry_flush_fallback) 28579a32a7e7SNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 28589a32a7e7SNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 28599a32a7e7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 28609a32a7e7SNicholas Piggin mfctr r9 28619a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2862f7964378SNicholas Piggin mtctr r9 2863f7964378SNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2864f7964378SNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2865f7964378SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 2866f7964378SNicholas Piggin blr 2867f7964378SNicholas Piggin 286808685be7SNicholas Piggin/* 286908685be7SNicholas Piggin * The SCV entry flush happens with interrupts enabled, so it must disable 287008685be7SNicholas Piggin * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10 287108685be7SNicholas Piggin * (containing LR) does not need to be preserved here because scv entry 287208685be7SNicholas Piggin * puts 0 in the pt_regs, CTR can be clobbered for the same reason. 287308685be7SNicholas Piggin */ 287408685be7SNicholas PigginTRAMP_REAL_BEGIN(scv_entry_flush_fallback) 287508685be7SNicholas Piggin li r10,0 287608685be7SNicholas Piggin mtmsrd r10,1 287708685be7SNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 287808685be7SNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 287908685be7SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 288008685be7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 288108685be7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 288208685be7SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 288308685be7SNicholas Piggin li r10,MSR_RI 288408685be7SNicholas Piggin mtmsrd r10,1 288508685be7SNicholas Piggin blr 288608685be7SNicholas Piggin 2887aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 2888aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2889aa8a5e00SMichael Ellerman GET_PACA(r13); 2890aa8a5e00SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 2891aa8a5e00SMichael Ellerman ld r1,PACAKSAVE(r13) 2892aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2893aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2894aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2895aa8a5e00SMichael Ellerman mfctr r9 28969a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2897aa8a5e00SMichael Ellerman mtctr r9 2898aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2899aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2900aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 290178ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2902aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2903aa8a5e00SMichael Ellerman rfid 2904aa8a5e00SMichael Ellerman 2905aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 2906aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2907aa8a5e00SMichael Ellerman GET_PACA(r13); 290878ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 290978ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2910aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2911aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2912aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2913aa8a5e00SMichael Ellerman mfctr r9 29149a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2915aa8a5e00SMichael Ellerman mtctr r9 2916aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2917aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2918aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 291978ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2920aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2921aa8a5e00SMichael Ellerman hrfid 2922aa8a5e00SMichael Ellerman 29237fa95f9aSNicholas PigginTRAMP_REAL_BEGIN(rfscv_flush_fallback) 29247fa95f9aSNicholas Piggin /* system call volatile */ 29257fa95f9aSNicholas Piggin mr r7,r13 29267fa95f9aSNicholas Piggin GET_PACA(r13); 29277fa95f9aSNicholas Piggin mr r8,r1 29287fa95f9aSNicholas Piggin ld r1,PACAKSAVE(r13) 29297fa95f9aSNicholas Piggin mfctr r9 29307fa95f9aSNicholas Piggin ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 29317fa95f9aSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 29327fa95f9aSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 29337fa95f9aSNicholas Piggin mtctr r11 29347fa95f9aSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 29357fa95f9aSNicholas Piggin 29367fa95f9aSNicholas Piggin /* order ld/st prior to dcbt stop all streams with flushing */ 29377fa95f9aSNicholas Piggin sync 29387fa95f9aSNicholas Piggin 29397fa95f9aSNicholas Piggin /* 29407fa95f9aSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 29417fa95f9aSNicholas Piggin * which suits some pipelines better (on others it should not 29427fa95f9aSNicholas Piggin * hurt). 29437fa95f9aSNicholas Piggin */ 29447fa95f9aSNicholas Piggin1: 29457fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 29467fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 29477fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 29487fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 29497fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 29507fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 29517fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 29527fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 29537fa95f9aSNicholas Piggin addi r10,r10,0x80*8 29547fa95f9aSNicholas Piggin bdnz 1b 29557fa95f9aSNicholas Piggin 29567fa95f9aSNicholas Piggin mtctr r9 29577fa95f9aSNicholas Piggin li r9,0 29587fa95f9aSNicholas Piggin li r10,0 29597fa95f9aSNicholas Piggin li r11,0 29607fa95f9aSNicholas Piggin mr r1,r8 29617fa95f9aSNicholas Piggin mr r13,r7 29627fa95f9aSNicholas Piggin RFSCV 29637fa95f9aSNicholas Piggin 29640eddf327SNicholas PigginUSE_TEXT_SECTION() 29659a32a7e7SNicholas Piggin 296669fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 296769fdd674SNicholas Pigginkvm_interrupt: 296869fdd674SNicholas Piggin /* 296969fdd674SNicholas Piggin * The conditional branch in KVMTEST can't reach all the way, 297069fdd674SNicholas Piggin * make a stub. 297169fdd674SNicholas Piggin */ 297269fdd674SNicholas Piggin b kvmppc_interrupt 297369fdd674SNicholas Piggin#endif 297469fdd674SNicholas Piggin 29759a32a7e7SNicholas Piggin_GLOBAL(do_uaccess_flush) 29769a32a7e7SNicholas Piggin UACCESS_FLUSH_FIXUP_SECTION 29779a32a7e7SNicholas Piggin nop 29789a32a7e7SNicholas Piggin nop 29799a32a7e7SNicholas Piggin nop 29809a32a7e7SNicholas Piggin blr 29819a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 29829a32a7e7SNicholas Piggin blr 29839a32a7e7SNicholas Piggin_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) 29849a32a7e7SNicholas PigginEXPORT_SYMBOL(do_uaccess_flush) 29859a32a7e7SNicholas Piggin 29869a32a7e7SNicholas Piggin 29873f7fbd97SNicholas PigginMASKED_INTERRUPT 29883f7fbd97SNicholas PigginMASKED_INTERRUPT hsrr=1 29897230c564SBenjamin Herrenschmidt 29900ebc4cdaSBenjamin Herrenschmidt /* 2991c1fb6816SMichael Neuling * Relocation-on interrupts: A subset of the interrupts can be delivered 2992c1fb6816SMichael Neuling * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 2993c1fb6816SMichael Neuling * it. Addresses are the same as the original interrupt addresses, but 2994c1fb6816SMichael Neuling * offset by 0xc000000000004000. 2995c1fb6816SMichael Neuling * It's impossible to receive interrupts below 0x300 via this mechanism. 2996c1fb6816SMichael Neuling * KVM: None of these traps are from the guest ; anything that escalated 2997c1fb6816SMichael Neuling * to HV=1 from HV=0 is delivered via real mode handlers. 2998c1fb6816SMichael Neuling */ 2999c1fb6816SMichael Neuling 3000c1fb6816SMichael Neuling /* 3001c1fb6816SMichael Neuling * This uses the standard macro, since the original 0x300 vector 3002c1fb6816SMichael Neuling * only has extra guff for STAB-based processors -- which never 3003c1fb6816SMichael Neuling * come here. 3004c1fb6816SMichael Neuling */ 3005da2bc464SMichael Ellerman 300657f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 30078ed8ab40SHari Bathini /* 30089d1988caSNicholas Piggin * All code below __end_soft_masked is treated as soft-masked. If 3009b2dc2977SNicholas Piggin * any code runs here with MSR[EE]=1, it must then cope with pending 3010b2dc2977SNicholas Piggin * soft interrupt being raised (i.e., by ensuring it is replayed). 3011b2dc2977SNicholas Piggin * 30128ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 30138ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 30148ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 30158ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 30168ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 30178ed8ab40SHari Bathini */ 30188ed8ab40SHari Bathini .align 7 30198ed8ab40SHari Bathini .globl __end_interrupts 30208ed8ab40SHari Bathini__end_interrupts: 302157f26649SNicholas PigginDEFINE_FIXED_SYMBOL(__end_interrupts) 302261383407SBenjamin Herrenschmidt 302357f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 302457f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 302557f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 302657f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 302757f26649SNicholas Piggin 302857f26649SNicholas PigginUSE_TEXT_SECTION() 302957f26649SNicholas Piggin 3030296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 3031296e753fSNicholas Pigginenable_machine_check: 3032296e753fSNicholas Piggin mflr r0 3033296e753fSNicholas Piggin bcl 20,31,$+4 3034296e753fSNicholas Piggin0: mflr r3 3035296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 3036296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 3037296e753fSNicholas Piggin mfmsr r3 3038296e753fSNicholas Piggin ori r3,r3,MSR_ME 3039296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 3040296e753fSNicholas Piggin RFI_TO_KERNEL 3041296e753fSNicholas Piggin1: mtlr r0 3042296e753fSNicholas Piggin blr 3043296e753fSNicholas Piggin 3044b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 3045b7d9ccecSNicholas Piggindisable_machine_check: 3046b7d9ccecSNicholas Piggin mflr r0 3047b7d9ccecSNicholas Piggin bcl 20,31,$+4 3048b7d9ccecSNicholas Piggin0: mflr r3 3049b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 3050b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 3051b7d9ccecSNicholas Piggin mfmsr r3 3052b7d9ccecSNicholas Piggin li r4,MSR_ME 3053b7d9ccecSNicholas Piggin andc r3,r3,r4 3054b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 3055b7d9ccecSNicholas Piggin RFI_TO_KERNEL 3056b7d9ccecSNicholas Piggin1: mtlr r0 3057b7d9ccecSNicholas Piggin blr 3058