1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 1629a011fcSSathvika Vasireddy#include <linux/linkage.h> 177230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 188aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1946f52210SStephen Rothwell#include <asm/ptrace.h> 207cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 21da2bc464SMichael Ellerman#include <asm/head-64.h> 222c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 23890274c2SMichael Ellerman#include <asm/kup.h> 248aa34ab8SBenjamin Herrenschmidt 257299417cSNicholas Piggin/* 267299417cSNicholas Piggin * Following are fixed section helper macros. 277299417cSNicholas Piggin * 287299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 297299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 307299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 317299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 327299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 337299417cSNicholas Piggin */ 347299417cSNicholas Piggin 35a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 36a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 37a2432811SNicholas Piggin 38a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 39a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 40a2432811SNicholas Piggin 41a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 42a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 43a2432811SNicholas Piggin 44a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 45a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 46a2432811SNicholas Piggin 47a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 48a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 49a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 50a2432811SNicholas Piggin .global name; \ 51a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 52d72c4a36SDaniel Axtens DEFINE_FIXED_SYMBOL(name, text); \ 53a2432811SNicholas Pigginname: 54a2432811SNicholas Piggin 55a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 56a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 57a2432811SNicholas Piggin 58a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 59a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 60a2432811SNicholas Piggin 61a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 62a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 63a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 64a2432811SNicholas Piggin 65a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 66a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 67a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 68a2432811SNicholas Piggin 690ebc4cdaSBenjamin Herrenschmidt/* 7012a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 7112a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 7212a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 7312a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 7412a04809SNicholas Piggin * that kernelbase be 64K aligned. 7512a04809SNicholas Piggin */ 7612a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 7712a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 7812a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 7912a04809SNicholas Piggin 80d72c4a36SDaniel Axtens#define __LOAD_HANDLER(reg, label, section) \ 8112a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 82d72c4a36SDaniel Axtens ori reg,reg,(ABS_ADDR(label, section))@l 8312a04809SNicholas Piggin 8412a04809SNicholas Piggin/* 8512a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 8612a04809SNicholas Piggin * head-y require >64K offsets. 8712a04809SNicholas Piggin */ 88d72c4a36SDaniel Axtens#define __LOAD_FAR_HANDLER(reg, label, section) \ 8912a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 90d72c4a36SDaniel Axtens ori reg,reg,(ABS_ADDR(label, section))@l; \ 91d72c4a36SDaniel Axtens addis reg,reg,(ABS_ADDR(label, section))@h 9212a04809SNicholas Piggin 9312a04809SNicholas Piggin/* 94a42a239dSNicholas Piggin * Interrupt code generation macros 95a42a239dSNicholas Piggin */ 9694325357SNicholas Piggin#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 9794325357SNicholas Piggin#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 9894325357SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 9994325357SNicholas Piggin#define IAREA .L_IAREA_\name\() /* PACA save area */ 10094325357SNicholas Piggin#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 10194325357SNicholas Piggin#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 102af47d79bSNicholas Piggin#define ICFAR .L_ICFAR_\name\() /* Uses CFAR */ 103af47d79bSNicholas Piggin#define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */ 10494325357SNicholas Piggin#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 10594325357SNicholas Piggin#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 10694325357SNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 10794325357SNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 10894325357SNicholas Piggin#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 10994325357SNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 1104f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 11194325357SNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 11294325357SNicholas Piggin#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 1134f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 11494325357SNicholas Piggin#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 1152487fd2eSRohan McLure#define IMSR_R12 .L_IMSR_R12_\name\() /* Assumes MSR saved to r12 */ 116a42a239dSNicholas Piggin 117a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 118a42a239dSNicholas Piggin.macro int_define_ ## n name 119a42a239dSNicholas Piggin 120a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 121a42a239dSNicholas Piggin.endm ; \ 122a42a239dSNicholas Pigginint_define_ ## n n ; \ 123a42a239dSNicholas Piggindo_define_int n 124a42a239dSNicholas Piggin 125a42a239dSNicholas Piggin.macro do_define_int name 126a42a239dSNicholas Piggin .ifndef IVEC 127a42a239dSNicholas Piggin .error "IVEC not defined" 128a42a239dSNicholas Piggin .endif 129a42a239dSNicholas Piggin .ifndef IHSRR 1303f7fbd97SNicholas Piggin IHSRR=0 1313f7fbd97SNicholas Piggin .endif 1323f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 1333f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 134a42a239dSNicholas Piggin .endif 135a42a239dSNicholas Piggin .ifndef IAREA 136a42a239dSNicholas Piggin IAREA=PACA_EXGEN 137a42a239dSNicholas Piggin .endif 1388729c26eSNicholas Piggin .ifndef IVIRT 1398729c26eSNicholas Piggin IVIRT=1 1408729c26eSNicholas Piggin .endif 141a3cd35beSNicholas Piggin .ifndef IISIDE 142a3cd35beSNicholas Piggin IISIDE=0 143a3cd35beSNicholas Piggin .endif 144af47d79bSNicholas Piggin .ifndef ICFAR 145af47d79bSNicholas Piggin ICFAR=1 146af47d79bSNicholas Piggin .endif 147af47d79bSNicholas Piggin .ifndef ICFAR_IF_HVMODE 148af47d79bSNicholas Piggin ICFAR_IF_HVMODE=0 149af47d79bSNicholas Piggin .endif 150a42a239dSNicholas Piggin .ifndef IDAR 151a42a239dSNicholas Piggin IDAR=0 152a42a239dSNicholas Piggin .endif 153a42a239dSNicholas Piggin .ifndef IDSISR 154a42a239dSNicholas Piggin IDSISR=0 155a42a239dSNicholas Piggin .endif 156d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 157d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 158d73a10cbSNicholas Piggin .endif 159d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 160d73a10cbSNicholas Piggin IREALMODE_COMMON=0 161d73a10cbSNicholas Piggin .else 162d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 163d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 164d73a10cbSNicholas Piggin .endif 165a42a239dSNicholas Piggin .endif 166a42a239dSNicholas Piggin .ifndef IMASK 167a42a239dSNicholas Piggin IMASK=0 168a42a239dSNicholas Piggin .endif 169a42a239dSNicholas Piggin .ifndef IKVM_REAL 170a42a239dSNicholas Piggin IKVM_REAL=0 171a42a239dSNicholas Piggin .endif 172a42a239dSNicholas Piggin .ifndef IKVM_VIRT 173a42a239dSNicholas Piggin IKVM_VIRT=0 174a42a239dSNicholas Piggin .endif 1757cb3a1a0SNicholas Piggin .ifndef ISTACK 1767cb3a1a0SNicholas Piggin ISTACK=1 1777cb3a1a0SNicholas Piggin .endif 1787cb3a1a0SNicholas Piggin .ifndef IKUAP 1797cb3a1a0SNicholas Piggin IKUAP=1 1807cb3a1a0SNicholas Piggin .endif 1812487fd2eSRohan McLure .ifndef IMSR_R12 1822487fd2eSRohan McLure IMSR_R12=0 1832487fd2eSRohan McLure .endif 184a42a239dSNicholas Piggin.endm 185a42a239dSNicholas Piggin 18612a04809SNicholas Piggin/* 1872284ffeaSNicholas Piggin * All interrupts which set HSRR registers, as well as SRESET and MCE and 1882284ffeaSNicholas Piggin * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 1892284ffeaSNicholas Piggin * so they all generally need to test whether they were taken in guest context. 1902284ffeaSNicholas Piggin * 1912284ffeaSNicholas Piggin * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 1922284ffeaSNicholas Piggin * taken with MSR[HV]=0. 1932284ffeaSNicholas Piggin * 1942284ffeaSNicholas Piggin * Interrupts which set SRR registers (with the above exceptions) do not 1952284ffeaSNicholas Piggin * elevate to MSR[HV]=1 mode, though most can be taken when running with 1962284ffeaSNicholas Piggin * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 1972284ffeaSNicholas Piggin * not need to test whether a guest is running because they get delivered to 1982284ffeaSNicholas Piggin * the guest directly, including nested HV KVM guests. 1992284ffeaSNicholas Piggin * 2002284ffeaSNicholas Piggin * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 2012284ffeaSNicholas Piggin * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 2022284ffeaSNicholas Piggin * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 2032284ffeaSNicholas Piggin * delivered to the real-mode entry point, therefore such interrupts only test 2042284ffeaSNicholas Piggin * KVM in their real mode handlers, and only when PR KVM is possible. 2052284ffeaSNicholas Piggin * 2062284ffeaSNicholas Piggin * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 2072284ffeaSNicholas Piggin * delivered in real-mode when the MMU is in hash mode because the MMU 2082284ffeaSNicholas Piggin * registers are not set appropriately to translate host addresses. In nested 2092284ffeaSNicholas Piggin * radix mode these can be delivered in virt-mode as the host translations are 2102284ffeaSNicholas Piggin * used implicitly (see: effective LPID, effective PID). 2112284ffeaSNicholas Piggin */ 2122284ffeaSNicholas Piggin 2132284ffeaSNicholas Piggin/* 2142284ffeaSNicholas Piggin * If an interrupt is taken while a guest is running, it is immediately routed 215f3601156SNicholas Piggin * to KVM to handle. 21612a04809SNicholas Piggin */ 21712a04809SNicholas Piggin 21869fdd674SNicholas Piggin.macro KVMTEST name handler 21969fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 22012a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 22112a04809SNicholas Piggin cmpwi r10,0 22212a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 2233f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 224def0db4fSNicholas Piggin BEGIN_FTR_SECTION 22569fdd674SNicholas Piggin li r10,(IVEC + 0x2) 226def0db4fSNicholas Piggin FTR_SECTION_ELSE 22769fdd674SNicholas Piggin li r10,(IVEC) 228def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 229b177ae2fSNicholas Piggin .elseif IHSRR 23069fdd674SNicholas Piggin li r10,(IVEC + 0x2) 23112a04809SNicholas Piggin .else 23269fdd674SNicholas Piggin li r10,(IVEC) 23312a04809SNicholas Piggin .endif 23469fdd674SNicholas Piggin bne \handler 23512a04809SNicholas Piggin#endif 23669fdd674SNicholas Piggin.endm 23712a04809SNicholas Piggin 238c7c5cbb4SNicholas Piggin/* 239c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 240c7c5cbb4SNicholas Piggin * 241c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 242c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 243c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 244c7c5cbb4SNicholas Piggin * the kernel. 245c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 246c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 247c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 248c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 249c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 250c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 251c7c5cbb4SNicholas Piggin * irq_soft_mask set. 252c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 253c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 254c7c5cbb4SNicholas Piggin * mode. 255c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 256c7c5cbb4SNicholas Piggin * This is done if early=2. 257c7c5cbb4SNicholas Piggin */ 2588729c26eSNicholas Piggin 2598729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 260d73a10cbSNicholas Piggin .if IREALMODE_COMMON 261d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 262d73a10cbSNicholas Piggin mtctr r10 263d73a10cbSNicholas Piggin bctr 264d73a10cbSNicholas Piggin .else 2658729c26eSNicholas Piggin .if \virt 2668729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 2678729c26eSNicholas Piggin b \name\()_common_virt 2688729c26eSNicholas Piggin#else 2698729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 2708729c26eSNicholas Piggin mtctr r10 2718729c26eSNicholas Piggin bctr 2728729c26eSNicholas Piggin#endif 2738729c26eSNicholas Piggin .else 2748729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 2758729c26eSNicholas Piggin mtctr r10 2768729c26eSNicholas Piggin bctr 2778729c26eSNicholas Piggin .endif 278d73a10cbSNicholas Piggin .endif 2798729c26eSNicholas Piggin.endm 2808729c26eSNicholas Piggin 281fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 282c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 283c7c5cbb4SNicholas Piggin GET_PACA(r13) 284fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 285931dc86bSNicholas PigginBEGIN_FTR_SECTION 286931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 287931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 288c7c5cbb4SNicholas Piggin HMT_MEDIUM 289620f5c59SRohan McLure std r10,IAREA+EX_R10(r13) /* save r10 */ 290af47d79bSNicholas Piggin .if ICFAR 291931dc86bSNicholas PigginBEGIN_FTR_SECTION 292931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 293931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 294af47d79bSNicholas Piggin .elseif ICFAR_IF_HVMODE 295af47d79bSNicholas PigginBEGIN_FTR_SECTION 296af47d79bSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 297af47d79bSNicholas Piggin mfspr r10,SPRN_CFAR 298af47d79bSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) 299af47d79bSNicholas PigginFTR_SECTION_ELSE 300af47d79bSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 301af47d79bSNicholas Piggin li r10,0 302af47d79bSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) 303af47d79bSNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 304af47d79bSNicholas Piggin .endif 305c7c5cbb4SNicholas Piggin .if \ool 306c7c5cbb4SNicholas Piggin .if !\virt 307c7c5cbb4SNicholas Piggin b tramp_real_\name 308c7c5cbb4SNicholas Piggin .pushsection .text 309c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 310c7c5cbb4SNicholas Piggin .else 311c7c5cbb4SNicholas Piggin b tramp_virt_\name 312c7c5cbb4SNicholas Piggin .pushsection .text 313c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 314c7c5cbb4SNicholas Piggin .endif 315c7c5cbb4SNicholas Piggin .endif 316c7c5cbb4SNicholas Piggin 317931dc86bSNicholas PigginBEGIN_FTR_SECTION 318931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 319931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 320af47d79bSNicholas Piggin .if ICFAR || ICFAR_IF_HVMODE 321931dc86bSNicholas PigginBEGIN_FTR_SECTION 322931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 323931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 324af47d79bSNicholas Piggin .endif 325c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 3268729c26eSNicholas Piggin mfctr r10 3278729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 328c7c5cbb4SNicholas Piggin mfcr r9 329620f5c59SRohan McLure std r11,IAREA+EX_R11(r13) /* save r11 - r12 */ 330fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 331c7c5cbb4SNicholas Piggin 332c7c5cbb4SNicholas Piggin /* 333c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 334c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 335c7c5cbb4SNicholas Piggin * not recoverable if they are live. 336c7c5cbb4SNicholas Piggin */ 337c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 338fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 339a3cd35beSNicholas Piggin .if IDAR && !IISIDE 340fc589ee4SNicholas Piggin .if IHSRR 341c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 342c7c5cbb4SNicholas Piggin .else 343c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 344c7c5cbb4SNicholas Piggin .endif 345fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 346c7c5cbb4SNicholas Piggin .endif 347a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 348fc589ee4SNicholas Piggin .if IHSRR 349c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 350c7c5cbb4SNicholas Piggin .else 351c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 352c7c5cbb4SNicholas Piggin .endif 353fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 354c7c5cbb4SNicholas Piggin .endif 355c7c5cbb4SNicholas Piggin 3563f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 3578729c26eSNicholas Piggin BEGIN_FTR_SECTION 3588729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3598729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3608729c26eSNicholas Piggin FTR_SECTION_ELSE 3618729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3628729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 3638729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 3648729c26eSNicholas Piggin .elseif IHSRR 3658729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3668729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3678729c26eSNicholas Piggin .else 3688729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3698729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 370c7c5cbb4SNicholas Piggin .endif 371d73a10cbSNicholas Piggin 372d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 3738729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 3748729c26eSNicholas Piggin .endif 3758729c26eSNicholas Piggin 376c7c5cbb4SNicholas Piggin .if \ool 377c7c5cbb4SNicholas Piggin .popsection 378c7c5cbb4SNicholas Piggin .endif 379c7c5cbb4SNicholas Piggin.endm 380c7c5cbb4SNicholas Piggin 381d064151fSNicholas Piggin/* 3828729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 3839600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 3849600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 3859600f261SNicholas Piggin * 3868729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 387d064151fSNicholas Piggin */ 3888729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 389d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_real, text) 3908729c26eSNicholas Piggin\name\()_common_real: 3919600f261SNicholas Piggin .if IKVM_REAL 39269fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 3939600f261SNicholas Piggin .endif 3949600f261SNicholas Piggin 3958729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 3968729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 397c080a173SDaniel Axtens .if IHSRR_IF_HVMODE 3988729c26eSNicholas Piggin BEGIN_FTR_SECTION 3998729c26eSNicholas Piggin xori r10,r10,MSR_RI 4008729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 4018729c26eSNicholas Piggin .elseif ! IHSRR 4028729c26eSNicholas Piggin xori r10,r10,MSR_RI 4038729c26eSNicholas Piggin .endif 4048729c26eSNicholas Piggin mtmsrd r10 4058729c26eSNicholas Piggin 4068729c26eSNicholas Piggin .if IVIRT 4079600f261SNicholas Piggin .if IKVM_VIRT 4089600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 4099600f261SNicholas Piggin .endif 4109600f261SNicholas Piggin 4118729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 412d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_virt, text) 4138729c26eSNicholas Piggin\name\()_common_virt: 4149600f261SNicholas Piggin .if IKVM_VIRT 41569fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4169600f261SNicholas Piggin1: 4179600f261SNicholas Piggin .endif 4188729c26eSNicholas Piggin .endif /* IVIRT */ 4198729c26eSNicholas Piggin.endm 4208729c26eSNicholas Piggin 4219600f261SNicholas Piggin/* 4229600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4239600f261SNicholas Piggin * want to run in real mode. 4249600f261SNicholas Piggin */ 4259600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 426d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_real, text) 4279600f261SNicholas Piggin\name\()_common_real: 4289600f261SNicholas Piggin .if IKVM_REAL 42969fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4309600f261SNicholas Piggin .endif 4319600f261SNicholas Piggin.endm 4329600f261SNicholas Piggin 4338729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 4340eddf327SNicholas Piggin .if IMASK 435b2dc2977SNicholas Piggin .if ! ISTACK 436b2dc2977SNicholas Piggin .error "No support for masked interrupt to use custom stack" 437b2dc2977SNicholas Piggin .endif 438b2dc2977SNicholas Piggin 439b2dc2977SNicholas Piggin /* If coming from user, skip soft-mask tests. */ 440b2dc2977SNicholas Piggin andi. r10,r12,MSR_PR 441325678fdSNicholas Piggin bne 3f 442b2dc2977SNicholas Piggin 4439d1988caSNicholas Piggin /* 444325678fdSNicholas Piggin * Kernel code running below __end_soft_masked may be 445325678fdSNicholas Piggin * implicitly soft-masked if it is within the regions 446325678fdSNicholas Piggin * in the soft mask table. 4479d1988caSNicholas Piggin */ 4489d1988caSNicholas Piggin LOAD_HANDLER(r10, __end_soft_masked) 449b2dc2977SNicholas Piggin cmpld r11,r10 450325678fdSNicholas Piggin bge+ 1f 4519d1988caSNicholas Piggin 452325678fdSNicholas Piggin /* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */ 453325678fdSNicholas Piggin mtctr r12 454325678fdSNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 455325678fdSNicholas Piggin SEARCH_SOFT_MASK_TABLE 456325678fdSNicholas Piggin cmpdi r12,0 457325678fdSNicholas Piggin mfctr r12 /* Restore r12 to SRR1 */ 458325678fdSNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 459325678fdSNicholas Piggin beq 1f /* Not in soft-mask table */ 460b2dc2977SNicholas Piggin li r10,IMASK 461325678fdSNicholas Piggin b 2f /* In soft-mask table, always mask */ 462b2dc2977SNicholas Piggin 463b2dc2977SNicholas Piggin /* Test the soft mask state against our interrupt's bit */ 464325678fdSNicholas Piggin1: lbz r10,PACAIRQSOFTMASK(r13) 465325678fdSNicholas Piggin2: andi. r10,r10,IMASK 4660eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 4670eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 4680eddf327SNicholas Piggin li r10,PACA_IRQ_EE 4690eddf327SNicholas Piggin .elseif IVEC == 0x900 4700eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 4710eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 4720eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 4730eddf327SNicholas Piggin .elseif IVEC == 0xe60 4740eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 4750eddf327SNicholas Piggin .elseif IVEC == 0xf00 4760eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 4770eddf327SNicholas Piggin .else 4780eddf327SNicholas Piggin .abort "Bad maskable vector" 4790eddf327SNicholas Piggin .endif 4800eddf327SNicholas Piggin 4813f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4820eddf327SNicholas Piggin BEGIN_FTR_SECTION 4830eddf327SNicholas Piggin bne masked_Hinterrupt 4840eddf327SNicholas Piggin FTR_SECTION_ELSE 4850eddf327SNicholas Piggin bne masked_interrupt 4860eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4870eddf327SNicholas Piggin .elseif IHSRR 4880eddf327SNicholas Piggin bne masked_Hinterrupt 4890eddf327SNicholas Piggin .else 4900eddf327SNicholas Piggin bne masked_interrupt 4910eddf327SNicholas Piggin .endif 4920eddf327SNicholas Piggin .endif 4930eddf327SNicholas Piggin 4946d71759aSNicholas Piggin .if ISTACK 4955d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 496325678fdSNicholas Piggin3: mr r10,r1 /* Save r1 */ 4975d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 4981b359982SNicholas Piggin beq- 100f 4995d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 5001b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 5011b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 5025d5e0edfSNicholas Piggin .endif 5038c9fb5d4SNicholas Piggin 5048c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 5058c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 5068c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 5078c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 5088c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 5098c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 5101df45d78SRohan McLure SANITIZE_GPR(0) 5115d5e0edfSNicholas Piggin 51259dc5bfcSNicholas Piggin /* Mark our [H]SRRs valid for return */ 51359dc5bfcSNicholas Piggin li r10,1 51459dc5bfcSNicholas Piggin .if IHSRR_IF_HVMODE 51559dc5bfcSNicholas Piggin BEGIN_FTR_SECTION 51659dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 51759dc5bfcSNicholas Piggin FTR_SECTION_ELSE 51859dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 51959dc5bfcSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 52059dc5bfcSNicholas Piggin .elseif IHSRR 52159dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 52259dc5bfcSNicholas Piggin .else 52359dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 52459dc5bfcSNicholas Piggin .endif 52559dc5bfcSNicholas Piggin 5266d71759aSNicholas Piggin .if ISTACK 5276d71759aSNicholas Piggin .if IKUAP 5285d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 5295d5e0edfSNicholas Piggin .endif 5301b359982SNicholas Piggin beq 101f /* if from kernel mode */ 531931dc86bSNicholas PigginBEGIN_FTR_SECTION 532931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 533931dc86bSNicholas Piggin std r9,_PPR(r1) 534931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5351b359982SNicholas Piggin101: 5365d5e0edfSNicholas Piggin .else 5376d71759aSNicholas Piggin .if IKUAP 538bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 539bcbceed4SNicholas Piggin .endif 5405d5e0edfSNicholas Piggin .endif 5415d5e0edfSNicholas Piggin 5428c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5436d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5446d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5458c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5468c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5476d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5486d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5496d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5508c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5518c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5528c9fb5d4SNicholas Piggin std r11,GPR13(r1) 5531df45d78SRohan McLure .if !IMSR_R12 5541df45d78SRohan McLure SANITIZE_GPRS(9, 12) 5551df45d78SRohan McLure .else 5561df45d78SRohan McLure SANITIZE_GPRS(9, 11) 5571df45d78SRohan McLure .endif 558a3cd35beSNicholas Piggin 5596cc0c16dSNicholas Piggin SAVE_NVGPRS(r1) 5601df45d78SRohan McLure SANITIZE_NVGPRS() 5616cc0c16dSNicholas Piggin 5626d71759aSNicholas Piggin .if IDAR 563a3cd35beSNicholas Piggin .if IISIDE 564d1a84718SNicholas Piggin ld r10,_NIP(r1) 565d1a84718SNicholas Piggin .else 5666d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 567d1a84718SNicholas Piggin .endif 568d1a84718SNicholas Piggin std r10,_DAR(r1) 569d1a84718SNicholas Piggin .endif 570a3cd35beSNicholas Piggin 5716d71759aSNicholas Piggin .if IDSISR 572a3cd35beSNicholas Piggin .if IISIDE 573d1a84718SNicholas Piggin ld r10,_MSR(r1) 574d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 575d1a84718SNicholas Piggin and r10,r10,r11 576d1a84718SNicholas Piggin .else 5776d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 578d1a84718SNicholas Piggin .endif 579d1a84718SNicholas Piggin std r10,_DSISR(r1) 580d1a84718SNicholas Piggin .endif 581a3cd35beSNicholas Piggin 582931dc86bSNicholas PigginBEGIN_FTR_SECTION 583af47d79bSNicholas Piggin .if ICFAR || ICFAR_IF_HVMODE 5846d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 585af47d79bSNicholas Piggin .else 586af47d79bSNicholas Piggin li r10,0 587af47d79bSNicholas Piggin .endif 5888c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 589931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 5908729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 5918c9fb5d4SNicholas Piggin std r10,_CTR(r1) 5921df45d78SRohan McLure SAVE_GPRS(2, 8, r1) /* save r2 - r8 in stackframe */ 5931df45d78SRohan McLure SANITIZE_GPRS(2, 8) 5948c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 5958e93fb33SNicholas Piggin LOAD_PACA_TOC() /* get kernel TOC into r2 */ 5968c9fb5d4SNicholas Piggin std r9,_LINK(r1) 5978c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5988c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 5998c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 6008c9fb5d4SNicholas Piggin std r11,_XER(r1) 6016cc0c16dSNicholas Piggin li r9,IVEC 6028c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 6038c9fb5d4SNicholas Piggin li r10,0 60417773afdSNicholas Piggin LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) 6058c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 606d2e8ff9fSNicholas Piggin std r11,STACK_INT_FRAME_MARKER(r1) /* mark the frame */ 607bcbceed4SNicholas Piggin.endm 608bcbceed4SNicholas Piggin 609391e941bSNicholas Piggin/* 6108729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 6118729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 6128729c26eSNicholas Piggin * SRR1, and relocation is on. 6138729c26eSNicholas Piggin * 6148729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 6158729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 6168729c26eSNicholas Piggin */ 6178729c26eSNicholas Piggin.macro GEN_COMMON name 6188729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 6198729c26eSNicholas Piggin __GEN_COMMON_BODY \name 6208729c26eSNicholas Piggin.endm 6218729c26eSNicholas Piggin 622f23699c9SNicholas Piggin.macro SEARCH_RESTART_TABLE 623f23699c9SNicholas Piggin#ifdef CONFIG_RELOCATABLE 624f23699c9SNicholas Piggin mr r12,r2 6258e93fb33SNicholas Piggin LOAD_PACA_TOC() 626f23699c9SNicholas Piggin LOAD_REG_ADDR(r9, __start___restart_table) 627f23699c9SNicholas Piggin LOAD_REG_ADDR(r10, __stop___restart_table) 628f23699c9SNicholas Piggin mr r2,r12 629f23699c9SNicholas Piggin#else 630f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___restart_table) 631f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___restart_table) 632f23699c9SNicholas Piggin#endif 633f23699c9SNicholas Piggin300: 634f23699c9SNicholas Piggin cmpd r9,r10 635f23699c9SNicholas Piggin beq 302f 636f23699c9SNicholas Piggin ld r12,0(r9) 637f23699c9SNicholas Piggin cmpld r11,r12 638f23699c9SNicholas Piggin blt 301f 639f23699c9SNicholas Piggin ld r12,8(r9) 640f23699c9SNicholas Piggin cmpld r11,r12 641f23699c9SNicholas Piggin bge 301f 642f23699c9SNicholas Piggin ld r12,16(r9) 643f23699c9SNicholas Piggin b 303f 644f23699c9SNicholas Piggin301: 645f23699c9SNicholas Piggin addi r9,r9,24 646f23699c9SNicholas Piggin b 300b 647f23699c9SNicholas Piggin302: 648f23699c9SNicholas Piggin li r12,0 649f23699c9SNicholas Piggin303: 650f23699c9SNicholas Piggin.endm 651f23699c9SNicholas Piggin 652325678fdSNicholas Piggin.macro SEARCH_SOFT_MASK_TABLE 653325678fdSNicholas Piggin#ifdef CONFIG_RELOCATABLE 654325678fdSNicholas Piggin mr r12,r2 6558e93fb33SNicholas Piggin LOAD_PACA_TOC() 656325678fdSNicholas Piggin LOAD_REG_ADDR(r9, __start___soft_mask_table) 657325678fdSNicholas Piggin LOAD_REG_ADDR(r10, __stop___soft_mask_table) 658325678fdSNicholas Piggin mr r2,r12 659325678fdSNicholas Piggin#else 660325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___soft_mask_table) 661325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___soft_mask_table) 662325678fdSNicholas Piggin#endif 663325678fdSNicholas Piggin300: 664325678fdSNicholas Piggin cmpd r9,r10 665325678fdSNicholas Piggin beq 302f 666325678fdSNicholas Piggin ld r12,0(r9) 667325678fdSNicholas Piggin cmpld r11,r12 668325678fdSNicholas Piggin blt 301f 669325678fdSNicholas Piggin ld r12,8(r9) 670325678fdSNicholas Piggin cmpld r11,r12 671325678fdSNicholas Piggin bge 301f 672325678fdSNicholas Piggin li r12,1 673325678fdSNicholas Piggin b 303f 674325678fdSNicholas Piggin301: 675325678fdSNicholas Piggin addi r9,r9,16 676325678fdSNicholas Piggin b 300b 677325678fdSNicholas Piggin302: 678325678fdSNicholas Piggin li r12,0 679325678fdSNicholas Piggin303: 680325678fdSNicholas Piggin.endm 681325678fdSNicholas Piggin 6828729c26eSNicholas Piggin/* 683391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 684391e941bSNicholas Piggin * standard exception. 685391e941bSNicholas Piggin */ 6863f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 687391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 688391e941bSNicholas Piggin ld r9,_MSR(r1) 68959dc5bfcSNicholas Piggin li r10,0 690391e941bSNicholas Piggin .if \hsrr 691391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 69259dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 693391e941bSNicholas Piggin .else 694391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 69559dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 696391e941bSNicholas Piggin .endif 697391e941bSNicholas Piggin ld r9,_NIP(r1) 698391e941bSNicholas Piggin .if \hsrr 699391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 700391e941bSNicholas Piggin .else 701391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 702391e941bSNicholas Piggin .endif 703391e941bSNicholas Piggin ld r9,_CTR(r1) 704391e941bSNicholas Piggin mtctr r9 705391e941bSNicholas Piggin ld r9,_XER(r1) 706391e941bSNicholas Piggin mtxer r9 707391e941bSNicholas Piggin ld r9,_LINK(r1) 708391e941bSNicholas Piggin mtlr r9 709391e941bSNicholas Piggin ld r9,_CCR(r1) 710391e941bSNicholas Piggin mtcr r9 7111df45d78SRohan McLure SANITIZE_RESTORE_NVGPRS() 712aebd1fb4SNicholas Piggin REST_GPRS(2, 13, r1) 713391e941bSNicholas Piggin REST_GPR(0, r1) 714391e941bSNicholas Piggin /* restore original r1. */ 715391e941bSNicholas Piggin ld r1,GPR1(r1) 716391e941bSNicholas Piggin.endm 717d064151fSNicholas Piggin 71812a04809SNicholas Piggin/* 719bf75a325SNicholas Piggin * EARLY_BOOT_FIXUP - Fix real-mode interrupt with wrong endian in early boot. 720bf75a325SNicholas Piggin * 721bf75a325SNicholas Piggin * There's a short window during boot where although the kernel is running 722bf75a325SNicholas Piggin * little endian, any exceptions will cause the CPU to switch back to big 723bf75a325SNicholas Piggin * endian. For example a WARN() boils down to a trap instruction, which will 724bf75a325SNicholas Piggin * cause a program check, and we end up here but with the CPU in big endian 725bf75a325SNicholas Piggin * mode. The first instruction of the program check handler (in GEN_INT_ENTRY 726bf75a325SNicholas Piggin * below) is an mtsprg, which when executed in the wrong endian is an lhzu with 727bf75a325SNicholas Piggin * a ~3GB displacement from r3. The content of r3 is random, so that is a load 728bf75a325SNicholas Piggin * from some random location, and depending on the system can easily lead to a 729bf75a325SNicholas Piggin * checkstop, or an infinitely recursive page fault. 730bf75a325SNicholas Piggin * 731bf75a325SNicholas Piggin * So to handle that case we have a trampoline here that can detect we are in 732bf75a325SNicholas Piggin * the wrong endian and flip us back to the correct endian. We can't flip 733bf75a325SNicholas Piggin * MSR[LE] using mtmsr, so we have to use rfid. That requires backing up SRR0/1 734bf75a325SNicholas Piggin * as well as a GPR. To do that we use SPRG0/2/3, as SPRG1 is already used for 735bf75a325SNicholas Piggin * the paca. SPRG3 is user readable, but this trampoline is only active very 736bf75a325SNicholas Piggin * early in boot, and SPRG3 will be reinitialised in vdso_getcpu_init() before 737bf75a325SNicholas Piggin * userspace starts. 738bf75a325SNicholas Piggin */ 739bf75a325SNicholas Piggin.macro EARLY_BOOT_FIXUP 740bf75a325SNicholas PigginBEGIN_FTR_SECTION 741e1100ceeSNicholas Piggin#ifdef CONFIG_CPU_LITTLE_ENDIAN 742bf75a325SNicholas Piggin tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 743bf75a325SNicholas Piggin b 2f // Skip trampoline if endian is correct 744bf75a325SNicholas Piggin .long 0xa643707d // mtsprg 0, r11 Backup r11 745bf75a325SNicholas Piggin .long 0xa6027a7d // mfsrr0 r11 746bf75a325SNicholas Piggin .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 747bf75a325SNicholas Piggin .long 0xa6027b7d // mfsrr1 r11 748bf75a325SNicholas Piggin .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 749bf75a325SNicholas Piggin .long 0xa600607d // mfmsr r11 750bf75a325SNicholas Piggin .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] 751bf75a325SNicholas Piggin .long 0xa6037b7d // mtsrr1 r11 752bf75a325SNicholas Piggin /* 753bf75a325SNicholas Piggin * This is 'li r11,1f' where 1f is the absolute address of that 754bf75a325SNicholas Piggin * label, byteswapped into the SI field of the instruction. 755bf75a325SNicholas Piggin */ 756bf75a325SNicholas Piggin .long 0x00006039 | \ 757bf75a325SNicholas Piggin ((ABS_ADDR(1f, real_vectors) & 0x00ff) << 24) | \ 758bf75a325SNicholas Piggin ((ABS_ADDR(1f, real_vectors) & 0xff00) << 8) 759bf75a325SNicholas Piggin .long 0xa6037a7d // mtsrr0 r11 760bf75a325SNicholas Piggin .long 0x2400004c // rfid 761bf75a325SNicholas Piggin1: 762bf75a325SNicholas Piggin mfsprg r11, 3 763bf75a325SNicholas Piggin mtsrr1 r11 // Restore SRR1 764bf75a325SNicholas Piggin mfsprg r11, 2 765bf75a325SNicholas Piggin mtsrr0 r11 // Restore SRR0 766bf75a325SNicholas Piggin mfsprg r11, 0 // Restore r11 767bf75a325SNicholas Piggin2: 768bf75a325SNicholas Piggin#endif 769e1100ceeSNicholas Piggin /* 770e1100ceeSNicholas Piggin * program check could hit at any time, and pseries can not block 771e1100ceeSNicholas Piggin * MSR[ME] in early boot. So check if there is anything useful in r13 772e1100ceeSNicholas Piggin * yet, and spin forever if not. 773e1100ceeSNicholas Piggin */ 774e1100ceeSNicholas Piggin mtsprg 0, r11 775e1100ceeSNicholas Piggin mfcr r11 776e1100ceeSNicholas Piggin cmpdi r13, 0 777e1100ceeSNicholas Piggin beq . 778e1100ceeSNicholas Piggin mtcr r11 779e1100ceeSNicholas Piggin mfsprg r11, 0 780e1100ceeSNicholas PigginEND_FTR_SECTION(0, 1) // nop out after boot 781bf75a325SNicholas Piggin.endm 782bf75a325SNicholas Piggin 783bf75a325SNicholas Piggin/* 78457f26649SNicholas Piggin * There are a few constraints to be concerned with. 78557f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 78657f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 78757f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 78857f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 78957f26649SNicholas Piggin * must be used. 79057f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 79157f26649SNicholas Piggin * virtual 0xc00... 79257f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 79357f26649SNicholas Piggin * 79457f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 79557f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 79657f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 79757f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 79857f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 79957f26649SNicholas Piggin * 80057f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 80157f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 80257f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 80357f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 80457f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 80557f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 80657f26649SNicholas Piggin * 8077fa95f9aSNicholas Piggin * The scv instructions are a special case. They get a 0x3000 offset applied. 8087fa95f9aSNicholas Piggin * scv exceptions have unique reentrancy properties, see below. 8097fa95f9aSNicholas Piggin * 81057f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 81157f26649SNicholas Piggin * 81257f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 81357f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 81457f26649SNicholas Piggin * 81557f26649SNicholas Piggin * 8160ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 8170ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 81857f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 8197fa95f9aSNicholas Piggin * 0x1900 - 0x2fff : Real mode trampolines 8207fa95f9aSNicholas Piggin * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 82157f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 8220ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 82357f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 82457f26649SNicholas Piggin * setup code, rest of kernel. 825e0319829SNicholas Piggin * 826e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 827e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 828e0319829SNicholas Piggin * vectors there. 8290ebc4cdaSBenjamin Herrenschmidt */ 83057f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 8317fa95f9aSNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000) 8327fa95f9aSNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900) 83357f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 834ccd47702SNicholas Piggin 835ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 836bd3524feSNicholas Piggin .globl start_real_trampolines 837bd3524feSNicholas Piggin .globl end_real_trampolines 838bd3524feSNicholas Piggin .globl start_virt_trampolines 839bd3524feSNicholas Piggin .globl end_virt_trampolines 840ccd47702SNicholas Piggin#endif 841ccd47702SNicholas Piggin 84257f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 84357f26649SNicholas Piggin/* 84457f26649SNicholas Piggin * Data area reserved for FWNMI option. 84557f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 84657f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 84757f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 84857f26649SNicholas Piggin */ 84957f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 85057f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 85157f26649SNicholas Piggin#else 85257f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 85357f26649SNicholas Piggin#endif 85457f26649SNicholas Piggin 85557f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 85657f26649SNicholas Piggin 8570ebc4cdaSBenjamin Herrenschmidt/* 8580ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 8590ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 8600ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 8610ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 8620ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 8630ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 8640ebc4cdaSBenjamin Herrenschmidt */ 8650ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 8660ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 8670ebc4cdaSBenjamin Herrenschmidt 8687fa95f9aSNicholas Piggin/** 8697fa95f9aSNicholas Piggin * Interrupt 0x3000 - System Call Vectored Interrupt (syscall). 8707fa95f9aSNicholas Piggin * This is a synchronous interrupt invoked with the "scv" instruction. The 8717fa95f9aSNicholas Piggin * system call does not alter the HV bit, so it is directed to the OS. 8727fa95f9aSNicholas Piggin * 8737fa95f9aSNicholas Piggin * Handling: 8747fa95f9aSNicholas Piggin * scv instructions enter the kernel without changing EE, RI, ME, or HV. 8757fa95f9aSNicholas Piggin * In particular, this means we can take a maskable interrupt at any point 8767fa95f9aSNicholas Piggin * in the scv handler, which is unlike any other interrupt. This is solved 877325678fdSNicholas Piggin * by treating the instruction addresses in the handler as being soft-masked, 878325678fdSNicholas Piggin * by adding a SOFT_MASK_TABLE entry for them. 8797fa95f9aSNicholas Piggin * 8807fa95f9aSNicholas Piggin * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and 8817fa95f9aSNicholas Piggin * ensure scv is never executed with relocation off, which means AIL-0 8827fa95f9aSNicholas Piggin * should never happen. 8837fa95f9aSNicholas Piggin * 8849d1988caSNicholas Piggin * Before leaving the following inside-__end_soft_masked text, at least of the 8859d1988caSNicholas Piggin * following must be true: 8867fa95f9aSNicholas Piggin * - MSR[PR]=1 (i.e., return to userspace) 8879d1988caSNicholas Piggin * - MSR_EE|MSR_RI is clear (no reentrant exceptions) 8887fa95f9aSNicholas Piggin * - Standard kernel environment is set up (stack, paca, etc) 8897fa95f9aSNicholas Piggin * 890b5149e22SNicholas Piggin * KVM: 891b5149e22SNicholas Piggin * These interrupts do not elevate HV 0->1, so HV is not involved. PR KVM 892b5149e22SNicholas Piggin * ensures that FSCR[SCV] is disabled whenever it has to force AIL off. 893b5149e22SNicholas Piggin * 8947fa95f9aSNicholas Piggin * Call convention: 8957fa95f9aSNicholas Piggin * 8967fa95f9aSNicholas Piggin * syscall register convention is in Documentation/powerpc/syscall64-abi.rst 8977fa95f9aSNicholas Piggin */ 8987fa95f9aSNicholas PigginEXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000) 8997fa95f9aSNicholas Piggin /* SCV 0 */ 9007fa95f9aSNicholas Piggin mr r9,r13 9017fa95f9aSNicholas Piggin GET_PACA(r13) 9027fa95f9aSNicholas Piggin mflr r11 9037fa95f9aSNicholas Piggin mfctr r12 9047fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 9057fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 9067fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 9077fa95f9aSNicholas Piggin b system_call_vectored_tramp 9087fa95f9aSNicholas Piggin#else 9097fa95f9aSNicholas Piggin b system_call_vectored_common 9107fa95f9aSNicholas Piggin#endif 9117fa95f9aSNicholas Piggin nop 9127fa95f9aSNicholas Piggin 9137fa95f9aSNicholas Piggin /* SCV 1 - 127 */ 9147fa95f9aSNicholas Piggin .rept 127 9157fa95f9aSNicholas Piggin mr r9,r13 9167fa95f9aSNicholas Piggin GET_PACA(r13) 9177fa95f9aSNicholas Piggin mflr r11 9187fa95f9aSNicholas Piggin mfctr r12 9197fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 9207fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 9217fa95f9aSNicholas Piggin li r0,-1 /* cause failure */ 9227fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 9237fa95f9aSNicholas Piggin b system_call_vectored_sigill_tramp 9247fa95f9aSNicholas Piggin#else 9257fa95f9aSNicholas Piggin b system_call_vectored_sigill 9267fa95f9aSNicholas Piggin#endif 9277fa95f9aSNicholas Piggin .endr 9287fa95f9aSNicholas PigginEXC_VIRT_END(system_call_vectored, 0x3000, 0x1000) 9297fa95f9aSNicholas Piggin 930787c70f2SNicholas Piggin// Treat scv vectors as soft-masked, see comment above. 931787c70f2SNicholas Piggin// Use absolute values rather than labels here, so they don't get relocated, 932787c70f2SNicholas Piggin// because this code runs unrelocated. 933787c70f2SNicholas PigginSOFT_MASK_TABLE(0xc000000000003000, 0xc000000000004000) 934325678fdSNicholas Piggin 9357fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 9367fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_tramp) 937d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_vectored_common, virt_trampolines) 9387fa95f9aSNicholas Piggin mtctr r10 9397fa95f9aSNicholas Piggin bctr 9407fa95f9aSNicholas Piggin 9417fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) 942d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_vectored_sigill, virt_trampolines) 9437fa95f9aSNicholas Piggin mtctr r10 9447fa95f9aSNicholas Piggin bctr 9457fa95f9aSNicholas Piggin#endif 9467fa95f9aSNicholas Piggin 9477fa95f9aSNicholas Piggin 948e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 9491a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 950e0319829SNicholas Piggin 951fb479e44SNicholas Piggin 95294325357SNicholas Piggin/** 95394325357SNicholas Piggin * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 95494325357SNicholas Piggin * This is a non-maskable, asynchronous interrupt always taken in real-mode. 95594325357SNicholas Piggin * It is caused by: 95694325357SNicholas Piggin * - Wake from power-saving state, on powernv. 95794325357SNicholas Piggin * - An NMI from another CPU, triggered by firmware or hypercall. 95894325357SNicholas Piggin * - As crash/debug signal injected from BMC, firmware or hypervisor. 95994325357SNicholas Piggin * 96094325357SNicholas Piggin * Handling: 96194325357SNicholas Piggin * Power-save wakeup is the only performance critical path, so this is 96294325357SNicholas Piggin * determined quickly as possible first. In this case volatile registers 96394325357SNicholas Piggin * can be discarded and SPRs like CFAR don't need to be read. 96494325357SNicholas Piggin * 96594325357SNicholas Piggin * If not a powersave wakeup, then it's run as a regular interrupt, however 96694325357SNicholas Piggin * it uses its own stack and PACA save area to preserve the regular kernel 96794325357SNicholas Piggin * environment for debugging. 96894325357SNicholas Piggin * 96994325357SNicholas Piggin * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 97094325357SNicholas Piggin * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 97194325357SNicholas Piggin * correct to switch to virtual mode to run the regular interrupt handler 97294325357SNicholas Piggin * because it might be interrupted when the MMU is in a bad state (e.g., SLB 97394325357SNicholas Piggin * is clear). 97494325357SNicholas Piggin * 97594325357SNicholas Piggin * FWNMI: 97694325357SNicholas Piggin * PAPR specifies a "fwnmi" facility which sends the sreset to a different 97794325357SNicholas Piggin * entry point with a different register set up. Some hypervisors will 97894325357SNicholas Piggin * send the sreset to 0x100 in the guest if it is not fwnmi capable. 97994325357SNicholas Piggin * 98094325357SNicholas Piggin * KVM: 98194325357SNicholas Piggin * Unlike most SRR interrupts, this may be taken by the host while executing 98294325357SNicholas Piggin * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 98394325357SNicholas Piggin * mode and then raise the sreset. 98494325357SNicholas Piggin */ 9854f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 9864f50541fSNicholas Piggin IVEC=0x100 9874f50541fSNicholas Piggin IAREA=PACA_EXNMI 9888729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 9894f50541fSNicholas Piggin ISTACK=0 9904f50541fSNicholas Piggin IKVM_REAL=1 9914f50541fSNicholas PigginINT_DEFINE_END(system_reset) 9924f50541fSNicholas Piggin 993a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 994fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 995fb479e44SNicholas Piggin /* 996fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 997ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 998ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 999ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 1000ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 10010e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 10020e10be2bSNicholas Piggin * 10030e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 10040e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 10050e10be2bSNicholas Piggin * 10060e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 10070e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 10080e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 10090e10be2bSNicholas Piggin * common exception macros to handle it. 1010fb479e44SNicholas Piggin */ 1011a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 10120e10be2bSNicholas Piggin SET_SCRATCH0(r13) 10130e10be2bSNicholas Piggin GET_PACA(r13) 10140e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 10150e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 10160e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 1017a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 10180e10be2bSNicholas Piggin mfocrf r4,0x80 10190e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 10200e10be2bSNicholas Piggin bne+ system_reset_idle_wake 10210e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 10220e10be2bSNicholas Piggin mtocrf 0x80,r4 10230e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 10240e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 10250e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 10260e10be2bSNicholas Piggin GET_SCRATCH0(r13) 1027a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1028fb479e44SNicholas Piggin#endif 1029fb479e44SNicholas Piggin 10304f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 1031c4f3b52cSNicholas Piggin /* 10320e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 10330e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 10340e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 10350e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 10360e10be2bSNicholas Piggin * be dangerous anyway. 1037c4f3b52cSNicholas Piggin */ 10381a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 10391a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 1040fb479e44SNicholas Piggin 1041fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 10420e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 10430e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 10440e10be2bSNicholas Piggin cmpwi cr1,r5,2 10450e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1046d72c4a36SDaniel Axtens __LOAD_FAR_HANDLER(r12, DOTSYM(idle_return_gpr_loss), real_trampolines) 1047fd983957SAlexey Kardashevskiy mtctr r12 1048fd983957SAlexey Kardashevskiy bctr 1049371fefd6SPaul Mackerras#endif 1050371fefd6SPaul Mackerras 1051acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1052acc8da44SNicholas Piggin/* 1053acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 1054acc8da44SNicholas Piggin */ 1055acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 10564f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 1057acc8da44SNicholas Piggin 1058acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 1059acc8da44SNicholas Piggin 1060a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 10618729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 1062c4f3b52cSNicholas Piggin /* 1063ff0b0d6eSNicholas Piggin * Increment paca->in_nmi. When the interrupt entry wrapper later 1064ff0b0d6eSNicholas Piggin * enable MSR_RI, then SLB or MCE will be able to recover, but a nested 1065ff0b0d6eSNicholas Piggin * NMI will notice in_nmi and not recover because of the use of the NMI 1066ff0b0d6eSNicholas Piggin * stack. in_nmi reentrancy is tested in system_reset_exception. 1067c4f3b52cSNicholas Piggin */ 1068c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 1069c4f3b52cSNicholas Piggin addi r10,r10,1 1070c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 1071aca79d2bSVaidyanathan Srinivasan 1072b1ee8a3dSNicholas Piggin mr r10,r1 1073b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 1074b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 10758729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 107647169fbaSNicholas Piggin 1077c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1078*4e991e3cSNicholas Piggin bl CFUNC(system_reset_exception) 107915b4dd79SNicholas Piggin 108015b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 1081fbc50063SNicholas Piggin li r9,0 108215b4dd79SNicholas Piggin mtmsrd r9,1 1083c4f3b52cSNicholas Piggin 1084c4f3b52cSNicholas Piggin /* 108515b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 1086c4f3b52cSNicholas Piggin */ 1087c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 1088c4f3b52cSNicholas Piggin subi r10,r10,1 1089c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 1090c4f3b52cSNicholas Piggin 10918e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 10923f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 109315b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 1094582baf44SNicholas Piggin 10950ebc4cdaSBenjamin Herrenschmidt 109694325357SNicholas Piggin/** 109794325357SNicholas Piggin * Interrupt 0x200 - Machine Check Interrupt (MCE). 109894325357SNicholas Piggin * This is a non-maskable interrupt always taken in real-mode. It can be 109994325357SNicholas Piggin * synchronous or asynchronous, caused by hardware or software, and it may be 110094325357SNicholas Piggin * taken in a power-saving state. 110194325357SNicholas Piggin * 110294325357SNicholas Piggin * Handling: 110394325357SNicholas Piggin * Similarly to system reset, this uses its own stack and PACA save area, 110494325357SNicholas Piggin * the difference is re-entrancy is allowed on the machine check stack. 110594325357SNicholas Piggin * 110694325357SNicholas Piggin * machine_check_early is run in real mode, and carefully decodes the 110794325357SNicholas Piggin * machine check and tries to handle it (e.g., flush the SLB if there was an 110894325357SNicholas Piggin * error detected there), determines if it was recoverable and logs the 110994325357SNicholas Piggin * event. 111094325357SNicholas Piggin * 1111b44fc96dSNicholas Piggin * This early code does not "reconcile" irq soft-mask state like SRESET or 1112b44fc96dSNicholas Piggin * regular interrupts do, so irqs_disabled() among other things may not work 1113b44fc96dSNicholas Piggin * properly (irq disable/enable already doesn't work because irq tracing can 1114b44fc96dSNicholas Piggin * not work in real mode). 1115b44fc96dSNicholas Piggin * 111694325357SNicholas Piggin * Then, depending on the execution context when the interrupt is taken, there 111794325357SNicholas Piggin * are 3 main actions: 111894325357SNicholas Piggin * - Executing in kernel mode. The event is queued with irq_work, which means 111994325357SNicholas Piggin * it is handled when it is next safe to do so (i.e., the kernel has enabled 112094325357SNicholas Piggin * interrupts), which could be immediately when the interrupt returns. This 112194325357SNicholas Piggin * avoids nasty issues like switching to virtual mode when the MMU is in a 112294325357SNicholas Piggin * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 112394325357SNicholas Piggin * but it has different priorities). Check to see if the CPU was in power 112494325357SNicholas Piggin * save, and return via the wake up code if it was. 112594325357SNicholas Piggin * 112694325357SNicholas Piggin * - Executing in user mode. machine_check_exception is run like a normal 112794325357SNicholas Piggin * interrupt handler, which processes the data generated by the early handler. 112894325357SNicholas Piggin * 112994325357SNicholas Piggin * - Executing in guest mode. The interrupt is run with its KVM test, and 113094325357SNicholas Piggin * branches to KVM to deal with. KVM may queue the event for the host 113194325357SNicholas Piggin * to report later. 113294325357SNicholas Piggin * 113394325357SNicholas Piggin * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 113494325357SNicholas Piggin * or SCRATCH0 is in use, it may cause a crash. 113594325357SNicholas Piggin * 113694325357SNicholas Piggin * KVM: 113794325357SNicholas Piggin * See SRESET. 113894325357SNicholas Piggin */ 11394f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 11404f50541fSNicholas Piggin IVEC=0x200 11414f50541fSNicholas Piggin IAREA=PACA_EXMC 11428729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 1143d73a10cbSNicholas Piggin IREALMODE_COMMON=1 11444f50541fSNicholas Piggin ISTACK=0 11454f50541fSNicholas Piggin IDAR=1 11464f50541fSNicholas Piggin IDSISR=1 11474f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 11484f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 11494f50541fSNicholas Piggin 11504f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 11514f50541fSNicholas Piggin IVEC=0x200 11524f50541fSNicholas Piggin IAREA=PACA_EXMC 11538729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 11544f50541fSNicholas Piggin IDAR=1 11554f50541fSNicholas Piggin IDSISR=1 11564f50541fSNicholas Piggin IKVM_REAL=1 11574f50541fSNicholas PigginINT_DEFINE_END(machine_check) 11584f50541fSNicholas Piggin 11594f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 11602f5182cfSNicholas Piggin EARLY_BOOT_FIXUP 11614f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 11621a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 11631a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 1164c8eb54dbSNicholas Piggin 1165abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1166abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 1167abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 11684f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 1169abd1f4caSNicholas Piggin#endif 1170abd1f4caSNicholas Piggin 1171fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 1172fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1173fce16d48SNicholas Piggin li r9,0; \ 1174fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 1175fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 1176fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 1177fce16d48SNicholas Piggin subi r12,r12,1; \ 1178fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 11793f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 1180fce16d48SNicholas Piggin 1181c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 11829600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 11839600f261SNicholas Piggin 1184afcf0095SNicholas Piggin /* 1185afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 1186afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 1187afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1188afcf0095SNicholas Piggin * 1189afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 1190afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 1191afcf0095SNicholas Piggin * machine checks. 1192afcf0095SNicholas Piggin * 1193afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 1194afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 1195afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 1196afcf0095SNicholas Piggin * 1197afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1198afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 1199afcf0095SNicholas Piggin * rfid with MSR_ME=1. 12001945bc45SNicholas Piggin * 12011945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 12021945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 12032bf1071aSNicholas Piggin * to restore state. 1204afcf0095SNicholas Piggin */ 1205afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 1206afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 1207c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1208afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 1209afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 1210c8eb54dbSNicholas Piggin 1211c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 1212c8eb54dbSNicholas Piggin bne 1f 1213c8eb54dbSNicholas Piggin /* First machine check entry */ 1214c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1215b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1216b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1217b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1218c8eb54dbSNicholas Piggin 12198729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1220c8eb54dbSNicholas Piggin 1221db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1222296e753fSNicholas Piggin bl enable_machine_check 1223db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1224c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 12252f5182cfSNicholas PigginBEGIN_FTR_SECTION 1226*4e991e3cSNicholas Piggin bl CFUNC(machine_check_early_boot) 12272f5182cfSNicholas PigginEND_FTR_SECTION(0, 1) // nop out after boot 1228*4e991e3cSNicholas Piggin bl CFUNC(machine_check_early) 1229afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1230afcf0095SNicholas Piggin ld r12,_MSR(r1) 12311945bc45SNicholas Piggin 1232afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1233afcf0095SNicholas Piggin /* 1234afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1235afcf0095SNicholas Piggin * of the following is true: 1236afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1237afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1238afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1239afcf0095SNicholas Piggin * 1240afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1241afcf0095SNicholas Piggin */ 12421945bc45SNicholas PigginBEGIN_FTR_SECTION 12431945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 12446102c005SNicholas Piggin bne machine_check_idle_common 12451945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1246afcf0095SNicholas Piggin#endif 12471945bc45SNicholas Piggin 1248afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1249afcf0095SNicholas Piggin /* 125019dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 125105f97d94SNicholas Piggin * exception handler which will take the 125269fdd674SNicholas Piggin * machine_check_kvm->kvm_interrupt branch to deliver the MC event 125305f97d94SNicholas Piggin * to guest. 1254afcf0095SNicholas Piggin */ 1255afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1256afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1257b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1258afcf0095SNicholas Piggin#endif 125919dbe673SNicholas Piggin 1260afcf0095SNicholas Piggin /* 126119dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 126219dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 126319dbe673SNicholas Piggin */ 126419dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1265b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 126619dbe673SNicholas Piggin 126719dbe673SNicholas Piggin /* 126819dbe673SNicholas Piggin * At this point we are coming from kernel context. 1269afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1270afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1271afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1272afcf0095SNicholas Piggin */ 1273afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1274b7d9ccecSNicholas Piggin beq unrecoverable_mce 1275b7d9ccecSNicholas Piggin 1276afcf0095SNicholas Piggin /* 1277afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1278afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1279afcf0095SNicholas Piggin */ 1280afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1281afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1282b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1283272f6364SNicholas Piggin 1284afcf0095SNicholas Piggin /* 1285afcf0095SNicholas Piggin * Return from MC interrupt. 1286afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1287afcf0095SNicholas Piggin * returning from kernel or opal call. 1288afcf0095SNicholas Piggin */ 1289*4e991e3cSNicholas Piggin bl CFUNC(machine_check_queue_event) 1290afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1291fe9d482bSNicholas Piggin RFI_TO_KERNEL 1292272f6364SNicholas Piggin 1293b3fe3526SNicholas Pigginmce_deliver: 1294b3fe3526SNicholas Piggin /* 1295b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1296b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1297b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1298b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1299b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1300b3fe3526SNicholas Piggin */ 13010b66370cSNicholas PigginBEGIN_FTR_SECTION 13020b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 13030b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 13040b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1305afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 13064f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1307afcf0095SNicholas Piggin 1308fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1309fce16d48SNicholas Piggin /* 1310fce16d48SNicholas Piggin * Machine check is different because we use a different 1311fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1312fce16d48SNicholas Piggin */ 13134f50541fSNicholas Piggin GEN_COMMON machine_check 1314c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1315*4e991e3cSNicholas Piggin bl CFUNC(machine_check_exception_async) 13161df7d5e4SNicholas Piggin b interrupt_return_srr 1317fce16d48SNicholas Piggin 13189600f261SNicholas Piggin 1319fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1320fce16d48SNicholas Piggin/* 1321fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1322fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1323fce16d48SNicholas Piggin */ 1324fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 1325*4e991e3cSNicholas Piggin bl CFUNC(machine_check_queue_event) 1326fce16d48SNicholas Piggin 1327fce16d48SNicholas Piggin /* 13288a5054d8SNicholas Piggin * GPR-loss wakeups are relatively straightforward, because the 13298a5054d8SNicholas Piggin * idle sleep code has saved all non-volatile registers on its 13308a5054d8SNicholas Piggin * own stack, and r1 in PACAR1. 1331fce16d48SNicholas Piggin * 13328a5054d8SNicholas Piggin * For no-loss wakeups the r1 and lr registers used by the 13338a5054d8SNicholas Piggin * early machine check handler have to be restored first. r2 is 13348a5054d8SNicholas Piggin * the kernel TOC, so no need to restore it. 1335fce16d48SNicholas Piggin * 1336fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1337fce16d48SNicholas Piggin */ 1338fce16d48SNicholas Piggin ld r3,_MSR(r1) 1339fce16d48SNicholas Piggin ld r4,_LINK(r1) 13408a5054d8SNicholas Piggin ld r1,GPR1(r1) 1341fce16d48SNicholas Piggin 1342fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1343fce16d48SNicholas Piggin subi r11,r11,1 1344fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1345fce16d48SNicholas Piggin 1346fce16d48SNicholas Piggin mtlr r4 1347fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1348fce16d48SNicholas Piggin cmpwi cr1,r10,2 13498a5054d8SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1350fce16d48SNicholas Piggin b idle_return_gpr_loss 1351fce16d48SNicholas Piggin#endif 1352fce16d48SNicholas Piggin 1353b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1354b7d9ccecSNicholas Piggin /* 1355b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1356b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1357b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1358b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1359b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1360b7d9ccecSNicholas Piggin */ 1361b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1362b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1363b7d9ccecSNicholas Piggin mtmsrd r10,1 1364*4e991e3cSNicholas Piggin bl CFUNC(disable_machine_check) 1365b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1366b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1367b7d9ccecSNicholas Piggin li r3,MSR_ME 1368b7d9ccecSNicholas Piggin andc r10,r10,r3 1369b7d9ccecSNicholas Piggin mtmsrd r10 1370b7d9ccecSNicholas Piggin 1371ac2a2a14SNicholas Piggin lhz r12,PACA_IN_MCE(r13) 1372ac2a2a14SNicholas Piggin subi r12,r12,1 1373ac2a2a14SNicholas Piggin sth r12,PACA_IN_MCE(r13) 1374ac2a2a14SNicholas Piggin 1375f08fb25bSNicholas Piggin /* 1376f08fb25bSNicholas Piggin * Invoke machine_check_exception to print MCE event and panic. 1377f08fb25bSNicholas Piggin * This is the NMI version of the handler because we are called from 1378f08fb25bSNicholas Piggin * the early handler which is a true NMI. 1379f08fb25bSNicholas Piggin */ 1380c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1381*4e991e3cSNicholas Piggin bl CFUNC(machine_check_exception) 1382b7d9ccecSNicholas Piggin 1383afcf0095SNicholas Piggin /* 1384b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1385b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1386afcf0095SNicholas Piggin */ 1387c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1388*4e991e3cSNicholas Piggin bl CFUNC(unrecoverable_exception) 1389b7d9ccecSNicholas Piggin b . 1390afcf0095SNicholas Piggin 13914f50541fSNicholas Piggin 13924f50541fSNicholas Piggin/** 139394325357SNicholas Piggin * Interrupt 0x300 - Data Storage Interrupt (DSI). 139494325357SNicholas Piggin * This is a synchronous interrupt generated due to a data access exception, 139594325357SNicholas Piggin * e.g., a load orstore which does not have a valid page table entry with 139694325357SNicholas Piggin * permissions. DAWR matches also fault here, as do RC updates, and minor misc 139794325357SNicholas Piggin * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 13984f50541fSNicholas Piggin * 139994325357SNicholas Piggin * Handling: 140094325357SNicholas Piggin * - Hash MMU 1401a4922f54SNicholas Piggin * Go to do_hash_fault, which attempts to fill the HPT from an entry in the 1402a4922f54SNicholas Piggin * Linux page table. Hash faults can hit in kernel mode in a fairly 140394325357SNicholas Piggin * arbitrary state (e.g., interrupts disabled, locks held) when accessing 140494325357SNicholas Piggin * "non-bolted" regions, e.g., vmalloc space. However these should always be 1405a4922f54SNicholas Piggin * backed by Linux page table entries. 14064f50541fSNicholas Piggin * 1407a4922f54SNicholas Piggin * If no entry is found the Linux page fault handler is invoked (by 1408a4922f54SNicholas Piggin * do_hash_fault). Linux page faults can happen in kernel mode due to user 1409a4922f54SNicholas Piggin * copy operations of course. 14104f50541fSNicholas Piggin * 1411cd81acc6SNicholas Piggin * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest 1412cd81acc6SNicholas Piggin * MMU context, which may cause a DSI in the host, which must go to the 1413cd81acc6SNicholas Piggin * KVM handler. MSR[IR] is not enabled, so the real-mode handler will 1414cd81acc6SNicholas Piggin * always be used regardless of AIL setting. 1415cd81acc6SNicholas Piggin * 141694325357SNicholas Piggin * - Radix MMU 141794325357SNicholas Piggin * The hardware loads from the Linux page table directly, so a fault goes 141894325357SNicholas Piggin * immediately to Linux page fault. 141994325357SNicholas Piggin * 142094325357SNicholas Piggin * Conditions like DAWR match are handled on the way in to Linux page fault. 14214f50541fSNicholas Piggin */ 1422a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1423a42a239dSNicholas Piggin IVEC=0x300 1424a42a239dSNicholas Piggin IDAR=1 1425a42a239dSNicholas Piggin IDSISR=1 1426a42a239dSNicholas Piggin IKVM_REAL=1 1427a42a239dSNicholas PigginINT_DEFINE_END(data_access) 14280ebc4cdaSBenjamin Herrenschmidt 1429e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1430689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1431e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1432e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1433a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1434e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 143580795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 14367cb3a1a0SNicholas Piggin GEN_COMMON data_access 1437a01a3f2dSNicholas Piggin ld r4,_DSISR(r1) 1438c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1439a01a3f2dSNicholas Piggin andis. r0,r4,DSISR_DABRMATCH@h 144036f01141SNicholas Piggin bne- 1f 1441387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 144280795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 1443*4e991e3cSNicholas Piggin bl CFUNC(do_hash_fault) 144480795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 1445*4e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 144680795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1447387e220aSNicholas Piggin#else 1448*4e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 1449387e220aSNicholas Piggin#endif 14501df7d5e4SNicholas Piggin b interrupt_return_srr 1451a4922f54SNicholas Piggin 1452*4e991e3cSNicholas Piggin1: bl CFUNC(do_break) 145336f01141SNicholas Piggin /* 145436f01141SNicholas Piggin * do_break() may have changed the NV GPRS while handling a breakpoint. 145536f01141SNicholas Piggin * If so, we need to restore them with their updated values. 145636f01141SNicholas Piggin */ 14571df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() 14581df7d5e4SNicholas Piggin b interrupt_return_srr 145980795e6cSNicholas Piggin 14600ebc4cdaSBenjamin Herrenschmidt 146194325357SNicholas Piggin/** 146294325357SNicholas Piggin * Interrupt 0x380 - Data Segment Interrupt (DSLB). 146394325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault missing SLB 146494325357SNicholas Piggin * entry for HPT, or an address outside RPT translation range. 146594325357SNicholas Piggin * 146694325357SNicholas Piggin * Handling: 146794325357SNicholas Piggin * - HPT: 146894325357SNicholas Piggin * This refills the SLB, or reports an access fault similarly to a bad page 146994325357SNicholas Piggin * fault. When coming from user-mode, the SLB handler may access any kernel 147094325357SNicholas Piggin * data, though it may itself take a DSLB. When coming from kernel mode, 147194325357SNicholas Piggin * recursive faults must be avoided so access is restricted to the kernel 147294325357SNicholas Piggin * image text/data, kernel stack, and any data allocated below 147394325357SNicholas Piggin * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 147494325357SNicholas Piggin * on user-handler data structures. 147594325357SNicholas Piggin * 1476cd81acc6SNicholas Piggin * KVM: Same as 0x300, DSLB must test for KVM guest. 147794325357SNicholas Piggin */ 14784f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 14794f50541fSNicholas Piggin IVEC=0x380 14804f50541fSNicholas Piggin IDAR=1 14814f50541fSNicholas Piggin IKVM_REAL=1 14824f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 14834f50541fSNicholas Piggin 14841a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1485689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 14861a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 14871a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 14884f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 14891a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 149048e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 14914f50541fSNicholas Piggin GEN_COMMON data_access_slb 1492387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 14937100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14947100e870SNicholas Piggin /* HPT case, do SLB fault */ 1495c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1496*4e991e3cSNicholas Piggin bl CFUNC(do_slb_fault) 149748e7b769SNicholas Piggin cmpdi r3,0 149848e7b769SNicholas Piggin bne- 1f 14991df7d5e4SNicholas Piggin b fast_interrupt_return_srr 150048e7b769SNicholas Piggin1: /* Error case */ 15017100e870SNicholas PigginMMU_FTR_SECTION_ELSE 15027100e870SNicholas Piggin /* Radix case, access is outside page table range */ 15037100e870SNicholas Piggin li r3,-EFAULT 15047100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1505387e220aSNicholas Piggin#else 1506387e220aSNicholas Piggin li r3,-EFAULT 1507387e220aSNicholas Piggin#endif 150848e7b769SNicholas Piggin std r3,RESULT(r1) 1509c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1510*4e991e3cSNicholas Piggin bl CFUNC(do_bad_segment_interrupt) 15111df7d5e4SNicholas Piggin b interrupt_return_srr 151248e7b769SNicholas Piggin 15132b9af6e4SNicholas Piggin 151494325357SNicholas Piggin/** 151594325357SNicholas Piggin * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 151694325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 151794325357SNicholas Piggin * instruction fetch. 151894325357SNicholas Piggin * 151994325357SNicholas Piggin * Handling: 152094325357SNicholas Piggin * Similar to DSI, though in response to fetch. The faulting address is found 152194325357SNicholas Piggin * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 152294325357SNicholas Piggin */ 15234f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 15244f50541fSNicholas Piggin IVEC=0x400 1525a3cd35beSNicholas Piggin IISIDE=1 1526a3cd35beSNicholas Piggin IDAR=1 1527a3cd35beSNicholas Piggin IDSISR=1 15282284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15294f50541fSNicholas Piggin IKVM_REAL=1 15302284ffeaSNicholas Piggin#endif 15314f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 15324f50541fSNicholas Piggin 15337299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 15344f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 15357299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 15367299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 15374f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 15387299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 153927ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 15404f50541fSNicholas Piggin GEN_COMMON instruction_access 1541c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1542387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 154327ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 1544*4e991e3cSNicholas Piggin bl CFUNC(do_hash_fault) 154527ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 1546*4e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 154727ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1548387e220aSNicholas Piggin#else 1549*4e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 1550387e220aSNicholas Piggin#endif 15511df7d5e4SNicholas Piggin b interrupt_return_srr 155227ce77dfSNicholas Piggin 15530ebc4cdaSBenjamin Herrenschmidt 155494325357SNicholas Piggin/** 155594325357SNicholas Piggin * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 155694325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 155794325357SNicholas Piggin * instruction fetch. 155894325357SNicholas Piggin * 155994325357SNicholas Piggin * Handling: 156094325357SNicholas Piggin * Similar to DSLB, though in response to fetch. The faulting address is found 156194325357SNicholas Piggin * in SRR0 (rather than DAR). 156294325357SNicholas Piggin */ 15634f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 15644f50541fSNicholas Piggin IVEC=0x480 1565a3cd35beSNicholas Piggin IISIDE=1 1566a3cd35beSNicholas Piggin IDAR=1 15672284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15684f50541fSNicholas Piggin IKVM_REAL=1 15692284ffeaSNicholas Piggin#endif 15704f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 15714f50541fSNicholas Piggin 15727299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 15734f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 15747299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 15757299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 15764f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 15777299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 157848e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 15794f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 1580387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 15817100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 15827100e870SNicholas Piggin /* HPT case, do SLB fault */ 1583c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1584*4e991e3cSNicholas Piggin bl CFUNC(do_slb_fault) 158548e7b769SNicholas Piggin cmpdi r3,0 158648e7b769SNicholas Piggin bne- 1f 15871df7d5e4SNicholas Piggin b fast_interrupt_return_srr 158848e7b769SNicholas Piggin1: /* Error case */ 15897100e870SNicholas PigginMMU_FTR_SECTION_ELSE 15907100e870SNicholas Piggin /* Radix case, access is outside page table range */ 15917100e870SNicholas Piggin li r3,-EFAULT 15927100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1593387e220aSNicholas Piggin#else 1594387e220aSNicholas Piggin li r3,-EFAULT 1595387e220aSNicholas Piggin#endif 159648e7b769SNicholas Piggin std r3,RESULT(r1) 1597c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1598*4e991e3cSNicholas Piggin bl CFUNC(do_bad_segment_interrupt) 15991df7d5e4SNicholas Piggin b interrupt_return_srr 16005e46e29eSNicholas Piggin 16019600f261SNicholas Piggin 160294325357SNicholas Piggin/** 160394325357SNicholas Piggin * Interrupt 0x500 - External Interrupt. 160494325357SNicholas Piggin * This is an asynchronous maskable interrupt in response to an "external 160594325357SNicholas Piggin * exception" from the interrupt controller or hypervisor (e.g., device 160694325357SNicholas Piggin * interrupt). It is maskable in hardware by clearing MSR[EE], and 160794325357SNicholas Piggin * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 160894325357SNicholas Piggin * 160994325357SNicholas Piggin * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 161094325357SNicholas Piggin * interrupts are delivered with HSRR registers, guests use SRRs, which 161194325357SNicholas Piggin * reqiures IHSRR_IF_HVMODE. 161294325357SNicholas Piggin * 161394325357SNicholas Piggin * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 161494325357SNicholas Piggin * external interrupts are delivered as Hypervisor Virtualization Interrupts 161594325357SNicholas Piggin * rather than External Interrupts. 161694325357SNicholas Piggin * 161794325357SNicholas Piggin * Handling: 161894325357SNicholas Piggin * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 161994325357SNicholas Piggin * because registers at the time of the interrupt are not so important as it is 162094325357SNicholas Piggin * asynchronous. 162194325357SNicholas Piggin * 162294325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 162394325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 1624af47d79bSNicholas Piggin * 1625af47d79bSNicholas Piggin * CFAR is not required because this is an asynchronous interrupt that in 1626af47d79bSNicholas Piggin * general won't have much bearing on the state of the CPU, with the possible 1627af47d79bSNicholas Piggin * exception of crash/debug IPIs, but those are generally moving to use SRESET 1628af47d79bSNicholas Piggin * IPIs. Unless this is an HV interrupt and KVM HV is possible, in which case 1629af47d79bSNicholas Piggin * it may be exiting the guest and need CFAR to be saved. 163094325357SNicholas Piggin */ 16314f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 16324f50541fSNicholas Piggin IVEC=0x500 16333f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 16344f50541fSNicholas Piggin IMASK=IRQS_DISABLED 16354f50541fSNicholas Piggin IKVM_REAL=1 16364f50541fSNicholas Piggin IKVM_VIRT=1 1637af47d79bSNicholas Piggin ICFAR=0 1638af47d79bSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1639af47d79bSNicholas Piggin ICFAR_IF_HVMODE=1 1640af47d79bSNicholas Piggin#endif 16414f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 16424f50541fSNicholas Piggin 16431a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 16444f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 16451a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 16461a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 16474f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 16481a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1649eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 16504f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1651c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1652*4e991e3cSNicholas Piggin bl CFUNC(do_IRQ) 16531df7d5e4SNicholas Piggin BEGIN_FTR_SECTION 16541df7d5e4SNicholas Piggin b interrupt_return_hsrr 16551df7d5e4SNicholas Piggin FTR_SECTION_ELSE 16561df7d5e4SNicholas Piggin b interrupt_return_srr 16571df7d5e4SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1658c138e588SNicholas Piggin 1659c138e588SNicholas Piggin 166094325357SNicholas Piggin/** 166194325357SNicholas Piggin * Interrupt 0x600 - Alignment Interrupt 166294325357SNicholas Piggin * This is a synchronous interrupt in response to data alignment fault. 166394325357SNicholas Piggin */ 16644f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 16654f50541fSNicholas Piggin IVEC=0x600 16664f50541fSNicholas Piggin IDAR=1 16674f50541fSNicholas Piggin IDSISR=1 16682284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16694f50541fSNicholas Piggin IKVM_REAL=1 16702284ffeaSNicholas Piggin#endif 16714f50541fSNicholas PigginINT_DEFINE_END(alignment) 16724f50541fSNicholas Piggin 1673e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 16744f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1675e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1676e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 16774f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1678e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1679f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 16804f50541fSNicholas Piggin GEN_COMMON alignment 1681c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1682*4e991e3cSNicholas Piggin bl CFUNC(alignment_exception) 16831df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 16841df7d5e4SNicholas Piggin b interrupt_return_srr 1685f9aa6714SNicholas Piggin 1686b01c8b54SPaul Mackerras 168794325357SNicholas Piggin/** 168894325357SNicholas Piggin * Interrupt 0x700 - Program Interrupt (program check). 168994325357SNicholas Piggin * This is a synchronous interrupt in response to various instruction faults: 169094325357SNicholas Piggin * traps, privilege errors, TM errors, floating point exceptions. 169194325357SNicholas Piggin * 169294325357SNicholas Piggin * Handling: 169394325357SNicholas Piggin * This interrupt may use the "emergency stack" in some cases when being taken 169494325357SNicholas Piggin * from kernel context, which complicates handling. 169594325357SNicholas Piggin */ 16964f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 16974f50541fSNicholas Piggin IVEC=0x700 16982284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16994f50541fSNicholas Piggin IKVM_REAL=1 17002284ffeaSNicholas Piggin#endif 17014f50541fSNicholas PigginINT_DEFINE_END(program_check) 17024f50541fSNicholas Piggin 17037299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 1704bf75a325SNicholas Piggin EARLY_BOOT_FIXUP 17054f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 17067299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 17077299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 17084f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 17097299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 171011e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 17118729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 17128729c26eSNicholas Piggin 1713265e60a1SCyril Bur /* 1714265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1715265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1716265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1717265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1718265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1719265e60a1SCyril Bur * the kernel. 1720265e60a1SCyril Bur */ 1721265e60a1SCyril Bur 17220a882e28SNicholas Piggin andi. r10,r12,MSR_PR 17233e607dc4SNicholas Piggin bne .Lnormal_stack /* If userspace, go normal path */ 17240a882e28SNicholas Piggin 17250a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 17263e607dc4SNicholas Piggin bne .Lemergency_stack /* If TM, emergency */ 17270a882e28SNicholas Piggin 17280a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 17293e607dc4SNicholas Piggin blt .Lnormal_stack /* normal path if not */ 17300a882e28SNicholas Piggin 17310a882e28SNicholas Piggin /* Use the emergency stack */ 17323e607dc4SNicholas Piggin.Lemergency_stack: 17333e607dc4SNicholas Piggin andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1734265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1735265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1736265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1737265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 17384f50541fSNicholas Piggin __ISTACK(program_check)=0 17398729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 17403e607dc4SNicholas Piggin b .Ldo_program_check 17413e607dc4SNicholas Piggin 17423e607dc4SNicholas Piggin.Lnormal_stack: 17434f50541fSNicholas Piggin __ISTACK(program_check)=1 17448729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 17453e607dc4SNicholas Piggin 17463e607dc4SNicholas Piggin.Ldo_program_check: 1747c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1748*4e991e3cSNicholas Piggin bl CFUNC(program_check_exception) 17491df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 17501df7d5e4SNicholas Piggin b interrupt_return_srr 175111e87346SNicholas Piggin 1752a485c709SPaul Mackerras 175394325357SNicholas Piggin/* 175494325357SNicholas Piggin * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 175594325357SNicholas Piggin * This is a synchronous interrupt in response to executing an fp instruction 175694325357SNicholas Piggin * with MSR[FP]=0. 175794325357SNicholas Piggin * 175894325357SNicholas Piggin * Handling: 175994325357SNicholas Piggin * This will load FP registers and enable the FP bit if coming from userspace, 176094325357SNicholas Piggin * otherwise report a bad kernel use of FP. 176194325357SNicholas Piggin */ 17624f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 17634f50541fSNicholas Piggin IVEC=0x800 17642284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17654f50541fSNicholas Piggin IKVM_REAL=1 17662284ffeaSNicholas Piggin#endif 17672487fd2eSRohan McLure IMSR_R12=1 17684f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 17694f50541fSNicholas Piggin 17707299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 17714f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 17727299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 17737299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 17744f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 17757299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1776c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 17774f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1778c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1779c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1780*4e991e3cSNicholas Piggin bl CFUNC(kernel_fp_unavailable_exception) 178163ce271bSChristophe Leroy0: trap 178263ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1783c78d9b97SNicholas Piggin1: 1784c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1785c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1786c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1787c78d9b97SNicholas Piggin * transaction), go do TM stuff 1788c78d9b97SNicholas Piggin */ 1789c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1790c78d9b97SNicholas Piggin bne- 2f 1791c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1792c78d9b97SNicholas Piggin#endif 1793*4e991e3cSNicholas Piggin bl CFUNC(load_up_fpu) 17941df7d5e4SNicholas Piggin b fast_interrupt_return_srr 1795c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1796c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1797c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1798*4e991e3cSNicholas Piggin bl CFUNC(fp_unavailable_tm) 17991df7d5e4SNicholas Piggin b interrupt_return_srr 1800c78d9b97SNicholas Piggin#endif 1801c78d9b97SNicholas Piggin 1802b01c8b54SPaul Mackerras 180394325357SNicholas Piggin/** 180494325357SNicholas Piggin * Interrupt 0x900 - Decrementer Interrupt. 180594325357SNicholas Piggin * This is an asynchronous interrupt in response to a decrementer exception 180694325357SNicholas Piggin * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 180794325357SNicholas Piggin * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 180894325357SNicholas Piggin * local_irq_disable()). 180994325357SNicholas Piggin * 181094325357SNicholas Piggin * Handling: 181194325357SNicholas Piggin * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 181294325357SNicholas Piggin * 181394325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 181494325357SNicholas Piggin * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 181594325357SNicholas Piggin * in the interrupted context. 181694325357SNicholas Piggin * If PPC_WATCHDOG is configured, the soft masked handler will actually set 181794325357SNicholas Piggin * things back up to run soft_nmi_interrupt as a regular interrupt handler 181894325357SNicholas Piggin * on the emergency stack. 1819af47d79bSNicholas Piggin * 1820af47d79bSNicholas Piggin * CFAR is not required because this is asynchronous (see hardware_interrupt). 1821af47d79bSNicholas Piggin * A watchdog interrupt may like to have CFAR, but usually the interesting 1822af47d79bSNicholas Piggin * branch is long gone by that point (e.g., infinite loop). 182394325357SNicholas Piggin */ 18244f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 18254f50541fSNicholas Piggin IVEC=0x900 18264f50541fSNicholas Piggin IMASK=IRQS_DISABLED 18272284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 18284f50541fSNicholas Piggin IKVM_REAL=1 18292284ffeaSNicholas Piggin#endif 1830af47d79bSNicholas Piggin ICFAR=0 18314f50541fSNicholas PigginINT_DEFINE_END(decrementer) 18324f50541fSNicholas Piggin 18337299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1834689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 18357299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 18367299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 18374f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 18387299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1839eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 18404f50541fSNicholas Piggin GEN_COMMON decrementer 1841c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1842*4e991e3cSNicholas Piggin bl CFUNC(timer_interrupt) 18431df7d5e4SNicholas Piggin b interrupt_return_srr 184439c0da57SNicholas Piggin 18450ebc4cdaSBenjamin Herrenschmidt 184694325357SNicholas Piggin/** 184794325357SNicholas Piggin * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 184894325357SNicholas Piggin * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 184994325357SNicholas Piggin * register. 185094325357SNicholas Piggin * 185194325357SNicholas Piggin * Handling: 185294325357SNicholas Piggin * Linux does not use this outside KVM where it's used to keep a host timer 185394325357SNicholas Piggin * while the guest is given control of DEC. It should normally be caught by 185494325357SNicholas Piggin * the KVM test and routed there. 185594325357SNicholas Piggin */ 18564f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 18574f50541fSNicholas Piggin IVEC=0x980 18583f7fbd97SNicholas Piggin IHSRR=1 18592babd6eaSNicholas Piggin ISTACK=0 18604f50541fSNicholas Piggin IKVM_REAL=1 18614f50541fSNicholas Piggin IKVM_VIRT=1 18624f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 18634f50541fSNicholas Piggin 18647299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 18654f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 18667299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 18677299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 18684f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 18697299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1870eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 18712babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 18722babd6eaSNicholas Piggin /* 18732babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 18742babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 18752babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 18762babd6eaSNicholas Piggin * edge-triggered. 18772babd6eaSNicholas Piggin * 18782babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 18792babd6eaSNicholas Piggin */ 188059dc5bfcSNicholas Piggin li r10,0 188159dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 18822babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 18832babd6eaSNicholas Piggin mtctr r10 18842babd6eaSNicholas Piggin mtcrf 0x80,r9 18852babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 18862babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 18872babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 18882babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 18892babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 18902babd6eaSNicholas Piggin HRFI_TO_KERNEL 1891facc6d74SNicholas Piggin 1892da2bc464SMichael Ellerman 189394325357SNicholas Piggin/** 189494325357SNicholas Piggin * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 189594325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsndp doorbell. 189694325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 189794325357SNicholas Piggin * IRQS_DISABLED mask (i.e., local_irq_disable()). 189894325357SNicholas Piggin * 189994325357SNicholas Piggin * Handling: 190094325357SNicholas Piggin * Guests may use this for IPIs between threads in a core if the 190194325357SNicholas Piggin * hypervisor supports it. NVGPRS are not saved (see 0x500). 190294325357SNicholas Piggin * 190394325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 190494325357SNicholas Piggin * replay, leaving MSR[EE] enabled in the interrupted context because the 190594325357SNicholas Piggin * doorbells are edge triggered. 1906af47d79bSNicholas Piggin * 1907af47d79bSNicholas Piggin * CFAR is not required, similarly to hardware_interrupt. 190894325357SNicholas Piggin */ 19094f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 19104f50541fSNicholas Piggin IVEC=0xa00 19114f50541fSNicholas Piggin IMASK=IRQS_DISABLED 19122284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 19134f50541fSNicholas Piggin IKVM_REAL=1 19142284ffeaSNicholas Piggin#endif 1915af47d79bSNicholas Piggin ICFAR=0 19164f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 19174f50541fSNicholas Piggin 19187299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 19194f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 19207299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 19217299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 19224f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 19237299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1924eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 19254f50541fSNicholas Piggin GEN_COMMON doorbell_super 1926c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1927ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1928*4e991e3cSNicholas Piggin bl CFUNC(doorbell_exception) 1929ca243163SNicholas Piggin#else 1930*4e991e3cSNicholas Piggin bl CFUNC(unknown_async_exception) 1931ca243163SNicholas Piggin#endif 19321df7d5e4SNicholas Piggin b interrupt_return_srr 1933ca243163SNicholas Piggin 1934da2bc464SMichael Ellerman 19355ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 19365ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1937341215dcSNicholas Piggin 193894325357SNicholas Piggin/** 193994325357SNicholas Piggin * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 194094325357SNicholas Piggin * This is a synchronous interrupt invoked with the "sc" instruction. The 194194325357SNicholas Piggin * system call is invoked with "sc 0" and does not alter the HV bit, so it 194294325357SNicholas Piggin * is directed to the currently running OS. The hypercall is invoked with 194394325357SNicholas Piggin * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1944acd7d8ceSNicholas Piggin * 1945acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1946acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1947acd7d8ceSNicholas Piggin * 194894325357SNicholas Piggin * Handling: 194994325357SNicholas Piggin * If the KVM test fires then it was due to a hypercall and is accordingly 195094325357SNicholas Piggin * routed to KVM. Otherwise this executes a normal Linux system call. 195194325357SNicholas Piggin * 1952acd7d8ceSNicholas Piggin * Call convention: 1953acd7d8ceSNicholas Piggin * 195458b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 195558b278f5SVaibhav Jain * Documentation/powerpc/syscall64-abi.rst and 195658b278f5SVaibhav Jain * Documentation/powerpc/papr_hcalls.rst respectively. 1957acd7d8ceSNicholas Piggin * 1958acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 195976fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 196076fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 196176fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1962acd7d8ceSNicholas Piggin */ 1963b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1964b177ae2fSNicholas Piggin IVEC=0xc00 1965b177ae2fSNicholas Piggin IKVM_REAL=1 1966b177ae2fSNicholas Piggin IKVM_VIRT=1 1967af47d79bSNicholas Piggin ICFAR=0 1968b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1969b177ae2fSNicholas Piggin 19701b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1971bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1972bc355125SPaul Mackerras /* 1973acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 197476fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 197576fc0cfcSNicholas Piggin * register. 1976acd7d8ceSNicholas Piggin * 1977acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1978acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1979bc355125SPaul Mackerras */ 19801b4d4a79SNicholas Piggin mtctr r13 19811b4d4a79SNicholas Piggin GET_PACA(r13) 19821b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 19831b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 198469fdd674SNicholas Piggin KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */ 19851b4d4a79SNicholas Piggin mfctr r9 1986bc355125SPaul Mackerras#else 19871b4d4a79SNicholas Piggin mr r9,r13 19881b4d4a79SNicholas Piggin GET_PACA(r13) 19891b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1990bc355125SPaul Mackerras#endif 1991bc355125SPaul Mackerras 1992727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 19931b4d4a79SNicholas PigginBEGIN_FTR_SECTION 19941b4d4a79SNicholas Piggin cmpdi r0,0x1ebe 19951b4d4a79SNicholas Piggin beq- 1f 19961b4d4a79SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 19971b4d4a79SNicholas Piggin#endif 19985c2511bfSMichael Ellerman 1999b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 20001b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 20011b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 2002b0b2a93dSNicholas Piggin 2003b0b2a93dSNicholas Piggin HMT_MEDIUM 2004b0b2a93dSNicholas Piggin 2005b0b2a93dSNicholas Piggin .if ! \virt 2006d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_common_real, real_vectors) 200714ad0e7dSNicholas Piggin mtctr r10 200814ad0e7dSNicholas Piggin bctr 20091b4d4a79SNicholas Piggin .else 2010b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 2011d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_common, virt_vectors) 2012b0b2a93dSNicholas Piggin mtctr r10 2013b0b2a93dSNicholas Piggin bctr 2014b0b2a93dSNicholas Piggin#else 20151b4d4a79SNicholas Piggin b system_call_common 2016d807ad37SNicholas Piggin#endif 20171b4d4a79SNicholas Piggin .endif 20181b4d4a79SNicholas Piggin 20191b4d4a79SNicholas Piggin#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 20201b4d4a79SNicholas Piggin /* Fast LE/BE switch system call */ 20211b4d4a79SNicholas Piggin1: mfspr r12,SPRN_SRR1 20221b4d4a79SNicholas Piggin xori r12,r12,MSR_LE 20231b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r12 20241b4d4a79SNicholas Piggin mr r13,r9 20251b4d4a79SNicholas Piggin RFI_TO_USER /* return to userspace */ 20261b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 20271b4d4a79SNicholas Piggin#endif 20281b4d4a79SNicholas Piggin.endm 2029d807ad37SNicholas Piggin 20301a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 20311b4d4a79SNicholas Piggin SYSTEM_CALL 0 20321a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 20331a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 20341b4d4a79SNicholas Piggin SYSTEM_CALL 1 20351a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 2036d807ad37SNicholas Piggin 2037acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 203869fdd674SNicholas PigginTRAMP_REAL_BEGIN(kvm_hcall) 2039e2762743SNicholas Piggin std r9,PACA_EXGEN+EX_R9(r13) 2040e2762743SNicholas Piggin std r11,PACA_EXGEN+EX_R11(r13) 2041e2762743SNicholas Piggin std r12,PACA_EXGEN+EX_R12(r13) 2042e2762743SNicholas Piggin mfcr r9 2043acd7d8ceSNicholas Piggin mfctr r10 2044e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_R13(r13) 2045e2762743SNicholas Piggin li r10,0 2046e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CFAR(r13) 2047e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CTR(r13) 2048e2762743SNicholas Piggin /* 2049e2762743SNicholas Piggin * Save the PPR (on systems that support it) before changing to 2050e2762743SNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 2051e2762743SNicholas Piggin * guest state (it is the guest's PPR value). 2052e2762743SNicholas Piggin */ 2053e2762743SNicholas PigginBEGIN_FTR_SECTION 2054e2762743SNicholas Piggin mfspr r10,SPRN_PPR 2055e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_PPR(r13) 2056e2762743SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2057e2762743SNicholas Piggin 2058e2762743SNicholas Piggin HMT_MEDIUM 2059e2762743SNicholas Piggin 20609600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 20619600f261SNicholas Piggin /* 206231c67cfeSNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives 20639600f261SNicholas Piggin * outside the head section. 20649600f261SNicholas Piggin */ 2065d72c4a36SDaniel Axtens __LOAD_FAR_HANDLER(r10, kvmppc_hcall, real_trampolines) 20669600f261SNicholas Piggin mtctr r10 20679600f261SNicholas Piggin bctr 20689600f261SNicholas Piggin#else 206931c67cfeSNicholas Piggin b kvmppc_hcall 20709600f261SNicholas Piggin#endif 2071acd7d8ceSNicholas Piggin#endif 2072da2bc464SMichael Ellerman 207394325357SNicholas Piggin/** 207494325357SNicholas Piggin * Interrupt 0xd00 - Trace Interrupt. 207594325357SNicholas Piggin * This is a synchronous interrupt in response to instruction step or 207694325357SNicholas Piggin * breakpoint faults. 207794325357SNicholas Piggin */ 20784f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 20794f50541fSNicholas Piggin IVEC=0xd00 20802284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 20814f50541fSNicholas Piggin IKVM_REAL=1 20822284ffeaSNicholas Piggin#endif 20834f50541fSNicholas PigginINT_DEFINE_END(single_step) 20844f50541fSNicholas Piggin 20857299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 20864f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 20877299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 20887299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 20894f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 20907299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 2091eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 20924f50541fSNicholas Piggin GEN_COMMON single_step 2093c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2094*4e991e3cSNicholas Piggin bl CFUNC(single_step_exception) 20951df7d5e4SNicholas Piggin b interrupt_return_srr 2096da2bc464SMichael Ellerman 20977299417cSNicholas Piggin 209894325357SNicholas Piggin/** 209994325357SNicholas Piggin * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 210094325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 210194325357SNicholas Piggin * guest data access. 210294325357SNicholas Piggin * 210394325357SNicholas Piggin * Handling: 210494325357SNicholas Piggin * This should always get routed to KVM. In radix MMU mode, this is caused 210594325357SNicholas Piggin * by a guest nested radix access that can't be performed due to the 210694325357SNicholas Piggin * partition scope page table. In hash mode, this can be caused by guests 210794325357SNicholas Piggin * running with translation disabled (virtual real mode) or with VPM enabled. 210894325357SNicholas Piggin * KVM will update the page table structures or disallow the access. 210994325357SNicholas Piggin */ 21104f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 21114f50541fSNicholas Piggin IVEC=0xe00 21123f7fbd97SNicholas Piggin IHSRR=1 21134f50541fSNicholas Piggin IDAR=1 21144f50541fSNicholas Piggin IDSISR=1 21154f50541fSNicholas Piggin IKVM_REAL=1 21164f50541fSNicholas Piggin IKVM_VIRT=1 21174f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 21184f50541fSNicholas Piggin 21197299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 21204f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 21217299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 21227299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 21234f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 21247299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2125f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 21264f50541fSNicholas Piggin GEN_COMMON h_data_storage 2127c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2128d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 2129*4e991e3cSNicholas Piggin bl CFUNC(do_bad_page_fault_segv) 2130d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 2131*4e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 2132d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 21331df7d5e4SNicholas Piggin b interrupt_return_hsrr 2134f5c32c1dSNicholas Piggin 21351707dd16SPaul Mackerras 213694325357SNicholas Piggin/** 213794325357SNicholas Piggin * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 213894325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 213994325357SNicholas Piggin * guest instruction fetch, similar to HDSI. 214094325357SNicholas Piggin */ 21414f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 21424f50541fSNicholas Piggin IVEC=0xe20 21433f7fbd97SNicholas Piggin IHSRR=1 21444f50541fSNicholas Piggin IKVM_REAL=1 21454f50541fSNicholas Piggin IKVM_VIRT=1 21464f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 21474f50541fSNicholas Piggin 21487299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 21494f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 21507299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 21517299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 21524f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 21537299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2154eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 21554f50541fSNicholas Piggin GEN_COMMON h_instr_storage 2156c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2157*4e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 21581df7d5e4SNicholas Piggin b interrupt_return_hsrr 215982517cabSNicholas Piggin 21601707dd16SPaul Mackerras 216194325357SNicholas Piggin/** 216294325357SNicholas Piggin * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 216394325357SNicholas Piggin */ 21644f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 21654f50541fSNicholas Piggin IVEC=0xe40 21663f7fbd97SNicholas Piggin IHSRR=1 21674f50541fSNicholas Piggin IKVM_REAL=1 21684f50541fSNicholas Piggin IKVM_VIRT=1 21694f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 21704f50541fSNicholas Piggin 21717299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 21724f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 21737299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 21747299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 21754f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 21767299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2177eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 21784f50541fSNicholas Piggin GEN_COMMON emulation_assist 2179c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2180*4e991e3cSNicholas Piggin bl CFUNC(emulation_assist_interrupt) 21811df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 21821df7d5e4SNicholas Piggin b interrupt_return_hsrr 2183031b4026SNicholas Piggin 21841707dd16SPaul Mackerras 218594325357SNicholas Piggin/** 218694325357SNicholas Piggin * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 218794325357SNicholas Piggin * This is an asynchronous interrupt caused by a Hypervisor Maintenance 218894325357SNicholas Piggin * Exception. It is always taken in real mode but uses HSRR registers 218994325357SNicholas Piggin * unlike SRESET and MCE. 219094325357SNicholas Piggin * 219194325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 219294325357SNicholas Piggin * with IRQS_DISABLED mask (i.e., local_irq_disable()). 219394325357SNicholas Piggin * 219494325357SNicholas Piggin * Handling: 219594325357SNicholas Piggin * This is a special case, this is handled similarly to machine checks, with an 219694325357SNicholas Piggin * initial real mode handler that is not soft-masked, which attempts to fix the 219794325357SNicholas Piggin * problem. Then a regular handler which is soft-maskable and reports the 219894325357SNicholas Piggin * problem. 219994325357SNicholas Piggin * 220094325357SNicholas Piggin * The emergency stack is used for the early real mode handler. 220194325357SNicholas Piggin * 220294325357SNicholas Piggin * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 220394325357SNicholas Piggin * either use soft-masking for the MCE, or use irq_work for the HMI. 220494325357SNicholas Piggin * 220594325357SNicholas Piggin * KVM: 220694325357SNicholas Piggin * Unlike MCE, this calls into KVM without calling the real mode handler 220794325357SNicholas Piggin * first. 2208e0319829SNicholas Piggin */ 22094f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 22104f50541fSNicholas Piggin IVEC=0xe60 22113f7fbd97SNicholas Piggin IHSRR=1 2212d73a10cbSNicholas Piggin IREALMODE_COMMON=1 22134f50541fSNicholas Piggin ISTACK=0 22144f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 22154f50541fSNicholas Piggin IKVM_REAL=1 22164f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 22174f50541fSNicholas Piggin 22184f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 22194f50541fSNicholas Piggin IVEC=0xe60 22203f7fbd97SNicholas Piggin IHSRR=1 22214f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22224f50541fSNicholas Piggin IKVM_REAL=1 22234f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 22244f50541fSNicholas Piggin 2225f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 22264f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2227f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 22281a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 22294f50541fSNicholas Piggin 2230293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 22319600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 22329600f261SNicholas Piggin 223362f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 2234a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 223562f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2236bcbceed4SNicholas Piggin 22378729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 2238bcbceed4SNicholas Piggin 2239c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2240*4e991e3cSNicholas Piggin bl CFUNC(hmi_exception_realmode) 22415080332cSMichael Neuling cmpdi cr0,r3,0 224267d4160aSNicholas Piggin bne 1f 22435080332cSMichael Neuling 22443f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 2245222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 22465080332cSMichael Neuling 224767d4160aSNicholas Piggin1: 224862f9b03bSNicholas Piggin /* 224962f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 225062f9b03bSNicholas Piggin * firmware. 225162f9b03bSNicholas Piggin */ 22523f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 22534f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 225462f9b03bSNicholas Piggin 22555080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 22564f50541fSNicholas Piggin GEN_COMMON hmi_exception 2257c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2258*4e991e3cSNicholas Piggin bl CFUNC(handle_hmi_exception) 22591df7d5e4SNicholas Piggin b interrupt_return_hsrr 22601707dd16SPaul Mackerras 22617299417cSNicholas Piggin 226294325357SNicholas Piggin/** 226394325357SNicholas Piggin * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 226494325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsnd doorbell. 226594325357SNicholas Piggin * Similar to the 0xa00 doorbell but for host rather than guest. 2266af47d79bSNicholas Piggin * 2267af47d79bSNicholas Piggin * CFAR is not required (similar to doorbell_interrupt), unless KVM HV 2268af47d79bSNicholas Piggin * is enabled, in which case it may be a guest exit. Most PowerNV kernels 2269af47d79bSNicholas Piggin * include KVM support so it would be nice if this could be dynamically 2270af47d79bSNicholas Piggin * patched out if KVM was not currently running any guests. 227194325357SNicholas Piggin */ 22724f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 22734f50541fSNicholas Piggin IVEC=0xe80 22743f7fbd97SNicholas Piggin IHSRR=1 22754f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22764f50541fSNicholas Piggin IKVM_REAL=1 22774f50541fSNicholas Piggin IKVM_VIRT=1 2278af47d79bSNicholas Piggin#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2279af47d79bSNicholas Piggin ICFAR=0 2280af47d79bSNicholas Piggin#endif 22814f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 22824f50541fSNicholas Piggin 22837299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 22844f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 22857299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 22867299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 22874f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 22887299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2289eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 22904f50541fSNicholas Piggin GEN_COMMON h_doorbell 2291c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 22929bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 2293*4e991e3cSNicholas Piggin bl CFUNC(doorbell_exception) 22949bcb81bfSNicholas Piggin#else 2295*4e991e3cSNicholas Piggin bl CFUNC(unknown_async_exception) 22969bcb81bfSNicholas Piggin#endif 22971df7d5e4SNicholas Piggin b interrupt_return_hsrr 22989bcb81bfSNicholas Piggin 22990ebc4cdaSBenjamin Herrenschmidt 230094325357SNicholas Piggin/** 230194325357SNicholas Piggin * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 230294325357SNicholas Piggin * This is an asynchronous interrupt in response to an "external exception". 230394325357SNicholas Piggin * Similar to 0x500 but for host only. 2304af47d79bSNicholas Piggin * 2305af47d79bSNicholas Piggin * Like h_doorbell, CFAR is only required for KVM HV because this can be 2306af47d79bSNicholas Piggin * a guest exit. 230794325357SNicholas Piggin */ 23084f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 23094f50541fSNicholas Piggin IVEC=0xea0 23103f7fbd97SNicholas Piggin IHSRR=1 23114f50541fSNicholas Piggin IMASK=IRQS_DISABLED 23124f50541fSNicholas Piggin IKVM_REAL=1 23134f50541fSNicholas Piggin IKVM_VIRT=1 2314af47d79bSNicholas Piggin#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2315af47d79bSNicholas Piggin ICFAR=0 2316af47d79bSNicholas Piggin#endif 23174f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 23184f50541fSNicholas Piggin 23197299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 23204f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 23217299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 23227299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 23234f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 23247299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2325eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 23264f50541fSNicholas Piggin GEN_COMMON h_virt_irq 2327c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2328*4e991e3cSNicholas Piggin bl CFUNC(do_IRQ) 23291df7d5e4SNicholas Piggin b interrupt_return_hsrr 233074408776SNicholas Piggin 23319baaef0aSBenjamin Herrenschmidt 23321a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 23331a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 23341a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 23351a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 2336bda7fea2SNicholas Piggin 23370ebc4cdaSBenjamin Herrenschmidt 233894325357SNicholas Piggin/* 233994325357SNicholas Piggin * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 234094325357SNicholas Piggin * This is an asynchronous interrupt in response to a PMU exception. 234194325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 234294325357SNicholas Piggin * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 234394325357SNicholas Piggin * 234494325357SNicholas Piggin * Handling: 234594325357SNicholas Piggin * This calls into the perf subsystem. 234694325357SNicholas Piggin * 234794325357SNicholas Piggin * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 234894325357SNicholas Piggin * runs under local_irq_disable. However it may be soft-masked in 234994325357SNicholas Piggin * powerpc-specific code. 235094325357SNicholas Piggin * 235194325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 235294325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 2353af47d79bSNicholas Piggin * 2354af47d79bSNicholas Piggin * CFAR is not used by perf interrupts so not required. 235594325357SNicholas Piggin */ 23564f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 23574f50541fSNicholas Piggin IVEC=0xf00 23584f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 23592284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23604f50541fSNicholas Piggin IKVM_REAL=1 23612284ffeaSNicholas Piggin#endif 2362af47d79bSNicholas Piggin ICFAR=0 23634f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 23644f50541fSNicholas Piggin 23657299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 23664f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 23677299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 23687299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 23694f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 23707299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2371eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 23724f50541fSNicholas Piggin GEN_COMMON performance_monitor 2373c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2374dc398a08SNicholas Piggin lbz r4,PACAIRQSOFTMASK(r13) 2375dc398a08SNicholas Piggin cmpdi r4,IRQS_ENABLED 2376dc398a08SNicholas Piggin bne 1f 2377*4e991e3cSNicholas Piggin bl CFUNC(performance_monitor_exception_async) 23781df7d5e4SNicholas Piggin b interrupt_return_srr 2379dc398a08SNicholas Piggin1: 2380*4e991e3cSNicholas Piggin bl CFUNC(performance_monitor_exception_nmi) 2381dc398a08SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 2382dc398a08SNicholas Piggin li r9,0 2383dc398a08SNicholas Piggin mtmsrd r9,1 2384b1c7f150SNicholas Piggin 2385dc398a08SNicholas Piggin kuap_kernel_restore r9, r10 2386dc398a08SNicholas Piggin 2387dc398a08SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 2388dc398a08SNicholas Piggin RFI_TO_KERNEL 23890ebc4cdaSBenjamin Herrenschmidt 239094325357SNicholas Piggin/** 239194325357SNicholas Piggin * Interrupt 0xf20 - Vector Unavailable Interrupt. 239294325357SNicholas Piggin * This is a synchronous interrupt in response to 239394325357SNicholas Piggin * executing a vector (or altivec) instruction with MSR[VEC]=0. 239494325357SNicholas Piggin * Similar to FP unavailable. 239594325357SNicholas Piggin */ 23964f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 23974f50541fSNicholas Piggin IVEC=0xf20 23982284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23994f50541fSNicholas Piggin IKVM_REAL=1 24002284ffeaSNicholas Piggin#endif 24012487fd2eSRohan McLure IMSR_R12=1 24024f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 24034f50541fSNicholas Piggin 24047299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 24054f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 24067299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 24077299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 24084f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 24097299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2410d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 24114f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 2412d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 2413d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 2414d1a0ca9cSNicholas Piggin beq 1f 2415d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2416d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2417d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2418d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 2419d1a0ca9cSNicholas Piggin */ 2420d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2421d1a0ca9cSNicholas Piggin bne- 2f 2422d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2423d1a0ca9cSNicholas Piggin#endif 2424*4e991e3cSNicholas Piggin bl CFUNC(load_up_altivec) 24251df7d5e4SNicholas Piggin b fast_interrupt_return_srr 2426d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2427d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 2428c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2429*4e991e3cSNicholas Piggin bl CFUNC(altivec_unavailable_tm) 24301df7d5e4SNicholas Piggin b interrupt_return_srr 2431d1a0ca9cSNicholas Piggin#endif 2432d1a0ca9cSNicholas Piggin1: 2433d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2434d1a0ca9cSNicholas Piggin#endif 2435c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2436*4e991e3cSNicholas Piggin bl CFUNC(altivec_unavailable_exception) 24371df7d5e4SNicholas Piggin b interrupt_return_srr 2438d1a0ca9cSNicholas Piggin 24390ebc4cdaSBenjamin Herrenschmidt 244094325357SNicholas Piggin/** 244194325357SNicholas Piggin * Interrupt 0xf40 - VSX Unavailable Interrupt. 244294325357SNicholas Piggin * This is a synchronous interrupt in response to 244394325357SNicholas Piggin * executing a VSX instruction with MSR[VSX]=0. 244494325357SNicholas Piggin * Similar to FP unavailable. 244594325357SNicholas Piggin */ 24464f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 24474f50541fSNicholas Piggin IVEC=0xf40 24482284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24494f50541fSNicholas Piggin IKVM_REAL=1 24502284ffeaSNicholas Piggin#endif 24512487fd2eSRohan McLure IMSR_R12=1 24524f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 24534f50541fSNicholas Piggin 24547299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 24554f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 24567299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 24577299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 24584f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 24597299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2460792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 24614f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2462792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2463792cbdddSNicholas PigginBEGIN_FTR_SECTION 2464792cbdddSNicholas Piggin beq 1f 2465792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2466792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2467792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2468792cbdddSNicholas Piggin * transaction), go do TM stuff 2469792cbdddSNicholas Piggin */ 2470792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2471792cbdddSNicholas Piggin bne- 2f 2472792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2473792cbdddSNicholas Piggin#endif 2474792cbdddSNicholas Piggin b load_up_vsx 2475792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2476792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2477c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2478*4e991e3cSNicholas Piggin bl CFUNC(vsx_unavailable_tm) 24791df7d5e4SNicholas Piggin b interrupt_return_srr 2480792cbdddSNicholas Piggin#endif 2481792cbdddSNicholas Piggin1: 2482792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2483792cbdddSNicholas Piggin#endif 2484c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2485*4e991e3cSNicholas Piggin bl CFUNC(vsx_unavailable_exception) 24861df7d5e4SNicholas Piggin b interrupt_return_srr 2487792cbdddSNicholas Piggin 2488d0c0c9a1SMichael Neuling 248994325357SNicholas Piggin/** 249094325357SNicholas Piggin * Interrupt 0xf60 - Facility Unavailable Interrupt. 249194325357SNicholas Piggin * This is a synchronous interrupt in response to 249294325357SNicholas Piggin * executing an instruction without access to the facility that can be 249394325357SNicholas Piggin * resolved by the OS (e.g., FSCR, MSR). 249494325357SNicholas Piggin * Similar to FP unavailable. 249594325357SNicholas Piggin */ 24964f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 24974f50541fSNicholas Piggin IVEC=0xf60 24982284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24994f50541fSNicholas Piggin IKVM_REAL=1 25002284ffeaSNicholas Piggin#endif 25014f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 25024f50541fSNicholas Piggin 25037299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 25044f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 25057299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 25067299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 25074f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 25087299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2509eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 25104f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2511c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2512*4e991e3cSNicholas Piggin bl CFUNC(facility_unavailable_exception) 25131df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 25141df7d5e4SNicholas Piggin b interrupt_return_srr 25151134713cSNicholas Piggin 2516da2bc464SMichael Ellerman 251794325357SNicholas Piggin/** 251894325357SNicholas Piggin * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 251994325357SNicholas Piggin * This is a synchronous interrupt in response to 252094325357SNicholas Piggin * executing an instruction without access to the facility that can only 252194325357SNicholas Piggin * be resolved in HV mode (e.g., HFSCR). 252294325357SNicholas Piggin * Similar to FP unavailable. 252394325357SNicholas Piggin */ 25244f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 25254f50541fSNicholas Piggin IVEC=0xf80 25263f7fbd97SNicholas Piggin IHSRR=1 25274f50541fSNicholas Piggin IKVM_REAL=1 25284f50541fSNicholas Piggin IKVM_VIRT=1 25294f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 25304f50541fSNicholas Piggin 25317299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 25324f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 25337299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 25347299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 25354f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 25367299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2537eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 25384f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2539c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2540*4e991e3cSNicholas Piggin bl CFUNC(facility_unavailable_exception) 25411df45d78SRohan McLure /* XXX Shouldn't be necessary in practice */ 25421df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() 25431df7d5e4SNicholas Piggin b interrupt_return_hsrr 254414b0072cSNicholas Piggin 2545da2bc464SMichael Ellerman 25461a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 25471a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 25481a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 25491a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 25501a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 25511a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 25521a6822d1SNicholas Piggin 25531a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 25541a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 25551a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 25561a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2557da2bc464SMichael Ellerman 25580ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 25594f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 25604f50541fSNicholas Piggin IVEC=0x1200 25613f7fbd97SNicholas Piggin IHSRR=1 25624f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 25634f50541fSNicholas Piggin 25647299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 25654f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 25667299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 25671a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2568eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 25694f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2570c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2571*4e991e3cSNicholas Piggin bl CFUNC(cbe_system_error_exception) 25721df7d5e4SNicholas Piggin b interrupt_return_hsrr 25739600f261SNicholas Piggin 2574da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 25751a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 25761a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2577da2bc464SMichael Ellerman#endif 2578da2bc464SMichael Ellerman 2579da487a5dSNicholas Piggin/** 2580da487a5dSNicholas Piggin * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. 2581da487a5dSNicholas Piggin * This has been removed from the ISA before 2.01, which is the earliest 2582da487a5dSNicholas Piggin * 64-bit BookS ISA supported, however the G5 / 970 implements this 2583da487a5dSNicholas Piggin * interrupt with a non-architected feature available through the support 2584da487a5dSNicholas Piggin * processor interface. 2585da487a5dSNicholas Piggin */ 25864f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 25874f50541fSNicholas Piggin IVEC=0x1300 25882284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 25894f50541fSNicholas Piggin IKVM_REAL=1 25902284ffeaSNicholas Piggin#endif 25914f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 25924f50541fSNicholas Piggin 25937299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 25944f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 25957299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 25967299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 25974f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 25987299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2599eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 26004f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2601c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2602*4e991e3cSNicholas Piggin bl CFUNC(instruction_breakpoint_exception) 26031df7d5e4SNicholas Piggin b interrupt_return_srr 26044e96dbbfSNicholas Piggin 26057299417cSNicholas Piggin 26061a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 26071a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2608da2bc464SMichael Ellerman 260994325357SNicholas Piggin/** 261094325357SNicholas Piggin * Interrupt 0x1500 - Soft Patch Interrupt 261194325357SNicholas Piggin * 261294325357SNicholas Piggin * Handling: 261394325357SNicholas Piggin * This is an implementation specific interrupt which can be used for a 261494325357SNicholas Piggin * range of exceptions. 261594325357SNicholas Piggin * 261694325357SNicholas Piggin * This interrupt handler is unique in that it runs the denormal assist 261794325357SNicholas Piggin * code even for guests (and even in guest context) without going to KVM, 261894325357SNicholas Piggin * for speed. POWER9 does not raise denorm exceptions, so this special case 261994325357SNicholas Piggin * could be phased out in future to reduce special cases. 262094325357SNicholas Piggin */ 26214f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 26224f50541fSNicholas Piggin IVEC=0x1500 26233f7fbd97SNicholas Piggin IHSRR=1 26244557ac6bSNicholas Piggin IBRANCH_TO_COMMON=0 26259600f261SNicholas Piggin IKVM_REAL=1 26264f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 26274f50541fSNicholas Piggin 26284f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 26294f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2630b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2631d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2632b92a66a6SMichael Neuling bne+ denorm_assist 2633b92a66a6SMichael Neuling#endif 26348729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 26354f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2636d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 26371a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 26384f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2639d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 264052b98923SNicholas Piggin bne+ denorm_assist 26418729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 26421a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2643d7e89849SNicholas Piggin#else 26441a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2645d7e89849SNicholas Piggin#endif 2646b92a66a6SMichael Neuling 2647b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2648da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2649b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2650b92a66a6SMichael Neuling/* 2651b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2652b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2653b92a66a6SMichael Neuling */ 2654b92a66a6SMichael Neuling mfmsr r10 2655b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2656b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2657b92a66a6SMichael Neuling mtmsrd r10 2658b92a66a6SMichael Neuling sync 2659d7c67fb1SMichael Neuling 2660f3c8b6c6SNicholas Piggin .Lreg=0 2661f3c8b6c6SNicholas Piggin .rept 32 2662f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2663f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2664f3c8b6c6SNicholas Piggin .endr 2665d7c67fb1SMichael Neuling 2666b92a66a6SMichael NeulingFTR_SECTION_ELSE 2667b92a66a6SMichael Neuling/* 2668b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2669b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2670b92a66a6SMichael Neuling */ 2671b92a66a6SMichael Neuling mfmsr r10 2672b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2673b92a66a6SMichael Neuling mtmsrd r10 2674b92a66a6SMichael Neuling sync 2675d7c67fb1SMichael Neuling 2676f3c8b6c6SNicholas Piggin .Lreg=0 2677f3c8b6c6SNicholas Piggin .rept 32 2678f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2679f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2680f3c8b6c6SNicholas Piggin .endr 2681d7c67fb1SMichael Neuling 2682b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2683fb0fce3eSMichael Neuling 2684fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2685fb0fce3eSMichael Neuling b denorm_done 2686fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2687fb0fce3eSMichael Neuling/* 2688fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2689fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2690fb0fce3eSMichael Neuling */ 2691f3c8b6c6SNicholas Piggin .Lreg=32 2692f3c8b6c6SNicholas Piggin .rept 32 2693f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2694f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2695f3c8b6c6SNicholas Piggin .endr 2696f3c8b6c6SNicholas Piggin 2697fb0fce3eSMichael Neulingdenorm_done: 2698f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2699f14040bcSMichael Neuling subi r11,r11,4 2700b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2701b92a66a6SMichael Neuling mtcrf 0x80,r9 2702b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2703931dc86bSNicholas PigginBEGIN_FTR_SECTION 2704931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2705931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2706931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2707630573c1SPaul MackerrasBEGIN_FTR_SECTION 2708630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2709630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2710630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 271159dc5bfcSNicholas Piggin li r10,0 271259dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 2713b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2714b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2715b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2716b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2717222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2718b92a66a6SMichael Neuling b . 2719b92a66a6SMichael Neuling#endif 2720b92a66a6SMichael Neuling 27214f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 27224f50541fSNicholas Piggin GEN_COMMON denorm_exception 2723c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2724*4e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 27251df7d5e4SNicholas Piggin b interrupt_return_hsrr 2726d7e89849SNicholas Piggin 2727d7e89849SNicholas Piggin 2728d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 27294f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 27304f50541fSNicholas Piggin IVEC=0x1600 27313f7fbd97SNicholas Piggin IHSRR=1 27324f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 27334f50541fSNicholas Piggin 27347299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 27354f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 27367299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 27371a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2738eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 27394f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2740c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2741*4e991e3cSNicholas Piggin bl CFUNC(cbe_maintenance_exception) 27421df7d5e4SNicholas Piggin b interrupt_return_hsrr 27439600f261SNicholas Piggin 2744d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 27451a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 27461a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2747d7e89849SNicholas Piggin#endif 2748d7e89849SNicholas Piggin 274969a79344SNicholas Piggin 27504f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 27514f50541fSNicholas Piggin IVEC=0x1700 27522284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 27534f50541fSNicholas Piggin IKVM_REAL=1 27542284ffeaSNicholas Piggin#endif 27554f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 27564f50541fSNicholas Piggin 27577299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 27584f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 27597299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 27607299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 27614f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 27627299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2763eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 27644f50541fSNicholas Piggin GEN_COMMON altivec_assist 2765c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2766b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 2767*4e991e3cSNicholas Piggin bl CFUNC(altivec_assist_exception) 27681df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 2769b51c079eSNicholas Piggin#else 2770*4e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 2771b51c079eSNicholas Piggin#endif 27721df7d5e4SNicholas Piggin b interrupt_return_srr 2773b51c079eSNicholas Piggin 2774d7e89849SNicholas Piggin 2775d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 27764f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 27774f50541fSNicholas Piggin IVEC=0x1800 27783f7fbd97SNicholas Piggin IHSRR=1 27794f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 27804f50541fSNicholas Piggin 27817299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 27824f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 27837299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 27841a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2785eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 27864f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2787c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2788*4e991e3cSNicholas Piggin bl CFUNC(cbe_thermal_exception) 27891df7d5e4SNicholas Piggin b interrupt_return_hsrr 27909600f261SNicholas Piggin 2791d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 27921a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 27931a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2794d7e89849SNicholas Piggin#endif 2795d7e89849SNicholas Piggin 27967299417cSNicholas Piggin 279775eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 27982104180aSNicholas Piggin 27990eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 28000eddf327SNicholas Piggin IVEC=0x900 28010eddf327SNicholas Piggin ISTACK=0 2802af47d79bSNicholas Piggin ICFAR=0 28030eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 28042104180aSNicholas Piggin 2805cc491f1dSNicholas Piggin/* 2806cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2807cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2808cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2809cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2810cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2811cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2812cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2813cc491f1dSNicholas Piggin */ 28142104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 28152104180aSNicholas Piggin mr r10,r1 28162104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 28172104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 28180eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 281971c3b05aSNicholas Piggin 2820c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2821*4e991e3cSNicholas Piggin bl CFUNC(soft_nmi_interrupt) 282271c3b05aSNicholas Piggin 282371c3b05aSNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 282471c3b05aSNicholas Piggin li r9,0 282571c3b05aSNicholas Piggin mtmsrd r9,1 282671c3b05aSNicholas Piggin 28278e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 2828f23699c9SNicholas Piggin 282971c3b05aSNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 283071c3b05aSNicholas Piggin RFI_TO_KERNEL 28312104180aSNicholas Piggin 283275eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2833d7e89849SNicholas Piggin 28340ebc4cdaSBenjamin Herrenschmidt/* 2835fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 28362b461880SMichael Ellerman * - If it was a decrementer interrupt, we bump the dec to max and return. 2837fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2838fe9e1d54SIan Munsie * triggered and won't automatically refire. 28390869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 28400869b6fdSMahesh Salgaonkar * and it won't refire. 28416cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2842fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 28430ebc4cdaSBenjamin Herrenschmidt */ 28443f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 28454508a74aSNicholas Piggin .if \hsrr 28464508a74aSNicholas Pigginmasked_Hinterrupt: 28474508a74aSNicholas Piggin .else 28484508a74aSNicholas Pigginmasked_interrupt: 28494508a74aSNicholas Piggin .endif 285063e40806SNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 2851c39fb71aSNicholas Piggin#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG 2852c39fb71aSNicholas Piggin /* 2853c39fb71aSNicholas Piggin * Ensure there was no previous MUST_HARD_MASK interrupt or 28548154850bSNicholas Piggin * HARD_DIS setting. If this does fire, the interrupt is still 28558154850bSNicholas Piggin * masked and MSR[EE] will be cleared on return, so no need to 28568154850bSNicholas Piggin * panic, but somebody probably enabled MSR[EE] under 28578154850bSNicholas Piggin * PACA_IRQ_HARD_DIS, mtmsr(mfmsr() | MSR_x) being a common 28588154850bSNicholas Piggin * cause. 2859c39fb71aSNicholas Piggin */ 2860c39fb71aSNicholas Piggin lbz r9,PACAIRQHAPPENED(r13) 2861c39fb71aSNicholas Piggin andi. r9,r9,(PACA_IRQ_MUST_HARD_MASK|PACA_IRQ_HARD_DIS) 2862c39fb71aSNicholas Piggin0: tdnei r9,0 28638154850bSNicholas Piggin EMIT_WARN_ENTRY 0b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) 2864c39fb71aSNicholas Piggin#endif 286563e40806SNicholas Piggin lbz r9,PACAIRQHAPPENED(r13) 286663e40806SNicholas Piggin or r9,r9,r10 286763e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 286863e40806SNicholas Piggin 286963e40806SNicholas Piggin .if ! \hsrr 28704508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 28714508a74aSNicholas Piggin bne 1f 287263e40806SNicholas Piggin LOAD_REG_IMMEDIATE(r9, 0x7fffffff) 287363e40806SNicholas Piggin mtspr SPRN_DEC,r9 28740eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 287563e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 28760eddf327SNicholas Piggin b soft_nmi_common 28770eddf327SNicholas Piggin#else 28780eddf327SNicholas Piggin b 2f 28790eddf327SNicholas Piggin#endif 288063e40806SNicholas Piggin .endif 288163e40806SNicholas Piggin 28824508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 28834508a74aSNicholas Piggin beq 2f 28840eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 28854508a74aSNicholas Piggin .if \hsrr 28860eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 28874508a74aSNicholas Piggin .else 28880eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 28894508a74aSNicholas Piggin .endif 289063e40806SNicholas Piggin ori r9,r9,PACA_IRQ_HARD_DIS 289163e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 28924508a74aSNicholas Piggin2: /* done */ 289363e40806SNicholas Piggin li r9,0 289459dc5bfcSNicholas Piggin .if \hsrr 289563e40806SNicholas Piggin stb r9,PACAHSRR_VALID(r13) 289659dc5bfcSNicholas Piggin .else 289763e40806SNicholas Piggin stb r9,PACASRR_VALID(r13) 289859dc5bfcSNicholas Piggin .endif 289963e40806SNicholas Piggin 2900f23699c9SNicholas Piggin SEARCH_RESTART_TABLE 2901f23699c9SNicholas Piggin cmpdi r12,0 2902f23699c9SNicholas Piggin beq 3f 2903f23699c9SNicholas Piggin .if \hsrr 2904f23699c9SNicholas Piggin mtspr SPRN_HSRR0,r12 2905f23699c9SNicholas Piggin .else 2906f23699c9SNicholas Piggin mtspr SPRN_SRR0,r12 2907f23699c9SNicholas Piggin .endif 2908f23699c9SNicholas Piggin3: 2909f23699c9SNicholas Piggin 291063e40806SNicholas Piggin ld r9,PACA_EXGEN+EX_CTR(r13) 291163e40806SNicholas Piggin mtctr r9 291263e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 29134508a74aSNicholas Piggin mtcrf 0x80,r9 29144508a74aSNicholas Piggin std r1,PACAR1(r13) 29154508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 29164508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 29174508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 29180eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 2919b2dc2977SNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 2920b2dc2977SNicholas Piggin /* May return to masked low address where r13 is not set up */ 29214508a74aSNicholas Piggin .if \hsrr 29224508a74aSNicholas Piggin HRFI_TO_KERNEL 29234508a74aSNicholas Piggin .else 29244508a74aSNicholas Piggin RFI_TO_KERNEL 29254508a74aSNicholas Piggin .endif 29264508a74aSNicholas Piggin b . 29274508a74aSNicholas Piggin.endm 29280ebc4cdaSBenjamin Herrenschmidt 2929a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2930a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2931a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2932a048a07dSNicholas Piggin sync 2933a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2934a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2935a048a07dSNicholas Piggin ori 31,31,0 2936a048a07dSNicholas Piggin .rept 14 2937a048a07dSNicholas Piggin b 1f 2938a048a07dSNicholas Piggin1: 2939a048a07dSNicholas Piggin .endr 2940a048a07dSNicholas Piggin blr 2941a048a07dSNicholas Piggin 29429a32a7e7SNicholas Piggin/* Clobbers r10, r11, ctr */ 29439a32a7e7SNicholas Piggin.macro L1D_DISPLACEMENT_FLUSH 2944aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2945bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2946bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2947aa8a5e00SMichael Ellerman mtctr r11 294815a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2949aa8a5e00SMichael Ellerman 2950aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2951aa8a5e00SMichael Ellerman sync 2952bdcb1aefSNicholas Piggin 2953bdcb1aefSNicholas Piggin /* 2954f7964378SNicholas Piggin * The load addresses are at staggered offsets within cachelines, 2955bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2956bdcb1aefSNicholas Piggin * hurt). 2957bdcb1aefSNicholas Piggin */ 2958bdcb1aefSNicholas Piggin1: 2959bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2960bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2961bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2962bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2963bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2964bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2965bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2966bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2967bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2968aa8a5e00SMichael Ellerman bdnz 1b 29699a32a7e7SNicholas Piggin.endm 2970aa8a5e00SMichael Ellerman 29719a32a7e7SNicholas PigginTRAMP_REAL_BEGIN(entry_flush_fallback) 29729a32a7e7SNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 29739a32a7e7SNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 29749a32a7e7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 29759a32a7e7SNicholas Piggin mfctr r9 29769a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2977f7964378SNicholas Piggin mtctr r9 2978f7964378SNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2979f7964378SNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2980f7964378SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 2981f7964378SNicholas Piggin blr 2982f7964378SNicholas Piggin 298308685be7SNicholas Piggin/* 298408685be7SNicholas Piggin * The SCV entry flush happens with interrupts enabled, so it must disable 298508685be7SNicholas Piggin * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10 298608685be7SNicholas Piggin * (containing LR) does not need to be preserved here because scv entry 298708685be7SNicholas Piggin * puts 0 in the pt_regs, CTR can be clobbered for the same reason. 298808685be7SNicholas Piggin */ 298908685be7SNicholas PigginTRAMP_REAL_BEGIN(scv_entry_flush_fallback) 299008685be7SNicholas Piggin li r10,0 299108685be7SNicholas Piggin mtmsrd r10,1 299208685be7SNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 299308685be7SNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 299408685be7SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 299508685be7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 299608685be7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 299708685be7SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 299808685be7SNicholas Piggin li r10,MSR_RI 299908685be7SNicholas Piggin mtmsrd r10,1 300008685be7SNicholas Piggin blr 300108685be7SNicholas Piggin 3002aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 3003aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 3004aa8a5e00SMichael Ellerman GET_PACA(r13); 3005aa8a5e00SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 3006aa8a5e00SMichael Ellerman ld r1,PACAKSAVE(r13) 3007aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 3008aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 3009aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 3010aa8a5e00SMichael Ellerman mfctr r9 30119a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 3012aa8a5e00SMichael Ellerman mtctr r9 3013aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 3014aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 3015aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 301678ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 3017aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 3018aa8a5e00SMichael Ellerman rfid 3019aa8a5e00SMichael Ellerman 3020aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 3021aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 3022aa8a5e00SMichael Ellerman GET_PACA(r13); 302378ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 302478ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 3025aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 3026aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 3027aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 3028aa8a5e00SMichael Ellerman mfctr r9 30299a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 3030aa8a5e00SMichael Ellerman mtctr r9 3031aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 3032aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 3033aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 303478ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 3035aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 3036aa8a5e00SMichael Ellerman hrfid 3037aa8a5e00SMichael Ellerman 30387fa95f9aSNicholas PigginTRAMP_REAL_BEGIN(rfscv_flush_fallback) 30397fa95f9aSNicholas Piggin /* system call volatile */ 30407fa95f9aSNicholas Piggin mr r7,r13 30417fa95f9aSNicholas Piggin GET_PACA(r13); 30427fa95f9aSNicholas Piggin mr r8,r1 30437fa95f9aSNicholas Piggin ld r1,PACAKSAVE(r13) 30447fa95f9aSNicholas Piggin mfctr r9 30457fa95f9aSNicholas Piggin ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 30467fa95f9aSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 30477fa95f9aSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 30487fa95f9aSNicholas Piggin mtctr r11 30497fa95f9aSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 30507fa95f9aSNicholas Piggin 30517fa95f9aSNicholas Piggin /* order ld/st prior to dcbt stop all streams with flushing */ 30527fa95f9aSNicholas Piggin sync 30537fa95f9aSNicholas Piggin 30547fa95f9aSNicholas Piggin /* 30557fa95f9aSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 30567fa95f9aSNicholas Piggin * which suits some pipelines better (on others it should not 30577fa95f9aSNicholas Piggin * hurt). 30587fa95f9aSNicholas Piggin */ 30597fa95f9aSNicholas Piggin1: 30607fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 30617fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 30627fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 30637fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 30647fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 30657fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 30667fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 30677fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 30687fa95f9aSNicholas Piggin addi r10,r10,0x80*8 30697fa95f9aSNicholas Piggin bdnz 1b 30707fa95f9aSNicholas Piggin 30717fa95f9aSNicholas Piggin mtctr r9 30727fa95f9aSNicholas Piggin li r9,0 30737fa95f9aSNicholas Piggin li r10,0 30747fa95f9aSNicholas Piggin li r11,0 30757fa95f9aSNicholas Piggin mr r1,r8 30767fa95f9aSNicholas Piggin mr r13,r7 30777fa95f9aSNicholas Piggin RFSCV 30787fa95f9aSNicholas Piggin 30790eddf327SNicholas PigginUSE_TEXT_SECTION() 30809a32a7e7SNicholas Piggin 308169fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 308269fdd674SNicholas Pigginkvm_interrupt: 308369fdd674SNicholas Piggin /* 308469fdd674SNicholas Piggin * The conditional branch in KVMTEST can't reach all the way, 308569fdd674SNicholas Piggin * make a stub. 308669fdd674SNicholas Piggin */ 308769fdd674SNicholas Piggin b kvmppc_interrupt 308869fdd674SNicholas Piggin#endif 308969fdd674SNicholas Piggin 30909a32a7e7SNicholas Piggin_GLOBAL(do_uaccess_flush) 30919a32a7e7SNicholas Piggin UACCESS_FLUSH_FIXUP_SECTION 30929a32a7e7SNicholas Piggin nop 30939a32a7e7SNicholas Piggin nop 30949a32a7e7SNicholas Piggin nop 30959a32a7e7SNicholas Piggin blr 30969a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 30979a32a7e7SNicholas Piggin blr 30989a32a7e7SNicholas Piggin_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) 30999a32a7e7SNicholas PigginEXPORT_SYMBOL(do_uaccess_flush) 31009a32a7e7SNicholas Piggin 31019a32a7e7SNicholas Piggin 31023f7fbd97SNicholas PigginMASKED_INTERRUPT 31033f7fbd97SNicholas PigginMASKED_INTERRUPT hsrr=1 31047230c564SBenjamin Herrenschmidt 310557f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 31068ed8ab40SHari Bathini /* 31079d1988caSNicholas Piggin * All code below __end_soft_masked is treated as soft-masked. If 3108b2dc2977SNicholas Piggin * any code runs here with MSR[EE]=1, it must then cope with pending 3109b2dc2977SNicholas Piggin * soft interrupt being raised (i.e., by ensuring it is replayed). 3110b2dc2977SNicholas Piggin * 31118ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 31128ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 31138ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 31148ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 31158ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 31168ed8ab40SHari Bathini */ 31178ed8ab40SHari Bathini .align 7 31188ed8ab40SHari Bathini .globl __end_interrupts 31198ed8ab40SHari Bathini__end_interrupts: 3120d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(__end_interrupts, virt_trampolines) 312161383407SBenjamin Herrenschmidt 312257f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 312357f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 312457f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 312557f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 312657f26649SNicholas Piggin 312757f26649SNicholas PigginUSE_TEXT_SECTION() 312857f26649SNicholas Piggin 3129296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 31302f5182cfSNicholas Piggin_GLOBAL(enable_machine_check) 3131296e753fSNicholas Piggin mflr r0 3132296e753fSNicholas Piggin bcl 20,31,$+4 3133296e753fSNicholas Piggin0: mflr r3 3134296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 3135296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 3136296e753fSNicholas Piggin mfmsr r3 3137296e753fSNicholas Piggin ori r3,r3,MSR_ME 3138296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 3139296e753fSNicholas Piggin RFI_TO_KERNEL 3140296e753fSNicholas Piggin1: mtlr r0 3141296e753fSNicholas Piggin blr 3142296e753fSNicholas Piggin 3143b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 314429a011fcSSathvika VasireddySYM_FUNC_START_LOCAL(disable_machine_check) 3145b7d9ccecSNicholas Piggin mflr r0 3146b7d9ccecSNicholas Piggin bcl 20,31,$+4 3147b7d9ccecSNicholas Piggin0: mflr r3 3148b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 3149b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 3150b7d9ccecSNicholas Piggin mfmsr r3 3151b7d9ccecSNicholas Piggin li r4,MSR_ME 3152b7d9ccecSNicholas Piggin andc r3,r3,r4 3153b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 3154b7d9ccecSNicholas Piggin RFI_TO_KERNEL 3155b7d9ccecSNicholas Piggin1: mtlr r0 3156b7d9ccecSNicholas Piggin blr 315729a011fcSSathvika VasireddySYM_FUNC_END(disable_machine_check) 3158