1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 1629a011fcSSathvika Vasireddy#include <linux/linkage.h> 177230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 188aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1946f52210SStephen Rothwell#include <asm/ptrace.h> 207cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 21da2bc464SMichael Ellerman#include <asm/head-64.h> 222c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 23890274c2SMichael Ellerman#include <asm/kup.h> 248aa34ab8SBenjamin Herrenschmidt 257299417cSNicholas Piggin/* 267299417cSNicholas Piggin * Following are fixed section helper macros. 277299417cSNicholas Piggin * 287299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 297299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 307299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 317299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 327299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 337299417cSNicholas Piggin */ 347299417cSNicholas Piggin 35a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 36a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 37a2432811SNicholas Piggin 38a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 39a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 40a2432811SNicholas Piggin 41a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 42a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 43a2432811SNicholas Piggin 44a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 45a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 46a2432811SNicholas Piggin 47a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 48a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 49a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 50a2432811SNicholas Piggin .global name; \ 51a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 52d72c4a36SDaniel Axtens DEFINE_FIXED_SYMBOL(name, text); \ 53a2432811SNicholas Pigginname: 54a2432811SNicholas Piggin 55a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 56a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 57a2432811SNicholas Piggin 58a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 59a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 60a2432811SNicholas Piggin 61a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 62a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 63a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 64a2432811SNicholas Piggin 65a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 66a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 67a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 68a2432811SNicholas Piggin 690ebc4cdaSBenjamin Herrenschmidt/* 7012a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 7112a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 7212a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 7312a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 7412a04809SNicholas Piggin * that kernelbase be 64K aligned. 7512a04809SNicholas Piggin */ 7612a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 7712a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 7812a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 7912a04809SNicholas Piggin 80d72c4a36SDaniel Axtens#define __LOAD_HANDLER(reg, label, section) \ 8112a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 82d72c4a36SDaniel Axtens ori reg,reg,(ABS_ADDR(label, section))@l 8312a04809SNicholas Piggin 8412a04809SNicholas Piggin/* 8512a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 8612a04809SNicholas Piggin * head-y require >64K offsets. 8712a04809SNicholas Piggin */ 88d72c4a36SDaniel Axtens#define __LOAD_FAR_HANDLER(reg, label, section) \ 8912a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 90d72c4a36SDaniel Axtens ori reg,reg,(ABS_ADDR(label, section))@l; \ 91d72c4a36SDaniel Axtens addis reg,reg,(ABS_ADDR(label, section))@h 9212a04809SNicholas Piggin 9312a04809SNicholas Piggin/* 94a42a239dSNicholas Piggin * Interrupt code generation macros 95a42a239dSNicholas Piggin */ 9694325357SNicholas Piggin#define IVEC .L_IVEC_\name\() /* Interrupt vector address */ 9794325357SNicholas Piggin#define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */ 9894325357SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */ 9994325357SNicholas Piggin#define IAREA .L_IAREA_\name\() /* PACA save area */ 10094325357SNicholas Piggin#define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ 10194325357SNicholas Piggin#define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ 102af47d79bSNicholas Piggin#define ICFAR .L_ICFAR_\name\() /* Uses CFAR */ 103af47d79bSNicholas Piggin#define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */ 10494325357SNicholas Piggin#define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ 10594325357SNicholas Piggin#define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ 10694325357SNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ 10794325357SNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ 10894325357SNicholas Piggin#define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ 10994325357SNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ 1104f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 11194325357SNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ 11294325357SNicholas Piggin#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ 1134f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 11494325357SNicholas Piggin#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ 1152487fd2eSRohan McLure#define IMSR_R12 .L_IMSR_R12_\name\() /* Assumes MSR saved to r12 */ 116a42a239dSNicholas Piggin 117a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 118a42a239dSNicholas Piggin.macro int_define_ ## n name 119a42a239dSNicholas Piggin 120a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 121a42a239dSNicholas Piggin.endm ; \ 122a42a239dSNicholas Pigginint_define_ ## n n ; \ 123a42a239dSNicholas Piggindo_define_int n 124a42a239dSNicholas Piggin 125a42a239dSNicholas Piggin.macro do_define_int name 126a42a239dSNicholas Piggin .ifndef IVEC 127a42a239dSNicholas Piggin .error "IVEC not defined" 128a42a239dSNicholas Piggin .endif 129a42a239dSNicholas Piggin .ifndef IHSRR 1303f7fbd97SNicholas Piggin IHSRR=0 1313f7fbd97SNicholas Piggin .endif 1323f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 1333f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 134a42a239dSNicholas Piggin .endif 135a42a239dSNicholas Piggin .ifndef IAREA 136a42a239dSNicholas Piggin IAREA=PACA_EXGEN 137a42a239dSNicholas Piggin .endif 1388729c26eSNicholas Piggin .ifndef IVIRT 1398729c26eSNicholas Piggin IVIRT=1 1408729c26eSNicholas Piggin .endif 141a3cd35beSNicholas Piggin .ifndef IISIDE 142a3cd35beSNicholas Piggin IISIDE=0 143a3cd35beSNicholas Piggin .endif 144af47d79bSNicholas Piggin .ifndef ICFAR 145af47d79bSNicholas Piggin ICFAR=1 146af47d79bSNicholas Piggin .endif 147af47d79bSNicholas Piggin .ifndef ICFAR_IF_HVMODE 148af47d79bSNicholas Piggin ICFAR_IF_HVMODE=0 149af47d79bSNicholas Piggin .endif 150a42a239dSNicholas Piggin .ifndef IDAR 151a42a239dSNicholas Piggin IDAR=0 152a42a239dSNicholas Piggin .endif 153a42a239dSNicholas Piggin .ifndef IDSISR 154a42a239dSNicholas Piggin IDSISR=0 155a42a239dSNicholas Piggin .endif 156d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 157d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 158d73a10cbSNicholas Piggin .endif 159d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 160d73a10cbSNicholas Piggin IREALMODE_COMMON=0 161d73a10cbSNicholas Piggin .else 162d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 163d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 164d73a10cbSNicholas Piggin .endif 165a42a239dSNicholas Piggin .endif 166a42a239dSNicholas Piggin .ifndef IMASK 167a42a239dSNicholas Piggin IMASK=0 168a42a239dSNicholas Piggin .endif 169a42a239dSNicholas Piggin .ifndef IKVM_REAL 170a42a239dSNicholas Piggin IKVM_REAL=0 171a42a239dSNicholas Piggin .endif 172a42a239dSNicholas Piggin .ifndef IKVM_VIRT 173a42a239dSNicholas Piggin IKVM_VIRT=0 174a42a239dSNicholas Piggin .endif 1757cb3a1a0SNicholas Piggin .ifndef ISTACK 1767cb3a1a0SNicholas Piggin ISTACK=1 1777cb3a1a0SNicholas Piggin .endif 1787cb3a1a0SNicholas Piggin .ifndef IKUAP 1797cb3a1a0SNicholas Piggin IKUAP=1 1807cb3a1a0SNicholas Piggin .endif 1812487fd2eSRohan McLure .ifndef IMSR_R12 1822487fd2eSRohan McLure IMSR_R12=0 1832487fd2eSRohan McLure .endif 184a42a239dSNicholas Piggin.endm 185a42a239dSNicholas Piggin 18612a04809SNicholas Piggin/* 1872284ffeaSNicholas Piggin * All interrupts which set HSRR registers, as well as SRESET and MCE and 1882284ffeaSNicholas Piggin * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, 1892284ffeaSNicholas Piggin * so they all generally need to test whether they were taken in guest context. 1902284ffeaSNicholas Piggin * 1912284ffeaSNicholas Piggin * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be 1922284ffeaSNicholas Piggin * taken with MSR[HV]=0. 1932284ffeaSNicholas Piggin * 1942284ffeaSNicholas Piggin * Interrupts which set SRR registers (with the above exceptions) do not 1952284ffeaSNicholas Piggin * elevate to MSR[HV]=1 mode, though most can be taken when running with 1962284ffeaSNicholas Piggin * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do 1972284ffeaSNicholas Piggin * not need to test whether a guest is running because they get delivered to 1982284ffeaSNicholas Piggin * the guest directly, including nested HV KVM guests. 1992284ffeaSNicholas Piggin * 2002284ffeaSNicholas Piggin * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host 2012284ffeaSNicholas Piggin * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the 2022284ffeaSNicholas Piggin * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be 2032284ffeaSNicholas Piggin * delivered to the real-mode entry point, therefore such interrupts only test 2042284ffeaSNicholas Piggin * KVM in their real mode handlers, and only when PR KVM is possible. 2052284ffeaSNicholas Piggin * 2062284ffeaSNicholas Piggin * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always 2072284ffeaSNicholas Piggin * delivered in real-mode when the MMU is in hash mode because the MMU 2082284ffeaSNicholas Piggin * registers are not set appropriately to translate host addresses. In nested 2092284ffeaSNicholas Piggin * radix mode these can be delivered in virt-mode as the host translations are 2102284ffeaSNicholas Piggin * used implicitly (see: effective LPID, effective PID). 2112284ffeaSNicholas Piggin */ 2122284ffeaSNicholas Piggin 2132284ffeaSNicholas Piggin/* 2142284ffeaSNicholas Piggin * If an interrupt is taken while a guest is running, it is immediately routed 215f3601156SNicholas Piggin * to KVM to handle. 21612a04809SNicholas Piggin */ 21712a04809SNicholas Piggin 21869fdd674SNicholas Piggin.macro KVMTEST name handler 21969fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 22012a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 22112a04809SNicholas Piggin cmpwi r10,0 22212a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 2233f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 224def0db4fSNicholas Piggin BEGIN_FTR_SECTION 22569fdd674SNicholas Piggin li r10,(IVEC + 0x2) 226def0db4fSNicholas Piggin FTR_SECTION_ELSE 22769fdd674SNicholas Piggin li r10,(IVEC) 228def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 229b177ae2fSNicholas Piggin .elseif IHSRR 23069fdd674SNicholas Piggin li r10,(IVEC + 0x2) 23112a04809SNicholas Piggin .else 23269fdd674SNicholas Piggin li r10,(IVEC) 23312a04809SNicholas Piggin .endif 23469fdd674SNicholas Piggin bne \handler 23512a04809SNicholas Piggin#endif 23669fdd674SNicholas Piggin.endm 23712a04809SNicholas Piggin 238c7c5cbb4SNicholas Piggin/* 239c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 240c7c5cbb4SNicholas Piggin * 241c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 242c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 243c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 244c7c5cbb4SNicholas Piggin * the kernel. 245c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 246c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 247c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 248c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 249c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 250c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 251c7c5cbb4SNicholas Piggin * irq_soft_mask set. 252c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 253c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 254c7c5cbb4SNicholas Piggin * mode. 255c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 256c7c5cbb4SNicholas Piggin * This is done if early=2. 257c7c5cbb4SNicholas Piggin */ 2588729c26eSNicholas Piggin 2598729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 260d73a10cbSNicholas Piggin .if IREALMODE_COMMON 261d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 262d73a10cbSNicholas Piggin mtctr r10 263d73a10cbSNicholas Piggin bctr 264d73a10cbSNicholas Piggin .else 2658729c26eSNicholas Piggin .if \virt 2668729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 2678729c26eSNicholas Piggin b \name\()_common_virt 2688729c26eSNicholas Piggin#else 2698729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 2708729c26eSNicholas Piggin mtctr r10 2718729c26eSNicholas Piggin bctr 2728729c26eSNicholas Piggin#endif 2738729c26eSNicholas Piggin .else 2748729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 2758729c26eSNicholas Piggin mtctr r10 2768729c26eSNicholas Piggin bctr 2778729c26eSNicholas Piggin .endif 278d73a10cbSNicholas Piggin .endif 2798729c26eSNicholas Piggin.endm 2808729c26eSNicholas Piggin 281fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 282c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 283c7c5cbb4SNicholas Piggin GET_PACA(r13) 284fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 285931dc86bSNicholas PigginBEGIN_FTR_SECTION 286931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 287931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 288c7c5cbb4SNicholas Piggin HMT_MEDIUM 289620f5c59SRohan McLure std r10,IAREA+EX_R10(r13) /* save r10 */ 290af47d79bSNicholas Piggin .if ICFAR 291931dc86bSNicholas PigginBEGIN_FTR_SECTION 292931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 293931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 294af47d79bSNicholas Piggin .elseif ICFAR_IF_HVMODE 295af47d79bSNicholas PigginBEGIN_FTR_SECTION 296af47d79bSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 297af47d79bSNicholas Piggin mfspr r10,SPRN_CFAR 298af47d79bSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) 299af47d79bSNicholas PigginFTR_SECTION_ELSE 300af47d79bSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 301af47d79bSNicholas Piggin li r10,0 302af47d79bSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) 303af47d79bSNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 304af47d79bSNicholas Piggin .endif 305c7c5cbb4SNicholas Piggin .if \ool 306c7c5cbb4SNicholas Piggin .if !\virt 307c7c5cbb4SNicholas Piggin b tramp_real_\name 308c7c5cbb4SNicholas Piggin .pushsection .text 309c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 310c7c5cbb4SNicholas Piggin .else 311c7c5cbb4SNicholas Piggin b tramp_virt_\name 312c7c5cbb4SNicholas Piggin .pushsection .text 313c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 314c7c5cbb4SNicholas Piggin .endif 315c7c5cbb4SNicholas Piggin .endif 316c7c5cbb4SNicholas Piggin 317931dc86bSNicholas PigginBEGIN_FTR_SECTION 318931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 319931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 320af47d79bSNicholas Piggin .if ICFAR || ICFAR_IF_HVMODE 321931dc86bSNicholas PigginBEGIN_FTR_SECTION 322931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 323931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 324af47d79bSNicholas Piggin .endif 325c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 3268729c26eSNicholas Piggin mfctr r10 3278729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 328c7c5cbb4SNicholas Piggin mfcr r9 329620f5c59SRohan McLure std r11,IAREA+EX_R11(r13) /* save r11 - r12 */ 330fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 331c7c5cbb4SNicholas Piggin 332c7c5cbb4SNicholas Piggin /* 333c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 334c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 335c7c5cbb4SNicholas Piggin * not recoverable if they are live. 336c7c5cbb4SNicholas Piggin */ 337c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 338fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 339a3cd35beSNicholas Piggin .if IDAR && !IISIDE 340fc589ee4SNicholas Piggin .if IHSRR 341c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 342c7c5cbb4SNicholas Piggin .else 343c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 344c7c5cbb4SNicholas Piggin .endif 345fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 346c7c5cbb4SNicholas Piggin .endif 347a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 348fc589ee4SNicholas Piggin .if IHSRR 349c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 350c7c5cbb4SNicholas Piggin .else 351c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 352c7c5cbb4SNicholas Piggin .endif 353fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 354c7c5cbb4SNicholas Piggin .endif 355c7c5cbb4SNicholas Piggin 3563f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 3578729c26eSNicholas Piggin BEGIN_FTR_SECTION 3588729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3598729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3608729c26eSNicholas Piggin FTR_SECTION_ELSE 3618729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3628729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 3638729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 3648729c26eSNicholas Piggin .elseif IHSRR 3658729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 3668729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 3678729c26eSNicholas Piggin .else 3688729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 3698729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 370c7c5cbb4SNicholas Piggin .endif 371d73a10cbSNicholas Piggin 372d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 3738729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 3748729c26eSNicholas Piggin .endif 3758729c26eSNicholas Piggin 376c7c5cbb4SNicholas Piggin .if \ool 377c7c5cbb4SNicholas Piggin .popsection 378c7c5cbb4SNicholas Piggin .endif 379c7c5cbb4SNicholas Piggin.endm 380c7c5cbb4SNicholas Piggin 381d064151fSNicholas Piggin/* 3828729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 3839600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 3849600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 3859600f261SNicholas Piggin * 3868729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 387d064151fSNicholas Piggin */ 3888729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 389d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_real, text) 3908729c26eSNicholas Piggin\name\()_common_real: 3919600f261SNicholas Piggin .if IKVM_REAL 39269fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 3939600f261SNicholas Piggin .endif 3949600f261SNicholas Piggin 3958729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 3968729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 397c080a173SDaniel Axtens .if IHSRR_IF_HVMODE 3988729c26eSNicholas Piggin BEGIN_FTR_SECTION 3998729c26eSNicholas Piggin xori r10,r10,MSR_RI 4008729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 4018729c26eSNicholas Piggin .elseif ! IHSRR 4028729c26eSNicholas Piggin xori r10,r10,MSR_RI 4038729c26eSNicholas Piggin .endif 4048729c26eSNicholas Piggin mtmsrd r10 4058729c26eSNicholas Piggin 4068729c26eSNicholas Piggin .if IVIRT 4079600f261SNicholas Piggin .if IKVM_VIRT 4089600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 4099600f261SNicholas Piggin .endif 4109600f261SNicholas Piggin 4118729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 412d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_virt, text) 4138729c26eSNicholas Piggin\name\()_common_virt: 4149600f261SNicholas Piggin .if IKVM_VIRT 41569fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4169600f261SNicholas Piggin1: 4179600f261SNicholas Piggin .endif 4188729c26eSNicholas Piggin .endif /* IVIRT */ 4198729c26eSNicholas Piggin.endm 4208729c26eSNicholas Piggin 4219600f261SNicholas Piggin/* 4229600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4239600f261SNicholas Piggin * want to run in real mode. 4249600f261SNicholas Piggin */ 4259600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 426d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(\name\()_common_real, text) 4279600f261SNicholas Piggin\name\()_common_real: 4289600f261SNicholas Piggin .if IKVM_REAL 42969fdd674SNicholas Piggin KVMTEST \name kvm_interrupt 4309600f261SNicholas Piggin .endif 4319600f261SNicholas Piggin.endm 4329600f261SNicholas Piggin 4338729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 4340eddf327SNicholas Piggin .if IMASK 435b2dc2977SNicholas Piggin .if ! ISTACK 436b2dc2977SNicholas Piggin .error "No support for masked interrupt to use custom stack" 437b2dc2977SNicholas Piggin .endif 438b2dc2977SNicholas Piggin 439b2dc2977SNicholas Piggin /* If coming from user, skip soft-mask tests. */ 440b2dc2977SNicholas Piggin andi. r10,r12,MSR_PR 441325678fdSNicholas Piggin bne 3f 442b2dc2977SNicholas Piggin 4439d1988caSNicholas Piggin /* 444325678fdSNicholas Piggin * Kernel code running below __end_soft_masked may be 445325678fdSNicholas Piggin * implicitly soft-masked if it is within the regions 446325678fdSNicholas Piggin * in the soft mask table. 4479d1988caSNicholas Piggin */ 4489d1988caSNicholas Piggin LOAD_HANDLER(r10, __end_soft_masked) 449b2dc2977SNicholas Piggin cmpld r11,r10 450325678fdSNicholas Piggin bge+ 1f 4519d1988caSNicholas Piggin 452325678fdSNicholas Piggin /* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */ 453325678fdSNicholas Piggin mtctr r12 454325678fdSNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 455325678fdSNicholas Piggin SEARCH_SOFT_MASK_TABLE 456325678fdSNicholas Piggin cmpdi r12,0 457325678fdSNicholas Piggin mfctr r12 /* Restore r12 to SRR1 */ 458325678fdSNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 459325678fdSNicholas Piggin beq 1f /* Not in soft-mask table */ 460b2dc2977SNicholas Piggin li r10,IMASK 461325678fdSNicholas Piggin b 2f /* In soft-mask table, always mask */ 462b2dc2977SNicholas Piggin 463b2dc2977SNicholas Piggin /* Test the soft mask state against our interrupt's bit */ 464325678fdSNicholas Piggin1: lbz r10,PACAIRQSOFTMASK(r13) 465325678fdSNicholas Piggin2: andi. r10,r10,IMASK 4660eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 4670eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 4680eddf327SNicholas Piggin li r10,PACA_IRQ_EE 4690eddf327SNicholas Piggin .elseif IVEC == 0x900 4700eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 4710eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 4720eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 4730eddf327SNicholas Piggin .elseif IVEC == 0xe60 4740eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 4750eddf327SNicholas Piggin .elseif IVEC == 0xf00 4760eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 4770eddf327SNicholas Piggin .else 4780eddf327SNicholas Piggin .abort "Bad maskable vector" 4790eddf327SNicholas Piggin .endif 4800eddf327SNicholas Piggin 4813f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4820eddf327SNicholas Piggin BEGIN_FTR_SECTION 4830eddf327SNicholas Piggin bne masked_Hinterrupt 4840eddf327SNicholas Piggin FTR_SECTION_ELSE 4850eddf327SNicholas Piggin bne masked_interrupt 4860eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4870eddf327SNicholas Piggin .elseif IHSRR 4880eddf327SNicholas Piggin bne masked_Hinterrupt 4890eddf327SNicholas Piggin .else 4900eddf327SNicholas Piggin bne masked_interrupt 4910eddf327SNicholas Piggin .endif 4920eddf327SNicholas Piggin .endif 4930eddf327SNicholas Piggin 4946d71759aSNicholas Piggin .if ISTACK 4955d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 496325678fdSNicholas Piggin3: mr r10,r1 /* Save r1 */ 4975d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 4981b359982SNicholas Piggin beq- 100f 4995d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 5001b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 5011b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 5025d5e0edfSNicholas Piggin .endif 5038c9fb5d4SNicholas Piggin 5048c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 5058c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 5068c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 5078c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 5088c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 5098c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 5101df45d78SRohan McLure SANITIZE_GPR(0) 5115d5e0edfSNicholas Piggin 51259dc5bfcSNicholas Piggin /* Mark our [H]SRRs valid for return */ 51359dc5bfcSNicholas Piggin li r10,1 51459dc5bfcSNicholas Piggin .if IHSRR_IF_HVMODE 51559dc5bfcSNicholas Piggin BEGIN_FTR_SECTION 51659dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 51759dc5bfcSNicholas Piggin FTR_SECTION_ELSE 51859dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 51959dc5bfcSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 52059dc5bfcSNicholas Piggin .elseif IHSRR 52159dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 52259dc5bfcSNicholas Piggin .else 52359dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 52459dc5bfcSNicholas Piggin .endif 52559dc5bfcSNicholas Piggin 5266d71759aSNicholas Piggin .if ISTACK 5276d71759aSNicholas Piggin .if IKUAP 5285d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 5295d5e0edfSNicholas Piggin .endif 5301b359982SNicholas Piggin beq 101f /* if from kernel mode */ 531931dc86bSNicholas PigginBEGIN_FTR_SECTION 532931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 533931dc86bSNicholas Piggin std r9,_PPR(r1) 534931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5351b359982SNicholas Piggin101: 5365d5e0edfSNicholas Piggin .else 5376d71759aSNicholas Piggin .if IKUAP 538bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 539bcbceed4SNicholas Piggin .endif 5405d5e0edfSNicholas Piggin .endif 5415d5e0edfSNicholas Piggin 5428c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5436d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5446d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5458c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5468c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5476d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5486d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5496d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5508c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5518c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5528c9fb5d4SNicholas Piggin std r11,GPR13(r1) 5531df45d78SRohan McLure .if !IMSR_R12 5541df45d78SRohan McLure SANITIZE_GPRS(9, 12) 5551df45d78SRohan McLure .else 5561df45d78SRohan McLure SANITIZE_GPRS(9, 11) 5571df45d78SRohan McLure .endif 558a3cd35beSNicholas Piggin 5596cc0c16dSNicholas Piggin SAVE_NVGPRS(r1) 5601df45d78SRohan McLure SANITIZE_NVGPRS() 5616cc0c16dSNicholas Piggin 5626d71759aSNicholas Piggin .if IDAR 563a3cd35beSNicholas Piggin .if IISIDE 564d1a84718SNicholas Piggin ld r10,_NIP(r1) 565d1a84718SNicholas Piggin .else 5666d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 567d1a84718SNicholas Piggin .endif 568d1a84718SNicholas Piggin std r10,_DAR(r1) 569d1a84718SNicholas Piggin .endif 570a3cd35beSNicholas Piggin 5716d71759aSNicholas Piggin .if IDSISR 572a3cd35beSNicholas Piggin .if IISIDE 573d1a84718SNicholas Piggin ld r10,_MSR(r1) 574d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 575d1a84718SNicholas Piggin and r10,r10,r11 576d1a84718SNicholas Piggin .else 5776d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 578d1a84718SNicholas Piggin .endif 579d1a84718SNicholas Piggin std r10,_DSISR(r1) 580d1a84718SNicholas Piggin .endif 581a3cd35beSNicholas Piggin 582931dc86bSNicholas PigginBEGIN_FTR_SECTION 583af47d79bSNicholas Piggin .if ICFAR || ICFAR_IF_HVMODE 5846d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 585af47d79bSNicholas Piggin .else 586af47d79bSNicholas Piggin li r10,0 587af47d79bSNicholas Piggin .endif 5888c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 589931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 5908729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 5918c9fb5d4SNicholas Piggin std r10,_CTR(r1) 5921df45d78SRohan McLure SAVE_GPRS(2, 8, r1) /* save r2 - r8 in stackframe */ 5931df45d78SRohan McLure SANITIZE_GPRS(2, 8) 5948c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 5958e93fb33SNicholas Piggin LOAD_PACA_TOC() /* get kernel TOC into r2 */ 5968c9fb5d4SNicholas Piggin std r9,_LINK(r1) 5978c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5988c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 5998c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 6008c9fb5d4SNicholas Piggin std r11,_XER(r1) 6016cc0c16dSNicholas Piggin li r9,IVEC 6028c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 6038c9fb5d4SNicholas Piggin li r10,0 60417773afdSNicholas Piggin LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) 6058c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 606d2e8ff9fSNicholas Piggin std r11,STACK_INT_FRAME_MARKER(r1) /* mark the frame */ 607bcbceed4SNicholas Piggin.endm 608bcbceed4SNicholas Piggin 609391e941bSNicholas Piggin/* 6108729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 6118729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 6128729c26eSNicholas Piggin * SRR1, and relocation is on. 6138729c26eSNicholas Piggin * 6148729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 6158729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 6168729c26eSNicholas Piggin */ 6178729c26eSNicholas Piggin.macro GEN_COMMON name 6188729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 6198729c26eSNicholas Piggin __GEN_COMMON_BODY \name 6208729c26eSNicholas Piggin.endm 6218729c26eSNicholas Piggin 622f23699c9SNicholas Piggin.macro SEARCH_RESTART_TABLE 623f23699c9SNicholas Piggin#ifdef CONFIG_RELOCATABLE 624f23699c9SNicholas Piggin mr r12,r2 6258e93fb33SNicholas Piggin LOAD_PACA_TOC() 626f23699c9SNicholas Piggin LOAD_REG_ADDR(r9, __start___restart_table) 627f23699c9SNicholas Piggin LOAD_REG_ADDR(r10, __stop___restart_table) 628f23699c9SNicholas Piggin mr r2,r12 629f23699c9SNicholas Piggin#else 630f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___restart_table) 631f23699c9SNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___restart_table) 632f23699c9SNicholas Piggin#endif 633f23699c9SNicholas Piggin300: 634f23699c9SNicholas Piggin cmpd r9,r10 635f23699c9SNicholas Piggin beq 302f 636f23699c9SNicholas Piggin ld r12,0(r9) 637f23699c9SNicholas Piggin cmpld r11,r12 638f23699c9SNicholas Piggin blt 301f 639f23699c9SNicholas Piggin ld r12,8(r9) 640f23699c9SNicholas Piggin cmpld r11,r12 641f23699c9SNicholas Piggin bge 301f 642f23699c9SNicholas Piggin ld r12,16(r9) 643f23699c9SNicholas Piggin b 303f 644f23699c9SNicholas Piggin301: 645f23699c9SNicholas Piggin addi r9,r9,24 646f23699c9SNicholas Piggin b 300b 647f23699c9SNicholas Piggin302: 648f23699c9SNicholas Piggin li r12,0 649f23699c9SNicholas Piggin303: 650f23699c9SNicholas Piggin.endm 651f23699c9SNicholas Piggin 652325678fdSNicholas Piggin.macro SEARCH_SOFT_MASK_TABLE 653325678fdSNicholas Piggin#ifdef CONFIG_RELOCATABLE 654325678fdSNicholas Piggin mr r12,r2 6558e93fb33SNicholas Piggin LOAD_PACA_TOC() 656325678fdSNicholas Piggin LOAD_REG_ADDR(r9, __start___soft_mask_table) 657325678fdSNicholas Piggin LOAD_REG_ADDR(r10, __stop___soft_mask_table) 658325678fdSNicholas Piggin mr r2,r12 659325678fdSNicholas Piggin#else 660325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___soft_mask_table) 661325678fdSNicholas Piggin LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___soft_mask_table) 662325678fdSNicholas Piggin#endif 663325678fdSNicholas Piggin300: 664325678fdSNicholas Piggin cmpd r9,r10 665325678fdSNicholas Piggin beq 302f 666325678fdSNicholas Piggin ld r12,0(r9) 667325678fdSNicholas Piggin cmpld r11,r12 668325678fdSNicholas Piggin blt 301f 669325678fdSNicholas Piggin ld r12,8(r9) 670325678fdSNicholas Piggin cmpld r11,r12 671325678fdSNicholas Piggin bge 301f 672325678fdSNicholas Piggin li r12,1 673325678fdSNicholas Piggin b 303f 674325678fdSNicholas Piggin301: 675325678fdSNicholas Piggin addi r9,r9,16 676325678fdSNicholas Piggin b 300b 677325678fdSNicholas Piggin302: 678325678fdSNicholas Piggin li r12,0 679325678fdSNicholas Piggin303: 680325678fdSNicholas Piggin.endm 681325678fdSNicholas Piggin 6828729c26eSNicholas Piggin/* 683391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 684391e941bSNicholas Piggin * standard exception. 685391e941bSNicholas Piggin */ 6863f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 687391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 688391e941bSNicholas Piggin ld r9,_MSR(r1) 68959dc5bfcSNicholas Piggin li r10,0 690391e941bSNicholas Piggin .if \hsrr 691391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 69259dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 693391e941bSNicholas Piggin .else 694391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 69559dc5bfcSNicholas Piggin stb r10,PACASRR_VALID(r13) 696391e941bSNicholas Piggin .endif 697391e941bSNicholas Piggin ld r9,_NIP(r1) 698391e941bSNicholas Piggin .if \hsrr 699391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 700391e941bSNicholas Piggin .else 701391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 702391e941bSNicholas Piggin .endif 703391e941bSNicholas Piggin ld r9,_CTR(r1) 704391e941bSNicholas Piggin mtctr r9 705391e941bSNicholas Piggin ld r9,_XER(r1) 706391e941bSNicholas Piggin mtxer r9 707391e941bSNicholas Piggin ld r9,_LINK(r1) 708391e941bSNicholas Piggin mtlr r9 709391e941bSNicholas Piggin ld r9,_CCR(r1) 710391e941bSNicholas Piggin mtcr r9 7111df45d78SRohan McLure SANITIZE_RESTORE_NVGPRS() 712aebd1fb4SNicholas Piggin REST_GPRS(2, 13, r1) 713391e941bSNicholas Piggin REST_GPR(0, r1) 714391e941bSNicholas Piggin /* restore original r1. */ 715391e941bSNicholas Piggin ld r1,GPR1(r1) 716391e941bSNicholas Piggin.endm 717d064151fSNicholas Piggin 71812a04809SNicholas Piggin/* 719bf75a325SNicholas Piggin * EARLY_BOOT_FIXUP - Fix real-mode interrupt with wrong endian in early boot. 720bf75a325SNicholas Piggin * 721bf75a325SNicholas Piggin * There's a short window during boot where although the kernel is running 722bf75a325SNicholas Piggin * little endian, any exceptions will cause the CPU to switch back to big 723bf75a325SNicholas Piggin * endian. For example a WARN() boils down to a trap instruction, which will 724bf75a325SNicholas Piggin * cause a program check, and we end up here but with the CPU in big endian 725bf75a325SNicholas Piggin * mode. The first instruction of the program check handler (in GEN_INT_ENTRY 726bf75a325SNicholas Piggin * below) is an mtsprg, which when executed in the wrong endian is an lhzu with 727bf75a325SNicholas Piggin * a ~3GB displacement from r3. The content of r3 is random, so that is a load 728bf75a325SNicholas Piggin * from some random location, and depending on the system can easily lead to a 729bf75a325SNicholas Piggin * checkstop, or an infinitely recursive page fault. 730bf75a325SNicholas Piggin * 731bf75a325SNicholas Piggin * So to handle that case we have a trampoline here that can detect we are in 732bf75a325SNicholas Piggin * the wrong endian and flip us back to the correct endian. We can't flip 733bf75a325SNicholas Piggin * MSR[LE] using mtmsr, so we have to use rfid. That requires backing up SRR0/1 734bf75a325SNicholas Piggin * as well as a GPR. To do that we use SPRG0/2/3, as SPRG1 is already used for 735bf75a325SNicholas Piggin * the paca. SPRG3 is user readable, but this trampoline is only active very 736bf75a325SNicholas Piggin * early in boot, and SPRG3 will be reinitialised in vdso_getcpu_init() before 737bf75a325SNicholas Piggin * userspace starts. 738bf75a325SNicholas Piggin */ 739bf75a325SNicholas Piggin.macro EARLY_BOOT_FIXUP 740bf75a325SNicholas PigginBEGIN_FTR_SECTION 741e1100ceeSNicholas Piggin#ifdef CONFIG_CPU_LITTLE_ENDIAN 742bf75a325SNicholas Piggin tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 743bf75a325SNicholas Piggin b 2f // Skip trampoline if endian is correct 744bf75a325SNicholas Piggin .long 0xa643707d // mtsprg 0, r11 Backup r11 745bf75a325SNicholas Piggin .long 0xa6027a7d // mfsrr0 r11 746bf75a325SNicholas Piggin .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 747bf75a325SNicholas Piggin .long 0xa6027b7d // mfsrr1 r11 748bf75a325SNicholas Piggin .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 749bf75a325SNicholas Piggin .long 0xa600607d // mfmsr r11 750bf75a325SNicholas Piggin .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] 751bf75a325SNicholas Piggin .long 0xa6037b7d // mtsrr1 r11 752bf75a325SNicholas Piggin /* 753bf75a325SNicholas Piggin * This is 'li r11,1f' where 1f is the absolute address of that 754bf75a325SNicholas Piggin * label, byteswapped into the SI field of the instruction. 755bf75a325SNicholas Piggin */ 756bf75a325SNicholas Piggin .long 0x00006039 | \ 757bf75a325SNicholas Piggin ((ABS_ADDR(1f, real_vectors) & 0x00ff) << 24) | \ 758bf75a325SNicholas Piggin ((ABS_ADDR(1f, real_vectors) & 0xff00) << 8) 759bf75a325SNicholas Piggin .long 0xa6037a7d // mtsrr0 r11 760bf75a325SNicholas Piggin .long 0x2400004c // rfid 761bf75a325SNicholas Piggin1: 762bf75a325SNicholas Piggin mfsprg r11, 3 763bf75a325SNicholas Piggin mtsrr1 r11 // Restore SRR1 764bf75a325SNicholas Piggin mfsprg r11, 2 765bf75a325SNicholas Piggin mtsrr0 r11 // Restore SRR0 766bf75a325SNicholas Piggin mfsprg r11, 0 // Restore r11 767bf75a325SNicholas Piggin2: 768bf75a325SNicholas Piggin#endif 769e1100ceeSNicholas Piggin /* 770e1100ceeSNicholas Piggin * program check could hit at any time, and pseries can not block 771e1100ceeSNicholas Piggin * MSR[ME] in early boot. So check if there is anything useful in r13 772e1100ceeSNicholas Piggin * yet, and spin forever if not. 773e1100ceeSNicholas Piggin */ 774e1100ceeSNicholas Piggin mtsprg 0, r11 775e1100ceeSNicholas Piggin mfcr r11 776e1100ceeSNicholas Piggin cmpdi r13, 0 777e1100ceeSNicholas Piggin beq . 778e1100ceeSNicholas Piggin mtcr r11 779e1100ceeSNicholas Piggin mfsprg r11, 0 780e1100ceeSNicholas PigginEND_FTR_SECTION(0, 1) // nop out after boot 781bf75a325SNicholas Piggin.endm 782bf75a325SNicholas Piggin 783bf75a325SNicholas Piggin/* 78457f26649SNicholas Piggin * There are a few constraints to be concerned with. 78557f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 78657f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 78757f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 78857f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 78957f26649SNicholas Piggin * must be used. 79057f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 79157f26649SNicholas Piggin * virtual 0xc00... 79257f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 79357f26649SNicholas Piggin * 79457f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 79557f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 79657f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 79757f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 79857f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 79957f26649SNicholas Piggin * 80057f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 80157f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 80257f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 80357f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 80457f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 80557f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 80657f26649SNicholas Piggin * 8077fa95f9aSNicholas Piggin * The scv instructions are a special case. They get a 0x3000 offset applied. 8087fa95f9aSNicholas Piggin * scv exceptions have unique reentrancy properties, see below. 8097fa95f9aSNicholas Piggin * 81057f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 81157f26649SNicholas Piggin * 81257f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 81357f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 81457f26649SNicholas Piggin * 81557f26649SNicholas Piggin * 8160ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 8170ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 81857f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 8197fa95f9aSNicholas Piggin * 0x1900 - 0x2fff : Real mode trampolines 8207fa95f9aSNicholas Piggin * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 82157f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 8220ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 82357f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 82457f26649SNicholas Piggin * setup code, rest of kernel. 825e0319829SNicholas Piggin * 826e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 827e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 828e0319829SNicholas Piggin * vectors there. 8290ebc4cdaSBenjamin Herrenschmidt */ 83057f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 8317fa95f9aSNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000) 8327fa95f9aSNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900) 83357f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 834ccd47702SNicholas Piggin 835ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 836bd3524feSNicholas Piggin .globl start_real_trampolines 837bd3524feSNicholas Piggin .globl end_real_trampolines 838bd3524feSNicholas Piggin .globl start_virt_trampolines 839bd3524feSNicholas Piggin .globl end_virt_trampolines 840ccd47702SNicholas Piggin#endif 841ccd47702SNicholas Piggin 84257f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 84357f26649SNicholas Piggin/* 84457f26649SNicholas Piggin * Data area reserved for FWNMI option. 84557f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 84657f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 84757f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 84857f26649SNicholas Piggin */ 84957f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 85057f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 85157f26649SNicholas Piggin#else 85257f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 85357f26649SNicholas Piggin#endif 85457f26649SNicholas Piggin 85557f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 85657f26649SNicholas Piggin 8570ebc4cdaSBenjamin Herrenschmidt/* 8580ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 8590ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 8600ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 8610ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 8620ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 8630ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 8640ebc4cdaSBenjamin Herrenschmidt */ 8650ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 8660ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 8670ebc4cdaSBenjamin Herrenschmidt 8687fa95f9aSNicholas Piggin/** 8697fa95f9aSNicholas Piggin * Interrupt 0x3000 - System Call Vectored Interrupt (syscall). 8707fa95f9aSNicholas Piggin * This is a synchronous interrupt invoked with the "scv" instruction. The 8717fa95f9aSNicholas Piggin * system call does not alter the HV bit, so it is directed to the OS. 8727fa95f9aSNicholas Piggin * 8737fa95f9aSNicholas Piggin * Handling: 8747fa95f9aSNicholas Piggin * scv instructions enter the kernel without changing EE, RI, ME, or HV. 8757fa95f9aSNicholas Piggin * In particular, this means we can take a maskable interrupt at any point 8767fa95f9aSNicholas Piggin * in the scv handler, which is unlike any other interrupt. This is solved 877325678fdSNicholas Piggin * by treating the instruction addresses in the handler as being soft-masked, 878325678fdSNicholas Piggin * by adding a SOFT_MASK_TABLE entry for them. 8797fa95f9aSNicholas Piggin * 8807fa95f9aSNicholas Piggin * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and 8817fa95f9aSNicholas Piggin * ensure scv is never executed with relocation off, which means AIL-0 8827fa95f9aSNicholas Piggin * should never happen. 8837fa95f9aSNicholas Piggin * 8849d1988caSNicholas Piggin * Before leaving the following inside-__end_soft_masked text, at least of the 8859d1988caSNicholas Piggin * following must be true: 8867fa95f9aSNicholas Piggin * - MSR[PR]=1 (i.e., return to userspace) 8879d1988caSNicholas Piggin * - MSR_EE|MSR_RI is clear (no reentrant exceptions) 8887fa95f9aSNicholas Piggin * - Standard kernel environment is set up (stack, paca, etc) 8897fa95f9aSNicholas Piggin * 890b5149e22SNicholas Piggin * KVM: 891b5149e22SNicholas Piggin * These interrupts do not elevate HV 0->1, so HV is not involved. PR KVM 892b5149e22SNicholas Piggin * ensures that FSCR[SCV] is disabled whenever it has to force AIL off. 893b5149e22SNicholas Piggin * 8947fa95f9aSNicholas Piggin * Call convention: 8957fa95f9aSNicholas Piggin * 896*17e02586SCosta Shulyupin * syscall register convention is in Documentation/arch/powerpc/syscall64-abi.rst 8977fa95f9aSNicholas Piggin */ 8987fa95f9aSNicholas PigginEXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000) 8997fa95f9aSNicholas Piggin /* SCV 0 */ 9007fa95f9aSNicholas Piggin mr r9,r13 9017fa95f9aSNicholas Piggin GET_PACA(r13) 9027fa95f9aSNicholas Piggin mflr r11 9037fa95f9aSNicholas Piggin mfctr r12 9047fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 9057fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 9067fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 9077fa95f9aSNicholas Piggin b system_call_vectored_tramp 9087fa95f9aSNicholas Piggin#else 9097fa95f9aSNicholas Piggin b system_call_vectored_common 9107fa95f9aSNicholas Piggin#endif 9117fa95f9aSNicholas Piggin nop 9127fa95f9aSNicholas Piggin 9137fa95f9aSNicholas Piggin /* SCV 1 - 127 */ 9147fa95f9aSNicholas Piggin .rept 127 9157fa95f9aSNicholas Piggin mr r9,r13 9167fa95f9aSNicholas Piggin GET_PACA(r13) 9177fa95f9aSNicholas Piggin mflr r11 9187fa95f9aSNicholas Piggin mfctr r12 9197fa95f9aSNicholas Piggin li r10,IRQS_ALL_DISABLED 9207fa95f9aSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 9217fa95f9aSNicholas Piggin li r0,-1 /* cause failure */ 9227fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 9237fa95f9aSNicholas Piggin b system_call_vectored_sigill_tramp 9247fa95f9aSNicholas Piggin#else 9257fa95f9aSNicholas Piggin b system_call_vectored_sigill 9267fa95f9aSNicholas Piggin#endif 9277fa95f9aSNicholas Piggin .endr 9287fa95f9aSNicholas PigginEXC_VIRT_END(system_call_vectored, 0x3000, 0x1000) 9297fa95f9aSNicholas Piggin 930787c70f2SNicholas Piggin// Treat scv vectors as soft-masked, see comment above. 931787c70f2SNicholas Piggin// Use absolute values rather than labels here, so they don't get relocated, 932787c70f2SNicholas Piggin// because this code runs unrelocated. 933787c70f2SNicholas PigginSOFT_MASK_TABLE(0xc000000000003000, 0xc000000000004000) 934325678fdSNicholas Piggin 9357fa95f9aSNicholas Piggin#ifdef CONFIG_RELOCATABLE 9367fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_tramp) 937d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_vectored_common, virt_trampolines) 9387fa95f9aSNicholas Piggin mtctr r10 9397fa95f9aSNicholas Piggin bctr 9407fa95f9aSNicholas Piggin 9417fa95f9aSNicholas PigginTRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) 942d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_vectored_sigill, virt_trampolines) 9437fa95f9aSNicholas Piggin mtctr r10 9447fa95f9aSNicholas Piggin bctr 9457fa95f9aSNicholas Piggin#endif 9467fa95f9aSNicholas Piggin 9477fa95f9aSNicholas Piggin 948e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 9491a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 950e0319829SNicholas Piggin 951fb479e44SNicholas Piggin 95294325357SNicholas Piggin/** 95394325357SNicholas Piggin * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI). 95494325357SNicholas Piggin * This is a non-maskable, asynchronous interrupt always taken in real-mode. 95594325357SNicholas Piggin * It is caused by: 95694325357SNicholas Piggin * - Wake from power-saving state, on powernv. 95794325357SNicholas Piggin * - An NMI from another CPU, triggered by firmware or hypercall. 95894325357SNicholas Piggin * - As crash/debug signal injected from BMC, firmware or hypervisor. 95994325357SNicholas Piggin * 96094325357SNicholas Piggin * Handling: 96194325357SNicholas Piggin * Power-save wakeup is the only performance critical path, so this is 96294325357SNicholas Piggin * determined quickly as possible first. In this case volatile registers 96394325357SNicholas Piggin * can be discarded and SPRs like CFAR don't need to be read. 96494325357SNicholas Piggin * 96594325357SNicholas Piggin * If not a powersave wakeup, then it's run as a regular interrupt, however 96694325357SNicholas Piggin * it uses its own stack and PACA save area to preserve the regular kernel 96794325357SNicholas Piggin * environment for debugging. 96894325357SNicholas Piggin * 96994325357SNicholas Piggin * This interrupt is not maskable, so triggering it when MSR[RI] is clear, 97094325357SNicholas Piggin * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely 97194325357SNicholas Piggin * correct to switch to virtual mode to run the regular interrupt handler 97294325357SNicholas Piggin * because it might be interrupted when the MMU is in a bad state (e.g., SLB 97394325357SNicholas Piggin * is clear). 97494325357SNicholas Piggin * 97594325357SNicholas Piggin * FWNMI: 97694325357SNicholas Piggin * PAPR specifies a "fwnmi" facility which sends the sreset to a different 97794325357SNicholas Piggin * entry point with a different register set up. Some hypervisors will 97894325357SNicholas Piggin * send the sreset to 0x100 in the guest if it is not fwnmi capable. 97994325357SNicholas Piggin * 98094325357SNicholas Piggin * KVM: 98194325357SNicholas Piggin * Unlike most SRR interrupts, this may be taken by the host while executing 98294325357SNicholas Piggin * in a guest, so a KVM test is required. KVM will pull the CPU out of guest 98394325357SNicholas Piggin * mode and then raise the sreset. 98494325357SNicholas Piggin */ 9854f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 9864f50541fSNicholas Piggin IVEC=0x100 9874f50541fSNicholas Piggin IAREA=PACA_EXNMI 9888729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 9894f50541fSNicholas Piggin ISTACK=0 9904f50541fSNicholas Piggin IKVM_REAL=1 9914f50541fSNicholas PigginINT_DEFINE_END(system_reset) 9924f50541fSNicholas Piggin 993a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 994fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 995fb479e44SNicholas Piggin /* 996fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 997ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 998ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 999ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 1000ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 10010e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 10020e10be2bSNicholas Piggin * 10030e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 10040e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 10050e10be2bSNicholas Piggin * 10060e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 10070e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 10080e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 10090e10be2bSNicholas Piggin * common exception macros to handle it. 1010fb479e44SNicholas Piggin */ 1011a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 10120e10be2bSNicholas Piggin SET_SCRATCH0(r13) 10130e10be2bSNicholas Piggin GET_PACA(r13) 10140e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 10150e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 10160e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 1017a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 10180e10be2bSNicholas Piggin mfocrf r4,0x80 10190e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 10200e10be2bSNicholas Piggin bne+ system_reset_idle_wake 10210e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 10220e10be2bSNicholas Piggin mtocrf 0x80,r4 10230e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 10240e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 10250e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 10260e10be2bSNicholas Piggin GET_SCRATCH0(r13) 1027a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1028fb479e44SNicholas Piggin#endif 1029fb479e44SNicholas Piggin 10304f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 1031c4f3b52cSNicholas Piggin /* 10320e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 10330e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 10340e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 10350e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 10360e10be2bSNicholas Piggin * be dangerous anyway. 1037c4f3b52cSNicholas Piggin */ 10381a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 10391a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 1040fb479e44SNicholas Piggin 1041fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 10420e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 10430e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 10440e10be2bSNicholas Piggin cmpwi cr1,r5,2 10450e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1046d72c4a36SDaniel Axtens __LOAD_FAR_HANDLER(r12, DOTSYM(idle_return_gpr_loss), real_trampolines) 1047fd983957SAlexey Kardashevskiy mtctr r12 1048fd983957SAlexey Kardashevskiy bctr 1049371fefd6SPaul Mackerras#endif 1050371fefd6SPaul Mackerras 1051acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1052acc8da44SNicholas Piggin/* 1053acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 1054acc8da44SNicholas Piggin */ 1055acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 10564f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 1057acc8da44SNicholas Piggin 1058acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 1059acc8da44SNicholas Piggin 1060a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 10618729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 1062c4f3b52cSNicholas Piggin /* 1063ff0b0d6eSNicholas Piggin * Increment paca->in_nmi. When the interrupt entry wrapper later 1064ff0b0d6eSNicholas Piggin * enable MSR_RI, then SLB or MCE will be able to recover, but a nested 1065ff0b0d6eSNicholas Piggin * NMI will notice in_nmi and not recover because of the use of the NMI 1066ff0b0d6eSNicholas Piggin * stack. in_nmi reentrancy is tested in system_reset_exception. 1067c4f3b52cSNicholas Piggin */ 1068c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 1069c4f3b52cSNicholas Piggin addi r10,r10,1 1070c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 1071aca79d2bSVaidyanathan Srinivasan 1072b1ee8a3dSNicholas Piggin mr r10,r1 1073b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 1074b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 10758729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 107647169fbaSNicholas Piggin 1077c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 10784e991e3cSNicholas Piggin bl CFUNC(system_reset_exception) 107915b4dd79SNicholas Piggin 108015b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 1081fbc50063SNicholas Piggin li r9,0 108215b4dd79SNicholas Piggin mtmsrd r9,1 1083c4f3b52cSNicholas Piggin 1084c4f3b52cSNicholas Piggin /* 108515b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 1086c4f3b52cSNicholas Piggin */ 1087c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 1088c4f3b52cSNicholas Piggin subi r10,r10,1 1089c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 1090c4f3b52cSNicholas Piggin 10918e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 10923f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 109315b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 1094582baf44SNicholas Piggin 10950ebc4cdaSBenjamin Herrenschmidt 109694325357SNicholas Piggin/** 109794325357SNicholas Piggin * Interrupt 0x200 - Machine Check Interrupt (MCE). 109894325357SNicholas Piggin * This is a non-maskable interrupt always taken in real-mode. It can be 109994325357SNicholas Piggin * synchronous or asynchronous, caused by hardware or software, and it may be 110094325357SNicholas Piggin * taken in a power-saving state. 110194325357SNicholas Piggin * 110294325357SNicholas Piggin * Handling: 110394325357SNicholas Piggin * Similarly to system reset, this uses its own stack and PACA save area, 110494325357SNicholas Piggin * the difference is re-entrancy is allowed on the machine check stack. 110594325357SNicholas Piggin * 110694325357SNicholas Piggin * machine_check_early is run in real mode, and carefully decodes the 110794325357SNicholas Piggin * machine check and tries to handle it (e.g., flush the SLB if there was an 110894325357SNicholas Piggin * error detected there), determines if it was recoverable and logs the 110994325357SNicholas Piggin * event. 111094325357SNicholas Piggin * 1111b44fc96dSNicholas Piggin * This early code does not "reconcile" irq soft-mask state like SRESET or 1112b44fc96dSNicholas Piggin * regular interrupts do, so irqs_disabled() among other things may not work 1113b44fc96dSNicholas Piggin * properly (irq disable/enable already doesn't work because irq tracing can 1114b44fc96dSNicholas Piggin * not work in real mode). 1115b44fc96dSNicholas Piggin * 111694325357SNicholas Piggin * Then, depending on the execution context when the interrupt is taken, there 111794325357SNicholas Piggin * are 3 main actions: 111894325357SNicholas Piggin * - Executing in kernel mode. The event is queued with irq_work, which means 111994325357SNicholas Piggin * it is handled when it is next safe to do so (i.e., the kernel has enabled 112094325357SNicholas Piggin * interrupts), which could be immediately when the interrupt returns. This 112194325357SNicholas Piggin * avoids nasty issues like switching to virtual mode when the MMU is in a 112294325357SNicholas Piggin * bad state, or when executing OPAL code. (SRESET is exposed to such issues, 112394325357SNicholas Piggin * but it has different priorities). Check to see if the CPU was in power 112494325357SNicholas Piggin * save, and return via the wake up code if it was. 112594325357SNicholas Piggin * 112694325357SNicholas Piggin * - Executing in user mode. machine_check_exception is run like a normal 112794325357SNicholas Piggin * interrupt handler, which processes the data generated by the early handler. 112894325357SNicholas Piggin * 112994325357SNicholas Piggin * - Executing in guest mode. The interrupt is run with its KVM test, and 113094325357SNicholas Piggin * branches to KVM to deal with. KVM may queue the event for the host 113194325357SNicholas Piggin * to report later. 113294325357SNicholas Piggin * 113394325357SNicholas Piggin * This interrupt is not maskable, so if it triggers when MSR[RI] is clear, 113494325357SNicholas Piggin * or SCRATCH0 is in use, it may cause a crash. 113594325357SNicholas Piggin * 113694325357SNicholas Piggin * KVM: 113794325357SNicholas Piggin * See SRESET. 113894325357SNicholas Piggin */ 11394f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 11404f50541fSNicholas Piggin IVEC=0x200 11414f50541fSNicholas Piggin IAREA=PACA_EXMC 11428729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 1143d73a10cbSNicholas Piggin IREALMODE_COMMON=1 11444f50541fSNicholas Piggin ISTACK=0 11454f50541fSNicholas Piggin IDAR=1 11464f50541fSNicholas Piggin IDSISR=1 11474f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 11484f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 11494f50541fSNicholas Piggin 11504f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 11514f50541fSNicholas Piggin IVEC=0x200 11524f50541fSNicholas Piggin IAREA=PACA_EXMC 11538729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 11544f50541fSNicholas Piggin IDAR=1 11554f50541fSNicholas Piggin IDSISR=1 11564f50541fSNicholas Piggin IKVM_REAL=1 11574f50541fSNicholas PigginINT_DEFINE_END(machine_check) 11584f50541fSNicholas Piggin 11594f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 11602f5182cfSNicholas Piggin EARLY_BOOT_FIXUP 11614f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 11621a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 11631a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 1164c8eb54dbSNicholas Piggin 1165abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 1166abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 1167abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 11684f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 1169abd1f4caSNicholas Piggin#endif 1170abd1f4caSNicholas Piggin 1171fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 1172fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 1173fce16d48SNicholas Piggin li r9,0; \ 1174fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 1175fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 1176fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 1177fce16d48SNicholas Piggin subi r12,r12,1; \ 1178fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 11793f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 1180fce16d48SNicholas Piggin 1181c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 11829600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 11839600f261SNicholas Piggin 1184afcf0095SNicholas Piggin /* 1185afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 1186afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 1187afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 1188afcf0095SNicholas Piggin * 1189afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 1190afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 1191afcf0095SNicholas Piggin * machine checks. 1192afcf0095SNicholas Piggin * 1193afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 1194afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 1195afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 1196afcf0095SNicholas Piggin * 1197afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 1198afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 1199afcf0095SNicholas Piggin * rfid with MSR_ME=1. 12001945bc45SNicholas Piggin * 12011945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 12021945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 12032bf1071aSNicholas Piggin * to restore state. 1204afcf0095SNicholas Piggin */ 1205afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 1206afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 1207c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 1208afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 1209afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 1210c8eb54dbSNicholas Piggin 1211c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 1212c8eb54dbSNicholas Piggin bne 1f 1213c8eb54dbSNicholas Piggin /* First machine check entry */ 1214c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1215b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1216b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1217b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1218c8eb54dbSNicholas Piggin 12198729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1220c8eb54dbSNicholas Piggin 1221db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1222296e753fSNicholas Piggin bl enable_machine_check 1223db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1224c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 12252f5182cfSNicholas PigginBEGIN_FTR_SECTION 12264e991e3cSNicholas Piggin bl CFUNC(machine_check_early_boot) 12272f5182cfSNicholas PigginEND_FTR_SECTION(0, 1) // nop out after boot 12284e991e3cSNicholas Piggin bl CFUNC(machine_check_early) 1229afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1230afcf0095SNicholas Piggin ld r12,_MSR(r1) 12311945bc45SNicholas Piggin 1232afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1233afcf0095SNicholas Piggin /* 1234afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1235afcf0095SNicholas Piggin * of the following is true: 1236afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1237afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1238afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1239afcf0095SNicholas Piggin * 1240afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1241afcf0095SNicholas Piggin */ 12421945bc45SNicholas PigginBEGIN_FTR_SECTION 12431945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 12446102c005SNicholas Piggin bne machine_check_idle_common 12451945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1246afcf0095SNicholas Piggin#endif 12471945bc45SNicholas Piggin 1248afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1249afcf0095SNicholas Piggin /* 125019dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 125105f97d94SNicholas Piggin * exception handler which will take the 125269fdd674SNicholas Piggin * machine_check_kvm->kvm_interrupt branch to deliver the MC event 125305f97d94SNicholas Piggin * to guest. 1254afcf0095SNicholas Piggin */ 1255afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1256afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1257b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1258afcf0095SNicholas Piggin#endif 125919dbe673SNicholas Piggin 1260afcf0095SNicholas Piggin /* 126119dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 126219dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 126319dbe673SNicholas Piggin */ 126419dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1265b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 126619dbe673SNicholas Piggin 126719dbe673SNicholas Piggin /* 126819dbe673SNicholas Piggin * At this point we are coming from kernel context. 1269afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1270afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1271afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1272afcf0095SNicholas Piggin */ 1273afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1274b7d9ccecSNicholas Piggin beq unrecoverable_mce 1275b7d9ccecSNicholas Piggin 1276afcf0095SNicholas Piggin /* 1277afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1278afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1279afcf0095SNicholas Piggin */ 1280afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1281afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1282b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1283272f6364SNicholas Piggin 1284afcf0095SNicholas Piggin /* 1285afcf0095SNicholas Piggin * Return from MC interrupt. 1286afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1287afcf0095SNicholas Piggin * returning from kernel or opal call. 1288afcf0095SNicholas Piggin */ 12894e991e3cSNicholas Piggin bl CFUNC(machine_check_queue_event) 1290afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1291fe9d482bSNicholas Piggin RFI_TO_KERNEL 1292272f6364SNicholas Piggin 1293b3fe3526SNicholas Pigginmce_deliver: 1294b3fe3526SNicholas Piggin /* 1295b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1296b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1297b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1298b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1299b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1300b3fe3526SNicholas Piggin */ 13010b66370cSNicholas PigginBEGIN_FTR_SECTION 13020b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 13030b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 13040b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1305afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 13064f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1307afcf0095SNicholas Piggin 1308fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1309fce16d48SNicholas Piggin /* 1310fce16d48SNicholas Piggin * Machine check is different because we use a different 1311fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1312fce16d48SNicholas Piggin */ 13134f50541fSNicholas Piggin GEN_COMMON machine_check 1314c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 13154e991e3cSNicholas Piggin bl CFUNC(machine_check_exception_async) 13161df7d5e4SNicholas Piggin b interrupt_return_srr 1317fce16d48SNicholas Piggin 13189600f261SNicholas Piggin 1319fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1320fce16d48SNicholas Piggin/* 1321fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1322fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1323fce16d48SNicholas Piggin */ 1324fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 13254e991e3cSNicholas Piggin bl CFUNC(machine_check_queue_event) 1326fce16d48SNicholas Piggin 1327fce16d48SNicholas Piggin /* 13288a5054d8SNicholas Piggin * GPR-loss wakeups are relatively straightforward, because the 13298a5054d8SNicholas Piggin * idle sleep code has saved all non-volatile registers on its 13308a5054d8SNicholas Piggin * own stack, and r1 in PACAR1. 1331fce16d48SNicholas Piggin * 13328a5054d8SNicholas Piggin * For no-loss wakeups the r1 and lr registers used by the 13338a5054d8SNicholas Piggin * early machine check handler have to be restored first. r2 is 13348a5054d8SNicholas Piggin * the kernel TOC, so no need to restore it. 1335fce16d48SNicholas Piggin * 1336fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1337fce16d48SNicholas Piggin */ 1338fce16d48SNicholas Piggin ld r3,_MSR(r1) 1339fce16d48SNicholas Piggin ld r4,_LINK(r1) 13408a5054d8SNicholas Piggin ld r1,GPR1(r1) 1341fce16d48SNicholas Piggin 1342fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1343fce16d48SNicholas Piggin subi r11,r11,1 1344fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1345fce16d48SNicholas Piggin 1346fce16d48SNicholas Piggin mtlr r4 1347fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1348fce16d48SNicholas Piggin cmpwi cr1,r10,2 13498a5054d8SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 1350fce16d48SNicholas Piggin b idle_return_gpr_loss 1351fce16d48SNicholas Piggin#endif 1352fce16d48SNicholas Piggin 1353b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1354b7d9ccecSNicholas Piggin /* 1355b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1356b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1357b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1358b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1359b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1360b7d9ccecSNicholas Piggin */ 1361b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1362b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1363b7d9ccecSNicholas Piggin mtmsrd r10,1 13644e991e3cSNicholas Piggin bl CFUNC(disable_machine_check) 1365b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1366b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1367b7d9ccecSNicholas Piggin li r3,MSR_ME 1368b7d9ccecSNicholas Piggin andc r10,r10,r3 1369b7d9ccecSNicholas Piggin mtmsrd r10 1370b7d9ccecSNicholas Piggin 1371ac2a2a14SNicholas Piggin lhz r12,PACA_IN_MCE(r13) 1372ac2a2a14SNicholas Piggin subi r12,r12,1 1373ac2a2a14SNicholas Piggin sth r12,PACA_IN_MCE(r13) 1374ac2a2a14SNicholas Piggin 1375f08fb25bSNicholas Piggin /* 1376f08fb25bSNicholas Piggin * Invoke machine_check_exception to print MCE event and panic. 1377f08fb25bSNicholas Piggin * This is the NMI version of the handler because we are called from 1378f08fb25bSNicholas Piggin * the early handler which is a true NMI. 1379f08fb25bSNicholas Piggin */ 1380c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 13814e991e3cSNicholas Piggin bl CFUNC(machine_check_exception) 1382b7d9ccecSNicholas Piggin 1383afcf0095SNicholas Piggin /* 1384b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1385b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1386afcf0095SNicholas Piggin */ 1387c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 13884e991e3cSNicholas Piggin bl CFUNC(unrecoverable_exception) 1389b7d9ccecSNicholas Piggin b . 1390afcf0095SNicholas Piggin 13914f50541fSNicholas Piggin 13924f50541fSNicholas Piggin/** 139394325357SNicholas Piggin * Interrupt 0x300 - Data Storage Interrupt (DSI). 139494325357SNicholas Piggin * This is a synchronous interrupt generated due to a data access exception, 139594325357SNicholas Piggin * e.g., a load orstore which does not have a valid page table entry with 139694325357SNicholas Piggin * permissions. DAWR matches also fault here, as do RC updates, and minor misc 139794325357SNicholas Piggin * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc. 13984f50541fSNicholas Piggin * 139994325357SNicholas Piggin * Handling: 140094325357SNicholas Piggin * - Hash MMU 1401a4922f54SNicholas Piggin * Go to do_hash_fault, which attempts to fill the HPT from an entry in the 1402a4922f54SNicholas Piggin * Linux page table. Hash faults can hit in kernel mode in a fairly 140394325357SNicholas Piggin * arbitrary state (e.g., interrupts disabled, locks held) when accessing 140494325357SNicholas Piggin * "non-bolted" regions, e.g., vmalloc space. However these should always be 1405a4922f54SNicholas Piggin * backed by Linux page table entries. 14064f50541fSNicholas Piggin * 1407a4922f54SNicholas Piggin * If no entry is found the Linux page fault handler is invoked (by 1408a4922f54SNicholas Piggin * do_hash_fault). Linux page faults can happen in kernel mode due to user 1409a4922f54SNicholas Piggin * copy operations of course. 14104f50541fSNicholas Piggin * 1411cd81acc6SNicholas Piggin * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest 1412cd81acc6SNicholas Piggin * MMU context, which may cause a DSI in the host, which must go to the 1413cd81acc6SNicholas Piggin * KVM handler. MSR[IR] is not enabled, so the real-mode handler will 1414cd81acc6SNicholas Piggin * always be used regardless of AIL setting. 1415cd81acc6SNicholas Piggin * 141694325357SNicholas Piggin * - Radix MMU 141794325357SNicholas Piggin * The hardware loads from the Linux page table directly, so a fault goes 141894325357SNicholas Piggin * immediately to Linux page fault. 141994325357SNicholas Piggin * 142094325357SNicholas Piggin * Conditions like DAWR match are handled on the way in to Linux page fault. 14214f50541fSNicholas Piggin */ 1422a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1423a42a239dSNicholas Piggin IVEC=0x300 1424a42a239dSNicholas Piggin IDAR=1 1425a42a239dSNicholas Piggin IDSISR=1 1426a42a239dSNicholas Piggin IKVM_REAL=1 1427a42a239dSNicholas PigginINT_DEFINE_END(data_access) 14280ebc4cdaSBenjamin Herrenschmidt 1429e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1430689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1431e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1432e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1433a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1434e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 143580795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 14367cb3a1a0SNicholas Piggin GEN_COMMON data_access 1437a01a3f2dSNicholas Piggin ld r4,_DSISR(r1) 1438c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1439a01a3f2dSNicholas Piggin andis. r0,r4,DSISR_DABRMATCH@h 144036f01141SNicholas Piggin bne- 1f 1441387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 144280795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 14434e991e3cSNicholas Piggin bl CFUNC(do_hash_fault) 144480795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 14454e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 144680795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1447387e220aSNicholas Piggin#else 14484e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 1449387e220aSNicholas Piggin#endif 14501df7d5e4SNicholas Piggin b interrupt_return_srr 1451a4922f54SNicholas Piggin 14524e991e3cSNicholas Piggin1: bl CFUNC(do_break) 145336f01141SNicholas Piggin /* 145436f01141SNicholas Piggin * do_break() may have changed the NV GPRS while handling a breakpoint. 145536f01141SNicholas Piggin * If so, we need to restore them with their updated values. 145636f01141SNicholas Piggin */ 14571df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() 14581df7d5e4SNicholas Piggin b interrupt_return_srr 145980795e6cSNicholas Piggin 14600ebc4cdaSBenjamin Herrenschmidt 146194325357SNicholas Piggin/** 146294325357SNicholas Piggin * Interrupt 0x380 - Data Segment Interrupt (DSLB). 146394325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault missing SLB 146494325357SNicholas Piggin * entry for HPT, or an address outside RPT translation range. 146594325357SNicholas Piggin * 146694325357SNicholas Piggin * Handling: 146794325357SNicholas Piggin * - HPT: 146894325357SNicholas Piggin * This refills the SLB, or reports an access fault similarly to a bad page 146994325357SNicholas Piggin * fault. When coming from user-mode, the SLB handler may access any kernel 147094325357SNicholas Piggin * data, though it may itself take a DSLB. When coming from kernel mode, 147194325357SNicholas Piggin * recursive faults must be avoided so access is restricted to the kernel 147294325357SNicholas Piggin * image text/data, kernel stack, and any data allocated below 147394325357SNicholas Piggin * ppc64_bolted_size (first segment). The kernel handler must avoid stomping 147494325357SNicholas Piggin * on user-handler data structures. 147594325357SNicholas Piggin * 1476cd81acc6SNicholas Piggin * KVM: Same as 0x300, DSLB must test for KVM guest. 147794325357SNicholas Piggin */ 14784f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 14794f50541fSNicholas Piggin IVEC=0x380 14804f50541fSNicholas Piggin IDAR=1 14814f50541fSNicholas Piggin IKVM_REAL=1 14824f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 14834f50541fSNicholas Piggin 14841a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1485689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 14861a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 14871a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 14884f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 14891a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 149048e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 14914f50541fSNicholas Piggin GEN_COMMON data_access_slb 1492387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 14937100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 14947100e870SNicholas Piggin /* HPT case, do SLB fault */ 1495c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 14964e991e3cSNicholas Piggin bl CFUNC(do_slb_fault) 149748e7b769SNicholas Piggin cmpdi r3,0 149848e7b769SNicholas Piggin bne- 1f 14991df7d5e4SNicholas Piggin b fast_interrupt_return_srr 150048e7b769SNicholas Piggin1: /* Error case */ 15017100e870SNicholas PigginMMU_FTR_SECTION_ELSE 15027100e870SNicholas Piggin /* Radix case, access is outside page table range */ 15037100e870SNicholas Piggin li r3,-EFAULT 15047100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1505387e220aSNicholas Piggin#else 1506387e220aSNicholas Piggin li r3,-EFAULT 1507387e220aSNicholas Piggin#endif 150848e7b769SNicholas Piggin std r3,RESULT(r1) 1509c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 15104e991e3cSNicholas Piggin bl CFUNC(do_bad_segment_interrupt) 15111df7d5e4SNicholas Piggin b interrupt_return_srr 151248e7b769SNicholas Piggin 15132b9af6e4SNicholas Piggin 151494325357SNicholas Piggin/** 151594325357SNicholas Piggin * Interrupt 0x400 - Instruction Storage Interrupt (ISI). 151694325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 151794325357SNicholas Piggin * instruction fetch. 151894325357SNicholas Piggin * 151994325357SNicholas Piggin * Handling: 152094325357SNicholas Piggin * Similar to DSI, though in response to fetch. The faulting address is found 152194325357SNicholas Piggin * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR). 152294325357SNicholas Piggin */ 15234f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 15244f50541fSNicholas Piggin IVEC=0x400 1525a3cd35beSNicholas Piggin IISIDE=1 1526a3cd35beSNicholas Piggin IDAR=1 1527a3cd35beSNicholas Piggin IDSISR=1 15282284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15294f50541fSNicholas Piggin IKVM_REAL=1 15302284ffeaSNicholas Piggin#endif 15314f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 15324f50541fSNicholas Piggin 15337299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 15344f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 15357299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 15367299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 15374f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 15387299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 153927ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 15404f50541fSNicholas Piggin GEN_COMMON instruction_access 1541c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1542387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 154327ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 15444e991e3cSNicholas Piggin bl CFUNC(do_hash_fault) 154527ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 15464e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 154727ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1548387e220aSNicholas Piggin#else 15494e991e3cSNicholas Piggin bl CFUNC(do_page_fault) 1550387e220aSNicholas Piggin#endif 15511df7d5e4SNicholas Piggin b interrupt_return_srr 155227ce77dfSNicholas Piggin 15530ebc4cdaSBenjamin Herrenschmidt 155494325357SNicholas Piggin/** 155594325357SNicholas Piggin * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). 155694325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault due to an 155794325357SNicholas Piggin * instruction fetch. 155894325357SNicholas Piggin * 155994325357SNicholas Piggin * Handling: 156094325357SNicholas Piggin * Similar to DSLB, though in response to fetch. The faulting address is found 156194325357SNicholas Piggin * in SRR0 (rather than DAR). 156294325357SNicholas Piggin */ 15634f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 15644f50541fSNicholas Piggin IVEC=0x480 1565a3cd35beSNicholas Piggin IISIDE=1 1566a3cd35beSNicholas Piggin IDAR=1 15672284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 15684f50541fSNicholas Piggin IKVM_REAL=1 15692284ffeaSNicholas Piggin#endif 15704f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 15714f50541fSNicholas Piggin 15727299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 15734f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 15747299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 15757299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 15764f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 15777299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 157848e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 15794f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 1580387e220aSNicholas Piggin#ifdef CONFIG_PPC_64S_HASH_MMU 15817100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 15827100e870SNicholas Piggin /* HPT case, do SLB fault */ 1583c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 15844e991e3cSNicholas Piggin bl CFUNC(do_slb_fault) 158548e7b769SNicholas Piggin cmpdi r3,0 158648e7b769SNicholas Piggin bne- 1f 15871df7d5e4SNicholas Piggin b fast_interrupt_return_srr 158848e7b769SNicholas Piggin1: /* Error case */ 15897100e870SNicholas PigginMMU_FTR_SECTION_ELSE 15907100e870SNicholas Piggin /* Radix case, access is outside page table range */ 15917100e870SNicholas Piggin li r3,-EFAULT 15927100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 1593387e220aSNicholas Piggin#else 1594387e220aSNicholas Piggin li r3,-EFAULT 1595387e220aSNicholas Piggin#endif 159648e7b769SNicholas Piggin std r3,RESULT(r1) 1597c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 15984e991e3cSNicholas Piggin bl CFUNC(do_bad_segment_interrupt) 15991df7d5e4SNicholas Piggin b interrupt_return_srr 16005e46e29eSNicholas Piggin 16019600f261SNicholas Piggin 160294325357SNicholas Piggin/** 160394325357SNicholas Piggin * Interrupt 0x500 - External Interrupt. 160494325357SNicholas Piggin * This is an asynchronous maskable interrupt in response to an "external 160594325357SNicholas Piggin * exception" from the interrupt controller or hypervisor (e.g., device 160694325357SNicholas Piggin * interrupt). It is maskable in hardware by clearing MSR[EE], and 160794325357SNicholas Piggin * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()). 160894325357SNicholas Piggin * 160994325357SNicholas Piggin * When running in HV mode, Linux sets up the LPCR[LPES] bit such that 161094325357SNicholas Piggin * interrupts are delivered with HSRR registers, guests use SRRs, which 161194325357SNicholas Piggin * reqiures IHSRR_IF_HVMODE. 161294325357SNicholas Piggin * 161394325357SNicholas Piggin * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that 161494325357SNicholas Piggin * external interrupts are delivered as Hypervisor Virtualization Interrupts 161594325357SNicholas Piggin * rather than External Interrupts. 161694325357SNicholas Piggin * 161794325357SNicholas Piggin * Handling: 161894325357SNicholas Piggin * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead, 161994325357SNicholas Piggin * because registers at the time of the interrupt are not so important as it is 162094325357SNicholas Piggin * asynchronous. 162194325357SNicholas Piggin * 162294325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 162394325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 1624af47d79bSNicholas Piggin * 1625af47d79bSNicholas Piggin * CFAR is not required because this is an asynchronous interrupt that in 1626af47d79bSNicholas Piggin * general won't have much bearing on the state of the CPU, with the possible 1627af47d79bSNicholas Piggin * exception of crash/debug IPIs, but those are generally moving to use SRESET 1628af47d79bSNicholas Piggin * IPIs. Unless this is an HV interrupt and KVM HV is possible, in which case 1629af47d79bSNicholas Piggin * it may be exiting the guest and need CFAR to be saved. 163094325357SNicholas Piggin */ 16314f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 16324f50541fSNicholas Piggin IVEC=0x500 16333f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 16344f50541fSNicholas Piggin IMASK=IRQS_DISABLED 16354f50541fSNicholas Piggin IKVM_REAL=1 16364f50541fSNicholas Piggin IKVM_VIRT=1 1637af47d79bSNicholas Piggin ICFAR=0 1638af47d79bSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1639af47d79bSNicholas Piggin ICFAR_IF_HVMODE=1 1640af47d79bSNicholas Piggin#endif 16414f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 16424f50541fSNicholas Piggin 16431a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 16444f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 16451a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 16461a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 16474f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 16481a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1649eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 16504f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1651c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 16524e991e3cSNicholas Piggin bl CFUNC(do_IRQ) 16531df7d5e4SNicholas Piggin BEGIN_FTR_SECTION 16541df7d5e4SNicholas Piggin b interrupt_return_hsrr 16551df7d5e4SNicholas Piggin FTR_SECTION_ELSE 16561df7d5e4SNicholas Piggin b interrupt_return_srr 16571df7d5e4SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1658c138e588SNicholas Piggin 1659c138e588SNicholas Piggin 166094325357SNicholas Piggin/** 166194325357SNicholas Piggin * Interrupt 0x600 - Alignment Interrupt 166294325357SNicholas Piggin * This is a synchronous interrupt in response to data alignment fault. 166394325357SNicholas Piggin */ 16644f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 16654f50541fSNicholas Piggin IVEC=0x600 16664f50541fSNicholas Piggin IDAR=1 16674f50541fSNicholas Piggin IDSISR=1 16682284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16694f50541fSNicholas Piggin IKVM_REAL=1 16702284ffeaSNicholas Piggin#endif 16714f50541fSNicholas PigginINT_DEFINE_END(alignment) 16724f50541fSNicholas Piggin 1673e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 16744f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1675e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1676e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 16774f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1678e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1679f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 16804f50541fSNicholas Piggin GEN_COMMON alignment 1681c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 16824e991e3cSNicholas Piggin bl CFUNC(alignment_exception) 16831df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 16841df7d5e4SNicholas Piggin b interrupt_return_srr 1685f9aa6714SNicholas Piggin 1686b01c8b54SPaul Mackerras 168794325357SNicholas Piggin/** 168894325357SNicholas Piggin * Interrupt 0x700 - Program Interrupt (program check). 168994325357SNicholas Piggin * This is a synchronous interrupt in response to various instruction faults: 169094325357SNicholas Piggin * traps, privilege errors, TM errors, floating point exceptions. 169194325357SNicholas Piggin * 169294325357SNicholas Piggin * Handling: 169394325357SNicholas Piggin * This interrupt may use the "emergency stack" in some cases when being taken 169494325357SNicholas Piggin * from kernel context, which complicates handling. 169594325357SNicholas Piggin */ 16964f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 16974f50541fSNicholas Piggin IVEC=0x700 16982284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 16994f50541fSNicholas Piggin IKVM_REAL=1 17002284ffeaSNicholas Piggin#endif 17014f50541fSNicholas PigginINT_DEFINE_END(program_check) 17024f50541fSNicholas Piggin 17037299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 1704bf75a325SNicholas Piggin EARLY_BOOT_FIXUP 17054f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 17067299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 17077299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 17084f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 17097299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 171011e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 17118729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 17128729c26eSNicholas Piggin 1713265e60a1SCyril Bur /* 1714265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1715265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1716265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1717265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1718265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1719265e60a1SCyril Bur * the kernel. 1720265e60a1SCyril Bur */ 1721265e60a1SCyril Bur 17220a882e28SNicholas Piggin andi. r10,r12,MSR_PR 17233e607dc4SNicholas Piggin bne .Lnormal_stack /* If userspace, go normal path */ 17240a882e28SNicholas Piggin 17250a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 17263e607dc4SNicholas Piggin bne .Lemergency_stack /* If TM, emergency */ 17270a882e28SNicholas Piggin 17280a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 17293e607dc4SNicholas Piggin blt .Lnormal_stack /* normal path if not */ 17300a882e28SNicholas Piggin 17310a882e28SNicholas Piggin /* Use the emergency stack */ 17323e607dc4SNicholas Piggin.Lemergency_stack: 17333e607dc4SNicholas Piggin andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1734265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1735265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1736265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1737265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 17384f50541fSNicholas Piggin __ISTACK(program_check)=0 17398729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 17403e607dc4SNicholas Piggin b .Ldo_program_check 17413e607dc4SNicholas Piggin 17423e607dc4SNicholas Piggin.Lnormal_stack: 17434f50541fSNicholas Piggin __ISTACK(program_check)=1 17448729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 17453e607dc4SNicholas Piggin 17463e607dc4SNicholas Piggin.Ldo_program_check: 1747c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 17484e991e3cSNicholas Piggin bl CFUNC(program_check_exception) 17491df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 17501df7d5e4SNicholas Piggin b interrupt_return_srr 175111e87346SNicholas Piggin 1752a485c709SPaul Mackerras 175394325357SNicholas Piggin/* 175494325357SNicholas Piggin * Interrupt 0x800 - Floating-Point Unavailable Interrupt. 175594325357SNicholas Piggin * This is a synchronous interrupt in response to executing an fp instruction 175694325357SNicholas Piggin * with MSR[FP]=0. 175794325357SNicholas Piggin * 175894325357SNicholas Piggin * Handling: 175994325357SNicholas Piggin * This will load FP registers and enable the FP bit if coming from userspace, 176094325357SNicholas Piggin * otherwise report a bad kernel use of FP. 176194325357SNicholas Piggin */ 17624f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 17634f50541fSNicholas Piggin IVEC=0x800 17642284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 17654f50541fSNicholas Piggin IKVM_REAL=1 17662284ffeaSNicholas Piggin#endif 17672487fd2eSRohan McLure IMSR_R12=1 17684f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 17694f50541fSNicholas Piggin 17707299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 17714f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 17727299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 17737299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 17744f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 17757299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1776c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 17774f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1778c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1779c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 17804e991e3cSNicholas Piggin bl CFUNC(kernel_fp_unavailable_exception) 178163ce271bSChristophe Leroy0: trap 178263ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1783c78d9b97SNicholas Piggin1: 1784c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1785c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1786c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1787c78d9b97SNicholas Piggin * transaction), go do TM stuff 1788c78d9b97SNicholas Piggin */ 1789c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1790c78d9b97SNicholas Piggin bne- 2f 1791c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1792c78d9b97SNicholas Piggin#endif 17934e991e3cSNicholas Piggin bl CFUNC(load_up_fpu) 17941df7d5e4SNicholas Piggin b fast_interrupt_return_srr 1795c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1796c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1797c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 17984e991e3cSNicholas Piggin bl CFUNC(fp_unavailable_tm) 17991df7d5e4SNicholas Piggin b interrupt_return_srr 1800c78d9b97SNicholas Piggin#endif 1801c78d9b97SNicholas Piggin 1802b01c8b54SPaul Mackerras 180394325357SNicholas Piggin/** 180494325357SNicholas Piggin * Interrupt 0x900 - Decrementer Interrupt. 180594325357SNicholas Piggin * This is an asynchronous interrupt in response to a decrementer exception 180694325357SNicholas Piggin * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing 180794325357SNicholas Piggin * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e., 180894325357SNicholas Piggin * local_irq_disable()). 180994325357SNicholas Piggin * 181094325357SNicholas Piggin * Handling: 181194325357SNicholas Piggin * This calls into Linux timer handler. NVGPRs are not saved (see 0x500). 181294325357SNicholas Piggin * 181394325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 181494325357SNicholas Piggin * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled 181594325357SNicholas Piggin * in the interrupted context. 181694325357SNicholas Piggin * If PPC_WATCHDOG is configured, the soft masked handler will actually set 181794325357SNicholas Piggin * things back up to run soft_nmi_interrupt as a regular interrupt handler 181894325357SNicholas Piggin * on the emergency stack. 1819af47d79bSNicholas Piggin * 1820af47d79bSNicholas Piggin * CFAR is not required because this is asynchronous (see hardware_interrupt). 1821af47d79bSNicholas Piggin * A watchdog interrupt may like to have CFAR, but usually the interesting 1822af47d79bSNicholas Piggin * branch is long gone by that point (e.g., infinite loop). 182394325357SNicholas Piggin */ 18244f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 18254f50541fSNicholas Piggin IVEC=0x900 18264f50541fSNicholas Piggin IMASK=IRQS_DISABLED 18272284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 18284f50541fSNicholas Piggin IKVM_REAL=1 18292284ffeaSNicholas Piggin#endif 1830af47d79bSNicholas Piggin ICFAR=0 18314f50541fSNicholas PigginINT_DEFINE_END(decrementer) 18324f50541fSNicholas Piggin 18337299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1834689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 18357299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 18367299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 18374f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 18387299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1839eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 18404f50541fSNicholas Piggin GEN_COMMON decrementer 1841c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 18424e991e3cSNicholas Piggin bl CFUNC(timer_interrupt) 18431df7d5e4SNicholas Piggin b interrupt_return_srr 184439c0da57SNicholas Piggin 18450ebc4cdaSBenjamin Herrenschmidt 184694325357SNicholas Piggin/** 184794325357SNicholas Piggin * Interrupt 0x980 - Hypervisor Decrementer Interrupt. 184894325357SNicholas Piggin * This is an asynchronous interrupt, similar to 0x900 but for the HDEC 184994325357SNicholas Piggin * register. 185094325357SNicholas Piggin * 185194325357SNicholas Piggin * Handling: 185294325357SNicholas Piggin * Linux does not use this outside KVM where it's used to keep a host timer 185394325357SNicholas Piggin * while the guest is given control of DEC. It should normally be caught by 185494325357SNicholas Piggin * the KVM test and routed there. 185594325357SNicholas Piggin */ 18564f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 18574f50541fSNicholas Piggin IVEC=0x980 18583f7fbd97SNicholas Piggin IHSRR=1 18592babd6eaSNicholas Piggin ISTACK=0 18604f50541fSNicholas Piggin IKVM_REAL=1 18614f50541fSNicholas Piggin IKVM_VIRT=1 18624f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 18634f50541fSNicholas Piggin 18647299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 18654f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 18667299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 18677299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 18684f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 18697299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1870eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 18712babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 18722babd6eaSNicholas Piggin /* 18732babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 18742babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 18752babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 18762babd6eaSNicholas Piggin * edge-triggered. 18772babd6eaSNicholas Piggin * 18782babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 18792babd6eaSNicholas Piggin */ 188059dc5bfcSNicholas Piggin li r10,0 188159dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 18822babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 18832babd6eaSNicholas Piggin mtctr r10 18842babd6eaSNicholas Piggin mtcrf 0x80,r9 18852babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 18862babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 18872babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 18882babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 18892babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 18902babd6eaSNicholas Piggin HRFI_TO_KERNEL 1891facc6d74SNicholas Piggin 1892da2bc464SMichael Ellerman 189394325357SNicholas Piggin/** 189494325357SNicholas Piggin * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. 189594325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsndp doorbell. 189694325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 189794325357SNicholas Piggin * IRQS_DISABLED mask (i.e., local_irq_disable()). 189894325357SNicholas Piggin * 189994325357SNicholas Piggin * Handling: 190094325357SNicholas Piggin * Guests may use this for IPIs between threads in a core if the 190194325357SNicholas Piggin * hypervisor supports it. NVGPRS are not saved (see 0x500). 190294325357SNicholas Piggin * 190394325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 190494325357SNicholas Piggin * replay, leaving MSR[EE] enabled in the interrupted context because the 190594325357SNicholas Piggin * doorbells are edge triggered. 1906af47d79bSNicholas Piggin * 1907af47d79bSNicholas Piggin * CFAR is not required, similarly to hardware_interrupt. 190894325357SNicholas Piggin */ 19094f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 19104f50541fSNicholas Piggin IVEC=0xa00 19114f50541fSNicholas Piggin IMASK=IRQS_DISABLED 19122284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 19134f50541fSNicholas Piggin IKVM_REAL=1 19142284ffeaSNicholas Piggin#endif 1915af47d79bSNicholas Piggin ICFAR=0 19164f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 19174f50541fSNicholas Piggin 19187299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 19194f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 19207299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 19217299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 19224f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 19237299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1924eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 19254f50541fSNicholas Piggin GEN_COMMON doorbell_super 1926c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 1927ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 19284e991e3cSNicholas Piggin bl CFUNC(doorbell_exception) 1929ca243163SNicholas Piggin#else 19304e991e3cSNicholas Piggin bl CFUNC(unknown_async_exception) 1931ca243163SNicholas Piggin#endif 19321df7d5e4SNicholas Piggin b interrupt_return_srr 1933ca243163SNicholas Piggin 1934da2bc464SMichael Ellerman 19355ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 19365ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1937341215dcSNicholas Piggin 193894325357SNicholas Piggin/** 193994325357SNicholas Piggin * Interrupt 0xc00 - System Call Interrupt (syscall, hcall). 194094325357SNicholas Piggin * This is a synchronous interrupt invoked with the "sc" instruction. The 194194325357SNicholas Piggin * system call is invoked with "sc 0" and does not alter the HV bit, so it 194294325357SNicholas Piggin * is directed to the currently running OS. The hypercall is invoked with 194394325357SNicholas Piggin * "sc 1" and it sets HV=1, so it elevates to hypervisor. 1944acd7d8ceSNicholas Piggin * 1945acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1946acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1947acd7d8ceSNicholas Piggin * 194894325357SNicholas Piggin * Handling: 194994325357SNicholas Piggin * If the KVM test fires then it was due to a hypercall and is accordingly 195094325357SNicholas Piggin * routed to KVM. Otherwise this executes a normal Linux system call. 195194325357SNicholas Piggin * 1952acd7d8ceSNicholas Piggin * Call convention: 1953acd7d8ceSNicholas Piggin * 195458b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 1955*17e02586SCosta Shulyupin * Documentation/arch/powerpc/syscall64-abi.rst and 1956*17e02586SCosta Shulyupin * Documentation/arch/powerpc/papr_hcalls.rst respectively. 1957acd7d8ceSNicholas Piggin * 1958acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 195976fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 196076fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 196176fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1962acd7d8ceSNicholas Piggin */ 1963b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1964b177ae2fSNicholas Piggin IVEC=0xc00 1965b177ae2fSNicholas Piggin IKVM_REAL=1 1966b177ae2fSNicholas Piggin IKVM_VIRT=1 1967af47d79bSNicholas Piggin ICFAR=0 1968b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1969b177ae2fSNicholas Piggin 19701b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1971bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1972bc355125SPaul Mackerras /* 1973acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 197476fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 197576fc0cfcSNicholas Piggin * register. 1976acd7d8ceSNicholas Piggin * 1977acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1978acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1979bc355125SPaul Mackerras */ 19801b4d4a79SNicholas Piggin mtctr r13 19811b4d4a79SNicholas Piggin GET_PACA(r13) 19821b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 19831b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 198469fdd674SNicholas Piggin KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */ 19851b4d4a79SNicholas Piggin mfctr r9 1986bc355125SPaul Mackerras#else 19871b4d4a79SNicholas Piggin mr r9,r13 19881b4d4a79SNicholas Piggin GET_PACA(r13) 19891b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1990bc355125SPaul Mackerras#endif 1991bc355125SPaul Mackerras 1992b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 19931b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 19941b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 1995b0b2a93dSNicholas Piggin 1996b0b2a93dSNicholas Piggin HMT_MEDIUM 1997b0b2a93dSNicholas Piggin 1998b0b2a93dSNicholas Piggin .if ! \virt 1999d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_common_real, real_vectors) 200014ad0e7dSNicholas Piggin mtctr r10 200114ad0e7dSNicholas Piggin bctr 20021b4d4a79SNicholas Piggin .else 2003b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 2004d72c4a36SDaniel Axtens __LOAD_HANDLER(r10, system_call_common, virt_vectors) 2005b0b2a93dSNicholas Piggin mtctr r10 2006b0b2a93dSNicholas Piggin bctr 2007b0b2a93dSNicholas Piggin#else 20081b4d4a79SNicholas Piggin b system_call_common 2009d807ad37SNicholas Piggin#endif 20101b4d4a79SNicholas Piggin .endif 20111b4d4a79SNicholas Piggin.endm 2012d807ad37SNicholas Piggin 20131a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 20141b4d4a79SNicholas Piggin SYSTEM_CALL 0 20151a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 20161a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 20171b4d4a79SNicholas Piggin SYSTEM_CALL 1 20181a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 2019d807ad37SNicholas Piggin 2020acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 202169fdd674SNicholas PigginTRAMP_REAL_BEGIN(kvm_hcall) 2022e2762743SNicholas Piggin std r9,PACA_EXGEN+EX_R9(r13) 2023e2762743SNicholas Piggin std r11,PACA_EXGEN+EX_R11(r13) 2024e2762743SNicholas Piggin std r12,PACA_EXGEN+EX_R12(r13) 2025e2762743SNicholas Piggin mfcr r9 2026acd7d8ceSNicholas Piggin mfctr r10 2027e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_R13(r13) 2028e2762743SNicholas Piggin li r10,0 2029e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CFAR(r13) 2030e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_CTR(r13) 2031e2762743SNicholas Piggin /* 2032e2762743SNicholas Piggin * Save the PPR (on systems that support it) before changing to 2033e2762743SNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 2034e2762743SNicholas Piggin * guest state (it is the guest's PPR value). 2035e2762743SNicholas Piggin */ 2036e2762743SNicholas PigginBEGIN_FTR_SECTION 2037e2762743SNicholas Piggin mfspr r10,SPRN_PPR 2038e2762743SNicholas Piggin std r10,PACA_EXGEN+EX_PPR(r13) 2039e2762743SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2040e2762743SNicholas Piggin 2041e2762743SNicholas Piggin HMT_MEDIUM 2042e2762743SNicholas Piggin 20439600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 20449600f261SNicholas Piggin /* 204531c67cfeSNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives 20469600f261SNicholas Piggin * outside the head section. 20479600f261SNicholas Piggin */ 2048d72c4a36SDaniel Axtens __LOAD_FAR_HANDLER(r10, kvmppc_hcall, real_trampolines) 20499600f261SNicholas Piggin mtctr r10 20509600f261SNicholas Piggin bctr 20519600f261SNicholas Piggin#else 205231c67cfeSNicholas Piggin b kvmppc_hcall 20539600f261SNicholas Piggin#endif 2054acd7d8ceSNicholas Piggin#endif 2055da2bc464SMichael Ellerman 205694325357SNicholas Piggin/** 205794325357SNicholas Piggin * Interrupt 0xd00 - Trace Interrupt. 205894325357SNicholas Piggin * This is a synchronous interrupt in response to instruction step or 205994325357SNicholas Piggin * breakpoint faults. 206094325357SNicholas Piggin */ 20614f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 20624f50541fSNicholas Piggin IVEC=0xd00 20632284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 20644f50541fSNicholas Piggin IKVM_REAL=1 20652284ffeaSNicholas Piggin#endif 20664f50541fSNicholas PigginINT_DEFINE_END(single_step) 20674f50541fSNicholas Piggin 20687299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 20694f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 20707299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 20717299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 20724f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 20737299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 2074eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 20754f50541fSNicholas Piggin GEN_COMMON single_step 2076c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 20774e991e3cSNicholas Piggin bl CFUNC(single_step_exception) 20781df7d5e4SNicholas Piggin b interrupt_return_srr 2079da2bc464SMichael Ellerman 20807299417cSNicholas Piggin 208194325357SNicholas Piggin/** 208294325357SNicholas Piggin * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). 208394325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 208494325357SNicholas Piggin * guest data access. 208594325357SNicholas Piggin * 208694325357SNicholas Piggin * Handling: 208794325357SNicholas Piggin * This should always get routed to KVM. In radix MMU mode, this is caused 208894325357SNicholas Piggin * by a guest nested radix access that can't be performed due to the 208994325357SNicholas Piggin * partition scope page table. In hash mode, this can be caused by guests 209094325357SNicholas Piggin * running with translation disabled (virtual real mode) or with VPM enabled. 209194325357SNicholas Piggin * KVM will update the page table structures or disallow the access. 209294325357SNicholas Piggin */ 20934f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 20944f50541fSNicholas Piggin IVEC=0xe00 20953f7fbd97SNicholas Piggin IHSRR=1 20964f50541fSNicholas Piggin IDAR=1 20974f50541fSNicholas Piggin IDSISR=1 20984f50541fSNicholas Piggin IKVM_REAL=1 20994f50541fSNicholas Piggin IKVM_VIRT=1 21004f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 21014f50541fSNicholas Piggin 21027299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 21034f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 21047299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 21057299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 21064f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 21077299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 2108f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 21094f50541fSNicholas Piggin GEN_COMMON h_data_storage 2110c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2111d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 21124e991e3cSNicholas Piggin bl CFUNC(do_bad_page_fault_segv) 2113d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 21144e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 2115d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 21161df7d5e4SNicholas Piggin b interrupt_return_hsrr 2117f5c32c1dSNicholas Piggin 21181707dd16SPaul Mackerras 211994325357SNicholas Piggin/** 212094325357SNicholas Piggin * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). 212194325357SNicholas Piggin * This is a synchronous interrupt in response to an MMU fault caused by a 212294325357SNicholas Piggin * guest instruction fetch, similar to HDSI. 212394325357SNicholas Piggin */ 21244f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 21254f50541fSNicholas Piggin IVEC=0xe20 21263f7fbd97SNicholas Piggin IHSRR=1 21274f50541fSNicholas Piggin IKVM_REAL=1 21284f50541fSNicholas Piggin IKVM_VIRT=1 21294f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 21304f50541fSNicholas Piggin 21317299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 21324f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 21337299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 21347299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 21354f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 21367299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 2137eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 21384f50541fSNicholas Piggin GEN_COMMON h_instr_storage 2139c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 21404e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 21411df7d5e4SNicholas Piggin b interrupt_return_hsrr 214282517cabSNicholas Piggin 21431707dd16SPaul Mackerras 214494325357SNicholas Piggin/** 214594325357SNicholas Piggin * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. 214694325357SNicholas Piggin */ 21474f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 21484f50541fSNicholas Piggin IVEC=0xe40 21493f7fbd97SNicholas Piggin IHSRR=1 21504f50541fSNicholas Piggin IKVM_REAL=1 21514f50541fSNicholas Piggin IKVM_VIRT=1 21524f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 21534f50541fSNicholas Piggin 21547299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 21554f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 21567299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 21577299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 21584f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 21597299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 2160eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 21614f50541fSNicholas Piggin GEN_COMMON emulation_assist 2162c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 21634e991e3cSNicholas Piggin bl CFUNC(emulation_assist_interrupt) 21641df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 21651df7d5e4SNicholas Piggin b interrupt_return_hsrr 2166031b4026SNicholas Piggin 21671707dd16SPaul Mackerras 216894325357SNicholas Piggin/** 216994325357SNicholas Piggin * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). 217094325357SNicholas Piggin * This is an asynchronous interrupt caused by a Hypervisor Maintenance 217194325357SNicholas Piggin * Exception. It is always taken in real mode but uses HSRR registers 217294325357SNicholas Piggin * unlike SRESET and MCE. 217394325357SNicholas Piggin * 217494325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable 217594325357SNicholas Piggin * with IRQS_DISABLED mask (i.e., local_irq_disable()). 217694325357SNicholas Piggin * 217794325357SNicholas Piggin * Handling: 217894325357SNicholas Piggin * This is a special case, this is handled similarly to machine checks, with an 217994325357SNicholas Piggin * initial real mode handler that is not soft-masked, which attempts to fix the 218094325357SNicholas Piggin * problem. Then a regular handler which is soft-maskable and reports the 218194325357SNicholas Piggin * problem. 218294325357SNicholas Piggin * 218394325357SNicholas Piggin * The emergency stack is used for the early real mode handler. 218494325357SNicholas Piggin * 218594325357SNicholas Piggin * XXX: unclear why MCE and HMI schemes could not be made common, e.g., 218694325357SNicholas Piggin * either use soft-masking for the MCE, or use irq_work for the HMI. 218794325357SNicholas Piggin * 218894325357SNicholas Piggin * KVM: 218994325357SNicholas Piggin * Unlike MCE, this calls into KVM without calling the real mode handler 219094325357SNicholas Piggin * first. 2191e0319829SNicholas Piggin */ 21924f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 21934f50541fSNicholas Piggin IVEC=0xe60 21943f7fbd97SNicholas Piggin IHSRR=1 2195d73a10cbSNicholas Piggin IREALMODE_COMMON=1 21964f50541fSNicholas Piggin ISTACK=0 21974f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 21984f50541fSNicholas Piggin IKVM_REAL=1 21994f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 22004f50541fSNicholas Piggin 22014f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 22024f50541fSNicholas Piggin IVEC=0xe60 22033f7fbd97SNicholas Piggin IHSRR=1 22044f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22054f50541fSNicholas Piggin IKVM_REAL=1 22064f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 22074f50541fSNicholas Piggin 2208f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 22094f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 2210f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 22111a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 22124f50541fSNicholas Piggin 2213293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 22149600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 22159600f261SNicholas Piggin 221662f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 2217a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 221862f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 2219bcbceed4SNicholas Piggin 22208729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 2221bcbceed4SNicholas Piggin 2222c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 22234e991e3cSNicholas Piggin bl CFUNC(hmi_exception_realmode) 22245080332cSMichael Neuling cmpdi cr0,r3,0 222567d4160aSNicholas Piggin bne 1f 22265080332cSMichael Neuling 22273f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 2228222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 22295080332cSMichael Neuling 223067d4160aSNicholas Piggin1: 223162f9b03bSNicholas Piggin /* 223262f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 223362f9b03bSNicholas Piggin * firmware. 223462f9b03bSNicholas Piggin */ 22353f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 22364f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 223762f9b03bSNicholas Piggin 22385080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 22394f50541fSNicholas Piggin GEN_COMMON hmi_exception 2240c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 22414e991e3cSNicholas Piggin bl CFUNC(handle_hmi_exception) 22421df7d5e4SNicholas Piggin b interrupt_return_hsrr 22431707dd16SPaul Mackerras 22447299417cSNicholas Piggin 224594325357SNicholas Piggin/** 224694325357SNicholas Piggin * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. 224794325357SNicholas Piggin * This is an asynchronous interrupt in response to a msgsnd doorbell. 224894325357SNicholas Piggin * Similar to the 0xa00 doorbell but for host rather than guest. 2249af47d79bSNicholas Piggin * 2250af47d79bSNicholas Piggin * CFAR is not required (similar to doorbell_interrupt), unless KVM HV 2251af47d79bSNicholas Piggin * is enabled, in which case it may be a guest exit. Most PowerNV kernels 2252af47d79bSNicholas Piggin * include KVM support so it would be nice if this could be dynamically 2253af47d79bSNicholas Piggin * patched out if KVM was not currently running any guests. 225494325357SNicholas Piggin */ 22554f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 22564f50541fSNicholas Piggin IVEC=0xe80 22573f7fbd97SNicholas Piggin IHSRR=1 22584f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22594f50541fSNicholas Piggin IKVM_REAL=1 22604f50541fSNicholas Piggin IKVM_VIRT=1 2261af47d79bSNicholas Piggin#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2262af47d79bSNicholas Piggin ICFAR=0 2263af47d79bSNicholas Piggin#endif 22644f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 22654f50541fSNicholas Piggin 22667299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 22674f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 22687299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 22697299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 22704f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 22717299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2272eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 22734f50541fSNicholas Piggin GEN_COMMON h_doorbell 2274c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 22759bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 22764e991e3cSNicholas Piggin bl CFUNC(doorbell_exception) 22779bcb81bfSNicholas Piggin#else 22784e991e3cSNicholas Piggin bl CFUNC(unknown_async_exception) 22799bcb81bfSNicholas Piggin#endif 22801df7d5e4SNicholas Piggin b interrupt_return_hsrr 22819bcb81bfSNicholas Piggin 22820ebc4cdaSBenjamin Herrenschmidt 228394325357SNicholas Piggin/** 228494325357SNicholas Piggin * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. 228594325357SNicholas Piggin * This is an asynchronous interrupt in response to an "external exception". 228694325357SNicholas Piggin * Similar to 0x500 but for host only. 2287af47d79bSNicholas Piggin * 2288af47d79bSNicholas Piggin * Like h_doorbell, CFAR is only required for KVM HV because this can be 2289af47d79bSNicholas Piggin * a guest exit. 229094325357SNicholas Piggin */ 22914f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 22924f50541fSNicholas Piggin IVEC=0xea0 22933f7fbd97SNicholas Piggin IHSRR=1 22944f50541fSNicholas Piggin IMASK=IRQS_DISABLED 22954f50541fSNicholas Piggin IKVM_REAL=1 22964f50541fSNicholas Piggin IKVM_VIRT=1 2297af47d79bSNicholas Piggin#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 2298af47d79bSNicholas Piggin ICFAR=0 2299af47d79bSNicholas Piggin#endif 23004f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 23014f50541fSNicholas Piggin 23027299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 23034f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 23047299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 23057299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 23064f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 23077299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2308eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 23094f50541fSNicholas Piggin GEN_COMMON h_virt_irq 2310c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 23114e991e3cSNicholas Piggin bl CFUNC(do_IRQ) 23121df7d5e4SNicholas Piggin b interrupt_return_hsrr 231374408776SNicholas Piggin 23149baaef0aSBenjamin Herrenschmidt 23151a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 23161a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 23171a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 23181a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 2319bda7fea2SNicholas Piggin 23200ebc4cdaSBenjamin Herrenschmidt 232194325357SNicholas Piggin/* 232294325357SNicholas Piggin * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU). 232394325357SNicholas Piggin * This is an asynchronous interrupt in response to a PMU exception. 232494325357SNicholas Piggin * It is maskable in hardware by clearing MSR[EE], and soft-maskable with 232594325357SNicholas Piggin * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()). 232694325357SNicholas Piggin * 232794325357SNicholas Piggin * Handling: 232894325357SNicholas Piggin * This calls into the perf subsystem. 232994325357SNicholas Piggin * 233094325357SNicholas Piggin * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it 233194325357SNicholas Piggin * runs under local_irq_disable. However it may be soft-masked in 233294325357SNicholas Piggin * powerpc-specific code. 233394325357SNicholas Piggin * 233494325357SNicholas Piggin * If soft masked, the masked handler will note the pending interrupt for 233594325357SNicholas Piggin * replay, and clear MSR[EE] in the interrupted context. 2336af47d79bSNicholas Piggin * 2337af47d79bSNicholas Piggin * CFAR is not used by perf interrupts so not required. 233894325357SNicholas Piggin */ 23394f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 23404f50541fSNicholas Piggin IVEC=0xf00 23414f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 23422284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23434f50541fSNicholas Piggin IKVM_REAL=1 23442284ffeaSNicholas Piggin#endif 2345af47d79bSNicholas Piggin ICFAR=0 23464f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 23474f50541fSNicholas Piggin 23487299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 23494f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 23507299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 23517299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 23524f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 23537299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2354eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 23554f50541fSNicholas Piggin GEN_COMMON performance_monitor 2356c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2357dc398a08SNicholas Piggin lbz r4,PACAIRQSOFTMASK(r13) 2358dc398a08SNicholas Piggin cmpdi r4,IRQS_ENABLED 2359dc398a08SNicholas Piggin bne 1f 23604e991e3cSNicholas Piggin bl CFUNC(performance_monitor_exception_async) 23611df7d5e4SNicholas Piggin b interrupt_return_srr 2362dc398a08SNicholas Piggin1: 23634e991e3cSNicholas Piggin bl CFUNC(performance_monitor_exception_nmi) 2364dc398a08SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 2365dc398a08SNicholas Piggin li r9,0 2366dc398a08SNicholas Piggin mtmsrd r9,1 2367b1c7f150SNicholas Piggin 2368dc398a08SNicholas Piggin kuap_kernel_restore r9, r10 2369dc398a08SNicholas Piggin 2370dc398a08SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 2371dc398a08SNicholas Piggin RFI_TO_KERNEL 23720ebc4cdaSBenjamin Herrenschmidt 237394325357SNicholas Piggin/** 237494325357SNicholas Piggin * Interrupt 0xf20 - Vector Unavailable Interrupt. 237594325357SNicholas Piggin * This is a synchronous interrupt in response to 237694325357SNicholas Piggin * executing a vector (or altivec) instruction with MSR[VEC]=0. 237794325357SNicholas Piggin * Similar to FP unavailable. 237894325357SNicholas Piggin */ 23794f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 23804f50541fSNicholas Piggin IVEC=0xf20 23812284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 23824f50541fSNicholas Piggin IKVM_REAL=1 23832284ffeaSNicholas Piggin#endif 23842487fd2eSRohan McLure IMSR_R12=1 23854f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 23864f50541fSNicholas Piggin 23877299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 23884f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 23897299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 23907299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 23914f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 23927299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 2393d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 23944f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 2395d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 2396d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 2397d1a0ca9cSNicholas Piggin beq 1f 2398d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2399d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2400d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2401d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 2402d1a0ca9cSNicholas Piggin */ 2403d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2404d1a0ca9cSNicholas Piggin bne- 2f 2405d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2406d1a0ca9cSNicholas Piggin#endif 24074e991e3cSNicholas Piggin bl CFUNC(load_up_altivec) 24081df7d5e4SNicholas Piggin b fast_interrupt_return_srr 2409d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2410d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 2411c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 24124e991e3cSNicholas Piggin bl CFUNC(altivec_unavailable_tm) 24131df7d5e4SNicholas Piggin b interrupt_return_srr 2414d1a0ca9cSNicholas Piggin#endif 2415d1a0ca9cSNicholas Piggin1: 2416d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2417d1a0ca9cSNicholas Piggin#endif 2418c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 24194e991e3cSNicholas Piggin bl CFUNC(altivec_unavailable_exception) 24201df7d5e4SNicholas Piggin b interrupt_return_srr 2421d1a0ca9cSNicholas Piggin 24220ebc4cdaSBenjamin Herrenschmidt 242394325357SNicholas Piggin/** 242494325357SNicholas Piggin * Interrupt 0xf40 - VSX Unavailable Interrupt. 242594325357SNicholas Piggin * This is a synchronous interrupt in response to 242694325357SNicholas Piggin * executing a VSX instruction with MSR[VSX]=0. 242794325357SNicholas Piggin * Similar to FP unavailable. 242894325357SNicholas Piggin */ 24294f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 24304f50541fSNicholas Piggin IVEC=0xf40 24312284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24324f50541fSNicholas Piggin IKVM_REAL=1 24332284ffeaSNicholas Piggin#endif 24342487fd2eSRohan McLure IMSR_R12=1 24354f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 24364f50541fSNicholas Piggin 24377299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 24384f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 24397299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 24407299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 24414f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 24427299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2443792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 24444f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2445792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2446792cbdddSNicholas PigginBEGIN_FTR_SECTION 2447792cbdddSNicholas Piggin beq 1f 2448792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2449792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2450792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2451792cbdddSNicholas Piggin * transaction), go do TM stuff 2452792cbdddSNicholas Piggin */ 2453792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2454792cbdddSNicholas Piggin bne- 2f 2455792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2456792cbdddSNicholas Piggin#endif 2457792cbdddSNicholas Piggin b load_up_vsx 2458792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2459792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2460c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 24614e991e3cSNicholas Piggin bl CFUNC(vsx_unavailable_tm) 24621df7d5e4SNicholas Piggin b interrupt_return_srr 2463792cbdddSNicholas Piggin#endif 2464792cbdddSNicholas Piggin1: 2465792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2466792cbdddSNicholas Piggin#endif 2467c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 24684e991e3cSNicholas Piggin bl CFUNC(vsx_unavailable_exception) 24691df7d5e4SNicholas Piggin b interrupt_return_srr 2470792cbdddSNicholas Piggin 2471d0c0c9a1SMichael Neuling 247294325357SNicholas Piggin/** 247394325357SNicholas Piggin * Interrupt 0xf60 - Facility Unavailable Interrupt. 247494325357SNicholas Piggin * This is a synchronous interrupt in response to 247594325357SNicholas Piggin * executing an instruction without access to the facility that can be 247694325357SNicholas Piggin * resolved by the OS (e.g., FSCR, MSR). 247794325357SNicholas Piggin * Similar to FP unavailable. 247894325357SNicholas Piggin */ 24794f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 24804f50541fSNicholas Piggin IVEC=0xf60 24812284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 24824f50541fSNicholas Piggin IKVM_REAL=1 24832284ffeaSNicholas Piggin#endif 24844f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 24854f50541fSNicholas Piggin 24867299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 24874f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 24887299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 24897299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 24904f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 24917299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2492eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 24934f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2494c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 24954e991e3cSNicholas Piggin bl CFUNC(facility_unavailable_exception) 24961df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 24971df7d5e4SNicholas Piggin b interrupt_return_srr 24981134713cSNicholas Piggin 2499da2bc464SMichael Ellerman 250094325357SNicholas Piggin/** 250194325357SNicholas Piggin * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. 250294325357SNicholas Piggin * This is a synchronous interrupt in response to 250394325357SNicholas Piggin * executing an instruction without access to the facility that can only 250494325357SNicholas Piggin * be resolved in HV mode (e.g., HFSCR). 250594325357SNicholas Piggin * Similar to FP unavailable. 250694325357SNicholas Piggin */ 25074f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 25084f50541fSNicholas Piggin IVEC=0xf80 25093f7fbd97SNicholas Piggin IHSRR=1 25104f50541fSNicholas Piggin IKVM_REAL=1 25114f50541fSNicholas Piggin IKVM_VIRT=1 25124f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 25134f50541fSNicholas Piggin 25147299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 25154f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 25167299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 25177299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 25184f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 25197299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2520eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 25214f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2522c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 25234e991e3cSNicholas Piggin bl CFUNC(facility_unavailable_exception) 25241df45d78SRohan McLure /* XXX Shouldn't be necessary in practice */ 25251df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() 25261df7d5e4SNicholas Piggin b interrupt_return_hsrr 252714b0072cSNicholas Piggin 2528da2bc464SMichael Ellerman 25291a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 25301a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 25311a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 25321a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 25331a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 25341a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 25351a6822d1SNicholas Piggin 25361a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 25371a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 25381a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 25391a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2540da2bc464SMichael Ellerman 25410ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 25424f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 25434f50541fSNicholas Piggin IVEC=0x1200 25443f7fbd97SNicholas Piggin IHSRR=1 25454f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 25464f50541fSNicholas Piggin 25477299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 25484f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 25497299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 25501a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2551eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 25524f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2553c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 25544e991e3cSNicholas Piggin bl CFUNC(cbe_system_error_exception) 25551df7d5e4SNicholas Piggin b interrupt_return_hsrr 25569600f261SNicholas Piggin 2557da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 25581a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 25591a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2560da2bc464SMichael Ellerman#endif 2561da2bc464SMichael Ellerman 2562da487a5dSNicholas Piggin/** 2563da487a5dSNicholas Piggin * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. 2564da487a5dSNicholas Piggin * This has been removed from the ISA before 2.01, which is the earliest 2565da487a5dSNicholas Piggin * 64-bit BookS ISA supported, however the G5 / 970 implements this 2566da487a5dSNicholas Piggin * interrupt with a non-architected feature available through the support 2567da487a5dSNicholas Piggin * processor interface. 2568da487a5dSNicholas Piggin */ 25694f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 25704f50541fSNicholas Piggin IVEC=0x1300 25712284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 25724f50541fSNicholas Piggin IKVM_REAL=1 25732284ffeaSNicholas Piggin#endif 25744f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 25754f50541fSNicholas Piggin 25767299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 25774f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 25787299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 25797299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 25804f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 25817299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2582eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 25834f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2584c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 25854e991e3cSNicholas Piggin bl CFUNC(instruction_breakpoint_exception) 25861df7d5e4SNicholas Piggin b interrupt_return_srr 25874e96dbbfSNicholas Piggin 25887299417cSNicholas Piggin 25891a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 25901a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2591da2bc464SMichael Ellerman 259294325357SNicholas Piggin/** 259394325357SNicholas Piggin * Interrupt 0x1500 - Soft Patch Interrupt 259494325357SNicholas Piggin * 259594325357SNicholas Piggin * Handling: 259694325357SNicholas Piggin * This is an implementation specific interrupt which can be used for a 259794325357SNicholas Piggin * range of exceptions. 259894325357SNicholas Piggin * 259994325357SNicholas Piggin * This interrupt handler is unique in that it runs the denormal assist 260094325357SNicholas Piggin * code even for guests (and even in guest context) without going to KVM, 260194325357SNicholas Piggin * for speed. POWER9 does not raise denorm exceptions, so this special case 260294325357SNicholas Piggin * could be phased out in future to reduce special cases. 260394325357SNicholas Piggin */ 26044f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 26054f50541fSNicholas Piggin IVEC=0x1500 26063f7fbd97SNicholas Piggin IHSRR=1 26074557ac6bSNicholas Piggin IBRANCH_TO_COMMON=0 26089600f261SNicholas Piggin IKVM_REAL=1 26094f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 26104f50541fSNicholas Piggin 26114f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 26124f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2613b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2614d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2615b92a66a6SMichael Neuling bne+ denorm_assist 2616b92a66a6SMichael Neuling#endif 26178729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 26184f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2619d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 26201a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 26214f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2622d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 262352b98923SNicholas Piggin bne+ denorm_assist 26248729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 26251a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2626d7e89849SNicholas Piggin#else 26271a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2628d7e89849SNicholas Piggin#endif 2629b92a66a6SMichael Neuling 2630b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2631da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2632b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2633b92a66a6SMichael Neuling/* 2634b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2635b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2636b92a66a6SMichael Neuling */ 2637b92a66a6SMichael Neuling mfmsr r10 2638b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2639b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2640b92a66a6SMichael Neuling mtmsrd r10 2641b92a66a6SMichael Neuling sync 2642d7c67fb1SMichael Neuling 2643f3c8b6c6SNicholas Piggin .Lreg=0 2644f3c8b6c6SNicholas Piggin .rept 32 2645f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2646f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2647f3c8b6c6SNicholas Piggin .endr 2648d7c67fb1SMichael Neuling 2649b92a66a6SMichael NeulingFTR_SECTION_ELSE 2650b92a66a6SMichael Neuling/* 2651b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2652b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2653b92a66a6SMichael Neuling */ 2654b92a66a6SMichael Neuling mfmsr r10 2655b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2656b92a66a6SMichael Neuling mtmsrd r10 2657b92a66a6SMichael Neuling sync 2658d7c67fb1SMichael Neuling 2659f3c8b6c6SNicholas Piggin .Lreg=0 2660f3c8b6c6SNicholas Piggin .rept 32 2661f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2662f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2663f3c8b6c6SNicholas Piggin .endr 2664d7c67fb1SMichael Neuling 2665b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2666fb0fce3eSMichael Neuling 2667fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2668fb0fce3eSMichael Neuling b denorm_done 2669fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2670fb0fce3eSMichael Neuling/* 2671fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2672fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2673fb0fce3eSMichael Neuling */ 2674f3c8b6c6SNicholas Piggin .Lreg=32 2675f3c8b6c6SNicholas Piggin .rept 32 2676f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2677f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2678f3c8b6c6SNicholas Piggin .endr 2679f3c8b6c6SNicholas Piggin 2680fb0fce3eSMichael Neulingdenorm_done: 2681f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2682f14040bcSMichael Neuling subi r11,r11,4 2683b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2684b92a66a6SMichael Neuling mtcrf 0x80,r9 2685b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2686931dc86bSNicholas PigginBEGIN_FTR_SECTION 2687931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2688931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2689931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2690630573c1SPaul MackerrasBEGIN_FTR_SECTION 2691630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2692630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2693630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 269459dc5bfcSNicholas Piggin li r10,0 269559dc5bfcSNicholas Piggin stb r10,PACAHSRR_VALID(r13) 2696b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2697b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2698b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2699b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2700222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2701b92a66a6SMichael Neuling b . 2702b92a66a6SMichael Neuling#endif 2703b92a66a6SMichael Neuling 27044f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 27054f50541fSNicholas Piggin GEN_COMMON denorm_exception 2706c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 27074e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 27081df7d5e4SNicholas Piggin b interrupt_return_hsrr 2709d7e89849SNicholas Piggin 2710d7e89849SNicholas Piggin 2711d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 27124f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 27134f50541fSNicholas Piggin IVEC=0x1600 27143f7fbd97SNicholas Piggin IHSRR=1 27154f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 27164f50541fSNicholas Piggin 27177299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 27184f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 27197299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 27201a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2721eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 27224f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2723c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 27244e991e3cSNicholas Piggin bl CFUNC(cbe_maintenance_exception) 27251df7d5e4SNicholas Piggin b interrupt_return_hsrr 27269600f261SNicholas Piggin 2727d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 27281a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 27291a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2730d7e89849SNicholas Piggin#endif 2731d7e89849SNicholas Piggin 273269a79344SNicholas Piggin 27334f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 27344f50541fSNicholas Piggin IVEC=0x1700 27352284ffeaSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 27364f50541fSNicholas Piggin IKVM_REAL=1 27372284ffeaSNicholas Piggin#endif 27384f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 27394f50541fSNicholas Piggin 27407299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 27414f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 27427299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 27437299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 27444f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 27457299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2746eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 27474f50541fSNicholas Piggin GEN_COMMON altivec_assist 2748c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 2749b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 27504e991e3cSNicholas Piggin bl CFUNC(altivec_assist_exception) 27511df45d78SRohan McLure HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */ 2752b51c079eSNicholas Piggin#else 27534e991e3cSNicholas Piggin bl CFUNC(unknown_exception) 2754b51c079eSNicholas Piggin#endif 27551df7d5e4SNicholas Piggin b interrupt_return_srr 2756b51c079eSNicholas Piggin 2757d7e89849SNicholas Piggin 2758d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 27594f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 27604f50541fSNicholas Piggin IVEC=0x1800 27613f7fbd97SNicholas Piggin IHSRR=1 27624f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 27634f50541fSNicholas Piggin 27647299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 27654f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 27667299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 27671a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2768eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 27694f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2770c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 27714e991e3cSNicholas Piggin bl CFUNC(cbe_thermal_exception) 27721df7d5e4SNicholas Piggin b interrupt_return_hsrr 27739600f261SNicholas Piggin 2774d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 27751a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 27761a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2777d7e89849SNicholas Piggin#endif 2778d7e89849SNicholas Piggin 27797299417cSNicholas Piggin 278075eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 27812104180aSNicholas Piggin 27820eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 27830eddf327SNicholas Piggin IVEC=0x900 27840eddf327SNicholas Piggin ISTACK=0 2785af47d79bSNicholas Piggin ICFAR=0 27860eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 27872104180aSNicholas Piggin 2788cc491f1dSNicholas Piggin/* 2789cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2790cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2791cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2792cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2793cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2794cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2795cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2796cc491f1dSNicholas Piggin */ 27972104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 27982104180aSNicholas Piggin mr r10,r1 27992104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 28002104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 28010eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 280271c3b05aSNicholas Piggin 2803c03be0a3SNicholas Piggin addi r3,r1,STACK_INT_FRAME_REGS 28044e991e3cSNicholas Piggin bl CFUNC(soft_nmi_interrupt) 280571c3b05aSNicholas Piggin 280671c3b05aSNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 280771c3b05aSNicholas Piggin li r9,0 280871c3b05aSNicholas Piggin mtmsrd r9,1 280971c3b05aSNicholas Piggin 28108e560921SAneesh Kumar K.V kuap_kernel_restore r9, r10 2811f23699c9SNicholas Piggin 281271c3b05aSNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=0 281371c3b05aSNicholas Piggin RFI_TO_KERNEL 28142104180aSNicholas Piggin 281575eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2816d7e89849SNicholas Piggin 28170ebc4cdaSBenjamin Herrenschmidt/* 2818fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 28192b461880SMichael Ellerman * - If it was a decrementer interrupt, we bump the dec to max and return. 2820fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2821fe9e1d54SIan Munsie * triggered and won't automatically refire. 28220869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 28230869b6fdSMahesh Salgaonkar * and it won't refire. 28246cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2825fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 28260ebc4cdaSBenjamin Herrenschmidt */ 28273f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 28284508a74aSNicholas Piggin .if \hsrr 28294508a74aSNicholas Pigginmasked_Hinterrupt: 28304508a74aSNicholas Piggin .else 28314508a74aSNicholas Pigginmasked_interrupt: 28324508a74aSNicholas Piggin .endif 283363e40806SNicholas Piggin stw r9,PACA_EXGEN+EX_CCR(r13) 2834c39fb71aSNicholas Piggin#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG 2835c39fb71aSNicholas Piggin /* 2836c39fb71aSNicholas Piggin * Ensure there was no previous MUST_HARD_MASK interrupt or 28378154850bSNicholas Piggin * HARD_DIS setting. If this does fire, the interrupt is still 28388154850bSNicholas Piggin * masked and MSR[EE] will be cleared on return, so no need to 28398154850bSNicholas Piggin * panic, but somebody probably enabled MSR[EE] under 28408154850bSNicholas Piggin * PACA_IRQ_HARD_DIS, mtmsr(mfmsr() | MSR_x) being a common 28418154850bSNicholas Piggin * cause. 2842c39fb71aSNicholas Piggin */ 2843c39fb71aSNicholas Piggin lbz r9,PACAIRQHAPPENED(r13) 2844c39fb71aSNicholas Piggin andi. r9,r9,(PACA_IRQ_MUST_HARD_MASK|PACA_IRQ_HARD_DIS) 2845c39fb71aSNicholas Piggin0: tdnei r9,0 28468154850bSNicholas Piggin EMIT_WARN_ENTRY 0b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) 2847c39fb71aSNicholas Piggin#endif 284863e40806SNicholas Piggin lbz r9,PACAIRQHAPPENED(r13) 284963e40806SNicholas Piggin or r9,r9,r10 285063e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 285163e40806SNicholas Piggin 285263e40806SNicholas Piggin .if ! \hsrr 28534508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 28544508a74aSNicholas Piggin bne 1f 285563e40806SNicholas Piggin LOAD_REG_IMMEDIATE(r9, 0x7fffffff) 285663e40806SNicholas Piggin mtspr SPRN_DEC,r9 28570eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 285863e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 28590eddf327SNicholas Piggin b soft_nmi_common 28600eddf327SNicholas Piggin#else 28610eddf327SNicholas Piggin b 2f 28620eddf327SNicholas Piggin#endif 286363e40806SNicholas Piggin .endif 286463e40806SNicholas Piggin 28654508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 28664508a74aSNicholas Piggin beq 2f 28670eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 28684508a74aSNicholas Piggin .if \hsrr 28690eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 28704508a74aSNicholas Piggin .else 28710eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 28724508a74aSNicholas Piggin .endif 287363e40806SNicholas Piggin ori r9,r9,PACA_IRQ_HARD_DIS 287463e40806SNicholas Piggin stb r9,PACAIRQHAPPENED(r13) 28754508a74aSNicholas Piggin2: /* done */ 287663e40806SNicholas Piggin li r9,0 287759dc5bfcSNicholas Piggin .if \hsrr 287863e40806SNicholas Piggin stb r9,PACAHSRR_VALID(r13) 287959dc5bfcSNicholas Piggin .else 288063e40806SNicholas Piggin stb r9,PACASRR_VALID(r13) 288159dc5bfcSNicholas Piggin .endif 288263e40806SNicholas Piggin 2883f23699c9SNicholas Piggin SEARCH_RESTART_TABLE 2884f23699c9SNicholas Piggin cmpdi r12,0 2885f23699c9SNicholas Piggin beq 3f 2886f23699c9SNicholas Piggin .if \hsrr 2887f23699c9SNicholas Piggin mtspr SPRN_HSRR0,r12 2888f23699c9SNicholas Piggin .else 2889f23699c9SNicholas Piggin mtspr SPRN_SRR0,r12 2890f23699c9SNicholas Piggin .endif 2891f23699c9SNicholas Piggin3: 2892f23699c9SNicholas Piggin 289363e40806SNicholas Piggin ld r9,PACA_EXGEN+EX_CTR(r13) 289463e40806SNicholas Piggin mtctr r9 289563e40806SNicholas Piggin lwz r9,PACA_EXGEN+EX_CCR(r13) 28964508a74aSNicholas Piggin mtcrf 0x80,r9 28974508a74aSNicholas Piggin std r1,PACAR1(r13) 28984508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 28994508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 29004508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 29010eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 2902b2dc2977SNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 2903b2dc2977SNicholas Piggin /* May return to masked low address where r13 is not set up */ 29044508a74aSNicholas Piggin .if \hsrr 29054508a74aSNicholas Piggin HRFI_TO_KERNEL 29064508a74aSNicholas Piggin .else 29074508a74aSNicholas Piggin RFI_TO_KERNEL 29084508a74aSNicholas Piggin .endif 29094508a74aSNicholas Piggin b . 29104508a74aSNicholas Piggin.endm 29110ebc4cdaSBenjamin Herrenschmidt 2912a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2913a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2914a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2915a048a07dSNicholas Piggin sync 2916a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2917a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2918a048a07dSNicholas Piggin ori 31,31,0 2919a048a07dSNicholas Piggin .rept 14 2920a048a07dSNicholas Piggin b 1f 2921a048a07dSNicholas Piggin1: 2922a048a07dSNicholas Piggin .endr 2923a048a07dSNicholas Piggin blr 2924a048a07dSNicholas Piggin 29259a32a7e7SNicholas Piggin/* Clobbers r10, r11, ctr */ 29269a32a7e7SNicholas Piggin.macro L1D_DISPLACEMENT_FLUSH 2927aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2928bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2929bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2930aa8a5e00SMichael Ellerman mtctr r11 293115a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2932aa8a5e00SMichael Ellerman 2933aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2934aa8a5e00SMichael Ellerman sync 2935bdcb1aefSNicholas Piggin 2936bdcb1aefSNicholas Piggin /* 2937f7964378SNicholas Piggin * The load addresses are at staggered offsets within cachelines, 2938bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2939bdcb1aefSNicholas Piggin * hurt). 2940bdcb1aefSNicholas Piggin */ 2941bdcb1aefSNicholas Piggin1: 2942bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2943bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2944bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2945bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2946bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2947bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2948bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2949bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2950bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2951aa8a5e00SMichael Ellerman bdnz 1b 29529a32a7e7SNicholas Piggin.endm 2953aa8a5e00SMichael Ellerman 29549a32a7e7SNicholas PigginTRAMP_REAL_BEGIN(entry_flush_fallback) 29559a32a7e7SNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 29569a32a7e7SNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 29579a32a7e7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 29589a32a7e7SNicholas Piggin mfctr r9 29599a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2960f7964378SNicholas Piggin mtctr r9 2961f7964378SNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2962f7964378SNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2963f7964378SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 2964f7964378SNicholas Piggin blr 2965f7964378SNicholas Piggin 296608685be7SNicholas Piggin/* 296708685be7SNicholas Piggin * The SCV entry flush happens with interrupts enabled, so it must disable 296808685be7SNicholas Piggin * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10 296908685be7SNicholas Piggin * (containing LR) does not need to be preserved here because scv entry 297008685be7SNicholas Piggin * puts 0 in the pt_regs, CTR can be clobbered for the same reason. 297108685be7SNicholas Piggin */ 297208685be7SNicholas PigginTRAMP_REAL_BEGIN(scv_entry_flush_fallback) 297308685be7SNicholas Piggin li r10,0 297408685be7SNicholas Piggin mtmsrd r10,1 297508685be7SNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 297608685be7SNicholas Piggin ori r10,r10,PACA_IRQ_HARD_DIS 297708685be7SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 297808685be7SNicholas Piggin std r11,PACA_EXRFI+EX_R11(r13) 297908685be7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 298008685be7SNicholas Piggin ld r11,PACA_EXRFI+EX_R11(r13) 298108685be7SNicholas Piggin li r10,MSR_RI 298208685be7SNicholas Piggin mtmsrd r10,1 298308685be7SNicholas Piggin blr 298408685be7SNicholas Piggin 2985aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 2986aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2987aa8a5e00SMichael Ellerman GET_PACA(r13); 2988aa8a5e00SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 2989aa8a5e00SMichael Ellerman ld r1,PACAKSAVE(r13) 2990aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2991aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2992aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2993aa8a5e00SMichael Ellerman mfctr r9 29949a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 2995aa8a5e00SMichael Ellerman mtctr r9 2996aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2997aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2998aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 299978ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 3000aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 3001aa8a5e00SMichael Ellerman rfid 3002aa8a5e00SMichael Ellerman 3003aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 3004aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 3005aa8a5e00SMichael Ellerman GET_PACA(r13); 300678ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 300778ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 3008aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 3009aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 3010aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 3011aa8a5e00SMichael Ellerman mfctr r9 30129a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 3013aa8a5e00SMichael Ellerman mtctr r9 3014aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 3015aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 3016aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 301778ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 3018aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 3019aa8a5e00SMichael Ellerman hrfid 3020aa8a5e00SMichael Ellerman 30217fa95f9aSNicholas PigginTRAMP_REAL_BEGIN(rfscv_flush_fallback) 30227fa95f9aSNicholas Piggin /* system call volatile */ 30237fa95f9aSNicholas Piggin mr r7,r13 30247fa95f9aSNicholas Piggin GET_PACA(r13); 30257fa95f9aSNicholas Piggin mr r8,r1 30267fa95f9aSNicholas Piggin ld r1,PACAKSAVE(r13) 30277fa95f9aSNicholas Piggin mfctr r9 30287fa95f9aSNicholas Piggin ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 30297fa95f9aSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 30307fa95f9aSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 30317fa95f9aSNicholas Piggin mtctr r11 30327fa95f9aSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 30337fa95f9aSNicholas Piggin 30347fa95f9aSNicholas Piggin /* order ld/st prior to dcbt stop all streams with flushing */ 30357fa95f9aSNicholas Piggin sync 30367fa95f9aSNicholas Piggin 30377fa95f9aSNicholas Piggin /* 30387fa95f9aSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 30397fa95f9aSNicholas Piggin * which suits some pipelines better (on others it should not 30407fa95f9aSNicholas Piggin * hurt). 30417fa95f9aSNicholas Piggin */ 30427fa95f9aSNicholas Piggin1: 30437fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 30447fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 30457fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 30467fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 30477fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 30487fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 30497fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 30507fa95f9aSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 30517fa95f9aSNicholas Piggin addi r10,r10,0x80*8 30527fa95f9aSNicholas Piggin bdnz 1b 30537fa95f9aSNicholas Piggin 30547fa95f9aSNicholas Piggin mtctr r9 30557fa95f9aSNicholas Piggin li r9,0 30567fa95f9aSNicholas Piggin li r10,0 30577fa95f9aSNicholas Piggin li r11,0 30587fa95f9aSNicholas Piggin mr r1,r8 30597fa95f9aSNicholas Piggin mr r13,r7 30607fa95f9aSNicholas Piggin RFSCV 30617fa95f9aSNicholas Piggin 30620eddf327SNicholas PigginUSE_TEXT_SECTION() 30639a32a7e7SNicholas Piggin 306469fdd674SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 306569fdd674SNicholas Pigginkvm_interrupt: 306669fdd674SNicholas Piggin /* 306769fdd674SNicholas Piggin * The conditional branch in KVMTEST can't reach all the way, 306869fdd674SNicholas Piggin * make a stub. 306969fdd674SNicholas Piggin */ 307069fdd674SNicholas Piggin b kvmppc_interrupt 307169fdd674SNicholas Piggin#endif 307269fdd674SNicholas Piggin 30739a32a7e7SNicholas Piggin_GLOBAL(do_uaccess_flush) 30749a32a7e7SNicholas Piggin UACCESS_FLUSH_FIXUP_SECTION 30759a32a7e7SNicholas Piggin nop 30769a32a7e7SNicholas Piggin nop 30779a32a7e7SNicholas Piggin nop 30789a32a7e7SNicholas Piggin blr 30799a32a7e7SNicholas Piggin L1D_DISPLACEMENT_FLUSH 30809a32a7e7SNicholas Piggin blr 30819a32a7e7SNicholas Piggin_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) 30829a32a7e7SNicholas PigginEXPORT_SYMBOL(do_uaccess_flush) 30839a32a7e7SNicholas Piggin 30849a32a7e7SNicholas Piggin 30853f7fbd97SNicholas PigginMASKED_INTERRUPT 30863f7fbd97SNicholas PigginMASKED_INTERRUPT hsrr=1 30877230c564SBenjamin Herrenschmidt 308857f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 30898ed8ab40SHari Bathini /* 30909d1988caSNicholas Piggin * All code below __end_soft_masked is treated as soft-masked. If 3091b2dc2977SNicholas Piggin * any code runs here with MSR[EE]=1, it must then cope with pending 3092b2dc2977SNicholas Piggin * soft interrupt being raised (i.e., by ensuring it is replayed). 3093b2dc2977SNicholas Piggin * 30948ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 30958ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 30968ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 30978ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 30988ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 30998ed8ab40SHari Bathini */ 31008ed8ab40SHari Bathini .align 7 31018ed8ab40SHari Bathini .globl __end_interrupts 31028ed8ab40SHari Bathini__end_interrupts: 3103d72c4a36SDaniel AxtensDEFINE_FIXED_SYMBOL(__end_interrupts, virt_trampolines) 310461383407SBenjamin Herrenschmidt 310557f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 310657f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 310757f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 310857f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 310957f26649SNicholas Piggin 311057f26649SNicholas PigginUSE_TEXT_SECTION() 311157f26649SNicholas Piggin 3112296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 31132f5182cfSNicholas Piggin_GLOBAL(enable_machine_check) 3114296e753fSNicholas Piggin mflr r0 3115296e753fSNicholas Piggin bcl 20,31,$+4 3116296e753fSNicholas Piggin0: mflr r3 3117296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 3118296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 3119296e753fSNicholas Piggin mfmsr r3 3120296e753fSNicholas Piggin ori r3,r3,MSR_ME 3121296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 3122296e753fSNicholas Piggin RFI_TO_KERNEL 3123296e753fSNicholas Piggin1: mtlr r0 3124296e753fSNicholas Piggin blr 3125296e753fSNicholas Piggin 3126b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 312729a011fcSSathvika VasireddySYM_FUNC_START_LOCAL(disable_machine_check) 3128b7d9ccecSNicholas Piggin mflr r0 3129b7d9ccecSNicholas Piggin bcl 20,31,$+4 3130b7d9ccecSNicholas Piggin0: mflr r3 3131b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 3132b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 3133b7d9ccecSNicholas Piggin mfmsr r3 3134b7d9ccecSNicholas Piggin li r4,MSR_ME 3135b7d9ccecSNicholas Piggin andc r3,r3,r4 3136b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 3137b7d9ccecSNicholas Piggin RFI_TO_KERNEL 3138b7d9ccecSNicholas Piggin1: mtlr r0 3139b7d9ccecSNicholas Piggin blr 314029a011fcSSathvika VasireddySYM_FUNC_END(disable_machine_check) 3141