1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 20ebc4cdaSBenjamin Herrenschmidt/* 30ebc4cdaSBenjamin Herrenschmidt * This file contains the 64-bit "server" PowerPC variant 40ebc4cdaSBenjamin Herrenschmidt * of the low level exception handling including exception 50ebc4cdaSBenjamin Herrenschmidt * vectors, exception return, part of the slb and stab 60ebc4cdaSBenjamin Herrenschmidt * handling and other fixed offset specific things. 70ebc4cdaSBenjamin Herrenschmidt * 80ebc4cdaSBenjamin Herrenschmidt * This file is meant to be #included from head_64.S due to 925985edcSLucas De Marchi * position dependent assembly. 100ebc4cdaSBenjamin Herrenschmidt * 110ebc4cdaSBenjamin Herrenschmidt * Most of this originates from head_64.S and thus has the same 120ebc4cdaSBenjamin Herrenschmidt * copyright history. 130ebc4cdaSBenjamin Herrenschmidt * 140ebc4cdaSBenjamin Herrenschmidt */ 150ebc4cdaSBenjamin Herrenschmidt 167230c564SBenjamin Herrenschmidt#include <asm/hw_irq.h> 178aa34ab8SBenjamin Herrenschmidt#include <asm/exception-64s.h> 1846f52210SStephen Rothwell#include <asm/ptrace.h> 197cba160aSShreyas B. Prabhu#include <asm/cpuidle.h> 20da2bc464SMichael Ellerman#include <asm/head-64.h> 212c86cd18SChristophe Leroy#include <asm/feature-fixups.h> 22890274c2SMichael Ellerman#include <asm/kup.h> 238aa34ab8SBenjamin Herrenschmidt 2415820091SNicholas Piggin/* PACA save area offsets (exgen, exmc, etc) */ 2515820091SNicholas Piggin#define EX_R9 0 2615820091SNicholas Piggin#define EX_R10 8 2715820091SNicholas Piggin#define EX_R11 16 2815820091SNicholas Piggin#define EX_R12 24 2915820091SNicholas Piggin#define EX_R13 32 3015820091SNicholas Piggin#define EX_DAR 40 3115820091SNicholas Piggin#define EX_DSISR 48 3215820091SNicholas Piggin#define EX_CCR 52 3315820091SNicholas Piggin#define EX_CFAR 56 3415820091SNicholas Piggin#define EX_PPR 64 3515820091SNicholas Piggin#define EX_CTR 72 3615820091SNicholas Piggin.if EX_SIZE != 10 3715820091SNicholas Piggin .error "EX_SIZE is wrong" 3815820091SNicholas Piggin.endif 3915820091SNicholas Piggin 407299417cSNicholas Piggin/* 417299417cSNicholas Piggin * Following are fixed section helper macros. 427299417cSNicholas Piggin * 437299417cSNicholas Piggin * EXC_REAL_BEGIN/END - real, unrelocated exception vectors 447299417cSNicholas Piggin * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors 457299417cSNicholas Piggin * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these) 467299417cSNicholas Piggin * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use) 477299417cSNicholas Piggin * EXC_COMMON - After switching to virtual, relocated mode. 487299417cSNicholas Piggin */ 497299417cSNicholas Piggin 50a2432811SNicholas Piggin#define EXC_REAL_BEGIN(name, start, size) \ 51a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 52a2432811SNicholas Piggin 53a2432811SNicholas Piggin#define EXC_REAL_END(name, start, size) \ 54a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 55a2432811SNicholas Piggin 56a2432811SNicholas Piggin#define EXC_VIRT_BEGIN(name, start, size) \ 57a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 58a2432811SNicholas Piggin 59a2432811SNicholas Piggin#define EXC_VIRT_END(name, start, size) \ 60a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 61a2432811SNicholas Piggin 62a2432811SNicholas Piggin#define EXC_COMMON_BEGIN(name) \ 63a2432811SNicholas Piggin USE_TEXT_SECTION(); \ 64a2432811SNicholas Piggin .balign IFETCH_ALIGN_BYTES; \ 65a2432811SNicholas Piggin .global name; \ 66a2432811SNicholas Piggin _ASM_NOKPROBE_SYMBOL(name); \ 67a2432811SNicholas Piggin DEFINE_FIXED_SYMBOL(name); \ 68a2432811SNicholas Pigginname: 69a2432811SNicholas Piggin 70a2432811SNicholas Piggin#define TRAMP_REAL_BEGIN(name) \ 71a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 72a2432811SNicholas Piggin 73a2432811SNicholas Piggin#define TRAMP_VIRT_BEGIN(name) \ 74a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 75a2432811SNicholas Piggin 76a2432811SNicholas Piggin#define EXC_REAL_NONE(start, size) \ 77a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 78a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 79a2432811SNicholas Piggin 80a2432811SNicholas Piggin#define EXC_VIRT_NONE(start, size) \ 81a2432811SNicholas Piggin FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 82a2432811SNicholas Piggin FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) 83a2432811SNicholas Piggin 840ebc4cdaSBenjamin Herrenschmidt/* 8512a04809SNicholas Piggin * We're short on space and time in the exception prolog, so we can't 8612a04809SNicholas Piggin * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. 8712a04809SNicholas Piggin * Instead we get the base of the kernel from paca->kernelbase and or in the low 8812a04809SNicholas Piggin * part of label. This requires that the label be within 64KB of kernelbase, and 8912a04809SNicholas Piggin * that kernelbase be 64K aligned. 9012a04809SNicholas Piggin */ 9112a04809SNicholas Piggin#define LOAD_HANDLER(reg, label) \ 9212a04809SNicholas Piggin ld reg,PACAKBASE(r13); /* get high part of &label */ \ 9312a04809SNicholas Piggin ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) 9412a04809SNicholas Piggin 9512a04809SNicholas Piggin#define __LOAD_HANDLER(reg, label) \ 9612a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 9712a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l 9812a04809SNicholas Piggin 9912a04809SNicholas Piggin/* 10012a04809SNicholas Piggin * Branches from unrelocated code (e.g., interrupts) to labels outside 10112a04809SNicholas Piggin * head-y require >64K offsets. 10212a04809SNicholas Piggin */ 10312a04809SNicholas Piggin#define __LOAD_FAR_HANDLER(reg, label) \ 10412a04809SNicholas Piggin ld reg,PACAKBASE(r13); \ 10512a04809SNicholas Piggin ori reg,reg,(ABS_ADDR(label))@l; \ 10612a04809SNicholas Piggin addis reg,reg,(ABS_ADDR(label))@h 10712a04809SNicholas Piggin 10812a04809SNicholas Piggin/* 10912a04809SNicholas Piggin * Branch to label using its 0xC000 address. This results in instruction 11012a04809SNicholas Piggin * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned 11112a04809SNicholas Piggin * on using mtmsr rather than rfid. 11212a04809SNicholas Piggin * 11312a04809SNicholas Piggin * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than 11412a04809SNicholas Piggin * load KBASE for a slight optimisation. 11512a04809SNicholas Piggin */ 11612a04809SNicholas Piggin#define BRANCH_TO_C000(reg, label) \ 1170e10be2bSNicholas Piggin __LOAD_FAR_HANDLER(reg, label); \ 11812a04809SNicholas Piggin mtctr reg; \ 11912a04809SNicholas Piggin bctr 12012a04809SNicholas Piggin 121a42a239dSNicholas Piggin/* 122a42a239dSNicholas Piggin * Interrupt code generation macros 123a42a239dSNicholas Piggin */ 124a42a239dSNicholas Piggin#define IVEC .L_IVEC_\name\() 125a42a239dSNicholas Piggin#define IHSRR .L_IHSRR_\name\() 126*3f7fbd97SNicholas Piggin#define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() 127a42a239dSNicholas Piggin#define IAREA .L_IAREA_\name\() 1288729c26eSNicholas Piggin#define IVIRT .L_IVIRT_\name\() 129a3cd35beSNicholas Piggin#define IISIDE .L_IISIDE_\name\() 130a42a239dSNicholas Piggin#define IDAR .L_IDAR_\name\() 131a42a239dSNicholas Piggin#define IDSISR .L_IDSISR_\name\() 132a42a239dSNicholas Piggin#define ISET_RI .L_ISET_RI_\name\() 133d73a10cbSNicholas Piggin#define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() 134d73a10cbSNicholas Piggin#define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() 135a42a239dSNicholas Piggin#define IMASK .L_IMASK_\name\() 136d52fd3d3SNicholas Piggin#define IKVM_SKIP .L_IKVM_SKIP_\name\() 137a42a239dSNicholas Piggin#define IKVM_REAL .L_IKVM_REAL_\name\() 1384f50541fSNicholas Piggin#define __IKVM_REAL(name) .L_IKVM_REAL_ ## name 139a42a239dSNicholas Piggin#define IKVM_VIRT .L_IKVM_VIRT_\name\() 1407cb3a1a0SNicholas Piggin#define ISTACK .L_ISTACK_\name\() 1414f50541fSNicholas Piggin#define __ISTACK(name) .L_ISTACK_ ## name 1427cb3a1a0SNicholas Piggin#define IRECONCILE .L_IRECONCILE_\name\() 1437cb3a1a0SNicholas Piggin#define IKUAP .L_IKUAP_\name\() 144a42a239dSNicholas Piggin 145a42a239dSNicholas Piggin#define INT_DEFINE_BEGIN(n) \ 146a42a239dSNicholas Piggin.macro int_define_ ## n name 147a42a239dSNicholas Piggin 148a42a239dSNicholas Piggin#define INT_DEFINE_END(n) \ 149a42a239dSNicholas Piggin.endm ; \ 150a42a239dSNicholas Pigginint_define_ ## n n ; \ 151a42a239dSNicholas Piggindo_define_int n 152a42a239dSNicholas Piggin 153a42a239dSNicholas Piggin.macro do_define_int name 154a42a239dSNicholas Piggin .ifndef IVEC 155a42a239dSNicholas Piggin .error "IVEC not defined" 156a42a239dSNicholas Piggin .endif 157a42a239dSNicholas Piggin .ifndef IHSRR 158*3f7fbd97SNicholas Piggin IHSRR=0 159*3f7fbd97SNicholas Piggin .endif 160*3f7fbd97SNicholas Piggin .ifndef IHSRR_IF_HVMODE 161*3f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=0 162a42a239dSNicholas Piggin .endif 163a42a239dSNicholas Piggin .ifndef IAREA 164a42a239dSNicholas Piggin IAREA=PACA_EXGEN 165a42a239dSNicholas Piggin .endif 1668729c26eSNicholas Piggin .ifndef IVIRT 1678729c26eSNicholas Piggin IVIRT=1 1688729c26eSNicholas Piggin .endif 169a3cd35beSNicholas Piggin .ifndef IISIDE 170a3cd35beSNicholas Piggin IISIDE=0 171a3cd35beSNicholas Piggin .endif 172a42a239dSNicholas Piggin .ifndef IDAR 173a42a239dSNicholas Piggin IDAR=0 174a42a239dSNicholas Piggin .endif 175a42a239dSNicholas Piggin .ifndef IDSISR 176a42a239dSNicholas Piggin IDSISR=0 177a42a239dSNicholas Piggin .endif 178a42a239dSNicholas Piggin .ifndef ISET_RI 179a42a239dSNicholas Piggin ISET_RI=1 180a42a239dSNicholas Piggin .endif 181d73a10cbSNicholas Piggin .ifndef IBRANCH_TO_COMMON 182d73a10cbSNicholas Piggin IBRANCH_TO_COMMON=1 183d73a10cbSNicholas Piggin .endif 184d73a10cbSNicholas Piggin .ifndef IREALMODE_COMMON 185d73a10cbSNicholas Piggin IREALMODE_COMMON=0 186d73a10cbSNicholas Piggin .else 187d73a10cbSNicholas Piggin .if ! IBRANCH_TO_COMMON 188d73a10cbSNicholas Piggin .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0" 189d73a10cbSNicholas Piggin .endif 190a42a239dSNicholas Piggin .endif 191a42a239dSNicholas Piggin .ifndef IMASK 192a42a239dSNicholas Piggin IMASK=0 193a42a239dSNicholas Piggin .endif 194d52fd3d3SNicholas Piggin .ifndef IKVM_SKIP 195d52fd3d3SNicholas Piggin IKVM_SKIP=0 196d52fd3d3SNicholas Piggin .endif 197a42a239dSNicholas Piggin .ifndef IKVM_REAL 198a42a239dSNicholas Piggin IKVM_REAL=0 199a42a239dSNicholas Piggin .endif 200a42a239dSNicholas Piggin .ifndef IKVM_VIRT 201a42a239dSNicholas Piggin IKVM_VIRT=0 202a42a239dSNicholas Piggin .endif 2037cb3a1a0SNicholas Piggin .ifndef ISTACK 2047cb3a1a0SNicholas Piggin ISTACK=1 2057cb3a1a0SNicholas Piggin .endif 2067cb3a1a0SNicholas Piggin .ifndef IRECONCILE 2077cb3a1a0SNicholas Piggin IRECONCILE=1 2087cb3a1a0SNicholas Piggin .endif 2097cb3a1a0SNicholas Piggin .ifndef IKUAP 2107cb3a1a0SNicholas Piggin IKUAP=1 2117cb3a1a0SNicholas Piggin .endif 212a42a239dSNicholas Piggin.endm 213a42a239dSNicholas Piggin 21412a04809SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 21512a04809SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 21612a04809SNicholas Piggin/* 21712a04809SNicholas Piggin * If hv is possible, interrupts come into to the hv version 21812a04809SNicholas Piggin * of the kvmppc_interrupt code, which then jumps to the PR handler, 21912a04809SNicholas Piggin * kvmppc_interrupt_pr, if the guest is a PR guest. 22012a04809SNicholas Piggin */ 22112a04809SNicholas Piggin#define kvmppc_interrupt kvmppc_interrupt_hv 22212a04809SNicholas Piggin#else 22312a04809SNicholas Piggin#define kvmppc_interrupt kvmppc_interrupt_pr 22412a04809SNicholas Piggin#endif 22512a04809SNicholas Piggin 2269d598f93SNicholas Piggin.macro KVMTEST name 22712a04809SNicholas Piggin lbz r10,HSTATE_IN_GUEST(r13) 22812a04809SNicholas Piggin cmpwi r10,0 22905f97d94SNicholas Piggin bne \name\()_kvm 23012a04809SNicholas Piggin.endm 23112a04809SNicholas Piggin 232b177ae2fSNicholas Piggin.macro GEN_KVM name 2339600f261SNicholas Piggin .balign IFETCH_ALIGN_BYTES 2349600f261SNicholas Piggin\name\()_kvm: 2359600f261SNicholas Piggin 236b177ae2fSNicholas Piggin .if IKVM_SKIP 23712a04809SNicholas Piggin cmpwi r10,KVM_GUEST_MODE_SKIP 23812a04809SNicholas Piggin beq 89f 23912a04809SNicholas Piggin .else 240931dc86bSNicholas PigginBEGIN_FTR_SECTION 241b177ae2fSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 24212a04809SNicholas Piggin std r10,HSTATE_CFAR(r13) 243931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 24412a04809SNicholas Piggin .endif 24512a04809SNicholas Piggin 2469600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 2479600f261SNicholas Piggin mtctr r10 248931dc86bSNicholas PigginBEGIN_FTR_SECTION 249b177ae2fSNicholas Piggin ld r10,IAREA+EX_PPR(r13) 25012a04809SNicholas Piggin std r10,HSTATE_PPR(r13) 251931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2529600f261SNicholas Piggin ld r11,IAREA+EX_R11(r13) 2539600f261SNicholas Piggin ld r12,IAREA+EX_R12(r13) 25412a04809SNicholas Piggin std r12,HSTATE_SCRATCH0(r13) 25512a04809SNicholas Piggin sldi r12,r9,32 2569600f261SNicholas Piggin ld r9,IAREA+EX_R9(r13) 2579600f261SNicholas Piggin ld r10,IAREA+EX_R10(r13) 25812a04809SNicholas Piggin /* HSRR variants have the 0x2 bit added to their trap number */ 259*3f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 260def0db4fSNicholas Piggin BEGIN_FTR_SECTION 261b177ae2fSNicholas Piggin ori r12,r12,(IVEC + 0x2) 262def0db4fSNicholas Piggin FTR_SECTION_ELSE 263b177ae2fSNicholas Piggin ori r12,r12,(IVEC) 264def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 265b177ae2fSNicholas Piggin .elseif IHSRR 266b177ae2fSNicholas Piggin ori r12,r12,(IVEC+ 0x2) 26712a04809SNicholas Piggin .else 268b177ae2fSNicholas Piggin ori r12,r12,(IVEC) 26912a04809SNicholas Piggin .endif 27064e41351SNicholas Piggin b kvmppc_interrupt 27112a04809SNicholas Piggin 272b177ae2fSNicholas Piggin .if IKVM_SKIP 27312a04809SNicholas Piggin89: mtocrf 0x80,r9 2749600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 2759600f261SNicholas Piggin mtctr r10 276b177ae2fSNicholas Piggin ld r9,IAREA+EX_R9(r13) 277b177ae2fSNicholas Piggin ld r10,IAREA+EX_R10(r13) 2789600f261SNicholas Piggin ld r11,IAREA+EX_R11(r13) 2799600f261SNicholas Piggin ld r12,IAREA+EX_R12(r13) 280*3f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 281def0db4fSNicholas Piggin BEGIN_FTR_SECTION 282def0db4fSNicholas Piggin b kvmppc_skip_Hinterrupt 283def0db4fSNicholas Piggin FTR_SECTION_ELSE 284def0db4fSNicholas Piggin b kvmppc_skip_interrupt 285def0db4fSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 286b177ae2fSNicholas Piggin .elseif IHSRR 28712a04809SNicholas Piggin b kvmppc_skip_Hinterrupt 28812a04809SNicholas Piggin .else 28912a04809SNicholas Piggin b kvmppc_skip_interrupt 29012a04809SNicholas Piggin .endif 29112a04809SNicholas Piggin .endif 29212a04809SNicholas Piggin.endm 29312a04809SNicholas Piggin 29412a04809SNicholas Piggin#else 2959d598f93SNicholas Piggin.macro KVMTEST name 29612a04809SNicholas Piggin.endm 297b177ae2fSNicholas Piggin.macro GEN_KVM name 29812a04809SNicholas Piggin.endm 29912a04809SNicholas Piggin#endif 30012a04809SNicholas Piggin 301c7c5cbb4SNicholas Piggin/* 302c7c5cbb4SNicholas Piggin * This is the BOOK3S interrupt entry code macro. 303c7c5cbb4SNicholas Piggin * 304c7c5cbb4SNicholas Piggin * This can result in one of several things happening: 305c7c5cbb4SNicholas Piggin * - Branch to the _common handler, relocated, in virtual mode. 306c7c5cbb4SNicholas Piggin * These are normal interrupts (synchronous and asynchronous) handled by 307c7c5cbb4SNicholas Piggin * the kernel. 308c7c5cbb4SNicholas Piggin * - Branch to KVM, relocated but real mode interrupts remain in real mode. 309c7c5cbb4SNicholas Piggin * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by 310c7c5cbb4SNicholas Piggin * / intended for host or guest kernel, but KVM must always be involved 311c7c5cbb4SNicholas Piggin * because the machine state is set for guest execution. 312c7c5cbb4SNicholas Piggin * - Branch to the masked handler, unrelocated. 313c7c5cbb4SNicholas Piggin * These occur when maskable asynchronous interrupts are taken with the 314c7c5cbb4SNicholas Piggin * irq_soft_mask set. 315c7c5cbb4SNicholas Piggin * - Branch to an "early" handler in real mode but relocated. 316c7c5cbb4SNicholas Piggin * This is done if early=1. MCE and HMI use these to handle errors in real 317c7c5cbb4SNicholas Piggin * mode. 318c7c5cbb4SNicholas Piggin * - Fall through and continue executing in real, unrelocated mode. 319c7c5cbb4SNicholas Piggin * This is done if early=2. 320c7c5cbb4SNicholas Piggin */ 3218729c26eSNicholas Piggin 3228729c26eSNicholas Piggin.macro GEN_BRANCH_TO_COMMON name, virt 323d73a10cbSNicholas Piggin .if IREALMODE_COMMON 324d73a10cbSNicholas Piggin LOAD_HANDLER(r10, \name\()_common) 325d73a10cbSNicholas Piggin mtctr r10 326d73a10cbSNicholas Piggin bctr 327d73a10cbSNicholas Piggin .else 3288729c26eSNicholas Piggin .if \virt 3298729c26eSNicholas Piggin#ifndef CONFIG_RELOCATABLE 3308729c26eSNicholas Piggin b \name\()_common_virt 3318729c26eSNicholas Piggin#else 3328729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_virt) 3338729c26eSNicholas Piggin mtctr r10 3348729c26eSNicholas Piggin bctr 3358729c26eSNicholas Piggin#endif 3368729c26eSNicholas Piggin .else 3378729c26eSNicholas Piggin LOAD_HANDLER(r10, \name\()_common_real) 3388729c26eSNicholas Piggin mtctr r10 3398729c26eSNicholas Piggin bctr 3408729c26eSNicholas Piggin .endif 341d73a10cbSNicholas Piggin .endif 3428729c26eSNicholas Piggin.endm 3438729c26eSNicholas Piggin 344fc589ee4SNicholas Piggin.macro GEN_INT_ENTRY name, virt, ool=0 345c7c5cbb4SNicholas Piggin SET_SCRATCH0(r13) /* save r13 */ 346c7c5cbb4SNicholas Piggin GET_PACA(r13) 347fc589ee4SNicholas Piggin std r9,IAREA+EX_R9(r13) /* save r9 */ 348931dc86bSNicholas PigginBEGIN_FTR_SECTION 349931dc86bSNicholas Piggin mfspr r9,SPRN_PPR 350931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 351c7c5cbb4SNicholas Piggin HMT_MEDIUM 352fc589ee4SNicholas Piggin std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ 353931dc86bSNicholas PigginBEGIN_FTR_SECTION 354931dc86bSNicholas Piggin mfspr r10,SPRN_CFAR 355931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 356c7c5cbb4SNicholas Piggin .if \ool 357c7c5cbb4SNicholas Piggin .if !\virt 358c7c5cbb4SNicholas Piggin b tramp_real_\name 359c7c5cbb4SNicholas Piggin .pushsection .text 360c7c5cbb4SNicholas Piggin TRAMP_REAL_BEGIN(tramp_real_\name) 361c7c5cbb4SNicholas Piggin .else 362c7c5cbb4SNicholas Piggin b tramp_virt_\name 363c7c5cbb4SNicholas Piggin .pushsection .text 364c7c5cbb4SNicholas Piggin TRAMP_VIRT_BEGIN(tramp_virt_\name) 365c7c5cbb4SNicholas Piggin .endif 366c7c5cbb4SNicholas Piggin .endif 367c7c5cbb4SNicholas Piggin 368931dc86bSNicholas PigginBEGIN_FTR_SECTION 369931dc86bSNicholas Piggin std r9,IAREA+EX_PPR(r13) 370931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 371931dc86bSNicholas PigginBEGIN_FTR_SECTION 372931dc86bSNicholas Piggin std r10,IAREA+EX_CFAR(r13) 373931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 374c7c5cbb4SNicholas Piggin INTERRUPT_TO_KERNEL 3758729c26eSNicholas Piggin mfctr r10 3768729c26eSNicholas Piggin std r10,IAREA+EX_CTR(r13) 377c7c5cbb4SNicholas Piggin mfcr r9 378fc589ee4SNicholas Piggin std r11,IAREA+EX_R11(r13) 379fc589ee4SNicholas Piggin std r12,IAREA+EX_R12(r13) 380c7c5cbb4SNicholas Piggin 381c7c5cbb4SNicholas Piggin /* 382c7c5cbb4SNicholas Piggin * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], 383c7c5cbb4SNicholas Piggin * because a d-side MCE will clobber those registers so is 384c7c5cbb4SNicholas Piggin * not recoverable if they are live. 385c7c5cbb4SNicholas Piggin */ 386c7c5cbb4SNicholas Piggin GET_SCRATCH0(r10) 387fc589ee4SNicholas Piggin std r10,IAREA+EX_R13(r13) 388a3cd35beSNicholas Piggin .if IDAR && !IISIDE 389fc589ee4SNicholas Piggin .if IHSRR 390c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDAR 391c7c5cbb4SNicholas Piggin .else 392c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DAR 393c7c5cbb4SNicholas Piggin .endif 394fc589ee4SNicholas Piggin std r10,IAREA+EX_DAR(r13) 395c7c5cbb4SNicholas Piggin .endif 396a3cd35beSNicholas Piggin .if IDSISR && !IISIDE 397fc589ee4SNicholas Piggin .if IHSRR 398c7c5cbb4SNicholas Piggin mfspr r10,SPRN_HDSISR 399c7c5cbb4SNicholas Piggin .else 400c7c5cbb4SNicholas Piggin mfspr r10,SPRN_DSISR 401c7c5cbb4SNicholas Piggin .endif 402fc589ee4SNicholas Piggin stw r10,IAREA+EX_DSISR(r13) 403c7c5cbb4SNicholas Piggin .endif 404c7c5cbb4SNicholas Piggin 405*3f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 4068729c26eSNicholas Piggin BEGIN_FTR_SECTION 4078729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 4088729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 4098729c26eSNicholas Piggin FTR_SECTION_ELSE 4108729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 4118729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 4128729c26eSNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 4138729c26eSNicholas Piggin .elseif IHSRR 4148729c26eSNicholas Piggin mfspr r11,SPRN_HSRR0 /* save HSRR0 */ 4158729c26eSNicholas Piggin mfspr r12,SPRN_HSRR1 /* and HSRR1 */ 4168729c26eSNicholas Piggin .else 4178729c26eSNicholas Piggin mfspr r11,SPRN_SRR0 /* save SRR0 */ 4188729c26eSNicholas Piggin mfspr r12,SPRN_SRR1 /* and SRR1 */ 419c7c5cbb4SNicholas Piggin .endif 420d73a10cbSNicholas Piggin 421d73a10cbSNicholas Piggin .if IBRANCH_TO_COMMON 4228729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON \name \virt 4238729c26eSNicholas Piggin .endif 4248729c26eSNicholas Piggin 425c7c5cbb4SNicholas Piggin .if \ool 426c7c5cbb4SNicholas Piggin .popsection 427c7c5cbb4SNicholas Piggin .endif 428c7c5cbb4SNicholas Piggin.endm 429c7c5cbb4SNicholas Piggin 430d064151fSNicholas Piggin/* 4318729c26eSNicholas Piggin * __GEN_COMMON_ENTRY is required to receive the branch from interrupt 4329600f261SNicholas Piggin * entry, except in the case of the real-mode handlers which require 4339600f261SNicholas Piggin * __GEN_REALMODE_COMMON_ENTRY. 4349600f261SNicholas Piggin * 4358729c26eSNicholas Piggin * This switches to virtual mode and sets MSR[RI]. 436d064151fSNicholas Piggin */ 4378729c26eSNicholas Piggin.macro __GEN_COMMON_ENTRY name 4388729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 4398729c26eSNicholas Piggin\name\()_common_real: 4409600f261SNicholas Piggin .if IKVM_REAL 4419d598f93SNicholas Piggin KVMTEST \name 4429600f261SNicholas Piggin .endif 4439600f261SNicholas Piggin 4448729c26eSNicholas Piggin ld r10,PACAKMSR(r13) /* get MSR value for kernel */ 4458729c26eSNicholas Piggin /* MSR[RI] is clear iff using SRR regs */ 4468729c26eSNicholas Piggin .if IHSRR == EXC_HV_OR_STD 4478729c26eSNicholas Piggin BEGIN_FTR_SECTION 4488729c26eSNicholas Piggin xori r10,r10,MSR_RI 4498729c26eSNicholas Piggin END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) 4508729c26eSNicholas Piggin .elseif ! IHSRR 4518729c26eSNicholas Piggin xori r10,r10,MSR_RI 4528729c26eSNicholas Piggin .endif 4538729c26eSNicholas Piggin mtmsrd r10 4548729c26eSNicholas Piggin 4558729c26eSNicholas Piggin .if IVIRT 4569600f261SNicholas Piggin .if IKVM_VIRT 4579600f261SNicholas Piggin b 1f /* skip the virt test coming from real */ 4589600f261SNicholas Piggin .endif 4599600f261SNicholas Piggin 4608729c26eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 4618729c26eSNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_virt) 4628729c26eSNicholas Piggin\name\()_common_virt: 4639600f261SNicholas Piggin .if IKVM_VIRT 4649d598f93SNicholas Piggin KVMTEST \name 4659600f261SNicholas Piggin1: 4669600f261SNicholas Piggin .endif 4678729c26eSNicholas Piggin .endif /* IVIRT */ 4688729c26eSNicholas Piggin.endm 4698729c26eSNicholas Piggin 4709600f261SNicholas Piggin/* 4719600f261SNicholas Piggin * Don't switch to virt mode. Used for early MCE and HMI handlers that 4729600f261SNicholas Piggin * want to run in real mode. 4739600f261SNicholas Piggin */ 4749600f261SNicholas Piggin.macro __GEN_REALMODE_COMMON_ENTRY name 4759600f261SNicholas PigginDEFINE_FIXED_SYMBOL(\name\()_common_real) 4769600f261SNicholas Piggin\name\()_common_real: 4779600f261SNicholas Piggin .if IKVM_REAL 4789d598f93SNicholas Piggin KVMTEST \name 4799600f261SNicholas Piggin .endif 4809600f261SNicholas Piggin.endm 4819600f261SNicholas Piggin 4828729c26eSNicholas Piggin.macro __GEN_COMMON_BODY name 4830eddf327SNicholas Piggin .if IMASK 4840eddf327SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 4850eddf327SNicholas Piggin andi. r10,r10,IMASK 4860eddf327SNicholas Piggin /* Associate vector numbers with bits in paca->irq_happened */ 4870eddf327SNicholas Piggin .if IVEC == 0x500 || IVEC == 0xea0 4880eddf327SNicholas Piggin li r10,PACA_IRQ_EE 4890eddf327SNicholas Piggin .elseif IVEC == 0x900 4900eddf327SNicholas Piggin li r10,PACA_IRQ_DEC 4910eddf327SNicholas Piggin .elseif IVEC == 0xa00 || IVEC == 0xe80 4920eddf327SNicholas Piggin li r10,PACA_IRQ_DBELL 4930eddf327SNicholas Piggin .elseif IVEC == 0xe60 4940eddf327SNicholas Piggin li r10,PACA_IRQ_HMI 4950eddf327SNicholas Piggin .elseif IVEC == 0xf00 4960eddf327SNicholas Piggin li r10,PACA_IRQ_PMI 4970eddf327SNicholas Piggin .else 4980eddf327SNicholas Piggin .abort "Bad maskable vector" 4990eddf327SNicholas Piggin .endif 5000eddf327SNicholas Piggin 501*3f7fbd97SNicholas Piggin .if IHSRR_IF_HVMODE 5020eddf327SNicholas Piggin BEGIN_FTR_SECTION 5030eddf327SNicholas Piggin bne masked_Hinterrupt 5040eddf327SNicholas Piggin FTR_SECTION_ELSE 5050eddf327SNicholas Piggin bne masked_interrupt 5060eddf327SNicholas Piggin ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 5070eddf327SNicholas Piggin .elseif IHSRR 5080eddf327SNicholas Piggin bne masked_Hinterrupt 5090eddf327SNicholas Piggin .else 5100eddf327SNicholas Piggin bne masked_interrupt 5110eddf327SNicholas Piggin .endif 5120eddf327SNicholas Piggin .endif 5130eddf327SNicholas Piggin 5146d71759aSNicholas Piggin .if ISTACK 5155d5e0edfSNicholas Piggin andi. r10,r12,MSR_PR /* See if coming from user */ 5165d5e0edfSNicholas Piggin mr r10,r1 /* Save r1 */ 5175d5e0edfSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */ 5181b359982SNicholas Piggin beq- 100f 5195d5e0edfSNicholas Piggin ld r1,PACAKSAVE(r13) /* kernel stack to use */ 5201b359982SNicholas Piggin100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */ 5211b359982SNicholas Piggin EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 5225d5e0edfSNicholas Piggin .endif 5238c9fb5d4SNicholas Piggin 5248c9fb5d4SNicholas Piggin std r9,_CCR(r1) /* save CR in stackframe */ 5258c9fb5d4SNicholas Piggin std r11,_NIP(r1) /* save SRR0 in stackframe */ 5268c9fb5d4SNicholas Piggin std r12,_MSR(r1) /* save SRR1 in stackframe */ 5278c9fb5d4SNicholas Piggin std r10,0(r1) /* make stack chain pointer */ 5288c9fb5d4SNicholas Piggin std r0,GPR0(r1) /* save r0 in stackframe */ 5298c9fb5d4SNicholas Piggin std r10,GPR1(r1) /* save r1 in stackframe */ 5305d5e0edfSNicholas Piggin 5318729c26eSNicholas Piggin .if ISET_RI 5328729c26eSNicholas Piggin li r10,MSR_RI 5338729c26eSNicholas Piggin mtmsrd r10,1 /* Set MSR_RI */ 5348729c26eSNicholas Piggin .endif 5358729c26eSNicholas Piggin 5366d71759aSNicholas Piggin .if ISTACK 5376d71759aSNicholas Piggin .if IKUAP 5385d5e0edfSNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1, cr0 5395d5e0edfSNicholas Piggin .endif 5401b359982SNicholas Piggin beq 101f /* if from kernel mode */ 5415d5e0edfSNicholas Piggin ACCOUNT_CPU_USER_ENTRY(r13, r9, r10) 542931dc86bSNicholas PigginBEGIN_FTR_SECTION 543931dc86bSNicholas Piggin ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ 544931dc86bSNicholas Piggin std r9,_PPR(r1) 545931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 5461b359982SNicholas Piggin101: 5475d5e0edfSNicholas Piggin .else 5486d71759aSNicholas Piggin .if IKUAP 549bcbceed4SNicholas Piggin kuap_save_amr_and_lock r9, r10, cr1 550bcbceed4SNicholas Piggin .endif 5515d5e0edfSNicholas Piggin .endif 5525d5e0edfSNicholas Piggin 5538c9fb5d4SNicholas Piggin /* Save original regs values from save area to stack frame. */ 5546d71759aSNicholas Piggin ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */ 5556d71759aSNicholas Piggin ld r10,IAREA+EX_R10(r13) 5568c9fb5d4SNicholas Piggin std r9,GPR9(r1) 5578c9fb5d4SNicholas Piggin std r10,GPR10(r1) 5586d71759aSNicholas Piggin ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */ 5596d71759aSNicholas Piggin ld r10,IAREA+EX_R12(r13) 5606d71759aSNicholas Piggin ld r11,IAREA+EX_R13(r13) 5618c9fb5d4SNicholas Piggin std r9,GPR11(r1) 5628c9fb5d4SNicholas Piggin std r10,GPR12(r1) 5638c9fb5d4SNicholas Piggin std r11,GPR13(r1) 564a3cd35beSNicholas Piggin 5656d71759aSNicholas Piggin .if IDAR 566a3cd35beSNicholas Piggin .if IISIDE 567d1a84718SNicholas Piggin ld r10,_NIP(r1) 568d1a84718SNicholas Piggin .else 5696d71759aSNicholas Piggin ld r10,IAREA+EX_DAR(r13) 570d1a84718SNicholas Piggin .endif 571d1a84718SNicholas Piggin std r10,_DAR(r1) 572d1a84718SNicholas Piggin .endif 573a3cd35beSNicholas Piggin 5746d71759aSNicholas Piggin .if IDSISR 575a3cd35beSNicholas Piggin .if IISIDE 576d1a84718SNicholas Piggin ld r10,_MSR(r1) 577d1a84718SNicholas Piggin lis r11,DSISR_SRR1_MATCH_64S@h 578d1a84718SNicholas Piggin and r10,r10,r11 579d1a84718SNicholas Piggin .else 5806d71759aSNicholas Piggin lwz r10,IAREA+EX_DSISR(r13) 581d1a84718SNicholas Piggin .endif 582d1a84718SNicholas Piggin std r10,_DSISR(r1) 583d1a84718SNicholas Piggin .endif 584a3cd35beSNicholas Piggin 585931dc86bSNicholas PigginBEGIN_FTR_SECTION 5866d71759aSNicholas Piggin ld r10,IAREA+EX_CFAR(r13) 5878c9fb5d4SNicholas Piggin std r10,ORIG_GPR3(r1) 588931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 5898729c26eSNicholas Piggin ld r10,IAREA+EX_CTR(r13) 5908c9fb5d4SNicholas Piggin std r10,_CTR(r1) 5918c9fb5d4SNicholas Piggin std r2,GPR2(r1) /* save r2 in stackframe */ 5928c9fb5d4SNicholas Piggin SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ 5938c9fb5d4SNicholas Piggin SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ 5948c9fb5d4SNicholas Piggin mflr r9 /* Get LR, later save to stack */ 5958c9fb5d4SNicholas Piggin ld r2,PACATOC(r13) /* get kernel TOC into r2 */ 5968c9fb5d4SNicholas Piggin std r9,_LINK(r1) 5978c9fb5d4SNicholas Piggin lbz r10,PACAIRQSOFTMASK(r13) 5988c9fb5d4SNicholas Piggin mfspr r11,SPRN_XER /* save XER in stackframe */ 5998c9fb5d4SNicholas Piggin std r10,SOFTE(r1) 6008c9fb5d4SNicholas Piggin std r11,_XER(r1) 6016d71759aSNicholas Piggin li r9,(IVEC)+1 6028c9fb5d4SNicholas Piggin std r9,_TRAP(r1) /* set trap number */ 6038c9fb5d4SNicholas Piggin li r10,0 6048c9fb5d4SNicholas Piggin ld r11,exception_marker@toc(r2) 6058c9fb5d4SNicholas Piggin std r10,RESULT(r1) /* clear regs->result */ 6068c9fb5d4SNicholas Piggin std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ 6075d5e0edfSNicholas Piggin 6086d71759aSNicholas Piggin .if ISTACK 6095d5e0edfSNicholas Piggin ACCOUNT_STOLEN_TIME 610bcbceed4SNicholas Piggin .endif 611d1a84718SNicholas Piggin 6126d71759aSNicholas Piggin .if IRECONCILE 613d1a84718SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 614d1a84718SNicholas Piggin .endif 615bcbceed4SNicholas Piggin.endm 616bcbceed4SNicholas Piggin 617391e941bSNicholas Piggin/* 6188729c26eSNicholas Piggin * On entry r13 points to the paca, r9-r13 are saved in the paca, 6198729c26eSNicholas Piggin * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and 6208729c26eSNicholas Piggin * SRR1, and relocation is on. 6218729c26eSNicholas Piggin * 6228729c26eSNicholas Piggin * If stack=0, then the stack is already set in r1, and r1 is saved in r10. 6238729c26eSNicholas Piggin * PPR save and CPU accounting is not done for the !stack case (XXX why not?) 6248729c26eSNicholas Piggin */ 6258729c26eSNicholas Piggin.macro GEN_COMMON name 6268729c26eSNicholas Piggin __GEN_COMMON_ENTRY \name 6278729c26eSNicholas Piggin __GEN_COMMON_BODY \name 6288729c26eSNicholas Piggin.endm 6298729c26eSNicholas Piggin 6308729c26eSNicholas Piggin/* 631391e941bSNicholas Piggin * Restore all registers including H/SRR0/1 saved in a stack frame of a 632391e941bSNicholas Piggin * standard exception. 633391e941bSNicholas Piggin */ 634*3f7fbd97SNicholas Piggin.macro EXCEPTION_RESTORE_REGS hsrr=0 635391e941bSNicholas Piggin /* Move original SRR0 and SRR1 into the respective regs */ 636391e941bSNicholas Piggin ld r9,_MSR(r1) 637391e941bSNicholas Piggin .if \hsrr 638391e941bSNicholas Piggin mtspr SPRN_HSRR1,r9 639391e941bSNicholas Piggin .else 640391e941bSNicholas Piggin mtspr SPRN_SRR1,r9 641391e941bSNicholas Piggin .endif 642391e941bSNicholas Piggin ld r9,_NIP(r1) 643391e941bSNicholas Piggin .if \hsrr 644391e941bSNicholas Piggin mtspr SPRN_HSRR0,r9 645391e941bSNicholas Piggin .else 646391e941bSNicholas Piggin mtspr SPRN_SRR0,r9 647391e941bSNicholas Piggin .endif 648391e941bSNicholas Piggin ld r9,_CTR(r1) 649391e941bSNicholas Piggin mtctr r9 650391e941bSNicholas Piggin ld r9,_XER(r1) 651391e941bSNicholas Piggin mtxer r9 652391e941bSNicholas Piggin ld r9,_LINK(r1) 653391e941bSNicholas Piggin mtlr r9 654391e941bSNicholas Piggin ld r9,_CCR(r1) 655391e941bSNicholas Piggin mtcr r9 656391e941bSNicholas Piggin REST_8GPRS(2, r1) 657391e941bSNicholas Piggin REST_4GPRS(10, r1) 658391e941bSNicholas Piggin REST_GPR(0, r1) 659391e941bSNicholas Piggin /* restore original r1. */ 660391e941bSNicholas Piggin ld r1,GPR1(r1) 661391e941bSNicholas Piggin.endm 662d064151fSNicholas Piggin 66312a04809SNicholas Piggin#define RUNLATCH_ON \ 66412a04809SNicholas PigginBEGIN_FTR_SECTION \ 66512a04809SNicholas Piggin ld r3, PACA_THREAD_INFO(r13); \ 66612a04809SNicholas Piggin ld r4,TI_LOCAL_FLAGS(r3); \ 66712a04809SNicholas Piggin andi. r0,r4,_TLF_RUNLATCH; \ 66812a04809SNicholas Piggin beql ppc64_runlatch_on_trampoline; \ 66912a04809SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CTRL) 67012a04809SNicholas Piggin 67112a04809SNicholas Piggin/* 67212a04809SNicholas Piggin * When the idle code in power4_idle puts the CPU into NAP mode, 67312a04809SNicholas Piggin * it has to do so in a loop, and relies on the external interrupt 67412a04809SNicholas Piggin * and decrementer interrupt entry code to get it out of the loop. 67512a04809SNicholas Piggin * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags 67612a04809SNicholas Piggin * to signal that it is in the loop and needs help to get out. 67712a04809SNicholas Piggin */ 67812a04809SNicholas Piggin#ifdef CONFIG_PPC_970_NAP 67912a04809SNicholas Piggin#define FINISH_NAP \ 68012a04809SNicholas PigginBEGIN_FTR_SECTION \ 68112a04809SNicholas Piggin ld r11, PACA_THREAD_INFO(r13); \ 68212a04809SNicholas Piggin ld r9,TI_LOCAL_FLAGS(r11); \ 68312a04809SNicholas Piggin andi. r10,r9,_TLF_NAPPING; \ 68412a04809SNicholas Piggin bnel power4_fixup_nap; \ 68512a04809SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 68612a04809SNicholas Piggin#else 68712a04809SNicholas Piggin#define FINISH_NAP 68812a04809SNicholas Piggin#endif 68912a04809SNicholas Piggin 69012a04809SNicholas Piggin/* 69157f26649SNicholas Piggin * There are a few constraints to be concerned with. 69257f26649SNicholas Piggin * - Real mode exceptions code/data must be located at their physical location. 69357f26649SNicholas Piggin * - Virtual mode exceptions must be mapped at their 0xc000... location. 69457f26649SNicholas Piggin * - Fixed location code must not call directly beyond the __end_interrupts 69557f26649SNicholas Piggin * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence 69657f26649SNicholas Piggin * must be used. 69757f26649SNicholas Piggin * - LOAD_HANDLER targets must be within first 64K of physical 0 / 69857f26649SNicholas Piggin * virtual 0xc00... 69957f26649SNicholas Piggin * - Conditional branch targets must be within +/-32K of caller. 70057f26649SNicholas Piggin * 70157f26649SNicholas Piggin * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and 70257f26649SNicholas Piggin * therefore don't have to run in physically located code or rfid to 70357f26649SNicholas Piggin * virtual mode kernel code. However on relocatable kernels they do have 70457f26649SNicholas Piggin * to branch to KERNELBASE offset because the rest of the kernel (outside 70557f26649SNicholas Piggin * the exception vectors) may be located elsewhere. 70657f26649SNicholas Piggin * 70757f26649SNicholas Piggin * Virtual exceptions correspond with physical, except their entry points 70857f26649SNicholas Piggin * are offset by 0xc000000000000000 and also tend to get an added 0x4000 70957f26649SNicholas Piggin * offset applied. Virtual exceptions are enabled with the Alternate 71057f26649SNicholas Piggin * Interrupt Location (AIL) bit set in the LPCR. However this does not 71157f26649SNicholas Piggin * guarantee they will be delivered virtually. Some conditions (see the ISA) 71257f26649SNicholas Piggin * cause exceptions to be delivered in real mode. 71357f26649SNicholas Piggin * 71457f26649SNicholas Piggin * It's impossible to receive interrupts below 0x300 via AIL. 71557f26649SNicholas Piggin * 71657f26649SNicholas Piggin * KVM: None of the virtual exceptions are from the guest. Anything that 71757f26649SNicholas Piggin * escalated to HV=1 from HV=0 is delivered via real mode handlers. 71857f26649SNicholas Piggin * 71957f26649SNicholas Piggin * 7200ebc4cdaSBenjamin Herrenschmidt * We layout physical memory as follows: 7210ebc4cdaSBenjamin Herrenschmidt * 0x0000 - 0x00ff : Secondary processor spin code 72257f26649SNicholas Piggin * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors 72357f26649SNicholas Piggin * 0x1900 - 0x3fff : Real mode trampolines 72457f26649SNicholas Piggin * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors 72557f26649SNicholas Piggin * 0x5900 - 0x6fff : Relon mode trampolines 7260ebc4cdaSBenjamin Herrenschmidt * 0x7000 - 0x7fff : FWNMI data area 72757f26649SNicholas Piggin * 0x8000 - .... : Common interrupt handlers, remaining early 72857f26649SNicholas Piggin * setup code, rest of kernel. 729e0319829SNicholas Piggin * 730e0319829SNicholas Piggin * We could reclaim 0x4000-0x42ff for real mode trampolines if the space 731e0319829SNicholas Piggin * is necessary. Until then it's more consistent to explicitly put VIRT_NONE 732e0319829SNicholas Piggin * vectors there. 7330ebc4cdaSBenjamin Herrenschmidt */ 73457f26649SNicholas PigginOPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900) 73557f26649SNicholas PigginOPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000) 73657f26649SNicholas PigginOPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900) 73757f26649SNicholas PigginOPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000) 738ccd47702SNicholas Piggin 739ccd47702SNicholas Piggin#ifdef CONFIG_PPC_POWERNV 740bd3524feSNicholas Piggin .globl start_real_trampolines 741bd3524feSNicholas Piggin .globl end_real_trampolines 742bd3524feSNicholas Piggin .globl start_virt_trampolines 743bd3524feSNicholas Piggin .globl end_virt_trampolines 744ccd47702SNicholas Piggin#endif 745ccd47702SNicholas Piggin 74657f26649SNicholas Piggin#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) 74757f26649SNicholas Piggin/* 74857f26649SNicholas Piggin * Data area reserved for FWNMI option. 74957f26649SNicholas Piggin * This address (0x7000) is fixed by the RPA. 75057f26649SNicholas Piggin * pseries and powernv need to keep the whole page from 75157f26649SNicholas Piggin * 0x7000 to 0x8000 free for use by the firmware 75257f26649SNicholas Piggin */ 75357f26649SNicholas PigginZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000) 75457f26649SNicholas PigginOPEN_TEXT_SECTION(0x8000) 75557f26649SNicholas Piggin#else 75657f26649SNicholas PigginOPEN_TEXT_SECTION(0x7000) 75757f26649SNicholas Piggin#endif 75857f26649SNicholas Piggin 75957f26649SNicholas PigginUSE_FIXED_SECTION(real_vectors) 76057f26649SNicholas Piggin 7610ebc4cdaSBenjamin Herrenschmidt/* 7620ebc4cdaSBenjamin Herrenschmidt * This is the start of the interrupt handlers for pSeries 7630ebc4cdaSBenjamin Herrenschmidt * This code runs with relocation off. 7640ebc4cdaSBenjamin Herrenschmidt * Code from here to __end_interrupts gets copied down to real 7650ebc4cdaSBenjamin Herrenschmidt * address 0x100 when we are running a relocatable kernel. 7660ebc4cdaSBenjamin Herrenschmidt * Therefore any relative branches in this section must only 7670ebc4cdaSBenjamin Herrenschmidt * branch to labels in this section. 7680ebc4cdaSBenjamin Herrenschmidt */ 7690ebc4cdaSBenjamin Herrenschmidt .globl __start_interrupts 7700ebc4cdaSBenjamin Herrenschmidt__start_interrupts: 7710ebc4cdaSBenjamin Herrenschmidt 772e0319829SNicholas Piggin/* No virt vectors corresponding with 0x0..0x100 */ 7731a6822d1SNicholas PigginEXC_VIRT_NONE(0x4000, 0x100) 774e0319829SNicholas Piggin 775fb479e44SNicholas Piggin 7764f50541fSNicholas PigginINT_DEFINE_BEGIN(system_reset) 7774f50541fSNicholas Piggin IVEC=0x100 7784f50541fSNicholas Piggin IAREA=PACA_EXNMI 7798729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 7804f50541fSNicholas Piggin /* 7814f50541fSNicholas Piggin * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is 7824f50541fSNicholas Piggin * being used, so a nested NMI exception would corrupt it. 7834f50541fSNicholas Piggin */ 7844f50541fSNicholas Piggin ISET_RI=0 7854f50541fSNicholas Piggin ISTACK=0 7864f50541fSNicholas Piggin IRECONCILE=0 7874f50541fSNicholas Piggin IKVM_REAL=1 7884f50541fSNicholas PigginINT_DEFINE_END(system_reset) 7894f50541fSNicholas Piggin 790a7c1ca19SNicholas PigginEXC_REAL_BEGIN(system_reset, 0x100, 0x100) 791fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 792fb479e44SNicholas Piggin /* 793fb479e44SNicholas Piggin * If running native on arch 2.06 or later, check if we are waking up 794ba6d334aSBenjamin Herrenschmidt * from nap/sleep/winkle, and branch to idle handler. This tests SRR1 795ba6d334aSBenjamin Herrenschmidt * bits 46:47. A non-0 value indicates that we are coming from a power 796ba6d334aSBenjamin Herrenschmidt * saving state. The idle wakeup handler initially runs in real mode, 797ba6d334aSBenjamin Herrenschmidt * but we branch to the 0xc000... address so we can turn on relocation 7980e10be2bSNicholas Piggin * with mtmsrd later, after SPRs are restored. 7990e10be2bSNicholas Piggin * 8000e10be2bSNicholas Piggin * Careful to minimise cost for the fast path (idle wakeup) while 8010e10be2bSNicholas Piggin * also avoiding clobbering CFAR for the debug path (non-idle). 8020e10be2bSNicholas Piggin * 8030e10be2bSNicholas Piggin * For the idle wake case volatile registers can be clobbered, which 8040e10be2bSNicholas Piggin * is why we use those initially. If it turns out to not be an idle 8050e10be2bSNicholas Piggin * wake, carefully put everything back the way it was, so we can use 8060e10be2bSNicholas Piggin * common exception macros to handle it. 807fb479e44SNicholas Piggin */ 808a7c1ca19SNicholas PigginBEGIN_FTR_SECTION 8090e10be2bSNicholas Piggin SET_SCRATCH0(r13) 8100e10be2bSNicholas Piggin GET_PACA(r13) 8110e10be2bSNicholas Piggin std r3,PACA_EXNMI+0*8(r13) 8120e10be2bSNicholas Piggin std r4,PACA_EXNMI+1*8(r13) 8130e10be2bSNicholas Piggin std r5,PACA_EXNMI+2*8(r13) 814a7c1ca19SNicholas Piggin mfspr r3,SPRN_SRR1 8150e10be2bSNicholas Piggin mfocrf r4,0x80 8160e10be2bSNicholas Piggin rlwinm. r5,r3,47-31,30,31 8170e10be2bSNicholas Piggin bne+ system_reset_idle_wake 8180e10be2bSNicholas Piggin /* Not powersave wakeup. Restore regs for regular interrupt handler. */ 8190e10be2bSNicholas Piggin mtocrf 0x80,r4 8200e10be2bSNicholas Piggin ld r3,PACA_EXNMI+0*8(r13) 8210e10be2bSNicholas Piggin ld r4,PACA_EXNMI+1*8(r13) 8220e10be2bSNicholas Piggin ld r5,PACA_EXNMI+2*8(r13) 8230e10be2bSNicholas Piggin GET_SCRATCH0(r13) 824a7c1ca19SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 825fb479e44SNicholas Piggin#endif 826fb479e44SNicholas Piggin 8274f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 828c4f3b52cSNicholas Piggin /* 8290e10be2bSNicholas Piggin * In theory, we should not enable relocation here if it was disabled 8300e10be2bSNicholas Piggin * in SRR1, because the MMU may not be configured to support it (e.g., 8310e10be2bSNicholas Piggin * SLB may have been cleared). In practice, there should only be a few 8320e10be2bSNicholas Piggin * small windows where that's the case, and sreset is considered to 8330e10be2bSNicholas Piggin * be dangerous anyway. 834c4f3b52cSNicholas Piggin */ 8351a6822d1SNicholas PigginEXC_REAL_END(system_reset, 0x100, 0x100) 8361a6822d1SNicholas PigginEXC_VIRT_NONE(0x4100, 0x100) 837fb479e44SNicholas Piggin 838fb479e44SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 8390e10be2bSNicholas PigginTRAMP_REAL_BEGIN(system_reset_idle_wake) 8400e10be2bSNicholas Piggin /* We are waking up from idle, so may clobber any volatile register */ 8410e10be2bSNicholas Piggin cmpwi cr1,r5,2 8420e10be2bSNicholas Piggin bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ 8430e10be2bSNicholas Piggin BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) 844371fefd6SPaul Mackerras#endif 845371fefd6SPaul Mackerras 846acc8da44SNicholas Piggin#ifdef CONFIG_PPC_PSERIES 847acc8da44SNicholas Piggin/* 848acc8da44SNicholas Piggin * Vectors for the FWNMI option. Share common code. 849acc8da44SNicholas Piggin */ 850acc8da44SNicholas PigginTRAMP_REAL_BEGIN(system_reset_fwnmi) 8514f50541fSNicholas Piggin __IKVM_REAL(system_reset)=0 8524f50541fSNicholas Piggin GEN_INT_ENTRY system_reset, virt=0 853acc8da44SNicholas Piggin 854acc8da44SNicholas Piggin#endif /* CONFIG_PPC_PSERIES */ 855acc8da44SNicholas Piggin 856a3d96f70SNicholas PigginEXC_COMMON_BEGIN(system_reset_common) 8578729c26eSNicholas Piggin __GEN_COMMON_ENTRY system_reset 858c4f3b52cSNicholas Piggin /* 859c4f3b52cSNicholas Piggin * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able 860c4f3b52cSNicholas Piggin * to recover, but nested NMI will notice in_nmi and not recover 861c4f3b52cSNicholas Piggin * because of the use of the NMI stack. in_nmi reentrancy is tested in 862c4f3b52cSNicholas Piggin * system_reset_exception. 863c4f3b52cSNicholas Piggin */ 864c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 865c4f3b52cSNicholas Piggin addi r10,r10,1 866c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 867c4f3b52cSNicholas Piggin li r10,MSR_RI 868c4f3b52cSNicholas Piggin mtmsrd r10,1 869aca79d2bSVaidyanathan Srinivasan 870b1ee8a3dSNicholas Piggin mr r10,r1 871b1ee8a3dSNicholas Piggin ld r1,PACA_NMI_EMERG_SP(r13) 872b1ee8a3dSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 8738729c26eSNicholas Piggin __GEN_COMMON_BODY system_reset 87447169fbaSNicholas Piggin bl save_nvgprs 87547169fbaSNicholas Piggin /* 87647169fbaSNicholas Piggin * Set IRQS_ALL_DISABLED unconditionally so arch_irqs_disabled does 87747169fbaSNicholas Piggin * the right thing. We do not want to reconcile because that goes 87847169fbaSNicholas Piggin * through irq tracing which we don't want in NMI. 87947169fbaSNicholas Piggin * 88047169fbaSNicholas Piggin * Save PACAIRQHAPPENED because some code will do a hard disable 88147169fbaSNicholas Piggin * (e.g., xmon). So we want to restore this back to where it was 88247169fbaSNicholas Piggin * when we return. DAR is unused in the stack, so save it there. 88347169fbaSNicholas Piggin */ 88447169fbaSNicholas Piggin li r10,IRQS_ALL_DISABLED 88547169fbaSNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 88647169fbaSNicholas Piggin lbz r10,PACAIRQHAPPENED(r13) 88747169fbaSNicholas Piggin std r10,_DAR(r1) 88847169fbaSNicholas Piggin 889c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 890c06075f3SNicholas Piggin bl system_reset_exception 89115b4dd79SNicholas Piggin 89215b4dd79SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */ 893fbc50063SNicholas Piggin li r9,0 89415b4dd79SNicholas Piggin mtmsrd r9,1 895c4f3b52cSNicholas Piggin 896c4f3b52cSNicholas Piggin /* 89715b4dd79SNicholas Piggin * MSR_RI is clear, now we can decrement paca->in_nmi. 898c4f3b52cSNicholas Piggin */ 899c4f3b52cSNicholas Piggin lhz r10,PACA_IN_NMI(r13) 900c4f3b52cSNicholas Piggin subi r10,r10,1 901c4f3b52cSNicholas Piggin sth r10,PACA_IN_NMI(r13) 902c4f3b52cSNicholas Piggin 90315b4dd79SNicholas Piggin /* 90415b4dd79SNicholas Piggin * Restore soft mask settings. 90515b4dd79SNicholas Piggin */ 90615b4dd79SNicholas Piggin ld r10,_DAR(r1) 90715b4dd79SNicholas Piggin stb r10,PACAIRQHAPPENED(r13) 90815b4dd79SNicholas Piggin ld r10,SOFTE(r1) 90915b4dd79SNicholas Piggin stb r10,PACAIRQSOFTMASK(r13) 91015b4dd79SNicholas Piggin 911*3f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 91215b4dd79SNicholas Piggin RFI_TO_USER_OR_KERNEL 913582baf44SNicholas Piggin 9149600f261SNicholas Piggin GEN_KVM system_reset 9159600f261SNicholas Piggin 9160ebc4cdaSBenjamin Herrenschmidt 9174f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check_early) 9184f50541fSNicholas Piggin IVEC=0x200 9194f50541fSNicholas Piggin IAREA=PACA_EXMC 9208729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 921d73a10cbSNicholas Piggin IREALMODE_COMMON=1 922c8eb54dbSNicholas Piggin /* 923c8eb54dbSNicholas Piggin * MSR_RI is not enabled, because PACA_EXMC is being used, so a 924c8eb54dbSNicholas Piggin * nested machine check corrupts it. machine_check_common enables 925c8eb54dbSNicholas Piggin * MSR_RI. 926c8eb54dbSNicholas Piggin */ 9274f50541fSNicholas Piggin ISET_RI=0 9284f50541fSNicholas Piggin ISTACK=0 9294f50541fSNicholas Piggin IDAR=1 9304f50541fSNicholas Piggin IDSISR=1 9314f50541fSNicholas Piggin IRECONCILE=0 9324f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 9334f50541fSNicholas PigginINT_DEFINE_END(machine_check_early) 9344f50541fSNicholas Piggin 9354f50541fSNicholas PigginINT_DEFINE_BEGIN(machine_check) 9364f50541fSNicholas Piggin IVEC=0x200 9374f50541fSNicholas Piggin IAREA=PACA_EXMC 9388729c26eSNicholas Piggin IVIRT=0 /* no virt entry point */ 9394f50541fSNicholas Piggin ISET_RI=0 9404f50541fSNicholas Piggin IDAR=1 9414f50541fSNicholas Piggin IDSISR=1 9424f50541fSNicholas Piggin IKVM_SKIP=1 9434f50541fSNicholas Piggin IKVM_REAL=1 9444f50541fSNicholas PigginINT_DEFINE_END(machine_check) 9454f50541fSNicholas Piggin 9464f50541fSNicholas PigginEXC_REAL_BEGIN(machine_check, 0x200, 0x100) 9474f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 9481a6822d1SNicholas PigginEXC_REAL_END(machine_check, 0x200, 0x100) 9491a6822d1SNicholas PigginEXC_VIRT_NONE(0x4200, 0x100) 950c8eb54dbSNicholas Piggin 951abd1f4caSNicholas Piggin#ifdef CONFIG_PPC_PSERIES 952abd1f4caSNicholas PigginTRAMP_REAL_BEGIN(machine_check_fwnmi) 953abd1f4caSNicholas Piggin /* See comment at machine_check exception, don't turn on RI */ 9544f50541fSNicholas Piggin GEN_INT_ENTRY machine_check_early, virt=0 955abd1f4caSNicholas Piggin#endif 956abd1f4caSNicholas Piggin 957fce16d48SNicholas Piggin#define MACHINE_CHECK_HANDLER_WINDUP \ 958fce16d48SNicholas Piggin /* Clear MSR_RI before setting SRR0 and SRR1. */\ 959fce16d48SNicholas Piggin li r9,0; \ 960fce16d48SNicholas Piggin mtmsrd r9,1; /* Clear MSR_RI */ \ 961fce16d48SNicholas Piggin /* Decrement paca->in_mce now RI is clear. */ \ 962fce16d48SNicholas Piggin lhz r12,PACA_IN_MCE(r13); \ 963fce16d48SNicholas Piggin subi r12,r12,1; \ 964fce16d48SNicholas Piggin sth r12,PACA_IN_MCE(r13); \ 965*3f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS 966fce16d48SNicholas Piggin 967c8eb54dbSNicholas PigginEXC_COMMON_BEGIN(machine_check_early_common) 9689600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY machine_check_early 9699600f261SNicholas Piggin 970afcf0095SNicholas Piggin /* 971afcf0095SNicholas Piggin * Switch to mc_emergency stack and handle re-entrancy (we limit 972afcf0095SNicholas Piggin * the nested MCE upto level 4 to avoid stack overflow). 973afcf0095SNicholas Piggin * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1 974afcf0095SNicholas Piggin * 975afcf0095SNicholas Piggin * We use paca->in_mce to check whether this is the first entry or 976afcf0095SNicholas Piggin * nested machine check. We increment paca->in_mce to track nested 977afcf0095SNicholas Piggin * machine checks. 978afcf0095SNicholas Piggin * 979afcf0095SNicholas Piggin * If this is the first entry then set stack pointer to 980afcf0095SNicholas Piggin * paca->mc_emergency_sp, otherwise r1 is already pointing to 981afcf0095SNicholas Piggin * stack frame on mc_emergency stack. 982afcf0095SNicholas Piggin * 983afcf0095SNicholas Piggin * NOTE: We are here with MSR_ME=0 (off), which means we risk a 984afcf0095SNicholas Piggin * checkstop if we get another machine check exception before we do 985afcf0095SNicholas Piggin * rfid with MSR_ME=1. 9861945bc45SNicholas Piggin * 9871945bc45SNicholas Piggin * This interrupt can wake directly from idle. If that is the case, 9881945bc45SNicholas Piggin * the machine check is handled then the idle wakeup code is called 9892bf1071aSNicholas Piggin * to restore state. 990afcf0095SNicholas Piggin */ 991afcf0095SNicholas Piggin lhz r10,PACA_IN_MCE(r13) 992afcf0095SNicholas Piggin cmpwi r10,0 /* Are we in nested machine check */ 993c8eb54dbSNicholas Piggin cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */ 994afcf0095SNicholas Piggin addi r10,r10,1 /* increment paca->in_mce */ 995afcf0095SNicholas Piggin sth r10,PACA_IN_MCE(r13) 996c8eb54dbSNicholas Piggin 997c8eb54dbSNicholas Piggin mr r10,r1 /* Save r1 */ 998c8eb54dbSNicholas Piggin bne 1f 999c8eb54dbSNicholas Piggin /* First machine check entry */ 1000c8eb54dbSNicholas Piggin ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */ 1001b7d9ccecSNicholas Piggin1: /* Limit nested MCE to level 4 to avoid stack overflow */ 1002b7d9ccecSNicholas Piggin bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */ 1003b7d9ccecSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1004c8eb54dbSNicholas Piggin 10058729c26eSNicholas Piggin __GEN_COMMON_BODY machine_check_early 1006c8eb54dbSNicholas Piggin 1007db7d31acSMahesh SalgaonkarBEGIN_FTR_SECTION 1008296e753fSNicholas Piggin bl enable_machine_check 1009db7d31acSMahesh SalgaonkarEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1010296e753fSNicholas Piggin li r10,MSR_RI 1011296e753fSNicholas Piggin mtmsrd r10,1 1012296e753fSNicholas Piggin 1013afcf0095SNicholas Piggin bl save_nvgprs 1014afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1015afcf0095SNicholas Piggin bl machine_check_early 1016afcf0095SNicholas Piggin std r3,RESULT(r1) /* Save result */ 1017afcf0095SNicholas Piggin ld r12,_MSR(r1) 10181945bc45SNicholas Piggin 1019afcf0095SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1020afcf0095SNicholas Piggin /* 1021afcf0095SNicholas Piggin * Check if thread was in power saving mode. We come here when any 1022afcf0095SNicholas Piggin * of the following is true: 1023afcf0095SNicholas Piggin * a. thread wasn't in power saving mode 1024afcf0095SNicholas Piggin * b. thread was in power saving mode with no state loss, 1025afcf0095SNicholas Piggin * supervisor state loss or hypervisor state loss. 1026afcf0095SNicholas Piggin * 1027afcf0095SNicholas Piggin * Go back to nap/sleep/winkle mode again if (b) is true. 1028afcf0095SNicholas Piggin */ 10291945bc45SNicholas PigginBEGIN_FTR_SECTION 10301945bc45SNicholas Piggin rlwinm. r11,r12,47-31,30,31 10316102c005SNicholas Piggin bne machine_check_idle_common 10321945bc45SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) 1033afcf0095SNicholas Piggin#endif 10341945bc45SNicholas Piggin 1035afcf0095SNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1036afcf0095SNicholas Piggin /* 103719dbe673SNicholas Piggin * Check if we are coming from guest. If yes, then run the normal 103805f97d94SNicholas Piggin * exception handler which will take the 103905f97d94SNicholas Piggin * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event 104005f97d94SNicholas Piggin * to guest. 1041afcf0095SNicholas Piggin */ 1042afcf0095SNicholas Piggin lbz r11,HSTATE_IN_GUEST(r13) 1043afcf0095SNicholas Piggin cmpwi r11,0 /* Check if coming from guest */ 1044b3fe3526SNicholas Piggin bne mce_deliver /* continue if we are. */ 1045afcf0095SNicholas Piggin#endif 104619dbe673SNicholas Piggin 1047afcf0095SNicholas Piggin /* 104819dbe673SNicholas Piggin * Check if we are coming from userspace. If yes, then run the normal 104919dbe673SNicholas Piggin * exception handler which will deliver the MC event to this kernel. 105019dbe673SNicholas Piggin */ 105119dbe673SNicholas Piggin andi. r11,r12,MSR_PR /* See if coming from user. */ 1052b3fe3526SNicholas Piggin bne mce_deliver /* continue in V mode if we are. */ 105319dbe673SNicholas Piggin 105419dbe673SNicholas Piggin /* 105519dbe673SNicholas Piggin * At this point we are coming from kernel context. 1056afcf0095SNicholas Piggin * Queue up the MCE event and return from the interrupt. 1057afcf0095SNicholas Piggin * But before that, check if this is an un-recoverable exception. 1058afcf0095SNicholas Piggin * If yes, then stay on emergency stack and panic. 1059afcf0095SNicholas Piggin */ 1060afcf0095SNicholas Piggin andi. r11,r12,MSR_RI 1061b7d9ccecSNicholas Piggin beq unrecoverable_mce 1062b7d9ccecSNicholas Piggin 1063afcf0095SNicholas Piggin /* 1064afcf0095SNicholas Piggin * Check if we have successfully handled/recovered from error, if not 1065afcf0095SNicholas Piggin * then stay on emergency stack and panic. 1066afcf0095SNicholas Piggin */ 1067afcf0095SNicholas Piggin ld r3,RESULT(r1) /* Load result */ 1068afcf0095SNicholas Piggin cmpdi r3,0 /* see if we handled MCE successfully */ 1069b7d9ccecSNicholas Piggin beq unrecoverable_mce /* if !handled then panic */ 1070272f6364SNicholas Piggin 1071afcf0095SNicholas Piggin /* 1072afcf0095SNicholas Piggin * Return from MC interrupt. 1073afcf0095SNicholas Piggin * Queue up the MCE event so that we can log it later, while 1074afcf0095SNicholas Piggin * returning from kernel or opal call. 1075afcf0095SNicholas Piggin */ 1076afcf0095SNicholas Piggin bl machine_check_queue_event 1077afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 1078fe9d482bSNicholas Piggin RFI_TO_KERNEL 1079272f6364SNicholas Piggin 1080b3fe3526SNicholas Pigginmce_deliver: 1081b3fe3526SNicholas Piggin /* 1082b3fe3526SNicholas Piggin * This is a host user or guest MCE. Restore all registers, then 1083b3fe3526SNicholas Piggin * run the "late" handler. For host user, this will run the 1084b3fe3526SNicholas Piggin * machine_check_exception handler in virtual mode like a normal 1085b3fe3526SNicholas Piggin * interrupt handler. For guest, this will trigger the KVM test 1086b3fe3526SNicholas Piggin * and branch to the KVM interrupt similarly to other interrupts. 1087b3fe3526SNicholas Piggin */ 10880b66370cSNicholas PigginBEGIN_FTR_SECTION 10890b66370cSNicholas Piggin ld r10,ORIG_GPR3(r1) 10900b66370cSNicholas Piggin mtspr SPRN_CFAR,r10 10910b66370cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1092afcf0095SNicholas Piggin MACHINE_CHECK_HANDLER_WINDUP 10934f50541fSNicholas Piggin GEN_INT_ENTRY machine_check, virt=0 1094afcf0095SNicholas Piggin 1095fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_common) 1096fce16d48SNicholas Piggin /* 1097fce16d48SNicholas Piggin * Machine check is different because we use a different 1098fce16d48SNicholas Piggin * save area: PACA_EXMC instead of PACA_EXGEN. 1099fce16d48SNicholas Piggin */ 11004f50541fSNicholas Piggin GEN_COMMON machine_check 11014f50541fSNicholas Piggin 1102fce16d48SNicholas Piggin FINISH_NAP 1103fce16d48SNicholas Piggin /* Enable MSR_RI when finished with PACA_EXMC */ 1104fce16d48SNicholas Piggin li r10,MSR_RI 1105fce16d48SNicholas Piggin mtmsrd r10,1 1106fce16d48SNicholas Piggin bl save_nvgprs 1107fce16d48SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1108fce16d48SNicholas Piggin bl machine_check_exception 1109fce16d48SNicholas Piggin b ret_from_except 1110fce16d48SNicholas Piggin 11119600f261SNicholas Piggin GEN_KVM machine_check 11129600f261SNicholas Piggin 11139600f261SNicholas Piggin 1114fce16d48SNicholas Piggin#ifdef CONFIG_PPC_P7_NAP 1115fce16d48SNicholas Piggin/* 1116fce16d48SNicholas Piggin * This is an idle wakeup. Low level machine check has already been 1117fce16d48SNicholas Piggin * done. Queue the event then call the idle code to do the wake up. 1118fce16d48SNicholas Piggin */ 1119fce16d48SNicholas PigginEXC_COMMON_BEGIN(machine_check_idle_common) 1120fce16d48SNicholas Piggin bl machine_check_queue_event 1121fce16d48SNicholas Piggin 1122fce16d48SNicholas Piggin /* 1123fce16d48SNicholas Piggin * We have not used any non-volatile GPRs here, and as a rule 1124fce16d48SNicholas Piggin * most exception code including machine check does not. 1125fce16d48SNicholas Piggin * Therefore PACA_NAPSTATELOST does not need to be set. Idle 1126fce16d48SNicholas Piggin * wakeup will restore volatile registers. 1127fce16d48SNicholas Piggin * 1128fce16d48SNicholas Piggin * Load the original SRR1 into r3 for pnv_powersave_wakeup_mce. 1129fce16d48SNicholas Piggin * 1130fce16d48SNicholas Piggin * Then decrement MCE nesting after finishing with the stack. 1131fce16d48SNicholas Piggin */ 1132fce16d48SNicholas Piggin ld r3,_MSR(r1) 1133fce16d48SNicholas Piggin ld r4,_LINK(r1) 1134fce16d48SNicholas Piggin 1135fce16d48SNicholas Piggin lhz r11,PACA_IN_MCE(r13) 1136fce16d48SNicholas Piggin subi r11,r11,1 1137fce16d48SNicholas Piggin sth r11,PACA_IN_MCE(r13) 1138fce16d48SNicholas Piggin 1139fce16d48SNicholas Piggin mtlr r4 1140fce16d48SNicholas Piggin rlwinm r10,r3,47-31,30,31 1141fce16d48SNicholas Piggin cmpwi cr1,r10,2 1142fce16d48SNicholas Piggin bltlr cr1 /* no state loss, return to idle caller */ 1143fce16d48SNicholas Piggin b idle_return_gpr_loss 1144fce16d48SNicholas Piggin#endif 1145fce16d48SNicholas Piggin 1146b7d9ccecSNicholas PigginEXC_COMMON_BEGIN(unrecoverable_mce) 1147b7d9ccecSNicholas Piggin /* 1148b7d9ccecSNicholas Piggin * We are going down. But there are chances that we might get hit by 1149b7d9ccecSNicholas Piggin * another MCE during panic path and we may run into unstable state 1150b7d9ccecSNicholas Piggin * with no way out. Hence, turn ME bit off while going down, so that 1151b7d9ccecSNicholas Piggin * when another MCE is hit during panic path, system will checkstop 1152b7d9ccecSNicholas Piggin * and hypervisor will get restarted cleanly by SP. 1153b7d9ccecSNicholas Piggin */ 1154b7d9ccecSNicholas PigginBEGIN_FTR_SECTION 1155b7d9ccecSNicholas Piggin li r10,0 /* clear MSR_RI */ 1156b7d9ccecSNicholas Piggin mtmsrd r10,1 1157b7d9ccecSNicholas Piggin bl disable_machine_check 1158b7d9ccecSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 1159b7d9ccecSNicholas Piggin ld r10,PACAKMSR(r13) 1160b7d9ccecSNicholas Piggin li r3,MSR_ME 1161b7d9ccecSNicholas Piggin andc r10,r10,r3 1162b7d9ccecSNicholas Piggin mtmsrd r10 1163b7d9ccecSNicholas Piggin 1164afcf0095SNicholas Piggin /* Invoke machine_check_exception to print MCE event and panic. */ 1165afcf0095SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1166afcf0095SNicholas Piggin bl machine_check_exception 1167b7d9ccecSNicholas Piggin 1168afcf0095SNicholas Piggin /* 1169b7d9ccecSNicholas Piggin * We will not reach here. Even if we did, there is no way out. 1170b7d9ccecSNicholas Piggin * Call unrecoverable_exception and die. 1171afcf0095SNicholas Piggin */ 1172b7d9ccecSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1173afcf0095SNicholas Piggin bl unrecoverable_exception 1174b7d9ccecSNicholas Piggin b . 1175afcf0095SNicholas Piggin 11764f50541fSNicholas Piggin 11774f50541fSNicholas Piggin/** 11784f50541fSNicholas Piggin * 0x300 - Data Storage Interrupt (DSI) 11794f50541fSNicholas Piggin * This interrupt is generated due to a data access which does not have a valid 11804f50541fSNicholas Piggin * page table entry with permissions to allow the data access to be performed. 11814f50541fSNicholas Piggin * DAWR matches also fault here, as do RC updates, and minor misc errors e.g., 11824f50541fSNicholas Piggin * copy/paste, AMO, certain invalid CI accesses, etc. 11834f50541fSNicholas Piggin * 11844f50541fSNicholas Piggin * This interrupt is delivered to the guest (HV bit unchanged). 11854f50541fSNicholas Piggin * 11864f50541fSNicholas Piggin * Linux HPT responds by first attempting to refill the hash table from the 11874f50541fSNicholas Piggin * Linux page table, then going to a full page fault if the Linux page table 11884f50541fSNicholas Piggin * entry was insufficient. RPT goes straight to full page fault. 11894f50541fSNicholas Piggin * 11904f50541fSNicholas Piggin * PR KVM ...? 11914f50541fSNicholas Piggin */ 1192a42a239dSNicholas PigginINT_DEFINE_BEGIN(data_access) 1193a42a239dSNicholas Piggin IVEC=0x300 1194a42a239dSNicholas Piggin IDAR=1 1195a42a239dSNicholas Piggin IDSISR=1 1196d52fd3d3SNicholas Piggin IKVM_SKIP=1 1197a42a239dSNicholas Piggin IKVM_REAL=1 1198a42a239dSNicholas PigginINT_DEFINE_END(data_access) 11990ebc4cdaSBenjamin Herrenschmidt 1200e779fc93SNicholas PigginEXC_REAL_BEGIN(data_access, 0x300, 0x80) 1201689e7322SNicholas Piggin GEN_INT_ENTRY data_access, virt=0 1202e779fc93SNicholas PigginEXC_REAL_END(data_access, 0x300, 0x80) 1203e779fc93SNicholas PigginEXC_VIRT_BEGIN(data_access, 0x4300, 0x80) 1204a42a239dSNicholas Piggin GEN_INT_ENTRY data_access, virt=1 1205e779fc93SNicholas PigginEXC_VIRT_END(data_access, 0x4300, 0x80) 120680795e6cSNicholas PigginEXC_COMMON_BEGIN(data_access_common) 12077cb3a1a0SNicholas Piggin GEN_COMMON data_access 12089b123d1eSNicholas Piggin ld r4,_DAR(r1) 12099b123d1eSNicholas Piggin ld r5,_DSISR(r1) 121080795e6cSNicholas PigginBEGIN_MMU_FTR_SECTION 12119b123d1eSNicholas Piggin ld r6,_MSR(r1) 12129b123d1eSNicholas Piggin li r3,0x300 121380795e6cSNicholas Piggin b do_hash_page /* Try to handle as hpte fault */ 121480795e6cSNicholas PigginMMU_FTR_SECTION_ELSE 121580795e6cSNicholas Piggin b handle_page_fault 121680795e6cSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 121780795e6cSNicholas Piggin 12189600f261SNicholas Piggin GEN_KVM data_access 12199600f261SNicholas Piggin 12200ebc4cdaSBenjamin Herrenschmidt 12214f50541fSNicholas PigginINT_DEFINE_BEGIN(data_access_slb) 12224f50541fSNicholas Piggin IVEC=0x380 12234f50541fSNicholas Piggin IAREA=PACA_EXSLB 12244f50541fSNicholas Piggin IRECONCILE=0 12254f50541fSNicholas Piggin IDAR=1 12264f50541fSNicholas Piggin IKVM_SKIP=1 12274f50541fSNicholas Piggin IKVM_REAL=1 12284f50541fSNicholas PigginINT_DEFINE_END(data_access_slb) 12294f50541fSNicholas Piggin 12301a6822d1SNicholas PigginEXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) 1231689e7322SNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=0 12321a6822d1SNicholas PigginEXC_REAL_END(data_access_slb, 0x380, 0x80) 12331a6822d1SNicholas PigginEXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) 12344f50541fSNicholas Piggin GEN_INT_ENTRY data_access_slb, virt=1 12351a6822d1SNicholas PigginEXC_VIRT_END(data_access_slb, 0x4380, 0x80) 123648e7b769SNicholas PigginEXC_COMMON_BEGIN(data_access_slb_common) 12374f50541fSNicholas Piggin GEN_COMMON data_access_slb 1238d1a84718SNicholas Piggin ld r4,_DAR(r1) 123948e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 12407100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 12417100e870SNicholas Piggin /* HPT case, do SLB fault */ 124248e7b769SNicholas Piggin bl do_slb_fault 124348e7b769SNicholas Piggin cmpdi r3,0 124448e7b769SNicholas Piggin bne- 1f 124548e7b769SNicholas Piggin b fast_exception_return 124648e7b769SNicholas Piggin1: /* Error case */ 12477100e870SNicholas PigginMMU_FTR_SECTION_ELSE 12487100e870SNicholas Piggin /* Radix case, access is outside page table range */ 12497100e870SNicholas Piggin li r3,-EFAULT 12507100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 125148e7b769SNicholas Piggin std r3,RESULT(r1) 125248e7b769SNicholas Piggin bl save_nvgprs 125348e7b769SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 125448e7b769SNicholas Piggin ld r4,_DAR(r1) 125548e7b769SNicholas Piggin ld r5,RESULT(r1) 125648e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 125748e7b769SNicholas Piggin bl do_bad_slb_fault 125848e7b769SNicholas Piggin b ret_from_except 125948e7b769SNicholas Piggin 12609600f261SNicholas Piggin GEN_KVM data_access_slb 12619600f261SNicholas Piggin 12622b9af6e4SNicholas Piggin 12634f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access) 12644f50541fSNicholas Piggin IVEC=0x400 1265a3cd35beSNicholas Piggin IISIDE=1 1266a3cd35beSNicholas Piggin IDAR=1 1267a3cd35beSNicholas Piggin IDSISR=1 12684f50541fSNicholas Piggin IKVM_REAL=1 12694f50541fSNicholas PigginINT_DEFINE_END(instruction_access) 12704f50541fSNicholas Piggin 12717299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access, 0x400, 0x80) 12724f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=0 12737299417cSNicholas PigginEXC_REAL_END(instruction_access, 0x400, 0x80) 12747299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) 12754f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access, virt=1 12767299417cSNicholas PigginEXC_VIRT_END(instruction_access, 0x4400, 0x80) 127727ce77dfSNicholas PigginEXC_COMMON_BEGIN(instruction_access_common) 12784f50541fSNicholas Piggin GEN_COMMON instruction_access 12799b123d1eSNicholas Piggin ld r4,_DAR(r1) 12809b123d1eSNicholas Piggin ld r5,_DSISR(r1) 128127ce77dfSNicholas PigginBEGIN_MMU_FTR_SECTION 12829b123d1eSNicholas Piggin ld r6,_MSR(r1) 12839b123d1eSNicholas Piggin li r3,0x400 128427ce77dfSNicholas Piggin b do_hash_page /* Try to handle as hpte fault */ 128527ce77dfSNicholas PigginMMU_FTR_SECTION_ELSE 128627ce77dfSNicholas Piggin b handle_page_fault 128727ce77dfSNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 128827ce77dfSNicholas Piggin 12899600f261SNicholas Piggin GEN_KVM instruction_access 12909600f261SNicholas Piggin 12910ebc4cdaSBenjamin Herrenschmidt 12924f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_access_slb) 12934f50541fSNicholas Piggin IVEC=0x480 12944f50541fSNicholas Piggin IAREA=PACA_EXSLB 12954f50541fSNicholas Piggin IRECONCILE=0 1296a3cd35beSNicholas Piggin IISIDE=1 1297a3cd35beSNicholas Piggin IDAR=1 12984f50541fSNicholas Piggin IKVM_REAL=1 12994f50541fSNicholas PigginINT_DEFINE_END(instruction_access_slb) 13004f50541fSNicholas Piggin 13017299417cSNicholas PigginEXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) 13024f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=0 13037299417cSNicholas PigginEXC_REAL_END(instruction_access_slb, 0x480, 0x80) 13047299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) 13054f50541fSNicholas Piggin GEN_INT_ENTRY instruction_access_slb, virt=1 13067299417cSNicholas PigginEXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) 130748e7b769SNicholas PigginEXC_COMMON_BEGIN(instruction_access_slb_common) 13084f50541fSNicholas Piggin GEN_COMMON instruction_access_slb 1309d1a84718SNicholas Piggin ld r4,_DAR(r1) 131054be0b9cSMichael Ellerman addi r3,r1,STACK_FRAME_OVERHEAD 13117100e870SNicholas PigginBEGIN_MMU_FTR_SECTION 13127100e870SNicholas Piggin /* HPT case, do SLB fault */ 131348e7b769SNicholas Piggin bl do_slb_fault 131448e7b769SNicholas Piggin cmpdi r3,0 131548e7b769SNicholas Piggin bne- 1f 131648e7b769SNicholas Piggin b fast_exception_return 131748e7b769SNicholas Piggin1: /* Error case */ 13187100e870SNicholas PigginMMU_FTR_SECTION_ELSE 13197100e870SNicholas Piggin /* Radix case, access is outside page table range */ 13207100e870SNicholas Piggin li r3,-EFAULT 13217100e870SNicholas PigginALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) 132248e7b769SNicholas Piggin std r3,RESULT(r1) 132348e7b769SNicholas Piggin bl save_nvgprs 132448e7b769SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1325d1a84718SNicholas Piggin ld r4,_DAR(r1) 132648e7b769SNicholas Piggin ld r5,RESULT(r1) 132748e7b769SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 132848e7b769SNicholas Piggin bl do_bad_slb_fault 132954be0b9cSMichael Ellerman b ret_from_except 13305e46e29eSNicholas Piggin 13319600f261SNicholas Piggin GEN_KVM instruction_access_slb 13329600f261SNicholas Piggin 13339600f261SNicholas Piggin 13344f50541fSNicholas PigginINT_DEFINE_BEGIN(hardware_interrupt) 13354f50541fSNicholas Piggin IVEC=0x500 1336*3f7fbd97SNicholas Piggin IHSRR_IF_HVMODE=1 13374f50541fSNicholas Piggin IMASK=IRQS_DISABLED 13384f50541fSNicholas Piggin IKVM_REAL=1 13394f50541fSNicholas Piggin IKVM_VIRT=1 13404f50541fSNicholas PigginINT_DEFINE_END(hardware_interrupt) 13414f50541fSNicholas Piggin 13421a6822d1SNicholas PigginEXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) 13434f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=0 13441a6822d1SNicholas PigginEXC_REAL_END(hardware_interrupt, 0x500, 0x100) 13451a6822d1SNicholas PigginEXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) 13464f50541fSNicholas Piggin GEN_INT_ENTRY hardware_interrupt, virt=1 13471a6822d1SNicholas PigginEXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1348eb204d86SNicholas PigginEXC_COMMON_BEGIN(hardware_interrupt_common) 13494f50541fSNicholas Piggin GEN_COMMON hardware_interrupt 1350eb204d86SNicholas Piggin FINISH_NAP 1351eb204d86SNicholas Piggin RUNLATCH_ON 1352eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1353eb204d86SNicholas Piggin bl do_IRQ 1354eb204d86SNicholas Piggin b ret_from_except_lite 1355c138e588SNicholas Piggin 13569600f261SNicholas Piggin GEN_KVM hardware_interrupt 13579600f261SNicholas Piggin 1358c138e588SNicholas Piggin 13594f50541fSNicholas PigginINT_DEFINE_BEGIN(alignment) 13604f50541fSNicholas Piggin IVEC=0x600 13614f50541fSNicholas Piggin IDAR=1 13624f50541fSNicholas Piggin IDSISR=1 13634f50541fSNicholas Piggin IKVM_REAL=1 13644f50541fSNicholas PigginINT_DEFINE_END(alignment) 13654f50541fSNicholas Piggin 1366e779fc93SNicholas PigginEXC_REAL_BEGIN(alignment, 0x600, 0x100) 13674f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=0 1368e779fc93SNicholas PigginEXC_REAL_END(alignment, 0x600, 0x100) 1369e779fc93SNicholas PigginEXC_VIRT_BEGIN(alignment, 0x4600, 0x100) 13704f50541fSNicholas Piggin GEN_INT_ENTRY alignment, virt=1 1371e779fc93SNicholas PigginEXC_VIRT_END(alignment, 0x4600, 0x100) 1372f9aa6714SNicholas PigginEXC_COMMON_BEGIN(alignment_common) 13734f50541fSNicholas Piggin GEN_COMMON alignment 1374f9aa6714SNicholas Piggin bl save_nvgprs 1375f9aa6714SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1376f9aa6714SNicholas Piggin bl alignment_exception 1377f9aa6714SNicholas Piggin b ret_from_except 1378f9aa6714SNicholas Piggin 13799600f261SNicholas Piggin GEN_KVM alignment 13809600f261SNicholas Piggin 1381b01c8b54SPaul Mackerras 13824f50541fSNicholas PigginINT_DEFINE_BEGIN(program_check) 13834f50541fSNicholas Piggin IVEC=0x700 13844f50541fSNicholas Piggin IKVM_REAL=1 13854f50541fSNicholas PigginINT_DEFINE_END(program_check) 13864f50541fSNicholas Piggin 13877299417cSNicholas PigginEXC_REAL_BEGIN(program_check, 0x700, 0x100) 13884f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=0 13897299417cSNicholas PigginEXC_REAL_END(program_check, 0x700, 0x100) 13907299417cSNicholas PigginEXC_VIRT_BEGIN(program_check, 0x4700, 0x100) 13914f50541fSNicholas Piggin GEN_INT_ENTRY program_check, virt=1 13927299417cSNicholas PigginEXC_VIRT_END(program_check, 0x4700, 0x100) 139311e87346SNicholas PigginEXC_COMMON_BEGIN(program_check_common) 13948729c26eSNicholas Piggin __GEN_COMMON_ENTRY program_check 13958729c26eSNicholas Piggin 1396265e60a1SCyril Bur /* 1397265e60a1SCyril Bur * It's possible to receive a TM Bad Thing type program check with 1398265e60a1SCyril Bur * userspace register values (in particular r1), but with SRR1 reporting 1399265e60a1SCyril Bur * that we came from the kernel. Normally that would confuse the bad 1400265e60a1SCyril Bur * stack logic, and we would report a bad kernel stack pointer. Instead 1401265e60a1SCyril Bur * we switch to the emergency stack if we're taking a TM Bad Thing from 1402265e60a1SCyril Bur * the kernel. 1403265e60a1SCyril Bur */ 1404265e60a1SCyril Bur 14050a882e28SNicholas Piggin andi. r10,r12,MSR_PR 14060a882e28SNicholas Piggin bne 2f /* If userspace, go normal path */ 14070a882e28SNicholas Piggin 14080a882e28SNicholas Piggin andis. r10,r12,(SRR1_PROGTM)@h 14090a882e28SNicholas Piggin bne 1f /* If TM, emergency */ 14100a882e28SNicholas Piggin 14110a882e28SNicholas Piggin cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ 14120a882e28SNicholas Piggin blt 2f /* normal path if not */ 14130a882e28SNicholas Piggin 14140a882e28SNicholas Piggin /* Use the emergency stack */ 14150a882e28SNicholas Piggin1: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ 1416265e60a1SCyril Bur /* 3 in EXCEPTION_PROLOG_COMMON */ 1417265e60a1SCyril Bur mr r10,r1 /* Save r1 */ 1418265e60a1SCyril Bur ld r1,PACAEMERGSP(r13) /* Use emergency stack */ 1419265e60a1SCyril Bur subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 14204f50541fSNicholas Piggin __ISTACK(program_check)=0 14218729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 14221b359982SNicholas Piggin b 3f 14230a882e28SNicholas Piggin2: 14244f50541fSNicholas Piggin __ISTACK(program_check)=1 14258729c26eSNicholas Piggin __GEN_COMMON_BODY program_check 14261b359982SNicholas Piggin3: 142711e87346SNicholas Piggin bl save_nvgprs 142811e87346SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 142911e87346SNicholas Piggin bl program_check_exception 143011e87346SNicholas Piggin b ret_from_except 143111e87346SNicholas Piggin 14329600f261SNicholas Piggin GEN_KVM program_check 14339600f261SNicholas Piggin 1434a485c709SPaul Mackerras 14354f50541fSNicholas PigginINT_DEFINE_BEGIN(fp_unavailable) 14364f50541fSNicholas Piggin IVEC=0x800 14374f50541fSNicholas Piggin IRECONCILE=0 14384f50541fSNicholas Piggin IKVM_REAL=1 14394f50541fSNicholas PigginINT_DEFINE_END(fp_unavailable) 14404f50541fSNicholas Piggin 14417299417cSNicholas PigginEXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100) 14424f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=0 14437299417cSNicholas PigginEXC_REAL_END(fp_unavailable, 0x800, 0x100) 14447299417cSNicholas PigginEXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100) 14454f50541fSNicholas Piggin GEN_INT_ENTRY fp_unavailable, virt=1 14467299417cSNicholas PigginEXC_VIRT_END(fp_unavailable, 0x4800, 0x100) 1447c78d9b97SNicholas PigginEXC_COMMON_BEGIN(fp_unavailable_common) 14484f50541fSNicholas Piggin GEN_COMMON fp_unavailable 1449c78d9b97SNicholas Piggin bne 1f /* if from user, just load it up */ 1450c78d9b97SNicholas Piggin bl save_nvgprs 1451c78d9b97SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1452c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1453c78d9b97SNicholas Piggin bl kernel_fp_unavailable_exception 145463ce271bSChristophe Leroy0: trap 145563ce271bSChristophe Leroy EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0 1456c78d9b97SNicholas Piggin1: 1457c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1458c78d9b97SNicholas PigginBEGIN_FTR_SECTION 1459c78d9b97SNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1460c78d9b97SNicholas Piggin * transaction), go do TM stuff 1461c78d9b97SNicholas Piggin */ 1462c78d9b97SNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1463c78d9b97SNicholas Piggin bne- 2f 1464c78d9b97SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_TM) 1465c78d9b97SNicholas Piggin#endif 1466c78d9b97SNicholas Piggin bl load_up_fpu 1467c78d9b97SNicholas Piggin b fast_exception_return 1468c78d9b97SNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1469c78d9b97SNicholas Piggin2: /* User process was in a transaction */ 1470c78d9b97SNicholas Piggin bl save_nvgprs 1471c78d9b97SNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1472c78d9b97SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1473c78d9b97SNicholas Piggin bl fp_unavailable_tm 1474c78d9b97SNicholas Piggin b ret_from_except 1475c78d9b97SNicholas Piggin#endif 1476c78d9b97SNicholas Piggin 14779600f261SNicholas Piggin GEN_KVM fp_unavailable 14789600f261SNicholas Piggin 1479b01c8b54SPaul Mackerras 14804f50541fSNicholas PigginINT_DEFINE_BEGIN(decrementer) 14814f50541fSNicholas Piggin IVEC=0x900 14824f50541fSNicholas Piggin IMASK=IRQS_DISABLED 14834f50541fSNicholas Piggin IKVM_REAL=1 14844f50541fSNicholas PigginINT_DEFINE_END(decrementer) 14854f50541fSNicholas Piggin 14867299417cSNicholas PigginEXC_REAL_BEGIN(decrementer, 0x900, 0x80) 1487689e7322SNicholas Piggin GEN_INT_ENTRY decrementer, virt=0 14887299417cSNicholas PigginEXC_REAL_END(decrementer, 0x900, 0x80) 14897299417cSNicholas PigginEXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) 14904f50541fSNicholas Piggin GEN_INT_ENTRY decrementer, virt=1 14917299417cSNicholas PigginEXC_VIRT_END(decrementer, 0x4900, 0x80) 1492eb204d86SNicholas PigginEXC_COMMON_BEGIN(decrementer_common) 14934f50541fSNicholas Piggin GEN_COMMON decrementer 1494eb204d86SNicholas Piggin FINISH_NAP 1495eb204d86SNicholas Piggin RUNLATCH_ON 1496eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1497eb204d86SNicholas Piggin bl timer_interrupt 1498eb204d86SNicholas Piggin b ret_from_except_lite 149939c0da57SNicholas Piggin 15009600f261SNicholas Piggin GEN_KVM decrementer 15019600f261SNicholas Piggin 15020ebc4cdaSBenjamin Herrenschmidt 15034f50541fSNicholas PigginINT_DEFINE_BEGIN(hdecrementer) 15044f50541fSNicholas Piggin IVEC=0x980 1505*3f7fbd97SNicholas Piggin IHSRR=1 15062babd6eaSNicholas Piggin ISTACK=0 15072babd6eaSNicholas Piggin IRECONCILE=0 15084f50541fSNicholas Piggin IKVM_REAL=1 15094f50541fSNicholas Piggin IKVM_VIRT=1 15104f50541fSNicholas PigginINT_DEFINE_END(hdecrementer) 15114f50541fSNicholas Piggin 15127299417cSNicholas PigginEXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) 15134f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=0 15147299417cSNicholas PigginEXC_REAL_END(hdecrementer, 0x980, 0x80) 15157299417cSNicholas PigginEXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) 15164f50541fSNicholas Piggin GEN_INT_ENTRY hdecrementer, virt=1 15177299417cSNicholas PigginEXC_VIRT_END(hdecrementer, 0x4980, 0x80) 1518eb204d86SNicholas PigginEXC_COMMON_BEGIN(hdecrementer_common) 15192babd6eaSNicholas Piggin __GEN_COMMON_ENTRY hdecrementer 15202babd6eaSNicholas Piggin /* 15212babd6eaSNicholas Piggin * Hypervisor decrementer interrupts not caught by the KVM test 15222babd6eaSNicholas Piggin * shouldn't occur but are sometimes left pending on exit from a KVM 15232babd6eaSNicholas Piggin * guest. We don't need to do anything to clear them, as they are 15242babd6eaSNicholas Piggin * edge-triggered. 15252babd6eaSNicholas Piggin * 15262babd6eaSNicholas Piggin * Be careful to avoid touching the kernel stack. 15272babd6eaSNicholas Piggin */ 15282babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 15292babd6eaSNicholas Piggin mtctr r10 15302babd6eaSNicholas Piggin mtcrf 0x80,r9 15312babd6eaSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 15322babd6eaSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 15332babd6eaSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 15342babd6eaSNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 15352babd6eaSNicholas Piggin ld r13,PACA_EXGEN+EX_R13(r13) 15362babd6eaSNicholas Piggin HRFI_TO_KERNEL 1537facc6d74SNicholas Piggin 15389600f261SNicholas Piggin GEN_KVM hdecrementer 15399600f261SNicholas Piggin 1540da2bc464SMichael Ellerman 15414f50541fSNicholas PigginINT_DEFINE_BEGIN(doorbell_super) 15424f50541fSNicholas Piggin IVEC=0xa00 15434f50541fSNicholas Piggin IMASK=IRQS_DISABLED 15444f50541fSNicholas Piggin IKVM_REAL=1 15454f50541fSNicholas PigginINT_DEFINE_END(doorbell_super) 15464f50541fSNicholas Piggin 15477299417cSNicholas PigginEXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) 15484f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=0 15497299417cSNicholas PigginEXC_REAL_END(doorbell_super, 0xa00, 0x100) 15507299417cSNicholas PigginEXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) 15514f50541fSNicholas Piggin GEN_INT_ENTRY doorbell_super, virt=1 15527299417cSNicholas PigginEXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1553eb204d86SNicholas PigginEXC_COMMON_BEGIN(doorbell_super_common) 15544f50541fSNicholas Piggin GEN_COMMON doorbell_super 1555eb204d86SNicholas Piggin FINISH_NAP 1556eb204d86SNicholas Piggin RUNLATCH_ON 1557eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1558ca243163SNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1559eb204d86SNicholas Piggin bl doorbell_exception 1560ca243163SNicholas Piggin#else 1561eb204d86SNicholas Piggin bl unknown_exception 1562ca243163SNicholas Piggin#endif 1563eb204d86SNicholas Piggin b ret_from_except_lite 1564ca243163SNicholas Piggin 15659600f261SNicholas Piggin GEN_KVM doorbell_super 15669600f261SNicholas Piggin 1567da2bc464SMichael Ellerman 15685ff79a5eSNicholas PigginEXC_REAL_NONE(0xb00, 0x100) 15695ff79a5eSNicholas PigginEXC_VIRT_NONE(0x4b00, 0x100) 1570341215dcSNicholas Piggin 1571acd7d8ceSNicholas Piggin/* 1572acd7d8ceSNicholas Piggin * system call / hypercall (0xc00, 0x4c00) 1573acd7d8ceSNicholas Piggin * 1574acd7d8ceSNicholas Piggin * The system call exception is invoked with "sc 0" and does not alter HV bit. 1575acd7d8ceSNicholas Piggin * 1576acd7d8ceSNicholas Piggin * The hypercall is invoked with "sc 1" and sets HV=1. 1577acd7d8ceSNicholas Piggin * 1578acd7d8ceSNicholas Piggin * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to 1579acd7d8ceSNicholas Piggin * 0x4c00 virtual mode. 1580acd7d8ceSNicholas Piggin * 1581acd7d8ceSNicholas Piggin * Call convention: 1582acd7d8ceSNicholas Piggin * 158358b278f5SVaibhav Jain * syscall and hypercalls register conventions are documented in 158458b278f5SVaibhav Jain * Documentation/powerpc/syscall64-abi.rst and 158558b278f5SVaibhav Jain * Documentation/powerpc/papr_hcalls.rst respectively. 1586acd7d8ceSNicholas Piggin * 1587acd7d8ceSNicholas Piggin * The intersection of volatile registers that don't contain possible 158876fc0cfcSNicholas Piggin * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry 158976fc0cfcSNicholas Piggin * without saving, though xer is not a good idea to use, as hardware may 159076fc0cfcSNicholas Piggin * interpret some bits so it may be costly to change them. 1591acd7d8ceSNicholas Piggin */ 1592b177ae2fSNicholas PigginINT_DEFINE_BEGIN(system_call) 1593b177ae2fSNicholas Piggin IVEC=0xc00 1594b177ae2fSNicholas Piggin IKVM_REAL=1 1595b177ae2fSNicholas Piggin IKVM_VIRT=1 1596b177ae2fSNicholas PigginINT_DEFINE_END(system_call) 1597b177ae2fSNicholas Piggin 15981b4d4a79SNicholas Piggin.macro SYSTEM_CALL virt 1599bc355125SPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 1600bc355125SPaul Mackerras /* 1601acd7d8ceSNicholas Piggin * There is a little bit of juggling to get syscall and hcall 160276fc0cfcSNicholas Piggin * working well. Save r13 in ctr to avoid using SPRG scratch 160376fc0cfcSNicholas Piggin * register. 1604acd7d8ceSNicholas Piggin * 1605acd7d8ceSNicholas Piggin * Userspace syscalls have already saved the PPR, hcalls must save 1606acd7d8ceSNicholas Piggin * it before setting HMT_MEDIUM. 1607bc355125SPaul Mackerras */ 16081b4d4a79SNicholas Piggin mtctr r13 16091b4d4a79SNicholas Piggin GET_PACA(r13) 16101b4d4a79SNicholas Piggin std r10,PACA_EXGEN+EX_R10(r13) 16111b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 16129d598f93SNicholas Piggin KVMTEST system_call /* uses r10, branch to system_call_kvm */ 16131b4d4a79SNicholas Piggin mfctr r9 1614bc355125SPaul Mackerras#else 16151b4d4a79SNicholas Piggin mr r9,r13 16161b4d4a79SNicholas Piggin GET_PACA(r13) 16171b4d4a79SNicholas Piggin INTERRUPT_TO_KERNEL 1618bc355125SPaul Mackerras#endif 1619bc355125SPaul Mackerras 1620727f1361SMichael Ellerman#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 16211b4d4a79SNicholas PigginBEGIN_FTR_SECTION 16221b4d4a79SNicholas Piggin cmpdi r0,0x1ebe 16231b4d4a79SNicholas Piggin beq- 1f 16241b4d4a79SNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 16251b4d4a79SNicholas Piggin#endif 16265c2511bfSMichael Ellerman 1627b0b2a93dSNicholas Piggin /* We reach here with PACA in r13, r13 in r9. */ 16281b4d4a79SNicholas Piggin mfspr r11,SPRN_SRR0 16291b4d4a79SNicholas Piggin mfspr r12,SPRN_SRR1 1630b0b2a93dSNicholas Piggin 1631b0b2a93dSNicholas Piggin HMT_MEDIUM 1632b0b2a93dSNicholas Piggin 1633b0b2a93dSNicholas Piggin .if ! \virt 16341b4d4a79SNicholas Piggin __LOAD_HANDLER(r10, system_call_common) 16351b4d4a79SNicholas Piggin mtspr SPRN_SRR0,r10 16361b4d4a79SNicholas Piggin ld r10,PACAKMSR(r13) 16371b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r10 16381b4d4a79SNicholas Piggin RFI_TO_KERNEL 16391b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 16401b4d4a79SNicholas Piggin .else 16411b4d4a79SNicholas Piggin li r10,MSR_RI 16421b4d4a79SNicholas Piggin mtmsrd r10,1 /* Set RI (EE=0) */ 1643b0b2a93dSNicholas Piggin#ifdef CONFIG_RELOCATABLE 1644b0b2a93dSNicholas Piggin __LOAD_HANDLER(r10, system_call_common) 1645b0b2a93dSNicholas Piggin mtctr r10 1646b0b2a93dSNicholas Piggin bctr 1647b0b2a93dSNicholas Piggin#else 16481b4d4a79SNicholas Piggin b system_call_common 1649d807ad37SNicholas Piggin#endif 16501b4d4a79SNicholas Piggin .endif 16511b4d4a79SNicholas Piggin 16521b4d4a79SNicholas Piggin#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH 16531b4d4a79SNicholas Piggin /* Fast LE/BE switch system call */ 16541b4d4a79SNicholas Piggin1: mfspr r12,SPRN_SRR1 16551b4d4a79SNicholas Piggin xori r12,r12,MSR_LE 16561b4d4a79SNicholas Piggin mtspr SPRN_SRR1,r12 16571b4d4a79SNicholas Piggin mr r13,r9 16581b4d4a79SNicholas Piggin RFI_TO_USER /* return to userspace */ 16591b4d4a79SNicholas Piggin b . /* prevent speculative execution */ 16601b4d4a79SNicholas Piggin#endif 16611b4d4a79SNicholas Piggin.endm 1662d807ad37SNicholas Piggin 16631a6822d1SNicholas PigginEXC_REAL_BEGIN(system_call, 0xc00, 0x100) 16641b4d4a79SNicholas Piggin SYSTEM_CALL 0 16651a6822d1SNicholas PigginEXC_REAL_END(system_call, 0xc00, 0x100) 16661a6822d1SNicholas PigginEXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) 16671b4d4a79SNicholas Piggin SYSTEM_CALL 1 16681a6822d1SNicholas PigginEXC_VIRT_END(system_call, 0x4c00, 0x100) 1669d807ad37SNicholas Piggin 1670acd7d8ceSNicholas Piggin#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 16719600f261SNicholas PigginTRAMP_REAL_BEGIN(system_call_kvm) 1672acd7d8ceSNicholas Piggin /* 1673acd7d8ceSNicholas Piggin * This is a hcall, so register convention is as above, with these 1674acd7d8ceSNicholas Piggin * differences: 1675acd7d8ceSNicholas Piggin * r13 = PACA 167676fc0cfcSNicholas Piggin * ctr = orig r13 167776fc0cfcSNicholas Piggin * orig r10 saved in PACA 1678acd7d8ceSNicholas Piggin */ 1679acd7d8ceSNicholas Piggin /* 1680acd7d8ceSNicholas Piggin * Save the PPR (on systems that support it) before changing to 1681acd7d8ceSNicholas Piggin * HMT_MEDIUM. That allows the KVM code to save that value into the 1682acd7d8ceSNicholas Piggin * guest state (it is the guest's PPR value). 1683acd7d8ceSNicholas Piggin */ 1684931dc86bSNicholas PigginBEGIN_FTR_SECTION 16859600f261SNicholas Piggin mfspr r10,SPRN_PPR 16869600f261SNicholas Piggin std r10,HSTATE_PPR(r13) 1687931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1688acd7d8ceSNicholas Piggin HMT_MEDIUM 1689acd7d8ceSNicholas Piggin mfctr r10 169076fc0cfcSNicholas Piggin SET_SCRATCH0(r10) 16919600f261SNicholas Piggin mfcr r10 16929600f261SNicholas Piggin std r12,HSTATE_SCRATCH0(r13) 16939600f261SNicholas Piggin sldi r12,r10,32 16949600f261SNicholas Piggin ori r12,r12,0xc00 16959600f261SNicholas Piggin#ifdef CONFIG_RELOCATABLE 16969600f261SNicholas Piggin /* 16979600f261SNicholas Piggin * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives 16989600f261SNicholas Piggin * outside the head section. 16999600f261SNicholas Piggin */ 17009600f261SNicholas Piggin __LOAD_FAR_HANDLER(r10, kvmppc_interrupt) 17019600f261SNicholas Piggin mtctr r10 17029600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 17039600f261SNicholas Piggin bctr 17049600f261SNicholas Piggin#else 17059600f261SNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 17069600f261SNicholas Piggin b kvmppc_interrupt 17079600f261SNicholas Piggin#endif 1708acd7d8ceSNicholas Piggin#endif 1709da2bc464SMichael Ellerman 1710d807ad37SNicholas Piggin 17114f50541fSNicholas PigginINT_DEFINE_BEGIN(single_step) 17124f50541fSNicholas Piggin IVEC=0xd00 17134f50541fSNicholas Piggin IKVM_REAL=1 17144f50541fSNicholas PigginINT_DEFINE_END(single_step) 17154f50541fSNicholas Piggin 17167299417cSNicholas PigginEXC_REAL_BEGIN(single_step, 0xd00, 0x100) 17174f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=0 17187299417cSNicholas PigginEXC_REAL_END(single_step, 0xd00, 0x100) 17197299417cSNicholas PigginEXC_VIRT_BEGIN(single_step, 0x4d00, 0x100) 17204f50541fSNicholas Piggin GEN_INT_ENTRY single_step, virt=1 17217299417cSNicholas PigginEXC_VIRT_END(single_step, 0x4d00, 0x100) 1722eb204d86SNicholas PigginEXC_COMMON_BEGIN(single_step_common) 17234f50541fSNicholas Piggin GEN_COMMON single_step 1724eb204d86SNicholas Piggin bl save_nvgprs 1725eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1726eb204d86SNicholas Piggin bl single_step_exception 1727eb204d86SNicholas Piggin b ret_from_except 1728da2bc464SMichael Ellerman 17299600f261SNicholas Piggin GEN_KVM single_step 17309600f261SNicholas Piggin 17317299417cSNicholas Piggin 17324f50541fSNicholas PigginINT_DEFINE_BEGIN(h_data_storage) 17334f50541fSNicholas Piggin IVEC=0xe00 1734*3f7fbd97SNicholas Piggin IHSRR=1 17354f50541fSNicholas Piggin IDAR=1 17364f50541fSNicholas Piggin IDSISR=1 17374f50541fSNicholas Piggin IKVM_SKIP=1 17384f50541fSNicholas Piggin IKVM_REAL=1 17394f50541fSNicholas Piggin IKVM_VIRT=1 17404f50541fSNicholas PigginINT_DEFINE_END(h_data_storage) 17414f50541fSNicholas Piggin 17427299417cSNicholas PigginEXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20) 17434f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=0, ool=1 17447299417cSNicholas PigginEXC_REAL_END(h_data_storage, 0xe00, 0x20) 17457299417cSNicholas PigginEXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20) 17464f50541fSNicholas Piggin GEN_INT_ENTRY h_data_storage, virt=1, ool=1 17477299417cSNicholas PigginEXC_VIRT_END(h_data_storage, 0x4e00, 0x20) 1748f5c32c1dSNicholas PigginEXC_COMMON_BEGIN(h_data_storage_common) 17494f50541fSNicholas Piggin GEN_COMMON h_data_storage 1750f5c32c1dSNicholas Piggin bl save_nvgprs 1751f5c32c1dSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1752d7b45615SSuraj Jitindar SinghBEGIN_MMU_FTR_SECTION 1753d1a84718SNicholas Piggin ld r4,_DAR(r1) 1754d7b45615SSuraj Jitindar Singh li r5,SIGSEGV 1755d7b45615SSuraj Jitindar Singh bl bad_page_fault 1756d7b45615SSuraj Jitindar SinghMMU_FTR_SECTION_ELSE 1757f5c32c1dSNicholas Piggin bl unknown_exception 1758d7b45615SSuraj Jitindar SinghALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) 1759f5c32c1dSNicholas Piggin b ret_from_except 1760f5c32c1dSNicholas Piggin 17619600f261SNicholas Piggin GEN_KVM h_data_storage 17629600f261SNicholas Piggin 17631707dd16SPaul Mackerras 17644f50541fSNicholas PigginINT_DEFINE_BEGIN(h_instr_storage) 17654f50541fSNicholas Piggin IVEC=0xe20 1766*3f7fbd97SNicholas Piggin IHSRR=1 17674f50541fSNicholas Piggin IKVM_REAL=1 17684f50541fSNicholas Piggin IKVM_VIRT=1 17694f50541fSNicholas PigginINT_DEFINE_END(h_instr_storage) 17704f50541fSNicholas Piggin 17717299417cSNicholas PigginEXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20) 17724f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=0, ool=1 17737299417cSNicholas PigginEXC_REAL_END(h_instr_storage, 0xe20, 0x20) 17747299417cSNicholas PigginEXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20) 17754f50541fSNicholas Piggin GEN_INT_ENTRY h_instr_storage, virt=1, ool=1 17767299417cSNicholas PigginEXC_VIRT_END(h_instr_storage, 0x4e20, 0x20) 1777eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_instr_storage_common) 17784f50541fSNicholas Piggin GEN_COMMON h_instr_storage 1779eb204d86SNicholas Piggin bl save_nvgprs 1780eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1781eb204d86SNicholas Piggin bl unknown_exception 1782eb204d86SNicholas Piggin b ret_from_except 178382517cabSNicholas Piggin 17849600f261SNicholas Piggin GEN_KVM h_instr_storage 17859600f261SNicholas Piggin 17861707dd16SPaul Mackerras 17874f50541fSNicholas PigginINT_DEFINE_BEGIN(emulation_assist) 17884f50541fSNicholas Piggin IVEC=0xe40 1789*3f7fbd97SNicholas Piggin IHSRR=1 17904f50541fSNicholas Piggin IKVM_REAL=1 17914f50541fSNicholas Piggin IKVM_VIRT=1 17924f50541fSNicholas PigginINT_DEFINE_END(emulation_assist) 17934f50541fSNicholas Piggin 17947299417cSNicholas PigginEXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20) 17954f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=0, ool=1 17967299417cSNicholas PigginEXC_REAL_END(emulation_assist, 0xe40, 0x20) 17977299417cSNicholas PigginEXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20) 17984f50541fSNicholas Piggin GEN_INT_ENTRY emulation_assist, virt=1, ool=1 17997299417cSNicholas PigginEXC_VIRT_END(emulation_assist, 0x4e40, 0x20) 1800eb204d86SNicholas PigginEXC_COMMON_BEGIN(emulation_assist_common) 18014f50541fSNicholas Piggin GEN_COMMON emulation_assist 1802eb204d86SNicholas Piggin bl save_nvgprs 1803eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1804eb204d86SNicholas Piggin bl emulation_assist_interrupt 1805eb204d86SNicholas Piggin b ret_from_except 1806031b4026SNicholas Piggin 18079600f261SNicholas Piggin GEN_KVM emulation_assist 18089600f261SNicholas Piggin 18091707dd16SPaul Mackerras 1810e0319829SNicholas Piggin/* 1811e0319829SNicholas Piggin * hmi_exception trampoline is a special case. It jumps to hmi_exception_early 1812e0319829SNicholas Piggin * first, and then eventaully from there to the trampoline to get into virtual 1813e0319829SNicholas Piggin * mode. 1814e0319829SNicholas Piggin */ 18154f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception_early) 18164f50541fSNicholas Piggin IVEC=0xe60 1817*3f7fbd97SNicholas Piggin IHSRR=1 1818d73a10cbSNicholas Piggin IREALMODE_COMMON=1 18194f50541fSNicholas Piggin ISTACK=0 18204f50541fSNicholas Piggin IRECONCILE=0 18214f50541fSNicholas Piggin IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ 18224f50541fSNicholas Piggin IKVM_REAL=1 18234f50541fSNicholas PigginINT_DEFINE_END(hmi_exception_early) 18244f50541fSNicholas Piggin 18254f50541fSNicholas PigginINT_DEFINE_BEGIN(hmi_exception) 18264f50541fSNicholas Piggin IVEC=0xe60 1827*3f7fbd97SNicholas Piggin IHSRR=1 18284f50541fSNicholas Piggin IMASK=IRQS_DISABLED 18294f50541fSNicholas Piggin IKVM_REAL=1 18304f50541fSNicholas PigginINT_DEFINE_END(hmi_exception) 18314f50541fSNicholas Piggin 1832f34c9675SNicholas PigginEXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) 18334f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1 1834f34c9675SNicholas PigginEXC_REAL_END(hmi_exception, 0xe60, 0x20) 18351a6822d1SNicholas PigginEXC_VIRT_NONE(0x4e60, 0x20) 18364f50541fSNicholas Piggin 1837293c2e27SNicholas PigginEXC_COMMON_BEGIN(hmi_exception_early_common) 18389600f261SNicholas Piggin __GEN_REALMODE_COMMON_ENTRY hmi_exception_early 18399600f261SNicholas Piggin 184062f9b03bSNicholas Piggin mr r10,r1 /* Save r1 */ 1841a4087a4dSNicholas Piggin ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ 184262f9b03bSNicholas Piggin subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ 1843bcbceed4SNicholas Piggin 18448729c26eSNicholas Piggin __GEN_COMMON_BODY hmi_exception_early 1845bcbceed4SNicholas Piggin 184662f9b03bSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1847293c2e27SNicholas Piggin bl hmi_exception_realmode 18485080332cSMichael Neuling cmpdi cr0,r3,0 184967d4160aSNicholas Piggin bne 1f 18505080332cSMichael Neuling 1851*3f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 1852222f20f1SNicholas Piggin HRFI_TO_USER_OR_KERNEL 18535080332cSMichael Neuling 185467d4160aSNicholas Piggin1: 185562f9b03bSNicholas Piggin /* 185662f9b03bSNicholas Piggin * Go to virtual mode and pull the HMI event information from 185762f9b03bSNicholas Piggin * firmware. 185862f9b03bSNicholas Piggin */ 1859*3f7fbd97SNicholas Piggin EXCEPTION_RESTORE_REGS hsrr=1 18604f50541fSNicholas Piggin GEN_INT_ENTRY hmi_exception, virt=0 186162f9b03bSNicholas Piggin 18629600f261SNicholas Piggin GEN_KVM hmi_exception_early 18639600f261SNicholas Piggin 18645080332cSMichael NeulingEXC_COMMON_BEGIN(hmi_exception_common) 18654f50541fSNicholas Piggin GEN_COMMON hmi_exception 186647169fbaSNicholas Piggin FINISH_NAP 186747169fbaSNicholas Piggin RUNLATCH_ON 1868d1a84718SNicholas Piggin bl save_nvgprs 1869c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1870c06075f3SNicholas Piggin bl handle_hmi_exception 1871c06075f3SNicholas Piggin b ret_from_except 18721707dd16SPaul Mackerras 18739600f261SNicholas Piggin GEN_KVM hmi_exception 18749600f261SNicholas Piggin 18757299417cSNicholas Piggin 18764f50541fSNicholas PigginINT_DEFINE_BEGIN(h_doorbell) 18774f50541fSNicholas Piggin IVEC=0xe80 1878*3f7fbd97SNicholas Piggin IHSRR=1 18794f50541fSNicholas Piggin IMASK=IRQS_DISABLED 18804f50541fSNicholas Piggin IKVM_REAL=1 18814f50541fSNicholas Piggin IKVM_VIRT=1 18824f50541fSNicholas PigginINT_DEFINE_END(h_doorbell) 18834f50541fSNicholas Piggin 18847299417cSNicholas PigginEXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) 18854f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=0, ool=1 18867299417cSNicholas PigginEXC_REAL_END(h_doorbell, 0xe80, 0x20) 18877299417cSNicholas PigginEXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) 18884f50541fSNicholas Piggin GEN_INT_ENTRY h_doorbell, virt=1, ool=1 18897299417cSNicholas PigginEXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 1890eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_doorbell_common) 18914f50541fSNicholas Piggin GEN_COMMON h_doorbell 1892eb204d86SNicholas Piggin FINISH_NAP 1893eb204d86SNicholas Piggin RUNLATCH_ON 1894eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 18959bcb81bfSNicholas Piggin#ifdef CONFIG_PPC_DOORBELL 1896eb204d86SNicholas Piggin bl doorbell_exception 18979bcb81bfSNicholas Piggin#else 1898eb204d86SNicholas Piggin bl unknown_exception 18999bcb81bfSNicholas Piggin#endif 1900eb204d86SNicholas Piggin b ret_from_except_lite 19019bcb81bfSNicholas Piggin 19029600f261SNicholas Piggin GEN_KVM h_doorbell 19039600f261SNicholas Piggin 19040ebc4cdaSBenjamin Herrenschmidt 19054f50541fSNicholas PigginINT_DEFINE_BEGIN(h_virt_irq) 19064f50541fSNicholas Piggin IVEC=0xea0 1907*3f7fbd97SNicholas Piggin IHSRR=1 19084f50541fSNicholas Piggin IMASK=IRQS_DISABLED 19094f50541fSNicholas Piggin IKVM_REAL=1 19104f50541fSNicholas Piggin IKVM_VIRT=1 19114f50541fSNicholas PigginINT_DEFINE_END(h_virt_irq) 19124f50541fSNicholas Piggin 19137299417cSNicholas PigginEXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) 19144f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=0, ool=1 19157299417cSNicholas PigginEXC_REAL_END(h_virt_irq, 0xea0, 0x20) 19167299417cSNicholas PigginEXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) 19174f50541fSNicholas Piggin GEN_INT_ENTRY h_virt_irq, virt=1, ool=1 19187299417cSNicholas PigginEXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 1919eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_virt_irq_common) 19204f50541fSNicholas Piggin GEN_COMMON h_virt_irq 1921eb204d86SNicholas Piggin FINISH_NAP 1922eb204d86SNicholas Piggin RUNLATCH_ON 1923eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1924eb204d86SNicholas Piggin bl do_IRQ 1925eb204d86SNicholas Piggin b ret_from_except_lite 192674408776SNicholas Piggin 19279600f261SNicholas Piggin GEN_KVM h_virt_irq 19289600f261SNicholas Piggin 19299baaef0aSBenjamin Herrenschmidt 19301a6822d1SNicholas PigginEXC_REAL_NONE(0xec0, 0x20) 19311a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ec0, 0x20) 19321a6822d1SNicholas PigginEXC_REAL_NONE(0xee0, 0x20) 19331a6822d1SNicholas PigginEXC_VIRT_NONE(0x4ee0, 0x20) 1934bda7fea2SNicholas Piggin 19350ebc4cdaSBenjamin Herrenschmidt 19364f50541fSNicholas PigginINT_DEFINE_BEGIN(performance_monitor) 19374f50541fSNicholas Piggin IVEC=0xf00 19384f50541fSNicholas Piggin IMASK=IRQS_PMI_DISABLED 19394f50541fSNicholas Piggin IKVM_REAL=1 19404f50541fSNicholas PigginINT_DEFINE_END(performance_monitor) 19414f50541fSNicholas Piggin 19427299417cSNicholas PigginEXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) 19434f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=0, ool=1 19447299417cSNicholas PigginEXC_REAL_END(performance_monitor, 0xf00, 0x20) 19457299417cSNicholas PigginEXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) 19464f50541fSNicholas Piggin GEN_INT_ENTRY performance_monitor, virt=1, ool=1 19477299417cSNicholas PigginEXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 1948eb204d86SNicholas PigginEXC_COMMON_BEGIN(performance_monitor_common) 19494f50541fSNicholas Piggin GEN_COMMON performance_monitor 1950eb204d86SNicholas Piggin FINISH_NAP 1951eb204d86SNicholas Piggin RUNLATCH_ON 1952eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1953eb204d86SNicholas Piggin bl performance_monitor_exception 1954eb204d86SNicholas Piggin b ret_from_except_lite 1955b1c7f150SNicholas Piggin 19569600f261SNicholas Piggin GEN_KVM performance_monitor 19579600f261SNicholas Piggin 19580ebc4cdaSBenjamin Herrenschmidt 19594f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_unavailable) 19604f50541fSNicholas Piggin IVEC=0xf20 19614f50541fSNicholas Piggin IRECONCILE=0 19624f50541fSNicholas Piggin IKVM_REAL=1 19634f50541fSNicholas PigginINT_DEFINE_END(altivec_unavailable) 19644f50541fSNicholas Piggin 19657299417cSNicholas PigginEXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20) 19664f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1 19677299417cSNicholas PigginEXC_REAL_END(altivec_unavailable, 0xf20, 0x20) 19687299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20) 19694f50541fSNicholas Piggin GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1 19707299417cSNicholas PigginEXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20) 1971d1a0ca9cSNicholas PigginEXC_COMMON_BEGIN(altivec_unavailable_common) 19724f50541fSNicholas Piggin GEN_COMMON altivec_unavailable 1973d1a0ca9cSNicholas Piggin#ifdef CONFIG_ALTIVEC 1974d1a0ca9cSNicholas PigginBEGIN_FTR_SECTION 1975d1a0ca9cSNicholas Piggin beq 1f 1976d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1977d1a0ca9cSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 1978d1a0ca9cSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 1979d1a0ca9cSNicholas Piggin * transaction), go do TM stuff 1980d1a0ca9cSNicholas Piggin */ 1981d1a0ca9cSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 1982d1a0ca9cSNicholas Piggin bne- 2f 1983d1a0ca9cSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 1984d1a0ca9cSNicholas Piggin#endif 1985d1a0ca9cSNicholas Piggin bl load_up_altivec 1986d1a0ca9cSNicholas Piggin b fast_exception_return 1987d1a0ca9cSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1988d1a0ca9cSNicholas Piggin2: /* User process was in a transaction */ 1989d1a0ca9cSNicholas Piggin bl save_nvgprs 1990d1a0ca9cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 1991d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 1992d1a0ca9cSNicholas Piggin bl altivec_unavailable_tm 1993d1a0ca9cSNicholas Piggin b ret_from_except 1994d1a0ca9cSNicholas Piggin#endif 1995d1a0ca9cSNicholas Piggin1: 1996d1a0ca9cSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 1997d1a0ca9cSNicholas Piggin#endif 1998d1a0ca9cSNicholas Piggin bl save_nvgprs 1999d1a0ca9cSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2000d1a0ca9cSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2001d1a0ca9cSNicholas Piggin bl altivec_unavailable_exception 2002d1a0ca9cSNicholas Piggin b ret_from_except 2003d1a0ca9cSNicholas Piggin 20049600f261SNicholas Piggin GEN_KVM altivec_unavailable 20059600f261SNicholas Piggin 20060ebc4cdaSBenjamin Herrenschmidt 20074f50541fSNicholas PigginINT_DEFINE_BEGIN(vsx_unavailable) 20084f50541fSNicholas Piggin IVEC=0xf40 20094f50541fSNicholas Piggin IRECONCILE=0 20104f50541fSNicholas Piggin IKVM_REAL=1 20114f50541fSNicholas PigginINT_DEFINE_END(vsx_unavailable) 20124f50541fSNicholas Piggin 20137299417cSNicholas PigginEXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20) 20144f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1 20157299417cSNicholas PigginEXC_REAL_END(vsx_unavailable, 0xf40, 0x20) 20167299417cSNicholas PigginEXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20) 20174f50541fSNicholas Piggin GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1 20187299417cSNicholas PigginEXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20) 2019792cbdddSNicholas PigginEXC_COMMON_BEGIN(vsx_unavailable_common) 20204f50541fSNicholas Piggin GEN_COMMON vsx_unavailable 2021792cbdddSNicholas Piggin#ifdef CONFIG_VSX 2022792cbdddSNicholas PigginBEGIN_FTR_SECTION 2023792cbdddSNicholas Piggin beq 1f 2024792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2025792cbdddSNicholas Piggin BEGIN_FTR_SECTION_NESTED(69) 2026792cbdddSNicholas Piggin /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in 2027792cbdddSNicholas Piggin * transaction), go do TM stuff 2028792cbdddSNicholas Piggin */ 2029792cbdddSNicholas Piggin rldicl. r0, r12, (64-MSR_TS_LG), (64-2) 2030792cbdddSNicholas Piggin bne- 2f 2031792cbdddSNicholas Piggin END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69) 2032792cbdddSNicholas Piggin#endif 2033792cbdddSNicholas Piggin b load_up_vsx 2034792cbdddSNicholas Piggin#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2035792cbdddSNicholas Piggin2: /* User process was in a transaction */ 2036792cbdddSNicholas Piggin bl save_nvgprs 2037792cbdddSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2038792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2039792cbdddSNicholas Piggin bl vsx_unavailable_tm 2040792cbdddSNicholas Piggin b ret_from_except 2041792cbdddSNicholas Piggin#endif 2042792cbdddSNicholas Piggin1: 2043792cbdddSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_VSX) 2044792cbdddSNicholas Piggin#endif 2045792cbdddSNicholas Piggin bl save_nvgprs 2046792cbdddSNicholas Piggin RECONCILE_IRQ_STATE(r10, r11) 2047792cbdddSNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2048792cbdddSNicholas Piggin bl vsx_unavailable_exception 2049792cbdddSNicholas Piggin b ret_from_except 2050792cbdddSNicholas Piggin 20519600f261SNicholas Piggin GEN_KVM vsx_unavailable 20529600f261SNicholas Piggin 2053d0c0c9a1SMichael Neuling 20544f50541fSNicholas PigginINT_DEFINE_BEGIN(facility_unavailable) 20554f50541fSNicholas Piggin IVEC=0xf60 20564f50541fSNicholas Piggin IKVM_REAL=1 20574f50541fSNicholas PigginINT_DEFINE_END(facility_unavailable) 20584f50541fSNicholas Piggin 20597299417cSNicholas PigginEXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20) 20604f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=0, ool=1 20617299417cSNicholas PigginEXC_REAL_END(facility_unavailable, 0xf60, 0x20) 20627299417cSNicholas PigginEXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20) 20634f50541fSNicholas Piggin GEN_INT_ENTRY facility_unavailable, virt=1, ool=1 20647299417cSNicholas PigginEXC_VIRT_END(facility_unavailable, 0x4f60, 0x20) 2065eb204d86SNicholas PigginEXC_COMMON_BEGIN(facility_unavailable_common) 20664f50541fSNicholas Piggin GEN_COMMON facility_unavailable 2067eb204d86SNicholas Piggin bl save_nvgprs 2068eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2069eb204d86SNicholas Piggin bl facility_unavailable_exception 2070eb204d86SNicholas Piggin b ret_from_except 20711134713cSNicholas Piggin 20729600f261SNicholas Piggin GEN_KVM facility_unavailable 20739600f261SNicholas Piggin 2074da2bc464SMichael Ellerman 20754f50541fSNicholas PigginINT_DEFINE_BEGIN(h_facility_unavailable) 20764f50541fSNicholas Piggin IVEC=0xf80 2077*3f7fbd97SNicholas Piggin IHSRR=1 20784f50541fSNicholas Piggin IKVM_REAL=1 20794f50541fSNicholas Piggin IKVM_VIRT=1 20804f50541fSNicholas PigginINT_DEFINE_END(h_facility_unavailable) 20814f50541fSNicholas Piggin 20827299417cSNicholas PigginEXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20) 20834f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1 20847299417cSNicholas PigginEXC_REAL_END(h_facility_unavailable, 0xf80, 0x20) 20857299417cSNicholas PigginEXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20) 20864f50541fSNicholas Piggin GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1 20877299417cSNicholas PigginEXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20) 2088eb204d86SNicholas PigginEXC_COMMON_BEGIN(h_facility_unavailable_common) 20894f50541fSNicholas Piggin GEN_COMMON h_facility_unavailable 2090eb204d86SNicholas Piggin bl save_nvgprs 2091eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2092eb204d86SNicholas Piggin bl facility_unavailable_exception 2093eb204d86SNicholas Piggin b ret_from_except 209414b0072cSNicholas Piggin 20959600f261SNicholas Piggin GEN_KVM h_facility_unavailable 20969600f261SNicholas Piggin 2097da2bc464SMichael Ellerman 20981a6822d1SNicholas PigginEXC_REAL_NONE(0xfa0, 0x20) 20991a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fa0, 0x20) 21001a6822d1SNicholas PigginEXC_REAL_NONE(0xfc0, 0x20) 21011a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fc0, 0x20) 21021a6822d1SNicholas PigginEXC_REAL_NONE(0xfe0, 0x20) 21031a6822d1SNicholas PigginEXC_VIRT_NONE(0x4fe0, 0x20) 21041a6822d1SNicholas Piggin 21051a6822d1SNicholas PigginEXC_REAL_NONE(0x1000, 0x100) 21061a6822d1SNicholas PigginEXC_VIRT_NONE(0x5000, 0x100) 21071a6822d1SNicholas PigginEXC_REAL_NONE(0x1100, 0x100) 21081a6822d1SNicholas PigginEXC_VIRT_NONE(0x5100, 0x100) 2109da2bc464SMichael Ellerman 21100ebc4cdaSBenjamin Herrenschmidt#ifdef CONFIG_CBE_RAS 21114f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_system_error) 21124f50541fSNicholas Piggin IVEC=0x1200 2113*3f7fbd97SNicholas Piggin IHSRR=1 21144f50541fSNicholas Piggin IKVM_SKIP=1 21154f50541fSNicholas Piggin IKVM_REAL=1 21164f50541fSNicholas PigginINT_DEFINE_END(cbe_system_error) 21174f50541fSNicholas Piggin 21187299417cSNicholas PigginEXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) 21194f50541fSNicholas Piggin GEN_INT_ENTRY cbe_system_error, virt=0 21207299417cSNicholas PigginEXC_REAL_END(cbe_system_error, 0x1200, 0x100) 21211a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2122eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_system_error_common) 21234f50541fSNicholas Piggin GEN_COMMON cbe_system_error 2124eb204d86SNicholas Piggin bl save_nvgprs 2125eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2126eb204d86SNicholas Piggin bl cbe_system_error_exception 2127eb204d86SNicholas Piggin b ret_from_except 21289600f261SNicholas Piggin 21299600f261SNicholas Piggin GEN_KVM cbe_system_error 21309600f261SNicholas Piggin 2131da2bc464SMichael Ellerman#else /* CONFIG_CBE_RAS */ 21321a6822d1SNicholas PigginEXC_REAL_NONE(0x1200, 0x100) 21331a6822d1SNicholas PigginEXC_VIRT_NONE(0x5200, 0x100) 2134da2bc464SMichael Ellerman#endif 2135da2bc464SMichael Ellerman 2136ff1b3206SNicholas Piggin 21374f50541fSNicholas PigginINT_DEFINE_BEGIN(instruction_breakpoint) 21384f50541fSNicholas Piggin IVEC=0x1300 21394f50541fSNicholas Piggin IKVM_SKIP=1 21404f50541fSNicholas Piggin IKVM_REAL=1 21414f50541fSNicholas PigginINT_DEFINE_END(instruction_breakpoint) 21424f50541fSNicholas Piggin 21437299417cSNicholas PigginEXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100) 21444f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=0 21457299417cSNicholas PigginEXC_REAL_END(instruction_breakpoint, 0x1300, 0x100) 21467299417cSNicholas PigginEXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100) 21474f50541fSNicholas Piggin GEN_INT_ENTRY instruction_breakpoint, virt=1 21487299417cSNicholas PigginEXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100) 2149eb204d86SNicholas PigginEXC_COMMON_BEGIN(instruction_breakpoint_common) 21504f50541fSNicholas Piggin GEN_COMMON instruction_breakpoint 2151eb204d86SNicholas Piggin bl save_nvgprs 2152eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2153eb204d86SNicholas Piggin bl instruction_breakpoint_exception 2154eb204d86SNicholas Piggin b ret_from_except 21554e96dbbfSNicholas Piggin 21569600f261SNicholas Piggin GEN_KVM instruction_breakpoint 21579600f261SNicholas Piggin 21587299417cSNicholas Piggin 21591a6822d1SNicholas PigginEXC_REAL_NONE(0x1400, 0x100) 21601a6822d1SNicholas PigginEXC_VIRT_NONE(0x5400, 0x100) 2161da2bc464SMichael Ellerman 21624f50541fSNicholas PigginINT_DEFINE_BEGIN(denorm_exception) 21634f50541fSNicholas Piggin IVEC=0x1500 2164*3f7fbd97SNicholas Piggin IHSRR=1 2165*3f7fbd97SNicholas Piggin IBRANCH_COMMON=0 21669600f261SNicholas Piggin IKVM_REAL=1 21674f50541fSNicholas PigginINT_DEFINE_END(denorm_exception) 21684f50541fSNicholas Piggin 21694f50541fSNicholas PigginEXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100) 21704f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=0 2171b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2172d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 2173b92a66a6SMichael Neuling bne+ denorm_assist 2174b92a66a6SMichael Neuling#endif 21758729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=0 21764f50541fSNicholas PigginEXC_REAL_END(denorm_exception, 0x1500, 0x100) 2177d7e89849SNicholas Piggin#ifdef CONFIG_PPC_DENORMALISATION 21781a6822d1SNicholas PigginEXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100) 21794f50541fSNicholas Piggin GEN_INT_ENTRY denorm_exception, virt=1 2180d73a10cbSNicholas Piggin andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */ 218152b98923SNicholas Piggin bne+ denorm_assist 21828729c26eSNicholas Piggin GEN_BRANCH_TO_COMMON denorm_exception, virt=1 21831a6822d1SNicholas PigginEXC_VIRT_END(denorm_exception, 0x5500, 0x100) 2184d7e89849SNicholas Piggin#else 21851a6822d1SNicholas PigginEXC_VIRT_NONE(0x5500, 0x100) 2186d7e89849SNicholas Piggin#endif 2187b92a66a6SMichael Neuling 2188b92a66a6SMichael Neuling#ifdef CONFIG_PPC_DENORMALISATION 2189da2bc464SMichael EllermanTRAMP_REAL_BEGIN(denorm_assist) 2190b92a66a6SMichael NeulingBEGIN_FTR_SECTION 2191b92a66a6SMichael Neuling/* 2192b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2193b92a66a6SMichael Neuling * For POWER6 do that here for all FP regs. 2194b92a66a6SMichael Neuling */ 2195b92a66a6SMichael Neuling mfmsr r10 2196b92a66a6SMichael Neuling ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1) 2197b92a66a6SMichael Neuling xori r10,r10,(MSR_FE0|MSR_FE1) 2198b92a66a6SMichael Neuling mtmsrd r10 2199b92a66a6SMichael Neuling sync 2200d7c67fb1SMichael Neuling 2201f3c8b6c6SNicholas Piggin .Lreg=0 2202f3c8b6c6SNicholas Piggin .rept 32 2203f3c8b6c6SNicholas Piggin fmr .Lreg,.Lreg 2204f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2205f3c8b6c6SNicholas Piggin .endr 2206d7c67fb1SMichael Neuling 2207b92a66a6SMichael NeulingFTR_SECTION_ELSE 2208b92a66a6SMichael Neuling/* 2209b92a66a6SMichael Neuling * To denormalise we need to move a copy of the register to itself. 2210b92a66a6SMichael Neuling * For POWER7 do that here for the first 32 VSX registers only. 2211b92a66a6SMichael Neuling */ 2212b92a66a6SMichael Neuling mfmsr r10 2213b92a66a6SMichael Neuling oris r10,r10,MSR_VSX@h 2214b92a66a6SMichael Neuling mtmsrd r10 2215b92a66a6SMichael Neuling sync 2216d7c67fb1SMichael Neuling 2217f3c8b6c6SNicholas Piggin .Lreg=0 2218f3c8b6c6SNicholas Piggin .rept 32 2219f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2220f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2221f3c8b6c6SNicholas Piggin .endr 2222d7c67fb1SMichael Neuling 2223b92a66a6SMichael NeulingALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) 2224fb0fce3eSMichael Neuling 2225fb0fce3eSMichael NeulingBEGIN_FTR_SECTION 2226fb0fce3eSMichael Neuling b denorm_done 2227fb0fce3eSMichael NeulingEND_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 2228fb0fce3eSMichael Neuling/* 2229fb0fce3eSMichael Neuling * To denormalise we need to move a copy of the register to itself. 2230fb0fce3eSMichael Neuling * For POWER8 we need to do that for all 64 VSX registers 2231fb0fce3eSMichael Neuling */ 2232f3c8b6c6SNicholas Piggin .Lreg=32 2233f3c8b6c6SNicholas Piggin .rept 32 2234f3c8b6c6SNicholas Piggin XVCPSGNDP(.Lreg,.Lreg,.Lreg) 2235f3c8b6c6SNicholas Piggin .Lreg=.Lreg+1 2236f3c8b6c6SNicholas Piggin .endr 2237f3c8b6c6SNicholas Piggin 2238fb0fce3eSMichael Neulingdenorm_done: 2239f14040bcSMichael Neuling mfspr r11,SPRN_HSRR0 2240f14040bcSMichael Neuling subi r11,r11,4 2241b92a66a6SMichael Neuling mtspr SPRN_HSRR0,r11 2242b92a66a6SMichael Neuling mtcrf 0x80,r9 2243b92a66a6SMichael Neuling ld r9,PACA_EXGEN+EX_R9(r13) 2244931dc86bSNicholas PigginBEGIN_FTR_SECTION 2245931dc86bSNicholas Piggin ld r10,PACA_EXGEN+EX_PPR(r13) 2246931dc86bSNicholas Piggin mtspr SPRN_PPR,r10 2247931dc86bSNicholas PigginEND_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 2248630573c1SPaul MackerrasBEGIN_FTR_SECTION 2249630573c1SPaul Mackerras ld r10,PACA_EXGEN+EX_CFAR(r13) 2250630573c1SPaul Mackerras mtspr SPRN_CFAR,r10 2251630573c1SPaul MackerrasEND_FTR_SECTION_IFSET(CPU_FTR_CFAR) 2252b92a66a6SMichael Neuling ld r10,PACA_EXGEN+EX_R10(r13) 2253b92a66a6SMichael Neuling ld r11,PACA_EXGEN+EX_R11(r13) 2254b92a66a6SMichael Neuling ld r12,PACA_EXGEN+EX_R12(r13) 2255b92a66a6SMichael Neuling ld r13,PACA_EXGEN+EX_R13(r13) 2256222f20f1SNicholas Piggin HRFI_TO_UNKNOWN 2257b92a66a6SMichael Neuling b . 2258b92a66a6SMichael Neuling#endif 2259b92a66a6SMichael Neuling 22604f50541fSNicholas PigginEXC_COMMON_BEGIN(denorm_exception_common) 22614f50541fSNicholas Piggin GEN_COMMON denorm_exception 2262eb204d86SNicholas Piggin bl save_nvgprs 2263eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2264eb204d86SNicholas Piggin bl unknown_exception 2265eb204d86SNicholas Piggin b ret_from_except 2266d7e89849SNicholas Piggin 22679600f261SNicholas Piggin GEN_KVM denorm_exception 22689600f261SNicholas Piggin 2269d7e89849SNicholas Piggin 2270d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 22714f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_maintenance) 22724f50541fSNicholas Piggin IVEC=0x1600 2273*3f7fbd97SNicholas Piggin IHSRR=1 22744f50541fSNicholas Piggin IKVM_SKIP=1 22754f50541fSNicholas Piggin IKVM_REAL=1 22764f50541fSNicholas PigginINT_DEFINE_END(cbe_maintenance) 22774f50541fSNicholas Piggin 22787299417cSNicholas PigginEXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) 22794f50541fSNicholas Piggin GEN_INT_ENTRY cbe_maintenance, virt=0 22807299417cSNicholas PigginEXC_REAL_END(cbe_maintenance, 0x1600, 0x100) 22811a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2282eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_maintenance_common) 22834f50541fSNicholas Piggin GEN_COMMON cbe_maintenance 2284eb204d86SNicholas Piggin bl save_nvgprs 2285eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2286eb204d86SNicholas Piggin bl cbe_maintenance_exception 2287eb204d86SNicholas Piggin b ret_from_except 22889600f261SNicholas Piggin 22899600f261SNicholas Piggin GEN_KVM cbe_maintenance 22909600f261SNicholas Piggin 2291d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 22921a6822d1SNicholas PigginEXC_REAL_NONE(0x1600, 0x100) 22931a6822d1SNicholas PigginEXC_VIRT_NONE(0x5600, 0x100) 2294d7e89849SNicholas Piggin#endif 2295d7e89849SNicholas Piggin 229669a79344SNicholas Piggin 22974f50541fSNicholas PigginINT_DEFINE_BEGIN(altivec_assist) 22984f50541fSNicholas Piggin IVEC=0x1700 22994f50541fSNicholas Piggin IKVM_REAL=1 23004f50541fSNicholas PigginINT_DEFINE_END(altivec_assist) 23014f50541fSNicholas Piggin 23027299417cSNicholas PigginEXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100) 23034f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=0 23047299417cSNicholas PigginEXC_REAL_END(altivec_assist, 0x1700, 0x100) 23057299417cSNicholas PigginEXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100) 23064f50541fSNicholas Piggin GEN_INT_ENTRY altivec_assist, virt=1 23077299417cSNicholas PigginEXC_VIRT_END(altivec_assist, 0x5700, 0x100) 2308eb204d86SNicholas PigginEXC_COMMON_BEGIN(altivec_assist_common) 23094f50541fSNicholas Piggin GEN_COMMON altivec_assist 2310eb204d86SNicholas Piggin bl save_nvgprs 2311eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2312b51c079eSNicholas Piggin#ifdef CONFIG_ALTIVEC 2313eb204d86SNicholas Piggin bl altivec_assist_exception 2314b51c079eSNicholas Piggin#else 2315eb204d86SNicholas Piggin bl unknown_exception 2316b51c079eSNicholas Piggin#endif 2317eb204d86SNicholas Piggin b ret_from_except 2318b51c079eSNicholas Piggin 23199600f261SNicholas Piggin GEN_KVM altivec_assist 23209600f261SNicholas Piggin 2321d7e89849SNicholas Piggin 2322d7e89849SNicholas Piggin#ifdef CONFIG_CBE_RAS 23234f50541fSNicholas PigginINT_DEFINE_BEGIN(cbe_thermal) 23244f50541fSNicholas Piggin IVEC=0x1800 2325*3f7fbd97SNicholas Piggin IHSRR=1 23264f50541fSNicholas Piggin IKVM_SKIP=1 23274f50541fSNicholas Piggin IKVM_REAL=1 23284f50541fSNicholas PigginINT_DEFINE_END(cbe_thermal) 23294f50541fSNicholas Piggin 23307299417cSNicholas PigginEXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) 23314f50541fSNicholas Piggin GEN_INT_ENTRY cbe_thermal, virt=0 23327299417cSNicholas PigginEXC_REAL_END(cbe_thermal, 0x1800, 0x100) 23331a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2334eb204d86SNicholas PigginEXC_COMMON_BEGIN(cbe_thermal_common) 23354f50541fSNicholas Piggin GEN_COMMON cbe_thermal 2336eb204d86SNicholas Piggin bl save_nvgprs 2337eb204d86SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2338eb204d86SNicholas Piggin bl cbe_thermal_exception 2339eb204d86SNicholas Piggin b ret_from_except 23409600f261SNicholas Piggin 23419600f261SNicholas Piggin GEN_KVM cbe_thermal 23429600f261SNicholas Piggin 2343d7e89849SNicholas Piggin#else /* CONFIG_CBE_RAS */ 23441a6822d1SNicholas PigginEXC_REAL_NONE(0x1800, 0x100) 23451a6822d1SNicholas PigginEXC_VIRT_NONE(0x5800, 0x100) 2346d7e89849SNicholas Piggin#endif 2347d7e89849SNicholas Piggin 23487299417cSNicholas Piggin 234975eb767eSNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 23502104180aSNicholas Piggin 23510eddf327SNicholas PigginINT_DEFINE_BEGIN(soft_nmi) 23520eddf327SNicholas Piggin IVEC=0x900 23530eddf327SNicholas Piggin ISTACK=0 23540eddf327SNicholas PigginINT_DEFINE_END(soft_nmi) 23552104180aSNicholas Piggin 2356cc491f1dSNicholas Piggin/* 2357cc491f1dSNicholas Piggin * Branch to soft_nmi_interrupt using the emergency stack. The emergency 2358cc491f1dSNicholas Piggin * stack is one that is usable by maskable interrupts so long as MSR_EE 2359cc491f1dSNicholas Piggin * remains off. It is used for recovery when something has corrupted the 2360cc491f1dSNicholas Piggin * normal kernel stack, for example. The "soft NMI" must not use the process 2361cc491f1dSNicholas Piggin * stack because we want irq disabled sections to avoid touching the stack 2362cc491f1dSNicholas Piggin * at all (other than PMU interrupts), so use the emergency stack for this, 2363cc491f1dSNicholas Piggin * and run it entirely with interrupts hard disabled. 2364cc491f1dSNicholas Piggin */ 23652104180aSNicholas PigginEXC_COMMON_BEGIN(soft_nmi_common) 23660eddf327SNicholas Piggin mfspr r11,SPRN_SRR0 23672104180aSNicholas Piggin mr r10,r1 23682104180aSNicholas Piggin ld r1,PACAEMERGSP(r13) 23692104180aSNicholas Piggin subi r1,r1,INT_FRAME_SIZE 23700eddf327SNicholas Piggin __GEN_COMMON_BODY soft_nmi 237147169fbaSNicholas Piggin bl save_nvgprs 2372c06075f3SNicholas Piggin addi r3,r1,STACK_FRAME_OVERHEAD 2373c06075f3SNicholas Piggin bl soft_nmi_interrupt 23742104180aSNicholas Piggin b ret_from_except 23752104180aSNicholas Piggin 237675eb767eSNicholas Piggin#endif /* CONFIG_PPC_WATCHDOG */ 2377d7e89849SNicholas Piggin 23780ebc4cdaSBenjamin Herrenschmidt/* 2379fe9e1d54SIan Munsie * An interrupt came in while soft-disabled. We set paca->irq_happened, then: 2380fe9e1d54SIan Munsie * - If it was a decrementer interrupt, we bump the dec to max and and return. 2381fe9e1d54SIan Munsie * - If it was a doorbell we return immediately since doorbells are edge 2382fe9e1d54SIan Munsie * triggered and won't automatically refire. 23830869b6fdSMahesh Salgaonkar * - If it was a HMI we return immediately since we handled it in realmode 23840869b6fdSMahesh Salgaonkar * and it won't refire. 23856cc3f91bSNicholas Piggin * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. 2386fe9e1d54SIan Munsie * This is called with r10 containing the value to OR to the paca field. 23870ebc4cdaSBenjamin Herrenschmidt */ 2388*3f7fbd97SNicholas Piggin.macro MASKED_INTERRUPT hsrr=0 23894508a74aSNicholas Piggin .if \hsrr 23904508a74aSNicholas Pigginmasked_Hinterrupt: 23914508a74aSNicholas Piggin .else 23924508a74aSNicholas Pigginmasked_interrupt: 23934508a74aSNicholas Piggin .endif 23944508a74aSNicholas Piggin lbz r11,PACAIRQHAPPENED(r13) 23954508a74aSNicholas Piggin or r11,r11,r10 23964508a74aSNicholas Piggin stb r11,PACAIRQHAPPENED(r13) 23974508a74aSNicholas Piggin cmpwi r10,PACA_IRQ_DEC 23984508a74aSNicholas Piggin bne 1f 23994508a74aSNicholas Piggin lis r10,0x7fff 24004508a74aSNicholas Piggin ori r10,r10,0xffff 24014508a74aSNicholas Piggin mtspr SPRN_DEC,r10 24020eddf327SNicholas Piggin#ifdef CONFIG_PPC_WATCHDOG 24030eddf327SNicholas Piggin b soft_nmi_common 24040eddf327SNicholas Piggin#else 24050eddf327SNicholas Piggin b 2f 24060eddf327SNicholas Piggin#endif 24074508a74aSNicholas Piggin1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK 24084508a74aSNicholas Piggin beq 2f 24090eddf327SNicholas Piggin xori r12,r12,MSR_EE /* clear MSR_EE */ 24104508a74aSNicholas Piggin .if \hsrr 24110eddf327SNicholas Piggin mtspr SPRN_HSRR1,r12 24124508a74aSNicholas Piggin .else 24130eddf327SNicholas Piggin mtspr SPRN_SRR1,r12 24144508a74aSNicholas Piggin .endif 24154508a74aSNicholas Piggin ori r11,r11,PACA_IRQ_HARD_DIS 24164508a74aSNicholas Piggin stb r11,PACAIRQHAPPENED(r13) 24174508a74aSNicholas Piggin2: /* done */ 24180eddf327SNicholas Piggin ld r10,PACA_EXGEN+EX_CTR(r13) 24190eddf327SNicholas Piggin mtctr r10 24204508a74aSNicholas Piggin mtcrf 0x80,r9 24214508a74aSNicholas Piggin std r1,PACAR1(r13) 24224508a74aSNicholas Piggin ld r9,PACA_EXGEN+EX_R9(r13) 24234508a74aSNicholas Piggin ld r10,PACA_EXGEN+EX_R10(r13) 24244508a74aSNicholas Piggin ld r11,PACA_EXGEN+EX_R11(r13) 24250eddf327SNicholas Piggin ld r12,PACA_EXGEN+EX_R12(r13) 24264508a74aSNicholas Piggin /* returns to kernel where r13 must be set up, so don't restore it */ 24274508a74aSNicholas Piggin .if \hsrr 24284508a74aSNicholas Piggin HRFI_TO_KERNEL 24294508a74aSNicholas Piggin .else 24304508a74aSNicholas Piggin RFI_TO_KERNEL 24314508a74aSNicholas Piggin .endif 24324508a74aSNicholas Piggin b . 24334508a74aSNicholas Piggin.endm 24340ebc4cdaSBenjamin Herrenschmidt 2435a048a07dSNicholas PigginTRAMP_REAL_BEGIN(stf_barrier_fallback) 2436a048a07dSNicholas Piggin std r9,PACA_EXRFI+EX_R9(r13) 2437a048a07dSNicholas Piggin std r10,PACA_EXRFI+EX_R10(r13) 2438a048a07dSNicholas Piggin sync 2439a048a07dSNicholas Piggin ld r9,PACA_EXRFI+EX_R9(r13) 2440a048a07dSNicholas Piggin ld r10,PACA_EXRFI+EX_R10(r13) 2441a048a07dSNicholas Piggin ori 31,31,0 2442a048a07dSNicholas Piggin .rept 14 2443a048a07dSNicholas Piggin b 1f 2444a048a07dSNicholas Piggin1: 2445a048a07dSNicholas Piggin .endr 2446a048a07dSNicholas Piggin blr 2447a048a07dSNicholas Piggin 2448aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(rfi_flush_fallback) 2449aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2450aa8a5e00SMichael Ellerman GET_PACA(r13); 245178ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 245278ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2453aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2454aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2455aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2456aa8a5e00SMichael Ellerman mfctr r9 2457aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2458bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2459bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2460aa8a5e00SMichael Ellerman mtctr r11 246115a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2462aa8a5e00SMichael Ellerman 2463aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2464aa8a5e00SMichael Ellerman sync 2465bdcb1aefSNicholas Piggin 2466bdcb1aefSNicholas Piggin /* 2467bdcb1aefSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 2468bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2469bdcb1aefSNicholas Piggin * hurt). 2470bdcb1aefSNicholas Piggin */ 2471bdcb1aefSNicholas Piggin1: 2472bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2473bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2474bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2475bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2476bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2477bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2478bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2479bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2480bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2481aa8a5e00SMichael Ellerman bdnz 1b 2482aa8a5e00SMichael Ellerman 2483aa8a5e00SMichael Ellerman mtctr r9 2484aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2485aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2486aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 248778ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2488aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2489aa8a5e00SMichael Ellerman rfid 2490aa8a5e00SMichael Ellerman 2491aa8a5e00SMichael EllermanTRAMP_REAL_BEGIN(hrfi_flush_fallback) 2492aa8a5e00SMichael Ellerman SET_SCRATCH0(r13); 2493aa8a5e00SMichael Ellerman GET_PACA(r13); 249478ee9946SMichael Ellerman std r1,PACA_EXRFI+EX_R12(r13) 249578ee9946SMichael Ellerman ld r1,PACAKSAVE(r13) 2496aa8a5e00SMichael Ellerman std r9,PACA_EXRFI+EX_R9(r13) 2497aa8a5e00SMichael Ellerman std r10,PACA_EXRFI+EX_R10(r13) 2498aa8a5e00SMichael Ellerman std r11,PACA_EXRFI+EX_R11(r13) 2499aa8a5e00SMichael Ellerman mfctr r9 2500aa8a5e00SMichael Ellerman ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) 2501bdcb1aefSNicholas Piggin ld r11,PACA_L1D_FLUSH_SIZE(r13) 2502bdcb1aefSNicholas Piggin srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ 2503aa8a5e00SMichael Ellerman mtctr r11 250415a3204dSNicholas Piggin DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ 2505aa8a5e00SMichael Ellerman 2506aa8a5e00SMichael Ellerman /* order ld/st prior to dcbt stop all streams with flushing */ 2507aa8a5e00SMichael Ellerman sync 2508bdcb1aefSNicholas Piggin 2509bdcb1aefSNicholas Piggin /* 2510bdcb1aefSNicholas Piggin * The load adresses are at staggered offsets within cachelines, 2511bdcb1aefSNicholas Piggin * which suits some pipelines better (on others it should not 2512bdcb1aefSNicholas Piggin * hurt). 2513bdcb1aefSNicholas Piggin */ 2514bdcb1aefSNicholas Piggin1: 2515bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*0(r10) 2516bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*1(r10) 2517bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*2(r10) 2518bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*3(r10) 2519bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*4(r10) 2520bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*5(r10) 2521bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*6(r10) 2522bdcb1aefSNicholas Piggin ld r11,(0x80 + 8)*7(r10) 2523bdcb1aefSNicholas Piggin addi r10,r10,0x80*8 2524aa8a5e00SMichael Ellerman bdnz 1b 2525aa8a5e00SMichael Ellerman 2526aa8a5e00SMichael Ellerman mtctr r9 2527aa8a5e00SMichael Ellerman ld r9,PACA_EXRFI+EX_R9(r13) 2528aa8a5e00SMichael Ellerman ld r10,PACA_EXRFI+EX_R10(r13) 2529aa8a5e00SMichael Ellerman ld r11,PACA_EXRFI+EX_R11(r13) 253078ee9946SMichael Ellerman ld r1,PACA_EXRFI+EX_R12(r13) 2531aa8a5e00SMichael Ellerman GET_SCRATCH0(r13); 2532aa8a5e00SMichael Ellerman hrfid 2533aa8a5e00SMichael Ellerman 25340eddf327SNicholas PigginUSE_TEXT_SECTION() 2535*3f7fbd97SNicholas Piggin MASKED_INTERRUPT 2536*3f7fbd97SNicholas Piggin MASKED_INTERRUPT hsrr=1 25377230c564SBenjamin Herrenschmidt 25384f6c11dbSPaul Mackerras#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 25399600f261SNicholas Pigginkvmppc_skip_interrupt: 25404f6c11dbSPaul Mackerras /* 25414f6c11dbSPaul Mackerras * Here all GPRs are unchanged from when the interrupt happened 25424f6c11dbSPaul Mackerras * except for r13, which is saved in SPRG_SCRATCH0. 25434f6c11dbSPaul Mackerras */ 25444f6c11dbSPaul Mackerras mfspr r13, SPRN_SRR0 25454f6c11dbSPaul Mackerras addi r13, r13, 4 25464f6c11dbSPaul Mackerras mtspr SPRN_SRR0, r13 25474f6c11dbSPaul Mackerras GET_SCRATCH0(r13) 2548222f20f1SNicholas Piggin RFI_TO_KERNEL 25494f6c11dbSPaul Mackerras b . 25504f6c11dbSPaul Mackerras 25519600f261SNicholas Pigginkvmppc_skip_Hinterrupt: 25524f6c11dbSPaul Mackerras /* 25534f6c11dbSPaul Mackerras * Here all GPRs are unchanged from when the interrupt happened 25544f6c11dbSPaul Mackerras * except for r13, which is saved in SPRG_SCRATCH0. 25554f6c11dbSPaul Mackerras */ 25564f6c11dbSPaul Mackerras mfspr r13, SPRN_HSRR0 25574f6c11dbSPaul Mackerras addi r13, r13, 4 25584f6c11dbSPaul Mackerras mtspr SPRN_HSRR0, r13 25594f6c11dbSPaul Mackerras GET_SCRATCH0(r13) 2560222f20f1SNicholas Piggin HRFI_TO_KERNEL 25614f6c11dbSPaul Mackerras b . 25624f6c11dbSPaul Mackerras#endif 25634f6c11dbSPaul Mackerras 25640ebc4cdaSBenjamin Herrenschmidt /* 2565c1fb6816SMichael Neuling * Relocation-on interrupts: A subset of the interrupts can be delivered 2566c1fb6816SMichael Neuling * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering 2567c1fb6816SMichael Neuling * it. Addresses are the same as the original interrupt addresses, but 2568c1fb6816SMichael Neuling * offset by 0xc000000000004000. 2569c1fb6816SMichael Neuling * It's impossible to receive interrupts below 0x300 via this mechanism. 2570c1fb6816SMichael Neuling * KVM: None of these traps are from the guest ; anything that escalated 2571c1fb6816SMichael Neuling * to HV=1 from HV=0 is delivered via real mode handlers. 2572c1fb6816SMichael Neuling */ 2573c1fb6816SMichael Neuling 2574c1fb6816SMichael Neuling /* 2575c1fb6816SMichael Neuling * This uses the standard macro, since the original 0x300 vector 2576c1fb6816SMichael Neuling * only has extra guff for STAB-based processors -- which never 2577c1fb6816SMichael Neuling * come here. 2578c1fb6816SMichael Neuling */ 2579da2bc464SMichael Ellerman 258057f26649SNicholas PigginEXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) 2581b1576fecSAnton Blanchard b __ppc64_runlatch_on 2582fe1952fcSBenjamin Herrenschmidt 258357f26649SNicholas PigginUSE_FIXED_SECTION(virt_trampolines) 25848ed8ab40SHari Bathini /* 25858ed8ab40SHari Bathini * The __end_interrupts marker must be past the out-of-line (OOL) 25868ed8ab40SHari Bathini * handlers, so that they are copied to real address 0x100 when running 25878ed8ab40SHari Bathini * a relocatable kernel. This ensures they can be reached from the short 25888ed8ab40SHari Bathini * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch 25898ed8ab40SHari Bathini * directly, without using LOAD_HANDLER(). 25908ed8ab40SHari Bathini */ 25918ed8ab40SHari Bathini .align 7 25928ed8ab40SHari Bathini .globl __end_interrupts 25938ed8ab40SHari Bathini__end_interrupts: 259457f26649SNicholas PigginDEFINE_FIXED_SYMBOL(__end_interrupts) 259561383407SBenjamin Herrenschmidt 2596087aa036SChen Gang#ifdef CONFIG_PPC_970_NAP 2597ed0bc98fSNicholas Piggin /* 2598ed0bc98fSNicholas Piggin * Called by exception entry code if _TLF_NAPPING was set, this clears 2599ed0bc98fSNicholas Piggin * the NAPPING flag, and redirects the exception exit to 2600ed0bc98fSNicholas Piggin * power4_fixup_nap_return. 2601ed0bc98fSNicholas Piggin */ 2602ed0bc98fSNicholas Piggin .globl power4_fixup_nap 26037c8cb4b5SNicholas PigginEXC_COMMON_BEGIN(power4_fixup_nap) 2604087aa036SChen Gang andc r9,r9,r10 2605087aa036SChen Gang std r9,TI_LOCAL_FLAGS(r11) 2606ed0bc98fSNicholas Piggin LOAD_REG_ADDR(r10, power4_idle_nap_return) 2607ed0bc98fSNicholas Piggin std r10,_NIP(r1) 2608ed0bc98fSNicholas Piggin blr 2609ed0bc98fSNicholas Piggin 2610ed0bc98fSNicholas Pigginpower4_idle_nap_return: 2611087aa036SChen Gang blr 2612087aa036SChen Gang#endif 2613087aa036SChen Gang 261457f26649SNicholas PigginCLOSE_FIXED_SECTION(real_vectors); 261557f26649SNicholas PigginCLOSE_FIXED_SECTION(real_trampolines); 261657f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_vectors); 261757f26649SNicholas PigginCLOSE_FIXED_SECTION(virt_trampolines); 261857f26649SNicholas Piggin 261957f26649SNicholas PigginUSE_TEXT_SECTION() 262057f26649SNicholas Piggin 2621296e753fSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 2622296e753fSNicholas Pigginenable_machine_check: 2623296e753fSNicholas Piggin mflr r0 2624296e753fSNicholas Piggin bcl 20,31,$+4 2625296e753fSNicholas Piggin0: mflr r3 2626296e753fSNicholas Piggin addi r3,r3,(1f - 0b) 2627296e753fSNicholas Piggin mtspr SPRN_SRR0,r3 2628296e753fSNicholas Piggin mfmsr r3 2629296e753fSNicholas Piggin ori r3,r3,MSR_ME 2630296e753fSNicholas Piggin mtspr SPRN_SRR1,r3 2631296e753fSNicholas Piggin RFI_TO_KERNEL 2632296e753fSNicholas Piggin1: mtlr r0 2633296e753fSNicholas Piggin blr 2634296e753fSNicholas Piggin 2635b7d9ccecSNicholas Piggin/* MSR[RI] should be clear because this uses SRR[01] */ 2636b7d9ccecSNicholas Piggindisable_machine_check: 2637b7d9ccecSNicholas Piggin mflr r0 2638b7d9ccecSNicholas Piggin bcl 20,31,$+4 2639b7d9ccecSNicholas Piggin0: mflr r3 2640b7d9ccecSNicholas Piggin addi r3,r3,(1f - 0b) 2641b7d9ccecSNicholas Piggin mtspr SPRN_SRR0,r3 2642b7d9ccecSNicholas Piggin mfmsr r3 2643b7d9ccecSNicholas Piggin li r4,MSR_ME 2644b7d9ccecSNicholas Piggin andc r3,r3,r4 2645b7d9ccecSNicholas Piggin mtspr SPRN_SRR1,r3 2646b7d9ccecSNicholas Piggin RFI_TO_KERNEL 2647b7d9ccecSNicholas Piggin1: mtlr r0 2648b7d9ccecSNicholas Piggin blr 2649b7d9ccecSNicholas Piggin 2650087aa036SChen Gang/* 26510ebc4cdaSBenjamin Herrenschmidt * Hash table stuff 26520ebc4cdaSBenjamin Herrenschmidt */ 2653f4329f2eSNicholas Piggin .balign IFETCH_ALIGN_BYTES 26546a3bab90SAnton Blancharddo_hash_page: 26554e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 2656e6c2a479SRam Pai lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h 2657398a719dSBenjamin Herrenschmidt ori r0,r0,DSISR_BAD_FAULT_64S@l 26589b123d1eSNicholas Piggin and. r0,r5,r0 /* weird error? */ 26590ebc4cdaSBenjamin Herrenschmidt bne- handle_page_fault /* if not, try to insert a HPTE */ 2660c911d2e1SChristophe Leroy ld r11, PACA_THREAD_INFO(r13) 26619c1e1052SPaul Mackerras lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */ 26629c1e1052SPaul Mackerras andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */ 26639c1e1052SPaul Mackerras bne 77f /* then don't call hash_page now */ 26640ebc4cdaSBenjamin Herrenschmidt 26650ebc4cdaSBenjamin Herrenschmidt /* 26669b123d1eSNicholas Piggin * r3 contains the trap number 26679b123d1eSNicholas Piggin * r4 contains the faulting address 26689b123d1eSNicholas Piggin * r5 contains dsisr 26699b123d1eSNicholas Piggin * r6 msr 26700ebc4cdaSBenjamin Herrenschmidt * 26717230c564SBenjamin Herrenschmidt * at return r3 = 0 for success, 1 for page fault, negative for error 26720ebc4cdaSBenjamin Herrenschmidt */ 2673106713a1SAneesh Kumar K.V bl __hash_page /* build HPTE if possible */ 2674106713a1SAneesh Kumar K.V cmpdi r3,0 /* see if __hash_page succeeded */ 26750ebc4cdaSBenjamin Herrenschmidt 26767230c564SBenjamin Herrenschmidt /* Success */ 26770ebc4cdaSBenjamin Herrenschmidt beq fast_exc_return_irq /* Return from exception on success */ 26780ebc4cdaSBenjamin Herrenschmidt 26797230c564SBenjamin Herrenschmidt /* Error */ 26807230c564SBenjamin Herrenschmidt blt- 13f 2681d89ba535SNaveen N. Rao 26829b123d1eSNicholas Piggin /* Reload DAR/DSISR into r4/r5 for the DABR check below */ 26839b123d1eSNicholas Piggin ld r4,_DAR(r1) 26849b123d1eSNicholas Piggin ld r5,_DSISR(r1) 26854e003747SMichael Ellerman#endif /* CONFIG_PPC_BOOK3S_64 */ 26860ebc4cdaSBenjamin Herrenschmidt 2687a546498fSBenjamin Herrenschmidt/* Here we have a page fault that hash_page can't handle. */ 2688a546498fSBenjamin Herrenschmidthandle_page_fault: 26899b123d1eSNicholas Piggin11: andis. r0,r5,DSISR_DABRMATCH@h 2690d89ba535SNaveen N. Rao bne- handle_dabr_fault 2691a546498fSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 2692b1576fecSAnton Blanchard bl do_page_fault 2693a546498fSBenjamin Herrenschmidt cmpdi r3,0 2694f474c28fSRavi Bangoria beq+ ret_from_except_lite 2695b1576fecSAnton Blanchard bl save_nvgprs 2696a546498fSBenjamin Herrenschmidt mr r5,r3 2697a546498fSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 2698c31f7134SNicholas Piggin ld r4,_DAR(r1) 2699b1576fecSAnton Blanchard bl bad_page_fault 2700b1576fecSAnton Blanchard b ret_from_except 27010ebc4cdaSBenjamin Herrenschmidt 27029c7cc234SK.Prasad/* We have a data breakpoint exception - handle it */ 27039c7cc234SK.Prasadhandle_dabr_fault: 2704b1576fecSAnton Blanchard bl save_nvgprs 27059c7cc234SK.Prasad ld r4,_DAR(r1) 27069c7cc234SK.Prasad ld r5,_DSISR(r1) 27079c7cc234SK.Prasad addi r3,r1,STACK_FRAME_OVERHEAD 2708b1576fecSAnton Blanchard bl do_break 2709f474c28fSRavi Bangoria /* 2710f474c28fSRavi Bangoria * do_break() may have changed the NV GPRS while handling a breakpoint. 2711f474c28fSRavi Bangoria * If so, we need to restore them with their updated values. Don't use 2712f474c28fSRavi Bangoria * ret_from_except_lite here. 2713f474c28fSRavi Bangoria */ 2714f474c28fSRavi Bangoria b ret_from_except 27159c7cc234SK.Prasad 27160ebc4cdaSBenjamin Herrenschmidt 27174e003747SMichael Ellerman#ifdef CONFIG_PPC_BOOK3S_64 27180ebc4cdaSBenjamin Herrenschmidt/* We have a page fault that hash_page could handle but HV refused 27190ebc4cdaSBenjamin Herrenschmidt * the PTE insertion 27200ebc4cdaSBenjamin Herrenschmidt */ 2721b1576fecSAnton Blanchard13: bl save_nvgprs 27220ebc4cdaSBenjamin Herrenschmidt mr r5,r3 27230ebc4cdaSBenjamin Herrenschmidt addi r3,r1,STACK_FRAME_OVERHEAD 27240ebc4cdaSBenjamin Herrenschmidt ld r4,_DAR(r1) 2725b1576fecSAnton Blanchard bl low_hash_fault 2726b1576fecSAnton Blanchard b ret_from_except 2727caca285eSAneesh Kumar K.V#endif 27280ebc4cdaSBenjamin Herrenschmidt 27299c1e1052SPaul Mackerras/* 27309c1e1052SPaul Mackerras * We come here as a result of a DSI at a point where we don't want 27319c1e1052SPaul Mackerras * to call hash_page, such as when we are accessing memory (possibly 27329c1e1052SPaul Mackerras * user memory) inside a PMU interrupt that occurred while interrupts 27339c1e1052SPaul Mackerras * were soft-disabled. We want to invoke the exception handler for 27349c1e1052SPaul Mackerras * the access, or panic if there isn't a handler. 27359c1e1052SPaul Mackerras */ 2736b1576fecSAnton Blanchard77: bl save_nvgprs 27379c1e1052SPaul Mackerras addi r3,r1,STACK_FRAME_OVERHEAD 27389c1e1052SPaul Mackerras li r5,SIGSEGV 2739b1576fecSAnton Blanchard bl bad_page_fault 2740b1576fecSAnton Blanchard b ret_from_except 27414e2bf01bSMichael Ellerman 27424e2bf01bSMichael Ellerman/* 2743a9af97aaSNicholas Piggin * When doorbell is triggered from system reset wakeup, the message is 2744a9af97aaSNicholas Piggin * not cleared, so it would fire again when EE is enabled. 2745a9af97aaSNicholas Piggin * 2746a9af97aaSNicholas Piggin * When coming from local_irq_enable, there may be the same problem if 2747a9af97aaSNicholas Piggin * we were hard disabled. 2748a9af97aaSNicholas Piggin * 2749a9af97aaSNicholas Piggin * Execute msgclr to clear pending exceptions before handling it. 2750a9af97aaSNicholas Piggin */ 2751a9af97aaSNicholas Pigginh_doorbell_common_msgclr: 2752a9af97aaSNicholas Piggin LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36)) 2753a9af97aaSNicholas Piggin PPC_MSGCLR(3) 27548729c26eSNicholas Piggin b h_doorbell_common_virt 2755a9af97aaSNicholas Piggin 2756a9af97aaSNicholas Piggindoorbell_super_common_msgclr: 2757a9af97aaSNicholas Piggin LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36)) 2758a9af97aaSNicholas Piggin PPC_MSGCLRP(3) 27598729c26eSNicholas Piggin b doorbell_super_common_virt 2760a9af97aaSNicholas Piggin 2761a9af97aaSNicholas Piggin/* 27620f0c6ca1SNicholas Piggin * Called from arch_local_irq_enable when an interrupt needs 27630f0c6ca1SNicholas Piggin * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate 27640f0c6ca1SNicholas Piggin * which kind of interrupt. MSR:EE is already off. We generate a 27650f0c6ca1SNicholas Piggin * stackframe like if a real interrupt had happened. 27660f0c6ca1SNicholas Piggin * 27670f0c6ca1SNicholas Piggin * Note: While MSR:EE is off, we need to make sure that _MSR 27680f0c6ca1SNicholas Piggin * in the generated frame has EE set to 1 or the exception 27690f0c6ca1SNicholas Piggin * handler will not properly re-enable them. 2770b48bbb82SNicholas Piggin * 2771b48bbb82SNicholas Piggin * Note that we don't specify LR as the NIP (return address) for 2772b48bbb82SNicholas Piggin * the interrupt because that would unbalance the return branch 2773b48bbb82SNicholas Piggin * predictor. 27740f0c6ca1SNicholas Piggin */ 27750f0c6ca1SNicholas Piggin_GLOBAL(__replay_interrupt) 27760f0c6ca1SNicholas Piggin /* We are going to jump to the exception common code which 27770f0c6ca1SNicholas Piggin * will retrieve various register values from the PACA which 27780f0c6ca1SNicholas Piggin * we don't give a damn about, so we don't bother storing them. 27790f0c6ca1SNicholas Piggin */ 27800f0c6ca1SNicholas Piggin mfmsr r12 27813e23a12bSMichael Ellerman LOAD_REG_ADDR(r11, replay_interrupt_return) 27820f0c6ca1SNicholas Piggin mfcr r9 27830f0c6ca1SNicholas Piggin ori r12,r12,MSR_EE 27840f0c6ca1SNicholas Piggin cmpwi r3,0x900 27858729c26eSNicholas Piggin beq decrementer_common_virt 27860f0c6ca1SNicholas Piggin cmpwi r3,0x500 2787e6c1203dSNicholas PigginBEGIN_FTR_SECTION 27888729c26eSNicholas Piggin beq h_virt_irq_common_virt 2789e6c1203dSNicholas PigginFTR_SECTION_ELSE 27908729c26eSNicholas Piggin beq hardware_interrupt_common_virt 2791e6c1203dSNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300) 2792f442d004SMadhavan Srinivasan cmpwi r3,0xf00 27938729c26eSNicholas Piggin beq performance_monitor_common_virt 27940f0c6ca1SNicholas PigginBEGIN_FTR_SECTION 2795d6f73fc6SNicholas Piggin cmpwi r3,0xa00 2796a9af97aaSNicholas Piggin beq h_doorbell_common_msgclr 27970f0c6ca1SNicholas Piggin cmpwi r3,0xe60 27988729c26eSNicholas Piggin beq hmi_exception_common_virt 27990f0c6ca1SNicholas PigginFTR_SECTION_ELSE 28000f0c6ca1SNicholas Piggin cmpwi r3,0xa00 2801a9af97aaSNicholas Piggin beq doorbell_super_common_msgclr 28020f0c6ca1SNicholas PigginALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) 28033e23a12bSMichael Ellermanreplay_interrupt_return: 28040f0c6ca1SNicholas Piggin blr 2805b48bbb82SNicholas Piggin 280615770a13SNaveen N. Rao_ASM_NOKPROBE_SYMBOL(__replay_interrupt) 2807