1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * S390 low-level entry points. 4 * 5 * Copyright IBM Corp. 1999, 2012 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 7 * Hartmut Penner (hp@de.ibm.com), 8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), 9 */ 10 11#include <linux/export.h> 12#include <linux/init.h> 13#include <linux/linkage.h> 14#include <asm/asm-extable.h> 15#include <asm/alternative.h> 16#include <asm/processor.h> 17#include <asm/cache.h> 18#include <asm/dwarf.h> 19#include <asm/errno.h> 20#include <asm/ptrace.h> 21#include <asm/thread_info.h> 22#include <asm/asm-offsets.h> 23#include <asm/unistd.h> 24#include <asm/page.h> 25#include <asm/sigp.h> 26#include <asm/bug.h> 27#include <asm/irq.h> 28#include <asm/fpu-insn.h> 29#include <asm/setup.h> 30#include <asm/nmi.h> 31#include <asm/nospec-insn.h> 32#include <asm/lowcore.h> 33#include <asm/machine.h> 34#include "entry.h" 35 36_LPP_OFFSET = __LC_LPP 37 38 .macro STBEAR address 39 ALTERNATIVE "nop", ".insn s,0xb2010000,\address", ALT_FACILITY(193) 40 .endm 41 42 .macro LBEAR address 43 ALTERNATIVE "nop", ".insn s,0xb2000000,\address", ALT_FACILITY(193) 44 .endm 45 46 .macro LPSWEY address, lpswe 47 ALTERNATIVE_2 "b \lpswe;nopr", \ 48 ".insn siy,0xeb0000000071,\address,0", ALT_FACILITY(193), \ 49 __stringify(.insn siy,0xeb0000000071,LOWCORE_ALT_ADDRESS+\address,0), \ 50 ALT_FEATURE(MFEATURE_LOWCORE) 51 .endm 52 53 .macro MBEAR reg, lowcore 54 ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK(\lowcore)),\ 55 ALT_FACILITY(193) 56 .endm 57 58 .macro CHECK_VMAP_STACK savearea, lowcore, oklabel 59 lgr %r14,%r15 60 nill %r14,0x10000 - THREAD_SIZE 61 oill %r14,STACK_INIT_OFFSET 62 clg %r14,__LC_KERNEL_STACK(\lowcore) 63 je \oklabel 64 clg %r14,__LC_ASYNC_STACK(\lowcore) 65 je \oklabel 66 clg %r14,__LC_MCCK_STACK(\lowcore) 67 je \oklabel 68 clg %r14,__LC_NODAT_STACK(\lowcore) 69 je \oklabel 70 clg %r14,__LC_RESTART_STACK(\lowcore) 71 je \oklabel 72 la %r14,\savearea(\lowcore) 73 j stack_invalid 74 .endm 75 76 /* 77 * The TSTMSK macro generates a test-under-mask instruction by 78 * calculating the memory offset for the specified mask value. 79 * Mask value can be any constant. The macro shifts the mask 80 * value to calculate the memory offset for the test-under-mask 81 * instruction. 82 */ 83 .macro TSTMSK addr, mask, size=8, bytepos=0 84 .if (\bytepos < \size) && (\mask >> 8) 85 .if (\mask & 0xff) 86 .error "Mask exceeds byte boundary" 87 .endif 88 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)" 89 .exitm 90 .endif 91 .ifeq \mask 92 .error "Mask must not be zero" 93 .endif 94 off = \size - \bytepos - 1 95 tm off+\addr, \mask 96 .endm 97 98 .macro BPOFF 99 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", ALT_SPEC(82) 100 .endm 101 102 .macro BPON 103 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82) 104 .endm 105 106 .macro BPENTER tif_ptr,tif_mask 107 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \ 108 "j .+12; nop; nop", ALT_SPEC(82) 109 .endm 110 111 .macro BPEXIT tif_ptr,tif_mask 112 TSTMSK \tif_ptr,\tif_mask 113 ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \ 114 "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82) 115 .endm 116 117#if IS_ENABLED(CONFIG_KVM) 118 .macro SIEEXIT sie_control,lowcore 119 lg %r9,\sie_control # get control block pointer 120 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE 121 lctlg %c1,%c1,__LC_USER_ASCE(\lowcore) # load primary asce 122 lg %r9,__LC_CURRENT(\lowcore) 123 mvi __TI_sie(%r9),0 124 larl %r9,sie_exit # skip forward to sie_exit 125 .endm 126#endif 127 128 .macro STACKLEAK_ERASE 129#ifdef CONFIG_KSTACK_ERASE 130 brasl %r14,stackleak_erase_on_task_stack 131#endif 132 .endm 133 134 GEN_BR_THUNK %r14 135 136 .section .kprobes.text, "ax" 137.Ldummy: 138 /* 139 * The following nop exists only in order to avoid that the next 140 * symbol starts at the beginning of the kprobes text section. 141 * In that case there would be several symbols at the same address. 142 * E.g. objdump would take an arbitrary symbol when disassembling 143 * the code. 144 * With the added nop in between this cannot happen. 145 */ 146 nop 0 147 148/* 149 * Scheduler resume function, called by __switch_to 150 * gpr2 = (task_struct *)prev 151 * gpr3 = (task_struct *)next 152 * Returns: 153 * gpr2 = prev 154 */ 155SYM_FUNC_START(__switch_to_asm) 156 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task 157 lghi %r4,__TASK_stack 158 lghi %r1,__TASK_thread 159 llill %r5,STACK_INIT_OFFSET 160 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev 161 lg %r15,0(%r4,%r3) # start of kernel stack of next 162 agr %r15,%r5 # end of kernel stack of next 163 GET_LC %r13 164 stg %r3,__LC_CURRENT(%r13) # store task struct of next 165 stg %r15,__LC_KERNEL_STACK(%r13) # store end of kernel stack 166 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next 167 lay %r4,__TASK_pid(%r3) 168 mvc __LC_CURRENT_PID(4,%r13),0(%r4) # store pid of next 169 ALTERNATIVE "nop", "lpp _LPP_OFFSET(%r13)", ALT_FACILITY(40) 170#ifdef CONFIG_STACKPROTECTOR 171 lg %r3,__TASK_stack_canary(%r3) 172 stg %r3,__LC_STACK_CANARY(%r13) 173#endif 174 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task 175 BR_EX %r14 176SYM_FUNC_END(__switch_to_asm) 177 178#if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS) 179 180SYM_FUNC_START(__WARN_trap) 181 mc MONCODE_BUG_ARG(%r0),0 182 BR_EX %r14 183SYM_FUNC_END(__WARN_trap) 184EXPORT_SYMBOL(__WARN_trap) 185 186#endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ 187 188#if IS_ENABLED(CONFIG_KVM) 189/* 190 * __sie64a calling convention: 191 * %r2 pointer to sie control block phys 192 * %r3 pointer to sie control block virt 193 * %r4 guest register save area 194 * %r5 guest asce 195 */ 196SYM_FUNC_START(__sie64a) 197 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers 198 GET_LC %r13 199 lg %r14,__LC_CURRENT(%r13) 200 stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical.. 201 stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses 202 stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area 203 stg %r5,__SF_SIE_GUEST_ASCE(%r15) # save guest asce 204 xc __SF_SIE_RETURN(8,%r15),__SF_SIE_RETURN(%r15) # return code = 0 205 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags 206 lmg %r0,%r13,0(%r4) # load guest gprs 0-13 207 mvi __TI_sie(%r14),1 208 stosm __SF_SIE_IRQ(%r15),0x03 # enable interrupts 209 lctlg %c1,%c1,__SF_SIE_GUEST_ASCE(%r15) # load primary asce 210 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 211 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now 212 tm __SIE_PROG20+3(%r14),3 # last exit... 213 jnz .Lsie_skip 214 lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr 215 BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST 216.Lsie_entry: 217 sie 0(%r14) 218# Let the next instruction be NOP to avoid triggering a machine check 219# and handling it in a guest as result of the instruction execution. 220 nopr 7 221.Lsie_leave: 222 BPOFF 223 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST 224.Lsie_skip: 225 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer 226 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE 227 GET_LC %r14 228 lctlg %c1,%c1,__LC_USER_ASCE(%r14) # load primary asce 229 lg %r14,__LC_CURRENT(%r14) 230 mvi __TI_sie(%r14),0 231SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL) 232 stnsm __SF_SIE_IRQ(%r15),0xfc # disable interrupts 233 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area 234 stmg %r0,%r13,0(%r14) # save guest gprs 0-13 235 xgr %r0,%r0 # clear guest registers to 236 xgr %r1,%r1 # prevent speculative use 237 xgr %r3,%r3 238 xgr %r4,%r4 239 xgr %r5,%r5 240 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers 241 lg %r2,__SF_SIE_RETURN(%r15) # return sie return code 242 BR_EX %r14 243SYM_FUNC_END(__sie64a) 244EXPORT_SYMBOL(__sie64a) 245EXPORT_SYMBOL(sie_exit) 246#endif 247 248#if IS_ENABLED(CONFIG_KVM) 249/* 250 * SAE calling convention: 251 * %r2 physical address of the SAE control block 252 */ 253SYM_FUNC_START(sae) 254 stmg %r6,%r14,__SF_GPRS(%r15) # store kernel registers 255 STBEAR __SF_SAE_BEAR(%r15) # save breaking event address register 256 .insn rre,0xb9a50000,%r2,0 # Start Arm Execution 257 LBEAR __SF_SAE_BEAR(%r15) # restore breaking event address register 258 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers 259 xgr %r0,%r0 # clear guest registers to 260 xgr %r1,%r1 # prevent speculative use 261 xgr %r2,%r2 262 xgr %r3,%r3 263 xgr %r4,%r4 264 xgr %r5,%r5 265 BR_EX %r14 266SYM_FUNC_END(sae) 267EXPORT_SYMBOL(sae) 268#endif 269 270/* 271 * SVC interrupt handler routine. System calls are synchronous events and 272 * are entered with interrupts disabled. 273 */ 274 275SYM_CODE_START(system_call) 276 STMG_LC %r8,%r15,__LC_SAVE_AREA 277 GET_LC %r13 278 stpt __LC_SYS_ENTER_TIMER(%r13) 279 BPOFF 280 lghi %r14,0 281.Lsysc_per: 282 STBEAR __LC_LAST_BREAK(%r13) 283 lg %r15,__LC_KERNEL_STACK(%r13) 284 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 285 stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 286 # clear user controlled register to prevent speculative use 287 xgr %r0,%r0 288 xgr %r1,%r1 289 xgr %r4,%r4 290 xgr %r5,%r5 291 xgr %r6,%r6 292 xgr %r7,%r7 293 xgr %r8,%r8 294 xgr %r9,%r9 295 xgr %r10,%r10 296 xgr %r11,%r11 297 xgr %r12,%r12 298 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs 299 mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) 300 MBEAR %r2,%r13 301 lgr %r3,%r14 302 brasl %r14,__do_syscall 303 STACKLEAK_ERASE 304 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 305 BPON 306 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15) 307 stpt __LC_EXIT_TIMER(%r13) 308 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 309 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE 310SYM_CODE_END(system_call) 311 312# 313# a new process exits the kernel with ret_from_fork 314# 315SYM_CODE_START(ret_from_fork) 316 lgr %r3,%r11 317 brasl %r14,__ret_from_fork 318 STACKLEAK_ERASE 319 GET_LC %r13 320 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 321 BPON 322 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15) 323 stpt __LC_EXIT_TIMER(%r13) 324 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 325 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE 326SYM_CODE_END(ret_from_fork) 327 328/* 329 * Program check handler routine 330 */ 331 332SYM_CODE_START(pgm_check_handler) 333 STMG_LC %r8,%r15,__LC_SAVE_AREA 334 GET_LC %r13 335 stpt __LC_SYS_ENTER_TIMER(%r13) 336 BPOFF 337 lmg %r8,%r9,__LC_PGM_OLD_PSW(%r13) 338 xgr %r10,%r10 339 tmhh %r8,0x0001 # coming from user space? 340 jo 3f # -> fault in user space 341#if IS_ENABLED(CONFIG_KVM) 342 lg %r11,__LC_CURRENT(%r13) 343 tm __TI_sie(%r11),0xff 344 jz 1f 345 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST 346 SIEEXIT __SF_SIE_CONTROL(%r15),%r13 347 lghi %r10,PGM_FLAG_GUEST_FAULT 348#endif 3491: tmhh %r8,0x4000 # PER bit set in old PSW ? 350 jnz 2f # -> enabled, can't be a double fault 351 tm __LC_PGM_ILC+3(%r13),0x80 # check for per exception 352 jnz .Lpgm_svcper # -> single stepped svc 3532: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 354 # CHECK_VMAP_STACK branches to stack_invalid or 4f 355 CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4f 3563: lg %r15,__LC_KERNEL_STACK(%r13) 3574: la %r11,STACK_FRAME_OVERHEAD(%r15) 358 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 359 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 360 stmg %r0,%r7,__PT_R0(%r11) 361 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) 362 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r13) 363 stmg %r8,%r9,__PT_PSW(%r11) 364 # clear user controlled registers to prevent speculative use 365 xgr %r0,%r0 366 xgr %r1,%r1 367 xgr %r4,%r4 368 xgr %r5,%r5 369 xgr %r6,%r6 370 xgr %r7,%r7 371 xgr %r12,%r12 372 lgr %r2,%r11 373 lgr %r3,%r10 374 brasl %r14,__do_pgm_check 375 tmhh %r8,0x0001 # returning to user space? 376 jno .Lpgm_exit_kernel 377 STACKLEAK_ERASE 378 BPON 379 stpt __LC_EXIT_TIMER(%r13) 380.Lpgm_exit_kernel: 381 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 382 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15) 383 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 384 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE 385 386# 387# single stepped system call 388# 389.Lpgm_svcper: 390 mvc __LC_RETURN_PSW(8,%r13),__LC_SVC_NEW_PSW(%r13) 391 larl %r14,.Lsysc_per 392 stg %r14,__LC_RETURN_PSW+8(%r13) 393 lghi %r14,SYSCALL_FLAG_PER_TRAP 394 LBEAR __LC_PGM_LAST_BREAK(%r13) 395 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per 396SYM_CODE_END(pgm_check_handler) 397 398/* 399 * Interrupt handler macro used for external and IO interrupts. 400 */ 401.macro INT_HANDLER name,lc_old_psw,handler 402SYM_CODE_START(\name) 403 STMG_LC %r8,%r15,__LC_SAVE_AREA 404 GET_LC %r13 405#ifdef CONFIG_NO_HZ_COMMON 406 larl %r12,kernel_cpustat 407 ag %r12,__LC_PERCPU_OFFSET(%r13) 408 asi __KCPUSTAT_SEQUENCE(%r12),1 409#endif 410 stcke __LC_INT_CLOCK(%r13) 411 stpt __LC_SYS_ENTER_TIMER(%r13) 412 larl %r10,s390_idle 413 ag %r10,__LC_PERCPU_OFFSET(%r13) 414 mvc __IDLE_CLOCK_EXIT(16,%r10),__LC_INT_CLOCK(%r13) 415#ifdef CONFIG_NO_HZ_COMMON 416 asi __KCPUSTAT_SEQUENCE(%r12),1 417#endif 418 STBEAR __LC_LAST_BREAK(%r13) 419 BPOFF 420 lmg %r8,%r9,\lc_old_psw(%r13) 421 tmhh %r8,0x0001 # interrupting from user ? 422 jnz 1f 423#if IS_ENABLED(CONFIG_KVM) 424 lg %r10,__LC_CURRENT(%r13) 425 tm __TI_sie(%r10),0xff 426 jz 0f 427 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST 428 SIEEXIT __SF_SIE_CONTROL(%r15),%r13 429#endif 4300: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) 431 j 2f 4321: lg %r15,__LC_KERNEL_STACK(%r13) 4332: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 434 la %r11,STACK_FRAME_OVERHEAD(%r15) 435 stmg %r0,%r7,__PT_R0(%r11) 436 # clear user controlled registers to prevent speculative use 437 xgr %r0,%r0 438 xgr %r1,%r1 439 xgr %r3,%r3 440 xgr %r4,%r4 441 xgr %r5,%r5 442 xgr %r6,%r6 443 xgr %r7,%r7 444 xgr %r12,%r12 445 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 446 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) 447 MBEAR %r11,%r13 448 stmg %r8,%r9,__PT_PSW(%r11) 449 lgr %r2,%r11 # pass pointer to pt_regs 450 brasl %r14,\handler 451 mvc __LC_RETURN_PSW(16,%r13),__PT_PSW(%r11) 452 tmhh %r8,0x0001 # returning to user ? 453 jno 2f 454 STACKLEAK_ERASE 455 BPON 456 stpt __LC_EXIT_TIMER(%r13) 4572: LBEAR __PT_LAST_BREAK(%r11) 458 lmg %r0,%r15,__PT_R0(%r11) 459 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE 460SYM_CODE_END(\name) 461.endm 462 463 .section .irqentry.text, "ax" 464 465INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq 466INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq 467 468 .section .kprobes.text, "ax" 469 470/* 471 * Machine check handler routines 472 */ 473SYM_CODE_START(mcck_int_handler) 474 BPOFF 475 GET_LC %r13 476 lmg %r8,%r9,__LC_MCK_OLD_PSW(%r13) 477 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_SYSTEM_DAMAGE 478 jo .Lmcck_panic # yes -> rest of mcck code invalid 479 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_CR_VALID 480 jno .Lmcck_panic # control registers invalid -> panic 481 ptlb 482 lay %r14,__LC_CPU_TIMER_SAVE_AREA(%r13) 483 mvc __LC_MCCK_ENTER_TIMER(8,%r13),0(%r14) 484 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_CPU_TIMER_VALID 485 jo 3f 486 la %r14,__LC_SYS_ENTER_TIMER(%r13) 487 clc 0(8,%r14),__LC_EXIT_TIMER(%r13) 488 jl 1f 489 la %r14,__LC_EXIT_TIMER(%r13) 4901: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER(%r13) 491 jl 2f 492 la %r14,__LC_LAST_UPDATE_TIMER(%r13) 4932: spt 0(%r14) 494 mvc __LC_MCCK_ENTER_TIMER(8,%r13),0(%r14) 4953: TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_PSW_MWP_VALID 496 jno .Lmcck_panic 497 tmhh %r8,0x0001 # interrupting from user ? 498 jnz .Lmcck_user 499 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_PSW_IA_VALID 500 jno .Lmcck_panic 501#if IS_ENABLED(CONFIG_KVM) 502 lg %r10,__LC_CURRENT(%r13) 503 tm __TI_sie(%r10),0xff 504 jz .Lmcck_user 505 # Need to compare the address instead of __TI_SIE flag. 506 # Otherwise there would be a race between setting the flag 507 # and entering SIE (or leaving and clearing the flag). This 508 # would cause machine checks targeted at the guest to be 509 # handled by the host. 510 larl %r14,.Lsie_entry 511 clgrjl %r9,%r14, 4f 512 larl %r14,.Lsie_leave 513 clgrjhe %r9,%r14, 4f 514 lg %r10,__LC_PCPU(%r13) 515 oi __PCPU_FLAGS+7(%r10), _CIF_MCCK_GUEST 5164: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST 517 SIEEXIT __SF_SIE_CONTROL(%r15),%r13 518#endif 519.Lmcck_user: 520 lg %r15,__LC_MCCK_STACK(%r13) 521 la %r11,STACK_FRAME_OVERHEAD(%r15) 522 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 523 lay %r14,__LC_GPREGS_SAVE_AREA(%r13) 524 mvc __PT_R0(128,%r11),0(%r14) 525 # clear user controlled registers to prevent speculative use 526 xgr %r0,%r0 527 xgr %r1,%r1 528 xgr %r3,%r3 529 xgr %r4,%r4 530 xgr %r5,%r5 531 xgr %r6,%r6 532 xgr %r7,%r7 533 xgr %r10,%r10 534 xgr %r12,%r12 535 stmg %r8,%r9,__PT_PSW(%r11) 536 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 537 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 538 lgr %r2,%r11 # pass pointer to pt_regs 539 brasl %r14,s390_do_machine_check 540 lmg %r0,%r10,__PT_R0(%r11) 541 mvc __LC_RETURN_MCCK_PSW(16,%r13),__PT_PSW(%r11) # move return PSW 542 tm __LC_RETURN_MCCK_PSW+1(%r13),0x01 # returning to user ? 543 jno 0f 544 BPON 545 stpt __LC_EXIT_TIMER(%r13) 5460: ALTERNATIVE "brcl 0,0", __stringify(lay %r12,__LC_LAST_BREAK_SAVE_AREA(%r13)),\ 547 ALT_FACILITY(193) 548 LBEAR 0(%r12) 549 lmg %r11,%r15,__PT_R11(%r11) 550 LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE 551 552.Lmcck_panic: 553 /* 554 * Iterate over all possible CPU addresses in the range 0..0xffff 555 * and stop each CPU using signal processor. Use compare and swap 556 * to allow just one CPU-stopper and prevent concurrent CPUs from 557 * stopping each other while leaving the others running. 558 */ 559 lhi %r5,0 560 lhi %r6,1 561 larl %r7,stop_lock 562 cs %r5,%r6,0(%r7) # single CPU-stopper only 563 jnz 4f 564 larl %r7,this_cpu 565 stap 0(%r7) # this CPU address 566 lh %r4,0(%r7) 567 nilh %r4,0 568 lhi %r0,1 569 sll %r0,16 # CPU counter 570 lhi %r3,0 # next CPU address 5710: cr %r3,%r4 572 je 2f 5731: sigp %r1,%r3,SIGP_STOP # stop next CPU 574 brc SIGP_CC_BUSY,1b 5752: ahi %r3,1 576 brct %r0,0b 5773: sigp %r1,%r4,SIGP_STOP # stop this CPU 578 brc SIGP_CC_BUSY,3b 5794: j 4b 580SYM_CODE_END(mcck_int_handler) 581 582SYM_CODE_START(restart_int_handler) 583 ALTERNATIVE "nop", "lpp _LPP_OFFSET", ALT_FACILITY(40) 584 stg %r15,__LC_SAVE_AREA_RESTART 585 TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4 586 jz 0f 587 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA 5880: larl %r15,daton_psw 589 lpswe 0(%r15) # turn dat on, keep irqs off 590.Ldaton: 591 GET_LC %r15 592 lg %r15,__LC_RESTART_STACK(%r15) 593 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15) 594 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 595 GET_LC %r13 596 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART(%r13) 597 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW(%r13) 598 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) 599 lg %r1,__LC_RESTART_FN(%r13) # load fn, parm & source cpu 600 lg %r2,__LC_RESTART_DATA(%r13) 601 lgf %r3,__LC_RESTART_SOURCE(%r13) 602 ltgr %r3,%r3 # test source cpu address 603 jm 1f # negative -> skip source stop 6040: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu 605 brc 10,0b # wait for status stored 6061: basr %r14,%r1 # call function 607 stap __SF_EMPTY(%r15) # store cpu address 608 llgh %r3,__SF_EMPTY(%r15) 6092: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu 610 brc 2,2b 6113: j 3b 612SYM_CODE_END(restart_int_handler) 613 614 __INIT 615SYM_CODE_START(early_pgm_check_handler) 616 STMG_LC %r8,%r15,__LC_SAVE_AREA 617 GET_LC %r13 618 aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) 619 la %r11,STACK_FRAME_OVERHEAD(%r15) 620 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 621 stmg %r0,%r7,__PT_R0(%r11) 622 mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW(%r13) 623 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) 624 lgr %r2,%r11 625 brasl %r14,__do_early_pgm_check 626 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) 627 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) 628 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE 629SYM_CODE_END(early_pgm_check_handler) 630 __FINIT 631 632 .section .kprobes.text, "ax" 633 634/* 635 * The synchronous or the asynchronous stack pointer is invalid. We are dead. 636 * No need to properly save the registers, we are going to panic anyway. 637 * Setup a pt_regs so that show_trace can provide a good call trace. 638 * Copy lowcore of the current CPU to stack_invalid_lowcore before it 639 * is modified by printing debug messages and crash handling. 640 */ 641SYM_CODE_START(stack_invalid) 642 GET_LC %r15 643 lg %r15,__LC_NODAT_STACK(%r15) # change to panic stack 644 la %r11,STACK_FRAME_OVERHEAD(%r15) 645 stmg %r0,%r7,__PT_R0(%r11) 646 stmg %r8,%r9,__PT_PSW(%r11) 647 mvc __PT_R8(64,%r11),0(%r14) 648 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) 649 GET_LC %r2 650 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r2) 651 /* copy lowcore; skip if used by another CPU */ 652 larl %r4,stack_invalid_lowcore_used 653 lhi %r1,0 654 lhi %r3,1 655 cs %r1,%r3,0(%r4) 656 jne .Lskip 657 larl %r0,stack_invalid_lowcore 658 lghi %r1,LC_PAGES * PAGE_SIZE 659 lgr %r4,%r2 660 lgr %r5,%r1 661 mvcl %r0,%r4 662.Lskip: 663 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 664 lgr %r2,%r11 # pass pointer to pt_regs 665 jg kernel_stack_invalid 666SYM_CODE_END(stack_invalid) 667 668 .section .data, "aw" 669 .balign 4 670SYM_DATA_LOCAL(stop_lock, .long 0) 671SYM_DATA_LOCAL(this_cpu, .short 0) 672 .balign 8 673SYM_DATA_START_LOCAL(daton_psw) 674 .quad PSW_KERNEL_BITS 675 .quad .Ldaton 676SYM_DATA_END(daton_psw) 677