xref: /linux/arch/s390/kernel/entry.S (revision 75b1a8f9d62e50f05d0e4e9f3c8bcde32527ffc1) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *    S390 low-level entry points.
4 *
5 *    Copyright IBM Corp. 1999, 2012
6 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 *		 Hartmut Penner (hp@de.ibm.com),
8 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 *		 Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
11
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/alternative-asm.h>
15#include <asm/processor.h>
16#include <asm/cache.h>
17#include <asm/ctl_reg.h>
18#include <asm/dwarf.h>
19#include <asm/errno.h>
20#include <asm/ptrace.h>
21#include <asm/thread_info.h>
22#include <asm/asm-offsets.h>
23#include <asm/unistd.h>
24#include <asm/page.h>
25#include <asm/sigp.h>
26#include <asm/irq.h>
27#include <asm/vx-insn.h>
28#include <asm/setup.h>
29#include <asm/nmi.h>
30#include <asm/export.h>
31#include <asm/nospec-insn.h>
32
33__PT_R0      =	__PT_GPRS
34__PT_R1      =	__PT_GPRS + 8
35__PT_R2      =	__PT_GPRS + 16
36__PT_R3      =	__PT_GPRS + 24
37__PT_R4      =	__PT_GPRS + 32
38__PT_R5      =	__PT_GPRS + 40
39__PT_R6      =	__PT_GPRS + 48
40__PT_R7      =	__PT_GPRS + 56
41__PT_R8      =	__PT_GPRS + 64
42__PT_R9      =	__PT_GPRS + 72
43__PT_R10     =	__PT_GPRS + 80
44__PT_R11     =	__PT_GPRS + 88
45__PT_R12     =	__PT_GPRS + 96
46__PT_R13     =	__PT_GPRS + 104
47__PT_R14     =	__PT_GPRS + 112
48__PT_R15     =	__PT_GPRS + 120
49
50STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
51STACK_SIZE  = 1 << STACK_SHIFT
52STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
53
54_TIF_WORK	= (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
55		   _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING | \
56		   _TIF_NOTIFY_SIGNAL)
57_TIF_TRACE	= (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
58		   _TIF_SYSCALL_TRACEPOINT)
59_CIF_WORK	= (_CIF_FPU)
60_PIF_WORK	= (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
61
62_LPP_OFFSET	= __LC_LPP
63
64	.macro	TRACE_IRQS_ON
65#ifdef CONFIG_TRACE_IRQFLAGS
66	basr	%r2,%r0
67	brasl	%r14,trace_hardirqs_on_caller
68#endif
69	.endm
70
71	.macro	TRACE_IRQS_OFF
72#ifdef CONFIG_TRACE_IRQFLAGS
73	basr	%r2,%r0
74	brasl	%r14,trace_hardirqs_off_caller
75#endif
76	.endm
77
78	.macro	LOCKDEP_SYS_EXIT
79#ifdef CONFIG_LOCKDEP
80	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
81	jz	.+10
82	brasl	%r14,lockdep_sys_exit
83#endif
84	.endm
85
86	.macro	CHECK_STACK savearea
87#ifdef CONFIG_CHECK_STACK
88	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
89	lghi	%r14,\savearea
90	jz	stack_overflow
91#endif
92	.endm
93
94	.macro	DEBUG_USER_ASCE
95#ifdef CONFIG_DEBUG_USER_ASCE
96	brasl	%r14,debug_user_asce
97#endif
98	.endm
99
100	.macro	CHECK_VMAP_STACK savearea,oklabel
101#ifdef CONFIG_VMAP_STACK
102	lgr	%r14,%r15
103	nill	%r14,0x10000 - STACK_SIZE
104	oill	%r14,STACK_INIT
105	clg	%r14,__LC_KERNEL_STACK
106	je	\oklabel
107	clg	%r14,__LC_ASYNC_STACK
108	je	\oklabel
109	clg	%r14,__LC_NODAT_STACK
110	je	\oklabel
111	clg	%r14,__LC_RESTART_STACK
112	je	\oklabel
113	lghi	%r14,\savearea
114	j	stack_overflow
115#else
116	j	\oklabel
117#endif
118	.endm
119
120	.macro	SWITCH_ASYNC savearea,timer,clock
121	tmhh	%r8,0x0001		# interrupting from user ?
122	jnz	4f
123#if IS_ENABLED(CONFIG_KVM)
124	lgr	%r14,%r9
125	larl	%r13,.Lsie_gmap
126	slgr	%r14,%r13
127	lghi	%r13,.Lsie_done - .Lsie_gmap
128	clgr	%r14,%r13
129	jhe	0f
130	lghi	%r11,\savearea		# inside critical section, do cleanup
131	brasl	%r14,.Lcleanup_sie
132#endif
1330:	larl	%r13,.Lpsw_idle_exit
134	cgr	%r13,%r9
135	jne	3f
136
137	larl	%r1,smp_cpu_mtid
138	llgf	%r1,0(%r1)
139	ltgr	%r1,%r1
140	jz	2f			# no SMT, skip mt_cycles calculation
141	.insn	rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
142	larl	%r3,mt_cycles
143	ag	%r3,__LC_PERCPU_OFFSET
144	la	%r4,__SF_EMPTY+16(%r15)
1451:	lg	%r0,0(%r3)
146	slg	%r0,0(%r4)
147	alg	%r0,64(%r4)
148	stg	%r0,0(%r3)
149	la	%r3,8(%r3)
150	la	%r4,8(%r4)
151	brct	%r1,1b
152
1532:	mvc	__CLOCK_IDLE_EXIT(8,%r2), \clock
154	mvc	__TIMER_IDLE_EXIT(8,%r2), \timer
155	# account system time going idle
156	ni	__LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
157
158	lg	%r13,__LC_STEAL_TIMER
159	alg	%r13,__CLOCK_IDLE_ENTER(%r2)
160	slg	%r13,__LC_LAST_UPDATE_CLOCK
161	stg	%r13,__LC_STEAL_TIMER
162
163	mvc	__LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
164
165	lg	%r13,__LC_SYSTEM_TIMER
166	alg	%r13,__LC_LAST_UPDATE_TIMER
167	slg	%r13,__TIMER_IDLE_ENTER(%r2)
168	stg	%r13,__LC_SYSTEM_TIMER
169	mvc	__LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
170
171	nihh	%r8,0xfcfd		# clear wait state and irq bits
1723:	lg	%r14,__LC_ASYNC_STACK	# are we already on the target stack?
173	slgr	%r14,%r15
174	srag	%r14,%r14,STACK_SHIFT
175	jnz	5f
176	CHECK_STACK \savearea
177	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
178	j	6f
1794:	UPDATE_VTIME %r14,%r15,\timer
180	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
1815:	lg	%r15,__LC_ASYNC_STACK	# load async stack
1826:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
183	.endm
184
185	.macro UPDATE_VTIME w1,w2,enter_timer
186	lg	\w1,__LC_EXIT_TIMER
187	lg	\w2,__LC_LAST_UPDATE_TIMER
188	slg	\w1,\enter_timer
189	slg	\w2,__LC_EXIT_TIMER
190	alg	\w1,__LC_USER_TIMER
191	alg	\w2,__LC_SYSTEM_TIMER
192	stg	\w1,__LC_USER_TIMER
193	stg	\w2,__LC_SYSTEM_TIMER
194	mvc	__LC_LAST_UPDATE_TIMER(8),\enter_timer
195	.endm
196
197	.macro RESTORE_SM_CLEAR_PER
198	stg	%r8,__LC_RETURN_PSW
199	ni	__LC_RETURN_PSW,0xbf
200	ssm	__LC_RETURN_PSW
201	.endm
202
203	.macro ENABLE_INTS
204	stosm	__SF_EMPTY(%r15),3
205	.endm
206
207	.macro ENABLE_INTS_TRACE
208	TRACE_IRQS_ON
209	ENABLE_INTS
210	.endm
211
212	.macro DISABLE_INTS
213	stnsm	__SF_EMPTY(%r15),0xfc
214	.endm
215
216	.macro DISABLE_INTS_TRACE
217	DISABLE_INTS
218	TRACE_IRQS_OFF
219	.endm
220
221	.macro STCK savearea
222#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
223	.insn	s,0xb27c0000,\savearea		# store clock fast
224#else
225	.insn	s,0xb2050000,\savearea		# store clock
226#endif
227	.endm
228
229	/*
230	 * The TSTMSK macro generates a test-under-mask instruction by
231	 * calculating the memory offset for the specified mask value.
232	 * Mask value can be any constant.  The macro shifts the mask
233	 * value to calculate the memory offset for the test-under-mask
234	 * instruction.
235	 */
236	.macro TSTMSK addr, mask, size=8, bytepos=0
237		.if (\bytepos < \size) && (\mask >> 8)
238			.if (\mask & 0xff)
239				.error "Mask exceeds byte boundary"
240			.endif
241			TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
242			.exitm
243		.endif
244		.ifeq \mask
245			.error "Mask must not be zero"
246		.endif
247		off = \size - \bytepos - 1
248		tm	off+\addr, \mask
249	.endm
250
251	.macro BPOFF
252	ALTERNATIVE "", ".long 0xb2e8c000", 82
253	.endm
254
255	.macro BPON
256	ALTERNATIVE "", ".long 0xb2e8d000", 82
257	.endm
258
259	.macro BPENTER tif_ptr,tif_mask
260	ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \
261		    "", 82
262	.endm
263
264	.macro BPEXIT tif_ptr,tif_mask
265	TSTMSK	\tif_ptr,\tif_mask
266	ALTERNATIVE "jz .+8;  .long 0xb2e8c000", \
267		    "jnz .+8; .long 0xb2e8d000", 82
268	.endm
269
270	GEN_BR_THUNK %r9
271	GEN_BR_THUNK %r14
272	GEN_BR_THUNK %r14,%r11
273
274	.section .kprobes.text, "ax"
275.Ldummy:
276	/*
277	 * This nop exists only in order to avoid that __switch_to starts at
278	 * the beginning of the kprobes text section. In that case we would
279	 * have several symbols at the same address. E.g. objdump would take
280	 * an arbitrary symbol name when disassembling this code.
281	 * With the added nop in between the __switch_to symbol is unique
282	 * again.
283	 */
284	nop	0
285
286ENTRY(__bpon)
287	.globl __bpon
288	BPON
289	BR_EX	%r14
290ENDPROC(__bpon)
291
292/*
293 * Scheduler resume function, called by switch_to
294 *  gpr2 = (task_struct *) prev
295 *  gpr3 = (task_struct *) next
296 * Returns:
297 *  gpr2 = prev
298 */
299ENTRY(__switch_to)
300	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
301	lghi	%r4,__TASK_stack
302	lghi	%r1,__TASK_thread
303	llill	%r5,STACK_INIT
304	stg	%r15,__THREAD_ksp(%r1,%r2)	# store kernel stack of prev
305	lg	%r15,0(%r4,%r3)			# start of kernel stack of next
306	agr	%r15,%r5			# end of kernel stack of next
307	stg	%r3,__LC_CURRENT		# store task struct of next
308	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
309	lg	%r15,__THREAD_ksp(%r1,%r3)	# load kernel stack of next
310	aghi	%r3,__TASK_pid
311	mvc	__LC_CURRENT_PID(4,%r0),0(%r3)	# store pid of next
312	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
313	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
314	BR_EX	%r14
315ENDPROC(__switch_to)
316
317#if IS_ENABLED(CONFIG_KVM)
318/*
319 * sie64a calling convention:
320 * %r2 pointer to sie control block
321 * %r3 guest register save area
322 */
323ENTRY(sie64a)
324	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
325	lg	%r12,__LC_CURRENT
326	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
327	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
328	xc	__SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
329	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
330	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU		# load guest fp/vx registers ?
331	jno	.Lsie_load_guest_gprs
332	brasl	%r14,load_fpu_regs		# load guest fp/vx regs
333.Lsie_load_guest_gprs:
334	lmg	%r0,%r13,0(%r3)			# load guest gprs 0-13
335	lg	%r14,__LC_GMAP			# get gmap pointer
336	ltgr	%r14,%r14
337	jz	.Lsie_gmap
338	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
339.Lsie_gmap:
340	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
341	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
342	tm	__SIE_PROG20+3(%r14),3		# last exit...
343	jnz	.Lsie_skip
344	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
345	jo	.Lsie_skip			# exit if fp/vx regs changed
346	BPEXIT	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
347.Lsie_entry:
348	sie	0(%r14)
349	BPOFF
350	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
351.Lsie_skip:
352	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
353	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
354.Lsie_done:
355# some program checks are suppressing. C code (e.g. do_protection_exception)
356# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
357# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
358# Other instructions between sie64a and .Lsie_done should not cause program
359# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
360# See also .Lcleanup_sie
361.Lrewind_pad6:
362	nopr	7
363.Lrewind_pad4:
364	nopr	7
365.Lrewind_pad2:
366	nopr	7
367	.globl sie_exit
368sie_exit:
369	lg	%r14,__SF_SIE_SAVEAREA(%r15)	# load guest register save area
370	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
371	xgr	%r0,%r0				# clear guest registers to
372	xgr	%r1,%r1				# prevent speculative use
373	xgr	%r2,%r2
374	xgr	%r3,%r3
375	xgr	%r4,%r4
376	xgr	%r5,%r5
377	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
378	lg	%r2,__SF_SIE_REASON(%r15)	# return exit reason code
379	BR_EX	%r14
380.Lsie_fault:
381	lghi	%r14,-EFAULT
382	stg	%r14,__SF_SIE_REASON(%r15)	# set exit reason code
383	j	sie_exit
384
385	EX_TABLE(.Lrewind_pad6,.Lsie_fault)
386	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
387	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
388	EX_TABLE(sie_exit,.Lsie_fault)
389ENDPROC(sie64a)
390EXPORT_SYMBOL(sie64a)
391EXPORT_SYMBOL(sie_exit)
392#endif
393
394/*
395 * SVC interrupt handler routine. System calls are synchronous events and
396 * are entered with interrupts disabled.
397 */
398
399ENTRY(system_call)
400	stpt	__LC_SYNC_ENTER_TIMER
401	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
402	BPOFF
403	lg	%r12,__LC_CURRENT
404	lghi	%r14,_PIF_SYSCALL
405.Lsysc_per:
406	lctlg	%c1,%c1,__LC_KERNEL_ASCE
407	lghi	%r13,__TASK_thread
408	lg	%r15,__LC_KERNEL_STACK
409	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
410	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
411	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
412	stmg	%r0,%r7,__PT_R0(%r11)
413	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
414	mvc	__PT_PSW(16,%r11),__LC_SVC_OLD_PSW
415	mvc	__PT_INT_CODE(4,%r11),__LC_SVC_ILC
416	stg	%r14,__PT_FLAGS(%r11)
417	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
418	ENABLE_INTS
419.Lsysc_do_svc:
420	# clear user controlled register to prevent speculative use
421	xgr	%r0,%r0
422	# load address of system call table
423	lg	%r10,__THREAD_sysc_table(%r13,%r12)
424	llgh	%r8,__PT_INT_CODE+2(%r11)
425	slag	%r8,%r8,3			# shift and test for svc 0
426	jnz	.Lsysc_nr_ok
427	# svc 0: system call number in %r1
428	llgfr	%r1,%r1				# clear high word in r1
429	sth	%r1,__PT_INT_CODE+2(%r11)
430	cghi	%r1,NR_syscalls
431	jnl	.Lsysc_nr_ok
432	slag	%r8,%r1,3
433.Lsysc_nr_ok:
434	stg	%r2,__PT_ORIG_GPR2(%r11)
435	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
436	lg	%r9,0(%r8,%r10)			# get system call add.
437	TSTMSK	__TI_flags(%r12),_TIF_TRACE
438	jnz	.Lsysc_tracesys
439	BASR_EX	%r14,%r9			# call sys_xxxx
440	stg	%r2,__PT_R2(%r11)		# store return value
441
442.Lsysc_return:
443#ifdef CONFIG_DEBUG_RSEQ
444	lgr	%r2,%r11
445	brasl	%r14,rseq_syscall
446#endif
447	LOCKDEP_SYS_EXIT
448.Lsysc_tif:
449	DISABLE_INTS
450	TSTMSK	__PT_FLAGS(%r11),_PIF_WORK
451	jnz	.Lsysc_work
452	TSTMSK	__TI_flags(%r12),_TIF_WORK
453	jnz	.Lsysc_work			# check for work
454	DEBUG_USER_ASCE
455	lctlg	%c1,%c1,__LC_USER_ASCE
456	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
457	TSTMSK	__LC_CPU_FLAGS, _CIF_FPU
458	jz	.Lsysc_skip_fpu
459	brasl	%r14,load_fpu_regs
460.Lsysc_skip_fpu:
461	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
462	stpt	__LC_EXIT_TIMER
463	lmg	%r0,%r15,__PT_R0(%r11)
464	b	__LC_RETURN_LPSWE
465
466#
467# One of the work bits is on. Find out which one.
468#
469.Lsysc_work:
470	ENABLE_INTS
471	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
472	jo	.Lsysc_reschedule
473	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
474	jo	.Lsysc_syscall_restart
475#ifdef CONFIG_UPROBES
476	TSTMSK	__TI_flags(%r12),_TIF_UPROBE
477	jo	.Lsysc_uprobe_notify
478#endif
479	TSTMSK	__TI_flags(%r12),_TIF_GUARDED_STORAGE
480	jo	.Lsysc_guarded_storage
481	TSTMSK	__PT_FLAGS(%r11),_PIF_PER_TRAP
482	jo	.Lsysc_singlestep
483#ifdef CONFIG_LIVEPATCH
484	TSTMSK	__TI_flags(%r12),_TIF_PATCH_PENDING
485	jo	.Lsysc_patch_pending	# handle live patching just before
486					# signals and possible syscall restart
487#endif
488	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
489	jo	.Lsysc_syscall_restart
490	TSTMSK	__TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL)
491	jnz	.Lsysc_sigpending
492	TSTMSK	__TI_flags(%r12),_TIF_NOTIFY_RESUME
493	jo	.Lsysc_notify_resume
494	j	.Lsysc_return
495
496#
497# _TIF_NEED_RESCHED is set, call schedule
498#
499.Lsysc_reschedule:
500	larl	%r14,.Lsysc_return
501	jg	schedule
502
503#
504# _TIF_SIGPENDING is set, call do_signal
505#
506.Lsysc_sigpending:
507	lgr	%r2,%r11		# pass pointer to pt_regs
508	brasl	%r14,do_signal
509	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL
510	jno	.Lsysc_return
511.Lsysc_do_syscall:
512	lghi	%r13,__TASK_thread
513	lmg	%r2,%r7,__PT_R2(%r11)	# load svc arguments
514	lghi	%r1,0			# svc 0 returns -ENOSYS
515	j	.Lsysc_do_svc
516
517#
518# _TIF_NOTIFY_RESUME is set, call do_notify_resume
519#
520.Lsysc_notify_resume:
521	lgr	%r2,%r11		# pass pointer to pt_regs
522	larl	%r14,.Lsysc_return
523	jg	do_notify_resume
524
525#
526# _TIF_UPROBE is set, call uprobe_notify_resume
527#
528#ifdef CONFIG_UPROBES
529.Lsysc_uprobe_notify:
530	lgr	%r2,%r11		# pass pointer to pt_regs
531	larl	%r14,.Lsysc_return
532	jg	uprobe_notify_resume
533#endif
534
535#
536# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
537#
538.Lsysc_guarded_storage:
539	lgr	%r2,%r11		# pass pointer to pt_regs
540	larl	%r14,.Lsysc_return
541	jg	gs_load_bc_cb
542#
543# _TIF_PATCH_PENDING is set, call klp_update_patch_state
544#
545#ifdef CONFIG_LIVEPATCH
546.Lsysc_patch_pending:
547	lg	%r2,__LC_CURRENT	# pass pointer to task struct
548	larl	%r14,.Lsysc_return
549	jg	klp_update_patch_state
550#endif
551
552#
553# _PIF_PER_TRAP is set, call do_per_trap
554#
555.Lsysc_singlestep:
556	ni	__PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
557	lgr	%r2,%r11		# pass pointer to pt_regs
558	larl	%r14,.Lsysc_return
559	jg	do_per_trap
560
561#
562# _PIF_SYSCALL_RESTART is set, repeat the current system call
563#
564.Lsysc_syscall_restart:
565	ni	__PT_FLAGS+7(%r11),255-_PIF_SYSCALL_RESTART
566	lmg	%r1,%r7,__PT_R1(%r11)	# load svc arguments
567	lg	%r2,__PT_ORIG_GPR2(%r11)
568	j	.Lsysc_do_svc
569
570#
571# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
572# and after the system call
573#
574.Lsysc_tracesys:
575	lgr	%r2,%r11		# pass pointer to pt_regs
576	la	%r3,0
577	llgh	%r0,__PT_INT_CODE+2(%r11)
578	stg	%r0,__PT_R2(%r11)
579	brasl	%r14,do_syscall_trace_enter
580	lghi	%r0,NR_syscalls
581	clgr	%r0,%r2
582	jnh	.Lsysc_tracenogo
583	sllg	%r8,%r2,3
584	lg	%r9,0(%r8,%r10)
585	lmg	%r3,%r7,__PT_R3(%r11)
586	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
587	lg	%r2,__PT_ORIG_GPR2(%r11)
588	BASR_EX	%r14,%r9		# call sys_xxx
589	stg	%r2,__PT_R2(%r11)	# store return value
590.Lsysc_tracenogo:
591	TSTMSK	__TI_flags(%r12),_TIF_TRACE
592	jz	.Lsysc_return
593	lgr	%r2,%r11		# pass pointer to pt_regs
594	larl	%r14,.Lsysc_return
595	jg	do_syscall_trace_exit
596ENDPROC(system_call)
597
598#
599# a new process exits the kernel with ret_from_fork
600#
601ENTRY(ret_from_fork)
602	la	%r11,STACK_FRAME_OVERHEAD(%r15)
603	lg	%r12,__LC_CURRENT
604	brasl	%r14,schedule_tail
605	tm	__PT_PSW+1(%r11),0x01	# forking a kernel thread ?
606	jne	.Lsysc_tracenogo
607	# it's a kernel thread
608	lmg	%r9,%r10,__PT_R9(%r11)	# load gprs
609	la	%r2,0(%r10)
610	BASR_EX	%r14,%r9
611	j	.Lsysc_tracenogo
612ENDPROC(ret_from_fork)
613
614ENTRY(kernel_thread_starter)
615	la	%r2,0(%r10)
616	BASR_EX	%r14,%r9
617	j	.Lsysc_tracenogo
618ENDPROC(kernel_thread_starter)
619
620/*
621 * Program check handler routine
622 */
623
624ENTRY(pgm_check_handler)
625	stpt	__LC_SYNC_ENTER_TIMER
626	BPOFF
627	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
628	lg	%r10,__LC_LAST_BREAK
629	srag	%r11,%r10,12
630	jnz	0f
631	/* if __LC_LAST_BREAK is < 4096, it contains one of
632	 * the lpswe addresses in lowcore. Set it to 1 (initial state)
633	 * to prevent leaking that address to userspace.
634	 */
635	lghi	%r10,1
6360:	lg	%r12,__LC_CURRENT
637	lghi	%r11,0
638	lmg	%r8,%r9,__LC_PGM_OLD_PSW
639	tmhh	%r8,0x0001		# coming from user space?
640	jno	.Lpgm_skip_asce
641	lctlg	%c1,%c1,__LC_KERNEL_ASCE
642	j	3f
643.Lpgm_skip_asce:
644#if IS_ENABLED(CONFIG_KVM)
645	# cleanup critical section for program checks in sie64a
646	lgr	%r14,%r9
647	larl	%r13,.Lsie_gmap
648	slgr	%r14,%r13
649	lghi	%r13,.Lsie_done - .Lsie_gmap
650	clgr	%r14,%r13
651	jhe	1f
652	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
653	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
654	lctlg	%c1,%c1,__LC_KERNEL_ASCE	# load primary asce
655	larl	%r9,sie_exit			# skip forward to sie_exit
656	lghi	%r11,_PIF_GUEST_FAULT
657#endif
6581:	tmhh	%r8,0x4000		# PER bit set in old PSW ?
659	jnz	2f			# -> enabled, can't be a double fault
660	tm	__LC_PGM_ILC+3,0x80	# check for per exception
661	jnz	.Lpgm_svcper		# -> single stepped svc
6622:	CHECK_STACK __LC_SAVE_AREA_SYNC
663	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
664	# CHECK_VMAP_STACK branches to stack_overflow or 5f
665	CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,5f
6663:	UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
667	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
668	lg	%r15,__LC_KERNEL_STACK
669	lgr	%r14,%r12
670	aghi	%r14,__TASK_thread	# pointer to thread_struct
671	lghi	%r13,__LC_PGM_TDB
672	tm	__LC_PGM_ILC+2,0x02	# check for transaction abort
673	jz	4f
674	mvc	__THREAD_trap_tdb(256,%r14),0(%r13)
6754:	stg	%r10,__THREAD_last_break(%r14)
6765:	lgr	%r13,%r11
677	la	%r11,STACK_FRAME_OVERHEAD(%r15)
678	stmg	%r0,%r7,__PT_R0(%r11)
679	# clear user controlled registers to prevent speculative use
680	xgr	%r0,%r0
681	xgr	%r1,%r1
682	xgr	%r2,%r2
683	xgr	%r3,%r3
684	xgr	%r4,%r4
685	xgr	%r5,%r5
686	xgr	%r6,%r6
687	xgr	%r7,%r7
688	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
689	stmg	%r8,%r9,__PT_PSW(%r11)
690	mvc	__PT_INT_CODE(4,%r11),__LC_PGM_ILC
691	mvc	__PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
692	stg	%r13,__PT_FLAGS(%r11)
693	stg	%r10,__PT_ARGS(%r11)
694	tm	__LC_PGM_ILC+3,0x80	# check for per exception
695	jz	6f
696	tmhh	%r8,0x0001		# kernel per event ?
697	jz	.Lpgm_kprobe
698	oi	__PT_FLAGS+7(%r11),_PIF_PER_TRAP
699	mvc	__THREAD_per_address(8,%r14),__LC_PER_ADDRESS
700	mvc	__THREAD_per_cause(2,%r14),__LC_PER_CODE
701	mvc	__THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
7026:	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
703	RESTORE_SM_CLEAR_PER
704	larl	%r1,pgm_check_table
705	llgh	%r10,__PT_INT_CODE+2(%r11)
706	nill	%r10,0x007f
707	sll	%r10,3
708	je	.Lpgm_return
709	lg	%r9,0(%r10,%r1)		# load address of handler routine
710	lgr	%r2,%r11		# pass pointer to pt_regs
711	BASR_EX	%r14,%r9		# branch to interrupt-handler
712.Lpgm_return:
713	LOCKDEP_SYS_EXIT
714	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
715	jno	.Lpgm_restore
716	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL
717	jo	.Lsysc_do_syscall
718	j	.Lsysc_tif
719.Lpgm_restore:
720	DISABLE_INTS
721	TSTMSK	__LC_CPU_FLAGS, _CIF_FPU
722	jz	.Lpgm_skip_fpu
723	brasl	%r14,load_fpu_regs
724.Lpgm_skip_fpu:
725	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
726	stpt	__LC_EXIT_TIMER
727	lmg	%r0,%r15,__PT_R0(%r11)
728	b	__LC_RETURN_LPSWE
729
730#
731# PER event in supervisor state, must be kprobes
732#
733.Lpgm_kprobe:
734	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
735	RESTORE_SM_CLEAR_PER
736	lgr	%r2,%r11		# pass pointer to pt_regs
737	brasl	%r14,do_per_trap
738	j	.Lpgm_return
739
740#
741# single stepped system call
742#
743.Lpgm_svcper:
744	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
745	larl	%r14,.Lsysc_per
746	stg	%r14,__LC_RETURN_PSW+8
747	lghi	%r14,_PIF_SYSCALL | _PIF_PER_TRAP
748	lpswe	__LC_RETURN_PSW		# branch to .Lsysc_per
749ENDPROC(pgm_check_handler)
750
751/*
752 * IO interrupt handler routine
753 */
754ENTRY(io_int_handler)
755	STCK	__LC_INT_CLOCK
756	stpt	__LC_ASYNC_ENTER_TIMER
757	BPOFF
758	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
759	lg	%r12,__LC_CURRENT
760	lmg	%r8,%r9,__LC_IO_OLD_PSW
761	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK
762	stmg	%r0,%r7,__PT_R0(%r11)
763	# clear user controlled registers to prevent speculative use
764	xgr	%r0,%r0
765	xgr	%r1,%r1
766	xgr	%r2,%r2
767	xgr	%r3,%r3
768	xgr	%r4,%r4
769	xgr	%r5,%r5
770	xgr	%r6,%r6
771	xgr	%r7,%r7
772	xgr	%r10,%r10
773	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
774	stmg	%r8,%r9,__PT_PSW(%r11)
775	tm	__PT_PSW+1(%r11),0x01	# coming from user space?
776	jno	.Lio_skip_asce
777	lctlg	%c1,%c1,__LC_KERNEL_ASCE
778.Lio_skip_asce:
779	mvc	__PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
780	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
781	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
782	TRACE_IRQS_OFF
783.Lio_loop:
784	lgr	%r2,%r11		# pass pointer to pt_regs
785	lghi	%r3,IO_INTERRUPT
786	tm	__PT_INT_CODE+8(%r11),0x80	# adapter interrupt ?
787	jz	.Lio_call
788	lghi	%r3,THIN_INTERRUPT
789.Lio_call:
790	brasl	%r14,do_IRQ
791	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR
792	jz	.Lio_return
793	tpi	0
794	jz	.Lio_return
795	mvc	__PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
796	j	.Lio_loop
797.Lio_return:
798	LOCKDEP_SYS_EXIT
799	TSTMSK	__TI_flags(%r12),_TIF_WORK
800	jnz	.Lio_work		# there is work to do (signals etc.)
801	TSTMSK	__LC_CPU_FLAGS,_CIF_WORK
802	jnz	.Lio_work
803.Lio_restore:
804	TRACE_IRQS_ON
805	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
806	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
807	jno	.Lio_exit_kernel
808	DEBUG_USER_ASCE
809	lctlg	%c1,%c1,__LC_USER_ASCE
810	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
811	stpt	__LC_EXIT_TIMER
812.Lio_exit_kernel:
813	lmg	%r0,%r15,__PT_R0(%r11)
814	b	__LC_RETURN_LPSWE
815.Lio_done:
816
817#
818# There is work todo, find out in which context we have been interrupted:
819# 1) if we return to user space we can do all _TIF_WORK work
820# 2) if we return to kernel code and kvm is enabled check if we need to
821#    modify the psw to leave SIE
822# 3) if we return to kernel code and preemptive scheduling is enabled check
823#    the preemption counter and if it is zero call preempt_schedule_irq
824# Before any work can be done, a switch to the kernel stack is required.
825#
826.Lio_work:
827	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
828	jo	.Lio_work_user		# yes -> do resched & signal
829#ifdef CONFIG_PREEMPTION
830	# check for preemptive scheduling
831	icm	%r0,15,__LC_PREEMPT_COUNT
832	jnz	.Lio_restore		# preemption is disabled
833	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
834	jno	.Lio_restore
835	# switch to kernel stack
836	lg	%r1,__PT_R15(%r11)
837	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
838	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
839	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
840	la	%r11,STACK_FRAME_OVERHEAD(%r1)
841	lgr	%r15,%r1
842	brasl	%r14,preempt_schedule_irq
843	j	.Lio_return
844#else
845	j	.Lio_restore
846#endif
847
848#
849# Need to do work before returning to userspace, switch to kernel stack
850#
851.Lio_work_user:
852	lg	%r1,__LC_KERNEL_STACK
853	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
854	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
855	la	%r11,STACK_FRAME_OVERHEAD(%r1)
856	lgr	%r15,%r1
857
858#
859# One of the work bits is on. Find out which one.
860#
861	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
862	jo	.Lio_reschedule
863#ifdef CONFIG_LIVEPATCH
864	TSTMSK	__TI_flags(%r12),_TIF_PATCH_PENDING
865	jo	.Lio_patch_pending
866#endif
867	TSTMSK	__TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL)
868	jnz	.Lio_sigpending
869	TSTMSK	__TI_flags(%r12),_TIF_NOTIFY_RESUME
870	jo	.Lio_notify_resume
871	TSTMSK	__TI_flags(%r12),_TIF_GUARDED_STORAGE
872	jo	.Lio_guarded_storage
873	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
874	jo	.Lio_vxrs
875	j	.Lio_return
876
877#
878# CIF_FPU is set, restore floating-point controls and floating-point registers.
879#
880.Lio_vxrs:
881	larl	%r14,.Lio_return
882	jg	load_fpu_regs
883
884#
885# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
886#
887.Lio_guarded_storage:
888	ENABLE_INTS_TRACE
889	lgr	%r2,%r11		# pass pointer to pt_regs
890	brasl	%r14,gs_load_bc_cb
891	DISABLE_INTS_TRACE
892	j	.Lio_return
893
894#
895# _TIF_NEED_RESCHED is set, call schedule
896#
897.Lio_reschedule:
898	ENABLE_INTS_TRACE
899	brasl	%r14,schedule		# call scheduler
900	DISABLE_INTS_TRACE
901	j	.Lio_return
902
903#
904# _TIF_PATCH_PENDING is set, call klp_update_patch_state
905#
906#ifdef CONFIG_LIVEPATCH
907.Lio_patch_pending:
908	lg	%r2,__LC_CURRENT	# pass pointer to task struct
909	larl	%r14,.Lio_return
910	jg	klp_update_patch_state
911#endif
912
913#
914# _TIF_SIGPENDING or is set, call do_signal
915#
916.Lio_sigpending:
917	ENABLE_INTS_TRACE
918	lgr	%r2,%r11		# pass pointer to pt_regs
919	brasl	%r14,do_signal
920	DISABLE_INTS_TRACE
921	j	.Lio_return
922
923#
924# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
925#
926.Lio_notify_resume:
927	ENABLE_INTS_TRACE
928	lgr	%r2,%r11		# pass pointer to pt_regs
929	brasl	%r14,do_notify_resume
930	DISABLE_INTS_TRACE
931	j	.Lio_return
932ENDPROC(io_int_handler)
933
934/*
935 * External interrupt handler routine
936 */
937ENTRY(ext_int_handler)
938	STCK	__LC_INT_CLOCK
939	stpt	__LC_ASYNC_ENTER_TIMER
940	BPOFF
941	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
942	lg	%r12,__LC_CURRENT
943	lmg	%r8,%r9,__LC_EXT_OLD_PSW
944	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK
945	stmg	%r0,%r7,__PT_R0(%r11)
946	# clear user controlled registers to prevent speculative use
947	xgr	%r0,%r0
948	xgr	%r1,%r1
949	xgr	%r2,%r2
950	xgr	%r3,%r3
951	xgr	%r4,%r4
952	xgr	%r5,%r5
953	xgr	%r6,%r6
954	xgr	%r7,%r7
955	xgr	%r10,%r10
956	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
957	stmg	%r8,%r9,__PT_PSW(%r11)
958	tm	__PT_PSW+1(%r11),0x01	# coming from user space?
959	jno	.Lext_skip_asce
960	lctlg	%c1,%c1,__LC_KERNEL_ASCE
961.Lext_skip_asce:
962	lghi	%r1,__LC_EXT_PARAMS2
963	mvc	__PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
964	mvc	__PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
965	mvc	__PT_INT_PARM_LONG(8,%r11),0(%r1)
966	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
967	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
968	TRACE_IRQS_OFF
969	lgr	%r2,%r11		# pass pointer to pt_regs
970	lghi	%r3,EXT_INTERRUPT
971	brasl	%r14,do_IRQ
972	j	.Lio_return
973ENDPROC(ext_int_handler)
974
975/*
976 * Load idle PSW.
977 */
978ENTRY(psw_idle)
979	stg	%r3,__SF_EMPTY(%r15)
980	larl	%r1,.Lpsw_idle_exit
981	stg	%r1,__SF_EMPTY+8(%r15)
982	larl	%r1,smp_cpu_mtid
983	llgf	%r1,0(%r1)
984	ltgr	%r1,%r1
985	jz	.Lpsw_idle_stcctm
986	.insn	rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
987.Lpsw_idle_stcctm:
988	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
989	BPON
990	STCK	__CLOCK_IDLE_ENTER(%r2)
991	stpt	__TIMER_IDLE_ENTER(%r2)
992	lpswe	__SF_EMPTY(%r15)
993.Lpsw_idle_exit:
994	BR_EX	%r14
995ENDPROC(psw_idle)
996
997/*
998 * Store floating-point controls and floating-point or vector register
999 * depending whether the vector facility is available.	A critical section
1000 * cleanup assures that the registers are stored even if interrupted for
1001 * some other work.  The CIF_FPU flag is set to trigger a lazy restore
1002 * of the register contents at return from io or a system call.
1003 */
1004ENTRY(save_fpu_regs)
1005	stnsm	__SF_EMPTY(%r15),0xfc
1006	lg	%r2,__LC_CURRENT
1007	aghi	%r2,__TASK_thread
1008	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
1009	jo	.Lsave_fpu_regs_exit
1010	stfpc	__THREAD_FPU_fpc(%r2)
1011	lg	%r3,__THREAD_FPU_regs(%r2)
1012	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1013	jz	.Lsave_fpu_regs_fp	  # no -> store FP regs
1014	VSTM	%v0,%v15,0,%r3		  # vstm 0,15,0(3)
1015	VSTM	%v16,%v31,256,%r3	  # vstm 16,31,256(3)
1016	j	.Lsave_fpu_regs_done	  # -> set CIF_FPU flag
1017.Lsave_fpu_regs_fp:
1018	std	0,0(%r3)
1019	std	1,8(%r3)
1020	std	2,16(%r3)
1021	std	3,24(%r3)
1022	std	4,32(%r3)
1023	std	5,40(%r3)
1024	std	6,48(%r3)
1025	std	7,56(%r3)
1026	std	8,64(%r3)
1027	std	9,72(%r3)
1028	std	10,80(%r3)
1029	std	11,88(%r3)
1030	std	12,96(%r3)
1031	std	13,104(%r3)
1032	std	14,112(%r3)
1033	std	15,120(%r3)
1034.Lsave_fpu_regs_done:
1035	oi	__LC_CPU_FLAGS+7,_CIF_FPU
1036.Lsave_fpu_regs_exit:
1037	ssm	__SF_EMPTY(%r15)
1038	BR_EX	%r14
1039.Lsave_fpu_regs_end:
1040ENDPROC(save_fpu_regs)
1041EXPORT_SYMBOL(save_fpu_regs)
1042
1043/*
1044 * Load floating-point controls and floating-point or vector registers.
1045 * A critical section cleanup assures that the register contents are
1046 * loaded even if interrupted for some other work.
1047 *
1048 * There are special calling conventions to fit into sysc and io return work:
1049 *	%r15:	<kernel stack>
1050 * The function requires:
1051 *	%r4
1052 */
1053load_fpu_regs:
1054	stnsm	__SF_EMPTY(%r15),0xfc
1055	lg	%r4,__LC_CURRENT
1056	aghi	%r4,__TASK_thread
1057	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
1058	jno	.Lload_fpu_regs_exit
1059	lfpc	__THREAD_FPU_fpc(%r4)
1060	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1061	lg	%r4,__THREAD_FPU_regs(%r4)	# %r4 <- reg save area
1062	jz	.Lload_fpu_regs_fp		# -> no VX, load FP regs
1063	VLM	%v0,%v15,0,%r4
1064	VLM	%v16,%v31,256,%r4
1065	j	.Lload_fpu_regs_done
1066.Lload_fpu_regs_fp:
1067	ld	0,0(%r4)
1068	ld	1,8(%r4)
1069	ld	2,16(%r4)
1070	ld	3,24(%r4)
1071	ld	4,32(%r4)
1072	ld	5,40(%r4)
1073	ld	6,48(%r4)
1074	ld	7,56(%r4)
1075	ld	8,64(%r4)
1076	ld	9,72(%r4)
1077	ld	10,80(%r4)
1078	ld	11,88(%r4)
1079	ld	12,96(%r4)
1080	ld	13,104(%r4)
1081	ld	14,112(%r4)
1082	ld	15,120(%r4)
1083.Lload_fpu_regs_done:
1084	ni	__LC_CPU_FLAGS+7,255-_CIF_FPU
1085.Lload_fpu_regs_exit:
1086	ssm	__SF_EMPTY(%r15)
1087	BR_EX	%r14
1088.Lload_fpu_regs_end:
1089ENDPROC(load_fpu_regs)
1090
1091/*
1092 * Machine check handler routines
1093 */
1094ENTRY(mcck_int_handler)
1095	STCK	__LC_MCCK_CLOCK
1096	BPOFF
1097	la	%r1,4095		# validate r1
1098	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# validate cpu timer
1099	sckc	__LC_CLOCK_COMPARATOR			# validate comparator
1100	lam	%a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs
1101	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
1102	lg	%r12,__LC_CURRENT
1103	lmg	%r8,%r9,__LC_MCK_OLD_PSW
1104	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
1105	jo	.Lmcck_panic		# yes -> rest of mcck code invalid
1106	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CR_VALID
1107	jno	.Lmcck_panic		# control registers invalid -> panic
1108	la	%r14,4095
1109	lctlg	%c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
1110	ptlb
1111	lg	%r11,__LC_MCESAD-4095(%r14) # extended machine check save area
1112	nill	%r11,0xfc00		# MCESA_ORIGIN_MASK
1113	TSTMSK	__LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE
1114	jno	0f
1115	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_GS_VALID
1116	jno	0f
1117	.insn	 rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC
11180:	l	%r14,__LC_FP_CREG_SAVE_AREA-4095(%r14)
1119	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_FC_VALID
1120	jo	0f
1121	sr	%r14,%r14
11220:	sfpc	%r14
1123	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1124	jo	0f
1125	lghi	%r14,__LC_FPREGS_SAVE_AREA
1126	ld	%f0,0(%r14)
1127	ld	%f1,8(%r14)
1128	ld	%f2,16(%r14)
1129	ld	%f3,24(%r14)
1130	ld	%f4,32(%r14)
1131	ld	%f5,40(%r14)
1132	ld	%f6,48(%r14)
1133	ld	%f7,56(%r14)
1134	ld	%f8,64(%r14)
1135	ld	%f9,72(%r14)
1136	ld	%f10,80(%r14)
1137	ld	%f11,88(%r14)
1138	ld	%f12,96(%r14)
1139	ld	%f13,104(%r14)
1140	ld	%f14,112(%r14)
1141	ld	%f15,120(%r14)
1142	j	1f
11430:	VLM	%v0,%v15,0,%r11
1144	VLM	%v16,%v31,256,%r11
11451:	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
1146	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
1147	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
1148	jo	3f
1149	la	%r14,__LC_SYNC_ENTER_TIMER
1150	clc	0(8,%r14),__LC_ASYNC_ENTER_TIMER
1151	jl	0f
1152	la	%r14,__LC_ASYNC_ENTER_TIMER
11530:	clc	0(8,%r14),__LC_EXIT_TIMER
1154	jl	1f
1155	la	%r14,__LC_EXIT_TIMER
11561:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
1157	jl	2f
1158	la	%r14,__LC_LAST_UPDATE_TIMER
11592:	spt	0(%r14)
1160	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
11613:	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
1162	jno	.Lmcck_panic
1163	tmhh	%r8,0x0001		# interrupting from user ?
1164	jnz	4f
1165	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
1166	jno	.Lmcck_panic
11674:	ssm	__LC_PGM_NEW_PSW	# turn dat on, keep irqs off
1168	SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER,__LC_MCCK_CLOCK
1169.Lmcck_skip:
1170	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
1171	stmg	%r0,%r7,__PT_R0(%r11)
1172	# clear user controlled registers to prevent speculative use
1173	xgr	%r0,%r0
1174	xgr	%r1,%r1
1175	xgr	%r2,%r2
1176	xgr	%r3,%r3
1177	xgr	%r4,%r4
1178	xgr	%r5,%r5
1179	xgr	%r6,%r6
1180	xgr	%r7,%r7
1181	xgr	%r10,%r10
1182	mvc	__PT_R8(64,%r11),0(%r14)
1183	stmg	%r8,%r9,__PT_PSW(%r11)
1184	la	%r14,4095
1185	mvc	__PT_CR1(8,%r11),__LC_CREGS_SAVE_AREA-4095+8(%r14)
1186	lctlg	%c1,%c1,__LC_KERNEL_ASCE
1187	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
1188	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1189	lgr	%r2,%r11		# pass pointer to pt_regs
1190	brasl	%r14,s390_do_machine_check
1191	cghi	%r2,0
1192	je	.Lmcck_return
1193	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack
1194	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
1195	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
1196	la	%r11,STACK_FRAME_OVERHEAD(%r1)
1197	lgr	%r15,%r1
1198	TRACE_IRQS_OFF
1199	brasl	%r14,s390_handle_mcck
1200	TRACE_IRQS_ON
1201.Lmcck_return:
1202	lctlg	%c1,%c1,__PT_CR1(%r11)
1203	lmg	%r0,%r10,__PT_R0(%r11)
1204	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
1205	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
1206	jno	0f
1207	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
1208	stpt	__LC_EXIT_TIMER
12090:	lmg	%r11,%r15,__PT_R11(%r11)
1210	b	__LC_RETURN_MCCK_LPSWE
1211
1212.Lmcck_panic:
1213	lg	%r15,__LC_NODAT_STACK
1214	la	%r11,STACK_FRAME_OVERHEAD(%r15)
1215	j	.Lmcck_skip
1216ENDPROC(mcck_int_handler)
1217
1218#
1219# PSW restart interrupt handler
1220#
1221ENTRY(restart_int_handler)
1222	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
1223	stg	%r15,__LC_SAVE_AREA_RESTART
1224	lg	%r15,__LC_RESTART_STACK
1225	xc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
1226	stmg	%r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
1227	mvc	STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
1228	mvc	STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
1229	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
1230	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
1231	lg	%r2,__LC_RESTART_DATA
1232	lg	%r3,__LC_RESTART_SOURCE
1233	ltgr	%r3,%r3				# test source cpu address
1234	jm	1f				# negative -> skip source stop
12350:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
1236	brc	10,0b				# wait for status stored
12371:	basr	%r14,%r1			# call function
1238	stap	__SF_EMPTY(%r15)		# store cpu address
1239	llgh	%r3,__SF_EMPTY(%r15)
12402:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
1241	brc	2,2b
12423:	j	3b
1243ENDPROC(restart_int_handler)
1244
1245	.section .kprobes.text, "ax"
1246
1247#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
1248/*
1249 * The synchronous or the asynchronous stack overflowed. We are dead.
1250 * No need to properly save the registers, we are going to panic anyway.
1251 * Setup a pt_regs so that show_trace can provide a good call trace.
1252 */
1253ENTRY(stack_overflow)
1254	lg	%r15,__LC_NODAT_STACK	# change to panic stack
1255	la	%r11,STACK_FRAME_OVERHEAD(%r15)
1256	stmg	%r0,%r7,__PT_R0(%r11)
1257	stmg	%r8,%r9,__PT_PSW(%r11)
1258	mvc	__PT_R8(64,%r11),0(%r14)
1259	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
1260	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1261	lgr	%r2,%r11		# pass pointer to pt_regs
1262	jg	kernel_stack_overflow
1263ENDPROC(stack_overflow)
1264#endif
1265
1266#if IS_ENABLED(CONFIG_KVM)
1267.Lcleanup_sie:
1268	cghi	%r11,__LC_SAVE_AREA_ASYNC	#Is this in normal interrupt?
1269	je	1f
1270	larl	%r13,.Lsie_entry
1271	slgr	%r9,%r13
1272	larl	%r13,.Lsie_skip
1273	clgr	%r9,%r13
1274	jh	1f
1275	oi	__LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
12761:	BPENTER	__SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
1277	lg	%r9,__SF_SIE_CONTROL(%r15)	# get control block pointer
1278	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
1279	lctlg	%c1,%c1,__LC_KERNEL_ASCE
1280	larl	%r9,sie_exit			# skip forward to sie_exit
1281	BR_EX	%r14,%r11
1282
1283#endif
1284	.section .rodata, "a"
1285#define SYSCALL(esame,emu)	.quad __s390x_ ## esame
1286	.globl	sys_call_table
1287sys_call_table:
1288#include "asm/syscall_table.h"
1289#undef SYSCALL
1290
1291#ifdef CONFIG_COMPAT
1292
1293#define SYSCALL(esame,emu)	.quad __s390_ ## emu
1294	.globl	sys_call_table_emu
1295sys_call_table_emu:
1296#include "asm/syscall_table.h"
1297#undef SYSCALL
1298#endif
1299