1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * S390 version
4 * Copyright IBM Corp. 1999, 2000
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
6 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
7 *
8 * Derived from "arch/i386/kernel/traps.c"
9 * Copyright (C) 1991, 1992 Linus Torvalds
10 */
11
12 #include <linux/capability.h>
13 #include <linux/cpufeature.h>
14 #include <linux/debugfs.h>
15 #include <linux/kprobes.h>
16 #include <linux/kdebug.h>
17 #include <linux/randomize_kstack.h>
18 #include <linux/extable.h>
19 #include <linux/ptrace.h>
20 #include <linux/sched.h>
21 #include <linux/sched/debug.h>
22 #include <linux/mm.h>
23 #include <linux/slab.h>
24 #include <linux/uaccess.h>
25 #include <linux/cpu.h>
26 #include <linux/entry-common.h>
27 #include <linux/kmsan.h>
28 #include <linux/bug.h>
29 #include <asm/pgmcheck-defs.h>
30 #include <asm/entry-percpu.h>
31 #include <asm/asm-extable.h>
32 #include <asm/irqflags.h>
33 #include <asm/ptrace.h>
34 #include <asm/vtime.h>
35 #include <asm/fpu.h>
36 #include <asm/fault.h>
37 #include "entry.h"
38
39 struct pgm_stat {
40 unsigned int count[128];
41 };
42
43 static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat);
44
get_trap_ip(struct pt_regs * regs)45 unsigned long get_trap_ip(struct pt_regs *regs)
46 {
47 static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED };
48 psw_t psw = regs->psw;
49
50 if (test_pt_regs_flag(regs, PIF_PSW_ADDR_ADJUSTED))
51 return psw.addr;
52 if (regs->int_code & 0x200)
53 psw.addr = current->thread.trap_tdb.data[3];
54 if (test_bit(regs->int_code & PGM_INT_CODE_MASK, nullified))
55 return psw.addr;
56 psw.addr = __rewind_psw(psw, regs->int_code >> 16);
57 return psw.addr;
58 }
59 NOKPROBE_SYMBOL(get_trap_ip);
60
61 #ifdef CONFIG_GENERIC_BUG
is_valid_bugaddr(unsigned long addr)62 int is_valid_bugaddr(unsigned long addr)
63 {
64 return 1;
65 }
66 #endif
67
do_report_trap(struct pt_regs * regs,int si_signo,int si_code,char * str)68 void do_report_trap(struct pt_regs *regs, int si_signo, int si_code, char *str)
69 {
70 if (user_mode(regs)) {
71 force_sig_fault(si_signo, si_code, (void __user *)get_trap_ip(regs));
72 report_user_fault(regs, si_signo, 0);
73 } else {
74 if (!fixup_exception(regs))
75 die(regs, str);
76 }
77 }
78
do_trap(struct pt_regs * regs,int si_signo,int si_code,char * str)79 static void do_trap(struct pt_regs *regs, int si_signo, int si_code, char *str)
80 {
81 if (notify_die(DIE_TRAP, str, regs, 0, regs->int_code, si_signo) == NOTIFY_STOP)
82 return;
83 do_report_trap(regs, si_signo, si_code, str);
84 }
85 NOKPROBE_SYMBOL(do_trap);
86
do_per_trap(struct pt_regs * regs)87 void do_per_trap(struct pt_regs *regs)
88 {
89 if (notify_die(DIE_SSTEP, "sstep", regs, 0, 0, SIGTRAP) == NOTIFY_STOP)
90 return;
91 if (!current->ptrace)
92 return;
93 force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __force __user *)current->thread.per_event.address);
94 }
95 NOKPROBE_SYMBOL(do_per_trap);
96
default_trap_handler(struct pt_regs * regs)97 static void default_trap_handler(struct pt_regs *regs)
98 {
99 if (user_mode(regs)) {
100 report_user_fault(regs, SIGSEGV, 0);
101 force_exit_sig(SIGSEGV);
102 } else
103 die(regs, "Unknown program exception");
104 }
105
106 #define DO_ERROR_INFO(name, signr, sicode, str) \
107 static void name(struct pt_regs *regs) \
108 { \
109 do_trap(regs, signr, sicode, str); \
110 }
111
112 DO_ERROR_INFO(addressing_exception, SIGILL, ILL_ILLADR, "addressing exception")
113 DO_ERROR_INFO(divide_exception, SIGFPE, FPE_INTDIV, "fixpoint divide exception")
114 DO_ERROR_INFO(execute_exception, SIGILL, ILL_ILLOPN, "execute exception")
115 DO_ERROR_INFO(hfp_divide_exception, SIGFPE, FPE_FLTDIV, "HFP divide exception")
116 DO_ERROR_INFO(hfp_overflow_exception, SIGFPE, FPE_FLTOVF, "HFP overflow exception")
117 DO_ERROR_INFO(hfp_significance_exception, SIGFPE, FPE_FLTRES, "HFP significance exception")
118 DO_ERROR_INFO(hfp_sqrt_exception, SIGFPE, FPE_FLTINV, "HFP square root exception")
119 DO_ERROR_INFO(hfp_underflow_exception, SIGFPE, FPE_FLTUND, "HFP underflow exception")
120 DO_ERROR_INFO(operand_exception, SIGILL, ILL_ILLOPN, "operand exception")
121 DO_ERROR_INFO(overflow_exception, SIGFPE, FPE_INTOVF, "fixpoint overflow exception")
122 DO_ERROR_INFO(privileged_op, SIGILL, ILL_PRVOPC, "privileged operation")
123 DO_ERROR_INFO(special_op_exception, SIGILL, ILL_ILLOPN, "special operation exception")
124 DO_ERROR_INFO(specification_exception, SIGILL, ILL_ILLOPN, "specification exception");
125 DO_ERROR_INFO(transaction_exception, SIGILL, ILL_ILLOPN, "transaction constraint exception")
126
do_fp_trap(struct pt_regs * regs,__u32 fpc)127 static inline void do_fp_trap(struct pt_regs *regs, __u32 fpc)
128 {
129 int si_code = 0;
130
131 /* FPC[2] is Data Exception Code */
132 if ((fpc & 0x00000300) == 0) {
133 /* bits 6 and 7 of DXC are 0 iff IEEE exception */
134 if (fpc & 0x8000) /* invalid fp operation */
135 si_code = FPE_FLTINV;
136 else if (fpc & 0x4000) /* div by 0 */
137 si_code = FPE_FLTDIV;
138 else if (fpc & 0x2000) /* overflow */
139 si_code = FPE_FLTOVF;
140 else if (fpc & 0x1000) /* underflow */
141 si_code = FPE_FLTUND;
142 else if (fpc & 0x0800) /* inexact */
143 si_code = FPE_FLTRES;
144 }
145 do_trap(regs, SIGFPE, si_code, "floating point exception");
146 }
147
translation_specification_exception(struct pt_regs * regs)148 static void translation_specification_exception(struct pt_regs *regs)
149 {
150 /* May never happen. */
151 panic("Translation-Specification Exception");
152 }
153
illegal_op(struct pt_regs * regs)154 static void illegal_op(struct pt_regs *regs)
155 {
156 int is_uprobe_insn = 0;
157 int signal = 0;
158 u16 opcode;
159
160 if (user_mode(regs)) {
161 u16 __user *location = (void __user *)get_trap_ip(regs);
162
163 if (get_user(opcode, location))
164 return;
165 if (opcode == S390_BREAKPOINT_U16) {
166 if (current->ptrace)
167 force_sig_fault(SIGTRAP, TRAP_BRKPT, location);
168 else
169 signal = SIGILL;
170 #ifdef CONFIG_UPROBES
171 } else if (opcode == UPROBE_SWBP_INSN) {
172 is_uprobe_insn = 1;
173 #endif
174 } else {
175 signal = SIGILL;
176 }
177 }
178 /*
179 * This is either an illegal op in kernel mode, or user space trapped
180 * on a uprobes illegal instruction. See if kprobes or uprobes picks
181 * it up. If not, SIGILL.
182 */
183 if (is_uprobe_insn || !user_mode(regs)) {
184 if (notify_die(DIE_BPT, "bpt", regs, 0, 3, SIGTRAP) != NOTIFY_STOP)
185 signal = SIGILL;
186 }
187 if (signal)
188 do_trap(regs, signal, ILL_ILLOPC, "illegal operation");
189 }
190 NOKPROBE_SYMBOL(illegal_op);
191
vector_exception(struct pt_regs * regs)192 static void vector_exception(struct pt_regs *regs)
193 {
194 int si_code, vic;
195
196 /* get vector interrupt code from fpc */
197 save_user_fpu_regs();
198 vic = (current->thread.ufpu.fpc & 0xf00) >> 8;
199 switch (vic) {
200 case 1: /* invalid vector operation */
201 si_code = FPE_FLTINV;
202 break;
203 case 2: /* division by zero */
204 si_code = FPE_FLTDIV;
205 break;
206 case 3: /* overflow */
207 si_code = FPE_FLTOVF;
208 break;
209 case 4: /* underflow */
210 si_code = FPE_FLTUND;
211 break;
212 case 5: /* inexact */
213 si_code = FPE_FLTRES;
214 break;
215 default: /* unknown cause */
216 si_code = 0;
217 }
218 do_trap(regs, SIGFPE, si_code, "vector exception");
219 }
220
data_exception(struct pt_regs * regs)221 static void data_exception(struct pt_regs *regs)
222 {
223 save_user_fpu_regs();
224 if (current->thread.ufpu.fpc & FPC_DXC_MASK)
225 do_fp_trap(regs, current->thread.ufpu.fpc);
226 else
227 do_trap(regs, SIGILL, ILL_ILLOPN, "data exception");
228 }
229
space_switch_exception(struct pt_regs * regs)230 static void space_switch_exception(struct pt_regs *regs)
231 {
232 /* Set user psw back to home space mode. */
233 if (user_mode(regs))
234 regs->psw.mask |= PSW_ASC_HOME;
235 /* Send SIGILL. */
236 do_trap(regs, SIGILL, ILL_PRVOPC, "space switch event");
237 }
238
239 #if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS)
240
__warn_args(struct arch_va_list * args,struct pt_regs * regs)241 void *__warn_args(struct arch_va_list *args, struct pt_regs *regs)
242 {
243 struct stack_frame *stack_frame;
244
245 /*
246 * Generate va_list from pt_regs. See ELF Application Binary Interface
247 * s390x Supplement documentation for details.
248 *
249 * - __overflow_arg_area needs to point to the parameter area, which
250 * is right above the standard stack frame (160 bytes)
251 *
252 * - __reg_save_area needs to point to a register save area where
253 * general registers (%r2 - %r6) can be found at offset 16. Which
254 * means that the gprs save area of pt_regs can be used
255 *
256 * - __gpr must be set to one, since the first parameter has been
257 * processed (pointer to bug_entry)
258 */
259 stack_frame = (struct stack_frame *)regs->gprs[15];
260 args->__overflow_arg_area = stack_frame + 1;
261 args->__reg_save_area = regs->gprs;
262 args->__gpr = 1;
263 return args;
264 }
265
266 #endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */
267
monitor_event_exception(struct pt_regs * regs)268 static void monitor_event_exception(struct pt_regs *regs)
269 {
270 enum bug_trap_type btt;
271
272 if (user_mode(regs))
273 return;
274 if (regs->monitor_code == MONCODE_BUG_ARG) {
275 regs->psw.addr = regs->gprs[14];
276 set_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED);
277 btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs);
278 } else {
279 btt = report_bug(regs->psw.addr - (regs->int_code >> 16), regs);
280 }
281 switch (btt) {
282 case BUG_TRAP_TYPE_NONE:
283 fixup_exception(regs);
284 break;
285 case BUG_TRAP_TYPE_WARN:
286 break;
287 case BUG_TRAP_TYPE_BUG:
288 die(regs, "monitor event");
289 break;
290 }
291 }
292
293 /*
294 * Content of the faulting CPU's lowcore, copied by stack_invalid().
295 */
296 struct lowcore stack_invalid_lowcore;
297 int stack_invalid_lowcore_used;
298
kernel_stack_invalid(struct pt_regs * regs)299 void kernel_stack_invalid(struct pt_regs *regs)
300 {
301 /*
302 * Normally regs are unpoisoned by the generic entry code, but
303 * kernel_stack_overflow() is a rare case that is called bypassing it.
304 */
305 kmsan_unpoison_entry_regs(regs);
306 bust_spinlocks(1);
307 pr_emerg("Kernel stack pointer invalid\n");
308 show_regs(regs);
309 bust_spinlocks(0);
310 panic("Invalid kernel stack pointer, cannot continue");
311 }
312 NOKPROBE_SYMBOL(kernel_stack_invalid);
313
test_monitor_call(void)314 static void __init test_monitor_call(void)
315 {
316 int val = 1;
317
318 if (!IS_ENABLED(CONFIG_BUG))
319 return;
320 asm_inline volatile(
321 "0: mc %[monc](%%r0),0\n"
322 " lhi %[val],0\n"
323 "1:\n"
324 EX_TABLE_INSN(0b, 1b)
325 : [val] "+d" (val)
326 : [monc] "i" (MONCODE_BUG));
327 if (!val)
328 panic("Monitor call doesn't work!\n");
329 }
330
trap_init(void)331 void __init trap_init(void)
332 {
333 struct lowcore *lc = get_lowcore();
334 unsigned long flags;
335 struct ctlreg cr0;
336
337 local_irq_save(flags);
338 cr0 = local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT);
339 psw_bits(lc->external_new_psw).mcheck = 1;
340 psw_bits(lc->program_new_psw).mcheck = 1;
341 psw_bits(lc->svc_new_psw).mcheck = 1;
342 psw_bits(lc->io_new_psw).mcheck = 1;
343 local_ctl_load(0, &cr0);
344 local_irq_restore(flags);
345 local_mcck_enable();
346 test_monitor_call();
347 }
348
349 static void (*pgm_check_table[128])(struct pt_regs *regs);
350
__do_pgm_check(struct pt_regs * regs,unsigned long flags)351 void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags)
352 {
353 struct lowcore *lc = get_lowcore();
354 irqentry_state_t state;
355 struct pgm_stat *stat;
356 unsigned int trapnr;
357 union teid teid;
358
359 teid.val = lc->trans_exc_code;
360 regs->int_code = lc->pgm_int_code;
361 regs->int_parm_long = teid.val;
362 regs->monitor_code = lc->monitor_code;
363 set_pt_regs_flag(regs, PIF_PGM_CHECK);
364
365 trapnr = regs->int_code & PGM_INT_CODE_MASK;
366 stat = this_cpu_ptr(&pgm_stat);
367 stat->count[trapnr]++;
368 /*
369 * In case of a guest fault, short-circuit the fault handler and return.
370 * This way the sie64a() function will return 0; fault address and
371 * other relevant bits are saved in current->thread.gmap_teid, and
372 * the fault number in current->thread.gmap_int_code. KVM will be
373 * able to use this information to handle the fault.
374 */
375 if (flags & PGM_FLAG_GUEST_FAULT) {
376 current->thread.gmap_teid.val = regs->int_parm_long;
377 current->thread.gmap_int_code = regs->int_code & 0xffff;
378 return;
379 }
380 percpu_entry(regs);
381 state = irqentry_enter(regs);
382 if (user_mode(regs)) {
383 update_timer_sys();
384 if (!cpu_has_bear()) {
385 if (regs->last_break < 4096)
386 regs->last_break = 1;
387 }
388 current->thread.last_break = regs->last_break;
389 }
390 if (lc->pgm_code & 0x0200) {
391 /* transaction abort */
392 current->thread.trap_tdb = lc->pgm_tdb;
393 }
394 if (lc->pgm_code & PGM_INT_CODE_PER) {
395 if (user_mode(regs)) {
396 struct per_event *ev = ¤t->thread.per_event;
397
398 set_thread_flag(TIF_PER_TRAP);
399 ev->address = lc->per_address;
400 ev->cause = lc->per_code_combined;
401 ev->paid = lc->per_access_id;
402 } else {
403 /* PER event in kernel is kprobes */
404 __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER);
405 do_per_trap(regs);
406 goto out;
407 }
408 }
409 if (!irqs_disabled_flags(regs->psw.mask))
410 trace_hardirqs_on();
411 __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER);
412 if (trapnr)
413 pgm_check_table[trapnr](regs);
414 out:
415 local_irq_disable();
416 irqentry_exit(regs, state);
417 percpu_exit(regs);
418 }
419
pgm_check_stat_show(struct seq_file * p,void * v)420 static int pgm_check_stat_show(struct seq_file *p, void *v)
421 {
422 int i, cpu;
423
424 cpus_read_lock();
425 seq_puts(p, " ");
426 for_each_online_cpu(cpu)
427 seq_printf(p, "CPU%-8d", cpu);
428 seq_putc(p, '\n');
429 for (i = 0; i < 128; i++) {
430 seq_printf(p, "%02x: ", i);
431 for_each_online_cpu(cpu)
432 seq_printf(p, "%10u ", per_cpu(pgm_stat, cpu).count[i]);
433 seq_putc(p, '\n');
434 }
435 cpus_read_unlock();
436 return 0;
437 }
438 DEFINE_SHOW_ATTRIBUTE(pgm_check_stat);
439
debugfs_pgm_check_init(void)440 static int __init debugfs_pgm_check_init(void)
441 {
442 debugfs_create_file("exceptions", 0400, arch_debugfs_dir, NULL, &pgm_check_stat_fops);
443 return 0;
444 }
445 late_initcall(debugfs_pgm_check_init);
446
447 /*
448 * The program check table contains exactly 128 (0x00-0x7f) entries. Each
449 * line defines the function to be called corresponding to the program check
450 * interruption code.
451 */
452 static void (*pgm_check_table[128])(struct pt_regs *regs) = {
453 [0x00] = default_trap_handler,
454 [0x01] = illegal_op,
455 [0x02] = privileged_op,
456 [0x03] = execute_exception,
457 [0x04] = do_protection_exception,
458 [0x05] = addressing_exception,
459 [0x06] = specification_exception,
460 [0x07] = data_exception,
461 [0x08] = overflow_exception,
462 [0x09] = divide_exception,
463 [0x0a] = overflow_exception,
464 [0x0b] = divide_exception,
465 [0x0c] = hfp_overflow_exception,
466 [0x0d] = hfp_underflow_exception,
467 [0x0e] = hfp_significance_exception,
468 [0x0f] = hfp_divide_exception,
469 [0x10] = do_dat_exception,
470 [0x11] = do_dat_exception,
471 [0x12] = translation_specification_exception,
472 [0x13] = special_op_exception,
473 [0x14] = default_trap_handler,
474 [0x15] = operand_exception,
475 [0x16] = default_trap_handler,
476 [0x17] = default_trap_handler,
477 [0x18] = transaction_exception,
478 [0x19] = default_trap_handler,
479 [0x1a] = default_trap_handler,
480 [0x1b] = vector_exception,
481 [0x1c] = space_switch_exception,
482 [0x1d] = hfp_sqrt_exception,
483 [0x1e ... 0x37] = default_trap_handler,
484 [0x38] = do_dat_exception,
485 [0x39] = do_dat_exception,
486 [0x3a] = do_dat_exception,
487 [0x3b] = do_dat_exception,
488 [0x3c] = default_trap_handler,
489 [0x3d] = do_secure_storage_access,
490 [0x3e] = default_trap_handler,
491 [0x3f] = default_trap_handler,
492 [0x40] = monitor_event_exception,
493 [0x41 ... 0x7f] = default_trap_handler,
494 };
495
496 #define COND_TRAP(x) asm( \
497 ".weak " __stringify(x) "\n\t" \
498 ".set " __stringify(x) "," \
499 __stringify(default_trap_handler))
500
501 COND_TRAP(do_secure_storage_access);
502