1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Based on arch/arm/mm/fault.c
4 *
5 * Copyright (C) 1995 Linus Torvalds
6 * Copyright (C) 1995-2004 Russell King
7 * Copyright (C) 2012 ARM Ltd.
8 */
9
10 #include <linux/acpi.h>
11 #include <linux/bitfield.h>
12 #include <linux/bpf_defs.h>
13 #include <linux/extable.h>
14 #include <linux/kfence.h>
15 #include <linux/signal.h>
16 #include <linux/mm.h>
17 #include <linux/hardirq.h>
18 #include <linux/init.h>
19 #include <linux/irqflags.h>
20 #include <linux/kasan.h>
21 #include <linux/kprobes.h>
22 #include <linux/uaccess.h>
23 #include <linux/page-flags.h>
24 #include <linux/sched/signal.h>
25 #include <linux/sched/debug.h>
26 #include <linux/highmem.h>
27 #include <linux/perf_event.h>
28 #include <linux/pkeys.h>
29 #include <linux/preempt.h>
30 #include <linux/hugetlb.h>
31
32 #include <asm/acpi.h>
33 #include <asm/bug.h>
34 #include <asm/cmpxchg.h>
35 #include <asm/cpufeature.h>
36 #include <asm/efi.h>
37 #include <asm/exception.h>
38 #include <asm/daifflags.h>
39 #include <asm/debug-monitors.h>
40 #include <asm/esr.h>
41 #include <asm/kprobes.h>
42 #include <asm/mte.h>
43 #include <asm/processor.h>
44 #include <asm/sysreg.h>
45 #include <asm/system_misc.h>
46 #include <asm/tlbflush.h>
47 #include <asm/traps.h>
48 #include <asm/virt.h>
49
50 struct fault_info {
51 int (*fn)(unsigned long far, unsigned long esr,
52 struct pt_regs *regs);
53 int sig;
54 int code;
55 const char *name;
56 };
57
58 static const struct fault_info fault_info[];
59
esr_to_fault_info(unsigned long esr)60 static inline const struct fault_info *esr_to_fault_info(unsigned long esr)
61 {
62 return fault_info + (esr & ESR_ELx_FSC);
63 }
64
data_abort_decode(unsigned long esr)65 static void data_abort_decode(unsigned long esr)
66 {
67 unsigned long iss2 = ESR_ELx_ISS2(esr);
68
69 pr_alert("Data abort info:\n");
70
71 if (esr & ESR_ELx_ISV) {
72 pr_alert(" Access size = %u byte(s)\n",
73 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
74 pr_alert(" SSE = %lu, SRT = %lu\n",
75 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
76 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
77 pr_alert(" SF = %lu, AR = %lu\n",
78 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
79 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
80 pr_alert(" Xs = %llu\n",
81 (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT);
82 } else {
83 pr_alert(" ISV = 0, ISS = 0x%08lx, ISS2 = 0x%08lx\n",
84 esr & ESR_ELx_ISS_MASK, iss2);
85 }
86
87 pr_alert(" CM = %lu, WnR = %lu, TnD = %lu, TagAccess = %lu\n",
88 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
89 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT,
90 (iss2 & ESR_ELx_TnD) >> ESR_ELx_TnD_SHIFT,
91 (iss2 & ESR_ELx_TagAccess) >> ESR_ELx_TagAccess_SHIFT);
92
93 pr_alert(" GCS = %ld, Overlay = %lu, DirtyBit = %lu\n",
94 (iss2 & ESR_ELx_GCS) >> ESR_ELx_GCS_SHIFT,
95 (iss2 & ESR_ELx_Overlay) >> ESR_ELx_Overlay_SHIFT,
96 (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT);
97 }
98
mem_abort_decode(unsigned long esr)99 static void mem_abort_decode(unsigned long esr)
100 {
101 pr_alert("Mem abort info:\n");
102
103 pr_alert(" ESR = 0x%016lx\n", esr);
104 pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
105 ESR_ELx_EC(esr), esr_get_class_string(esr),
106 (esr & ESR_ELx_IL) ? 32 : 16);
107 pr_alert(" SET = %lu, FnV = %lu\n",
108 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
109 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
110 pr_alert(" EA = %lu, S1PTW = %lu\n",
111 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
112 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
113 pr_alert(" FSC = 0x%02lx: %s\n", (esr & ESR_ELx_FSC),
114 esr_to_fault_info(esr)->name);
115
116 if (esr_is_data_abort(esr))
117 data_abort_decode(esr);
118 }
119
mm_to_pgd_phys(struct mm_struct * mm)120 static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
121 {
122 /* Either init_pg_dir or swapper_pg_dir */
123 if (mm == &init_mm)
124 return __pa_symbol(mm->pgd);
125
126 return (unsigned long)virt_to_phys(mm->pgd);
127 }
128
129 /*
130 * Dump out the page tables associated with 'addr' in the currently active mm.
131 */
show_pte(unsigned long addr)132 static void show_pte(unsigned long addr)
133 {
134 struct mm_struct *mm;
135 pgd_t *pgdp;
136 pgd_t pgd;
137
138 if (is_ttbr0_addr(addr)) {
139 /* TTBR0 */
140 mm = current->active_mm;
141 if (mm == &init_mm) {
142 pr_alert("[%016lx] user address but active_mm is swapper\n",
143 addr);
144 return;
145 }
146 } else if (is_ttbr1_addr(addr)) {
147 /* TTBR1 */
148 mm = &init_mm;
149 } else {
150 pr_alert("[%016lx] address between user and kernel address ranges\n",
151 addr);
152 return;
153 }
154
155 pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n",
156 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
157 vabits_actual, mm_to_pgd_phys(mm));
158
159 guard(irqsave)();
160
161 pgdp = pgd_offset(mm, addr);
162 pgd = READ_ONCE(*pgdp);
163 pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
164
165 do {
166 p4d_t *p4dp, p4d;
167 pud_t *pudp, pud;
168 pmd_t *pmdp, pmd;
169 pte_t *ptep, pte;
170
171 if (pgd_none(pgd) || pgd_bad(pgd))
172 break;
173
174 p4dp = p4d_offset_lockless(pgdp, pgd, addr);
175 p4d = READ_ONCE(*p4dp);
176 pr_cont(", p4d=%016llx", p4d_val(p4d));
177 if (p4d_none(p4d) || p4d_bad(p4d))
178 break;
179
180 pudp = pud_offset_lockless(p4dp, p4d, addr);
181 pud = READ_ONCE(*pudp);
182 pr_cont(", pud=%016llx", pud_val(pud));
183 if (pud_none(pud) || pud_bad(pud))
184 break;
185
186 pmdp = pmd_offset_lockless(pudp, pud, addr);
187 pmd = READ_ONCE(*pmdp);
188 pr_cont(", pmd=%016llx", pmd_val(pmd));
189 if (pmd_none(pmd) || pmd_bad(pmd))
190 break;
191
192 ptep = pte_offset_map(&pmd, addr);
193 if (!ptep)
194 break;
195
196 pte = __ptep_get(ptep);
197 pr_cont(", pte=%016llx", pte_val(pte));
198 pte_unmap(ptep);
199 } while(0);
200
201 pr_cont("\n");
202 }
203
204 /*
205 * This function sets the access flags (dirty, accessed), as well as write
206 * permission, and only to a more permissive setting.
207 *
208 * It needs to cope with hardware update of the accessed/dirty state by other
209 * agents in the system and can safely skip the __sync_icache_dcache() call as,
210 * like __set_ptes(), the PTE is never changed from no-exec to exec here.
211 *
212 * Returns whether or not the PTE actually changed.
213 */
__ptep_set_access_flags_anysz(struct vm_area_struct * vma,unsigned long address,pte_t * ptep,pte_t entry,int dirty,unsigned long pgsize)214 int __ptep_set_access_flags_anysz(struct vm_area_struct *vma,
215 unsigned long address, pte_t *ptep,
216 pte_t entry, int dirty, unsigned long pgsize)
217 {
218 pteval_t old_pteval, pteval;
219 pte_t pte = __ptep_get(ptep);
220 int level;
221
222 if (pte_same(pte, entry))
223 return 0;
224
225 /* only preserve the access flags and write permission */
226 pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
227
228 /*
229 * Setting the flags must be done atomically to avoid racing with the
230 * hardware update of the access/dirty state. The PTE_RDONLY bit must
231 * be set to the most permissive (lowest value) of *ptep and entry
232 * (calculated as: a & b == ~(~a | ~b)).
233 */
234 pte_val(entry) ^= PTE_RDONLY;
235 pteval = pte_val(pte);
236 do {
237 old_pteval = pteval;
238 pteval ^= PTE_RDONLY;
239 pteval |= pte_val(entry);
240 pteval ^= PTE_RDONLY;
241 pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
242 } while (pteval != old_pteval);
243
244 /*
245 * Invalidate the local stale read-only entry. Remote stale entries
246 * may still cause page faults and be invalidated via
247 * flush_tlb_fix_spurious_fault().
248 */
249 if (dirty) {
250 switch (pgsize) {
251 case PAGE_SIZE:
252 level = 3;
253 break;
254 case PMD_SIZE:
255 level = 2;
256 break;
257 #ifndef __PAGETABLE_PMD_FOLDED
258 case PUD_SIZE:
259 level = 1;
260 break;
261 #endif
262 default:
263 level = TLBI_TTL_UNKNOWN;
264 WARN_ON(1);
265 }
266
267 __flush_tlb_range(vma, address, address + pgsize, pgsize, level,
268 TLBF_NOWALKCACHE | TLBF_NOBROADCAST);
269 }
270 return 1;
271 }
272
is_el1_instruction_abort(unsigned long esr)273 static bool is_el1_instruction_abort(unsigned long esr)
274 {
275 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
276 }
277
is_el1_data_abort(unsigned long esr)278 static bool is_el1_data_abort(unsigned long esr)
279 {
280 return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
281 }
282
is_el1_permission_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)283 static inline bool is_el1_permission_fault(unsigned long addr, unsigned long esr,
284 struct pt_regs *regs)
285 {
286 if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
287 return false;
288
289 if (esr_fsc_is_permission_fault(esr))
290 return true;
291
292 if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
293 return esr_fsc_is_translation_fault(esr) &&
294 (regs->pstate & PSR_PAN_BIT);
295
296 return false;
297 }
298
is_pkvm_stage2_abort(unsigned int esr)299 static bool is_pkvm_stage2_abort(unsigned int esr)
300 {
301 /*
302 * S1PTW should only ever be set in ESR_EL1 if the pkvm hypervisor
303 * injected a stage-2 abort -- see host_inject_mem_abort().
304 */
305 return is_pkvm_initialized() && (esr & ESR_ELx_S1PTW);
306 }
307
is_spurious_el1_translation_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)308 static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
309 unsigned long esr,
310 struct pt_regs *regs)
311 {
312 unsigned long flags;
313 u64 par, dfsc;
314
315 if (!is_el1_data_abort(esr) || !esr_fsc_is_translation_fault(esr))
316 return false;
317
318 local_irq_save(flags);
319 asm volatile("at s1e1r, %0" :: "r" (addr));
320 isb();
321 par = read_sysreg_par();
322 local_irq_restore(flags);
323
324 /*
325 * If we now have a valid translation, treat the translation fault as
326 * spurious.
327 */
328 if (!(par & SYS_PAR_EL1_F)) {
329 if (is_pkvm_stage2_abort(esr)) {
330 par &= SYS_PAR_EL1_PA;
331 return pkvm_force_reclaim_guest_page(par);
332 }
333
334 return true;
335 }
336
337 /*
338 * If we got a different type of fault from the AT instruction,
339 * treat the translation fault as spurious.
340 */
341 dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
342 return !esr_fsc_is_translation_fault(dfsc);
343 }
344
die_kernel_fault(const char * msg,unsigned long addr,unsigned long esr,struct pt_regs * regs)345 static void die_kernel_fault(const char *msg, unsigned long addr,
346 unsigned long esr, struct pt_regs *regs)
347 {
348 bust_spinlocks(1);
349
350 pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
351 addr);
352
353 kasan_non_canonical_hook(addr);
354
355 mem_abort_decode(esr);
356
357 show_pte(addr);
358 die("Oops", regs, esr);
359 bust_spinlocks(0);
360 make_task_dead(SIGKILL);
361 }
362
363 #ifdef CONFIG_KASAN_HW_TAGS
report_tag_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)364 static void report_tag_fault(unsigned long addr, unsigned long esr,
365 struct pt_regs *regs)
366 {
367 /*
368 * SAS bits aren't set for all faults reported in EL1, so we can't
369 * find out access size.
370 */
371 bool is_write = !!(esr & ESR_ELx_WNR);
372 kasan_report((void *)addr, 0, is_write, regs->pc);
373 }
374 #else
375 /* Tag faults aren't enabled without CONFIG_KASAN_HW_TAGS. */
report_tag_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)376 static inline void report_tag_fault(unsigned long addr, unsigned long esr,
377 struct pt_regs *regs) { }
378 #endif
379
do_tag_recovery(unsigned long addr,unsigned long esr,struct pt_regs * regs)380 static void do_tag_recovery(unsigned long addr, unsigned long esr,
381 struct pt_regs *regs)
382 {
383
384 report_tag_fault(addr, esr, regs);
385
386 /*
387 * Disable MTE Tag Checking on the local CPU for the current EL.
388 * It will be done lazily on the other CPUs when they will hit a
389 * tag fault.
390 */
391 sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK,
392 SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE));
393 isb();
394 }
395
is_el1_mte_sync_tag_check_fault(unsigned long esr)396 static bool is_el1_mte_sync_tag_check_fault(unsigned long esr)
397 {
398 unsigned long fsc = esr & ESR_ELx_FSC;
399
400 if (!is_el1_data_abort(esr))
401 return false;
402
403 if (fsc == ESR_ELx_FSC_MTE)
404 return true;
405
406 return false;
407 }
408
__do_kernel_fault(unsigned long addr,unsigned long esr,struct pt_regs * regs)409 static void __do_kernel_fault(unsigned long addr, unsigned long esr,
410 struct pt_regs *regs)
411 {
412 const char *msg;
413
414 /*
415 * Are we prepared to handle this kernel fault?
416 * We are almost certainly not prepared to handle instruction faults.
417 */
418 if (!is_el1_instruction_abort(esr) && fixup_exception(regs, esr))
419 return;
420
421 if (is_spurious_el1_translation_fault(addr, esr, regs)) {
422 WARN_RATELIMIT(!is_pkvm_stage2_abort(esr),
423 "Ignoring spurious kernel translation fault at virtual address %016lx\n", addr);
424 return;
425 }
426
427 if (is_el1_mte_sync_tag_check_fault(esr)) {
428 do_tag_recovery(addr, esr, regs);
429
430 return;
431 }
432
433 if (is_el1_permission_fault(addr, esr, regs)) {
434 if (esr & ESR_ELx_WNR)
435 msg = "write to read-only memory";
436 else if (is_el1_instruction_abort(esr))
437 msg = "execute from non-executable memory";
438 else
439 msg = "read from unreadable memory";
440 } else if (addr < PAGE_SIZE) {
441 msg = "NULL pointer dereference";
442 } else if (is_pkvm_stage2_abort(esr)) {
443 msg = "access to hypervisor-protected memory";
444 } else {
445 if (esr_fsc_is_translation_fault(esr)) {
446 if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
447 return;
448 if (bpf_arena_handle_page_fault(addr, esr & ESR_ELx_WNR, regs->pc))
449 return;
450 }
451
452 msg = "paging request";
453 }
454
455 if (efi_runtime_fixup_exception(regs, msg))
456 return;
457
458 die_kernel_fault(msg, addr, esr, regs);
459 }
460
set_thread_esr(unsigned long address,unsigned long esr)461 static void set_thread_esr(unsigned long address, unsigned long esr)
462 {
463 current->thread.fault_address = address;
464
465 /*
466 * If the faulting address is in the kernel, we must sanitize the ESR.
467 * From userspace's point of view, kernel-only mappings don't exist
468 * at all, so we report them as level 0 translation faults.
469 * (This is not quite the way that "no mapping there at all" behaves:
470 * an alignment fault not caused by the memory type would take
471 * precedence over translation fault for a real access to empty
472 * space. Unfortunately we can't easily distinguish "alignment fault
473 * not caused by memory type" from "alignment fault caused by memory
474 * type", so we ignore this wrinkle and just return the translation
475 * fault.)
476 */
477 if (!is_ttbr0_addr(current->thread.fault_address)) {
478 switch (ESR_ELx_EC(esr)) {
479 case ESR_ELx_EC_DABT_LOW:
480 /*
481 * These bits provide only information about the
482 * faulting instruction, which userspace knows already.
483 * We explicitly clear bits which are architecturally
484 * RES0 in case they are given meanings in future.
485 * We always report the ESR as if the fault was taken
486 * to EL1 and so ISV and the bits in ISS[23:14] are
487 * clear. (In fact it always will be a fault to EL1.)
488 */
489 esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
490 ESR_ELx_CM | ESR_ELx_WNR;
491 esr |= ESR_ELx_FSC_FAULT;
492 break;
493 case ESR_ELx_EC_IABT_LOW:
494 /*
495 * Claim a level 0 translation fault.
496 * All other bits are architecturally RES0 for faults
497 * reported with that DFSC value, so we clear them.
498 */
499 esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
500 esr |= ESR_ELx_FSC_FAULT;
501 break;
502 default:
503 /*
504 * This should never happen (entry.S only brings us
505 * into this code for insn and data aborts from a lower
506 * exception level). Fail safe by not providing an ESR
507 * context record at all.
508 */
509 WARN(1, "ESR 0x%lx is not DABT or IABT from EL0\n", esr);
510 esr = 0;
511 break;
512 }
513 }
514
515 current->thread.fault_code = esr;
516 }
517
do_bad_area(unsigned long far,unsigned long esr,struct pt_regs * regs)518 static void do_bad_area(unsigned long far, unsigned long esr,
519 struct pt_regs *regs)
520 {
521 unsigned long addr = untagged_addr(far);
522
523 /*
524 * If we are in kernel mode at this point, we have no context to
525 * handle this fault with.
526 */
527 if (user_mode(regs)) {
528 const struct fault_info *inf = esr_to_fault_info(esr);
529
530 set_thread_esr(addr, esr);
531 arm64_force_sig_fault(inf->sig, inf->code, far, inf->name);
532 } else {
533 __do_kernel_fault(addr, esr, regs);
534 }
535 }
536
fault_from_pkey(struct vm_area_struct * vma,unsigned int mm_flags)537 static bool fault_from_pkey(struct vm_area_struct *vma, unsigned int mm_flags)
538 {
539 if (!system_supports_poe())
540 return false;
541
542 /*
543 * We do not check whether an Overlay fault has occurred because we
544 * cannot make a decision based solely on its value:
545 *
546 * - If Overlay is set, a fault did occur due to POE, but it may be
547 * spurious in those cases where we update POR_EL0 without ISB (e.g.
548 * on context-switch). We would then need to manually check POR_EL0
549 * against vma_pkey(vma), which is exactly what
550 * arch_vma_access_permitted() does.
551 *
552 * - If Overlay is not set, we may still need to report a pkey fault.
553 * This is the case if an access was made within a mapping but with no
554 * page mapped, and POR_EL0 forbids the access (according to
555 * vma_pkey()). Such access will result in a SIGSEGV regardless
556 * because core code checks arch_vma_access_permitted(), but in order
557 * to report the correct error code - SEGV_PKUERR - we must handle
558 * that case here.
559 */
560 return !arch_vma_access_permitted(vma,
561 mm_flags & FAULT_FLAG_WRITE,
562 mm_flags & FAULT_FLAG_INSTRUCTION,
563 false);
564 }
565
is_gcs_fault(unsigned long esr)566 static bool is_gcs_fault(unsigned long esr)
567 {
568 if (!esr_is_data_abort(esr))
569 return false;
570
571 return ESR_ELx_ISS2(esr) & ESR_ELx_GCS;
572 }
573
is_el0_instruction_abort(unsigned long esr)574 static bool is_el0_instruction_abort(unsigned long esr)
575 {
576 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
577 }
578
579 /*
580 * Note: not valid for EL1 DC IVAC, but we never use that such that it
581 * should fault. EL0 cannot issue DC IVAC (undef).
582 */
is_write_abort(unsigned long esr)583 static bool is_write_abort(unsigned long esr)
584 {
585 return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
586 }
587
is_invalid_gcs_access(struct vm_area_struct * vma,u64 esr)588 static bool is_invalid_gcs_access(struct vm_area_struct *vma, u64 esr)
589 {
590 if (!system_supports_gcs())
591 return false;
592
593 if (unlikely(is_gcs_fault(esr))) {
594 /* GCS accesses must be performed on a GCS page */
595 if (!(vma->vm_flags & VM_SHADOW_STACK))
596 return true;
597 } else if (unlikely(vma->vm_flags & VM_SHADOW_STACK)) {
598 /* Only GCS operations can write to a GCS page */
599 return esr_is_data_abort(esr) && is_write_abort(esr);
600 }
601
602 return false;
603 }
604
do_page_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)605 static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
606 struct pt_regs *regs)
607 {
608 const struct fault_info *inf;
609 struct mm_struct *mm = current->mm;
610 vm_fault_t fault;
611 vm_flags_t vm_flags;
612 unsigned int mm_flags = FAULT_FLAG_DEFAULT;
613 unsigned long addr = untagged_addr(far);
614 struct vm_area_struct *vma;
615 int si_code;
616 int pkey = -1;
617
618 if (kprobe_page_fault(regs, esr))
619 return 0;
620
621 /*
622 * If we're in an interrupt or have no user context, we must not take
623 * the fault.
624 */
625 if (faulthandler_disabled() || !mm)
626 goto no_context;
627
628 if (user_mode(regs))
629 mm_flags |= FAULT_FLAG_USER;
630
631 /*
632 * vm_flags tells us what bits we must have in vma->vm_flags
633 * for the fault to be benign, __do_page_fault() would check
634 * vma->vm_flags & vm_flags and returns an error if the
635 * intersection is empty
636 */
637 if (is_el0_instruction_abort(esr)) {
638 /* It was exec fault */
639 vm_flags = VM_EXEC;
640 mm_flags |= FAULT_FLAG_INSTRUCTION;
641 } else if (is_gcs_fault(esr)) {
642 /*
643 * The GCS permission on a page implies both read and
644 * write so always handle any GCS fault as a write fault,
645 * we need to trigger CoW even for GCS reads.
646 */
647 vm_flags = VM_WRITE;
648 mm_flags |= FAULT_FLAG_WRITE;
649 } else if (is_write_abort(esr)) {
650 /* It was write fault */
651 vm_flags = VM_WRITE;
652 mm_flags |= FAULT_FLAG_WRITE;
653 } else {
654 /* It was read fault */
655 vm_flags = VM_READ;
656 /* Write implies read */
657 vm_flags |= VM_WRITE;
658 /* If EPAN is absent then exec implies read */
659 if (!alternative_has_cap_unlikely(ARM64_HAS_EPAN))
660 vm_flags |= VM_EXEC;
661 }
662
663 if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
664 if (is_el1_instruction_abort(esr))
665 die_kernel_fault("execution of user memory",
666 addr, esr, regs);
667
668 if (!insn_may_access_user(regs->pc, esr))
669 die_kernel_fault("access to user memory outside uaccess routines",
670 addr, esr, regs);
671 }
672
673 if (is_pkvm_stage2_abort(esr)) {
674 if (!user_mode(regs))
675 goto no_context;
676 arm64_force_sig_fault(SIGSEGV, SEGV_ACCERR, far, "stage-2 fault");
677 return 0;
678 }
679
680 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
681
682 if (!(mm_flags & FAULT_FLAG_USER))
683 goto lock_mmap;
684
685 vma = lock_vma_under_rcu(mm, addr);
686 if (!vma)
687 goto lock_mmap;
688
689 if (is_invalid_gcs_access(vma, esr)) {
690 vma_end_read(vma);
691 fault = 0;
692 si_code = SEGV_ACCERR;
693 goto bad_area;
694 }
695
696 if (!(vma->vm_flags & vm_flags)) {
697 vma_end_read(vma);
698 fault = 0;
699 si_code = SEGV_ACCERR;
700 count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
701 goto bad_area;
702 }
703
704 if (fault_from_pkey(vma, mm_flags)) {
705 pkey = vma_pkey(vma);
706 vma_end_read(vma);
707 fault = 0;
708 si_code = SEGV_PKUERR;
709 count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
710 goto bad_area;
711 }
712
713 fault = handle_mm_fault(vma, addr, mm_flags | FAULT_FLAG_VMA_LOCK, regs);
714 if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
715 vma_end_read(vma);
716
717 if (!(fault & VM_FAULT_RETRY)) {
718 count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
719 goto done;
720 }
721 count_vm_vma_lock_event(VMA_LOCK_RETRY);
722 if (fault & VM_FAULT_MAJOR)
723 mm_flags |= FAULT_FLAG_TRIED;
724
725 /* Quick path to respond to signals */
726 if (fault_signal_pending(fault, regs)) {
727 if (!user_mode(regs))
728 goto no_context;
729 return 0;
730 }
731 lock_mmap:
732
733 retry:
734 vma = lock_mm_and_find_vma(mm, addr, regs);
735 if (unlikely(!vma)) {
736 fault = 0;
737 si_code = SEGV_MAPERR;
738 goto bad_area;
739 }
740
741 if (!(vma->vm_flags & vm_flags)) {
742 mmap_read_unlock(mm);
743 fault = 0;
744 si_code = SEGV_ACCERR;
745 goto bad_area;
746 }
747
748 if (fault_from_pkey(vma, mm_flags)) {
749 pkey = vma_pkey(vma);
750 mmap_read_unlock(mm);
751 fault = 0;
752 si_code = SEGV_PKUERR;
753 goto bad_area;
754 }
755
756 fault = handle_mm_fault(vma, addr, mm_flags, regs);
757
758 /* Quick path to respond to signals */
759 if (fault_signal_pending(fault, regs)) {
760 if (!user_mode(regs))
761 goto no_context;
762 return 0;
763 }
764
765 /* The fault is fully completed (including releasing mmap lock) */
766 if (fault & VM_FAULT_COMPLETED)
767 return 0;
768
769 if (fault & VM_FAULT_RETRY) {
770 mm_flags |= FAULT_FLAG_TRIED;
771 goto retry;
772 }
773 mmap_read_unlock(mm);
774
775 done:
776 /* Handle the "normal" (no error) case first. */
777 if (likely(!(fault & VM_FAULT_ERROR)))
778 return 0;
779
780 si_code = SEGV_MAPERR;
781 bad_area:
782 /*
783 * If we are in kernel mode at this point, we have no context to
784 * handle this fault with.
785 */
786 if (!user_mode(regs))
787 goto no_context;
788
789 if (fault & VM_FAULT_OOM) {
790 /*
791 * We ran out of memory, call the OOM killer, and return to
792 * userspace (which will retry the fault, or kill us if we got
793 * oom-killed).
794 */
795 pagefault_out_of_memory();
796 return 0;
797 }
798
799 inf = esr_to_fault_info(esr);
800 set_thread_esr(addr, esr);
801 if (fault & VM_FAULT_SIGBUS) {
802 /*
803 * We had some memory, but were unable to successfully fix up
804 * this page fault.
805 */
806 arm64_force_sig_fault(SIGBUS, BUS_ADRERR, far, inf->name);
807 } else if (fault & (VM_FAULT_HWPOISON_LARGE | VM_FAULT_HWPOISON)) {
808 unsigned int lsb;
809
810 lsb = PAGE_SHIFT;
811 if (fault & VM_FAULT_HWPOISON_LARGE)
812 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
813
814 arm64_force_sig_mceerr(BUS_MCEERR_AR, far, lsb, inf->name);
815 } else {
816 /*
817 * The pkey value that we return to userspace can be different
818 * from the pkey that caused the fault.
819 *
820 * 1. T1 : mprotect_key(foo, PAGE_SIZE, pkey=4);
821 * 2. T1 : set POR_EL0 to deny access to pkey=4, touches, page
822 * 3. T1 : faults...
823 * 4. T2: mprotect_key(foo, PAGE_SIZE, pkey=5);
824 * 5. T1 : enters fault handler, takes mmap_lock, etc...
825 * 6. T1 : reaches here, sees vma_pkey(vma)=5, when we really
826 * faulted on a pte with its pkey=4.
827 */
828 /* Something tried to access memory that out of memory map */
829 if (si_code == SEGV_PKUERR)
830 arm64_force_sig_fault_pkey(far, inf->name, pkey);
831 else
832 arm64_force_sig_fault(SIGSEGV, si_code, far, inf->name);
833 }
834
835 return 0;
836
837 no_context:
838 __do_kernel_fault(addr, esr, regs);
839 return 0;
840 }
841
do_translation_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)842 static int __kprobes do_translation_fault(unsigned long far,
843 unsigned long esr,
844 struct pt_regs *regs)
845 {
846 unsigned long addr = untagged_addr(far);
847
848 if (is_ttbr0_addr(addr))
849 return do_page_fault(far, esr, regs);
850
851 do_bad_area(far, esr, regs);
852 return 0;
853 }
854
do_alignment_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)855 static int do_alignment_fault(unsigned long far, unsigned long esr,
856 struct pt_regs *regs)
857 {
858 if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS) &&
859 compat_user_mode(regs))
860 return do_compat_alignment_fixup(far, regs);
861 do_bad_area(far, esr, regs);
862 return 0;
863 }
864
do_bad(unsigned long far,unsigned long esr,struct pt_regs * regs)865 static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
866 {
867 return 1; /* "fault" */
868 }
869
do_sea(unsigned long far,unsigned long esr,struct pt_regs * regs)870 static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
871 {
872 const struct fault_info *inf;
873 unsigned long siaddr;
874
875 inf = esr_to_fault_info(esr);
876
877 if (user_mode(regs) && apei_claim_sea(regs) == 0) {
878 /*
879 * APEI claimed this as a firmware-first notification.
880 * Some processing deferred to task_work before ret_to_user().
881 */
882 return 0;
883 }
884
885 if (esr & ESR_ELx_FnV) {
886 siaddr = 0;
887 } else {
888 /*
889 * The architecture specifies that the tag bits of FAR_EL1 are
890 * UNKNOWN for synchronous external aborts. Mask them out now
891 * so that userspace doesn't see them.
892 */
893 siaddr = untagged_addr(far);
894 }
895 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
896 arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
897
898 return 0;
899 }
900
do_tag_check_fault(unsigned long far,unsigned long esr,struct pt_regs * regs)901 static int do_tag_check_fault(unsigned long far, unsigned long esr,
902 struct pt_regs *regs)
903 {
904 /*
905 * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN
906 * for tag check faults. Set them to corresponding bits in the untagged
907 * address if ARM64_MTE_FAR isn't supported.
908 * Otherwise, bits 63:60 of FAR_EL1 are not UNKNOWN.
909 */
910 if (!cpus_have_cap(ARM64_MTE_FAR))
911 far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK);
912
913 do_bad_area(far, esr, regs);
914 return 0;
915 }
916
917 static const struct fault_info fault_info[] = {
918 { do_bad, SIGKILL, SI_KERNEL, "ttbr address size fault" },
919 { do_bad, SIGKILL, SI_KERNEL, "level 1 address size fault" },
920 { do_bad, SIGKILL, SI_KERNEL, "level 2 address size fault" },
921 { do_bad, SIGKILL, SI_KERNEL, "level 3 address size fault" },
922 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
923 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
924 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
925 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
926 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 0 access flag fault" },
927 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
928 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
929 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
930 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 0 permission fault" },
931 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
932 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
933 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
934 { do_sea, SIGBUS, BUS_OBJERR, "synchronous external abort" },
935 { do_tag_check_fault, SIGSEGV, SEGV_MTESERR, "synchronous tag check fault" },
936 { do_bad, SIGKILL, SI_KERNEL, "unknown 18" },
937 { do_sea, SIGKILL, SI_KERNEL, "level -1 (translation table walk)" },
938 { do_sea, SIGKILL, SI_KERNEL, "level 0 (translation table walk)" },
939 { do_sea, SIGKILL, SI_KERNEL, "level 1 (translation table walk)" },
940 { do_sea, SIGKILL, SI_KERNEL, "level 2 (translation table walk)" },
941 { do_sea, SIGKILL, SI_KERNEL, "level 3 (translation table walk)" },
942 { do_sea, SIGBUS, BUS_OBJERR, "synchronous parity or ECC error" }, // Reserved when RAS is implemented
943 { do_bad, SIGKILL, SI_KERNEL, "unknown 25" },
944 { do_bad, SIGKILL, SI_KERNEL, "unknown 26" },
945 { do_sea, SIGKILL, SI_KERNEL, "level -1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
946 { do_sea, SIGKILL, SI_KERNEL, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
947 { do_sea, SIGKILL, SI_KERNEL, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
948 { do_sea, SIGKILL, SI_KERNEL, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
949 { do_sea, SIGKILL, SI_KERNEL, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
950 { do_bad, SIGKILL, SI_KERNEL, "unknown 32" },
951 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
952 { do_bad, SIGKILL, SI_KERNEL, "unknown 34" },
953 { do_bad, SIGKILL, SI_KERNEL, "unknown 35" },
954 { do_bad, SIGKILL, SI_KERNEL, "unknown 36" },
955 { do_bad, SIGKILL, SI_KERNEL, "unknown 37" },
956 { do_bad, SIGKILL, SI_KERNEL, "unknown 38" },
957 { do_bad, SIGKILL, SI_KERNEL, "unknown 39" },
958 { do_bad, SIGKILL, SI_KERNEL, "unknown 40" },
959 { do_bad, SIGKILL, SI_KERNEL, "level -1 address size fault" },
960 { do_bad, SIGKILL, SI_KERNEL, "unknown 42" },
961 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level -1 translation fault" },
962 { do_bad, SIGKILL, SI_KERNEL, "unknown 44" },
963 { do_bad, SIGKILL, SI_KERNEL, "unknown 45" },
964 { do_bad, SIGKILL, SI_KERNEL, "unknown 46" },
965 { do_bad, SIGKILL, SI_KERNEL, "unknown 47" },
966 { do_bad, SIGKILL, SI_KERNEL, "TLB conflict abort" },
967 { do_bad, SIGKILL, SI_KERNEL, "Unsupported atomic hardware update fault" },
968 { do_bad, SIGKILL, SI_KERNEL, "unknown 50" },
969 { do_bad, SIGKILL, SI_KERNEL, "unknown 51" },
970 { do_bad, SIGKILL, SI_KERNEL, "implementation fault (lockdown abort)" },
971 { do_bad, SIGBUS, BUS_OBJERR, "implementation fault (unsupported exclusive)" },
972 { do_bad, SIGKILL, SI_KERNEL, "unknown 54" },
973 { do_bad, SIGKILL, SI_KERNEL, "unknown 55" },
974 { do_bad, SIGKILL, SI_KERNEL, "unknown 56" },
975 { do_bad, SIGKILL, SI_KERNEL, "unknown 57" },
976 { do_bad, SIGKILL, SI_KERNEL, "unknown 58" },
977 { do_bad, SIGKILL, SI_KERNEL, "unknown 59" },
978 { do_bad, SIGKILL, SI_KERNEL, "unknown 60" },
979 { do_bad, SIGKILL, SI_KERNEL, "section domain fault" },
980 { do_bad, SIGKILL, SI_KERNEL, "page domain fault" },
981 { do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
982 };
983
do_mem_abort(unsigned long far,unsigned long esr,struct pt_regs * regs)984 void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs)
985 {
986 const struct fault_info *inf = esr_to_fault_info(esr);
987 unsigned long addr = untagged_addr(far);
988
989 if (!inf->fn(far, esr, regs))
990 return;
991
992 if (!user_mode(regs))
993 die_kernel_fault(inf->name, addr, esr, regs);
994
995 /*
996 * At this point we have an unrecognized fault type whose tag bits may
997 * have been defined as UNKNOWN. Therefore we only expose the untagged
998 * address to the signal handler.
999 */
1000 arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
1001 }
1002 NOKPROBE_SYMBOL(do_mem_abort);
1003
do_sp_pc_abort(unsigned long addr,unsigned long esr,struct pt_regs * regs)1004 void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
1005 {
1006 arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
1007 addr, esr);
1008 }
1009 NOKPROBE_SYMBOL(do_sp_pc_abort);
1010
1011 /*
1012 * Used during anonymous page fault handling.
1013 */
vma_alloc_zeroed_movable_folio(struct vm_area_struct * vma,unsigned long vaddr)1014 struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
1015 unsigned long vaddr)
1016 {
1017 gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO;
1018
1019 /*
1020 * If the page is mapped with PROT_MTE, initialise the tags at the
1021 * point of allocation and page zeroing as this is usually faster than
1022 * separate DC ZVA and STGM.
1023 */
1024 if (vma->vm_flags & VM_MTE)
1025 flags |= __GFP_ZEROTAGS;
1026
1027 return vma_alloc_folio(flags, 0, vma, vaddr);
1028 }
1029
tag_clear_highpages(struct page * page,int numpages,bool clear_pages)1030 bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages)
1031 {
1032 /*
1033 * Check if MTE is supported and fall back to clear_highpage().
1034 * get_huge_zero_folio() unconditionally passes __GFP_ZEROTAGS and
1035 * post_alloc_hook() will invoke tag_clear_highpages().
1036 */
1037 if (!system_supports_mte())
1038 return clear_pages;
1039
1040 /* Newly allocated pages, shouldn't have been tagged yet */
1041 for (int i = 0; i < numpages; i++, page++) {
1042 WARN_ON_ONCE(!try_page_mte_tagging(page));
1043 if (clear_pages)
1044 mte_zero_clear_page_tags(page_address(page));
1045 else
1046 mte_clear_page_tags(page_address(page));
1047 set_page_mte_tagged(page);
1048 }
1049 return false;
1050 }
1051