1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/fault.c 4 * 5 * Copyright (C) 1995 Linus Torvalds 6 * Modifications for ARM processor (c) 1995-2004 Russell King 7 */ 8 #include <linux/extable.h> 9 #include <linux/signal.h> 10 #include <linux/mm.h> 11 #include <linux/hardirq.h> 12 #include <linux/init.h> 13 #include <linux/kprobes.h> 14 #include <linux/uaccess.h> 15 #include <linux/page-flags.h> 16 #include <linux/sched/signal.h> 17 #include <linux/sched/debug.h> 18 #include <linux/highmem.h> 19 #include <linux/perf_event.h> 20 #include <linux/kfence.h> 21 22 #include <asm/system_misc.h> 23 #include <asm/system_info.h> 24 #include <asm/tlbflush.h> 25 26 #include "fault.h" 27 28 bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) 29 { 30 unsigned long addr = (unsigned long)unsafe_src; 31 32 return addr >= TASK_SIZE && ULONG_MAX - addr >= size; 33 } 34 35 #ifdef CONFIG_MMU 36 37 /* 38 * This is useful to dump out the page tables associated with 39 * 'addr' in mm 'mm'. 40 */ 41 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) 42 { 43 pgd_t *pgd; 44 45 if (!mm) 46 mm = &init_mm; 47 48 pgd = pgd_offset(mm, addr); 49 printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd)); 50 51 do { 52 p4d_t *p4d; 53 pud_t *pud; 54 pmd_t *pmd; 55 pte_t *pte; 56 57 p4d = p4d_offset(pgd, addr); 58 if (p4d_none(*p4d)) 59 break; 60 61 if (p4d_bad(*p4d)) { 62 pr_cont("(bad)"); 63 break; 64 } 65 66 pud = pud_offset(p4d, addr); 67 if (PTRS_PER_PUD != 1) 68 pr_cont(", *pud=%08llx", (long long)pud_val(*pud)); 69 70 if (pud_none(*pud)) 71 break; 72 73 if (pud_bad(*pud)) { 74 pr_cont("(bad)"); 75 break; 76 } 77 78 pmd = pmd_offset(pud, addr); 79 if (PTRS_PER_PMD != 1) 80 pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd)); 81 82 if (pmd_none(*pmd)) 83 break; 84 85 if (pmd_bad(*pmd)) { 86 pr_cont("(bad)"); 87 break; 88 } 89 90 /* We must not map this if we have highmem enabled */ 91 if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT))) 92 break; 93 94 pte = pte_offset_map(pmd, addr); 95 if (!pte) 96 break; 97 98 pr_cont(", *pte=%08llx", (long long)pte_val(*pte)); 99 #ifndef CONFIG_ARM_LPAE 100 pr_cont(", *ppte=%08llx", 101 (long long)pte_val(pte[PTE_HWTABLE_PTRS])); 102 #endif 103 pte_unmap(pte); 104 } while(0); 105 106 pr_cont("\n"); 107 } 108 #else /* CONFIG_MMU */ 109 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) 110 { } 111 #endif /* CONFIG_MMU */ 112 113 static inline bool is_write_fault(unsigned int fsr) 114 { 115 return (fsr & FSR_WRITE) && !(fsr & FSR_CM); 116 } 117 118 static inline bool is_translation_fault(unsigned int fsr) 119 { 120 int fs = fsr_fs(fsr); 121 #ifdef CONFIG_ARM_LPAE 122 if ((fs & FS_MMU_NOLL_MASK) == FS_TRANS_NOLL) 123 return true; 124 #else 125 if (fs == FS_L1_TRANS || fs == FS_L2_TRANS) 126 return true; 127 #endif 128 return false; 129 } 130 131 static void die_kernel_fault(const char *msg, struct mm_struct *mm, 132 unsigned long addr, unsigned int fsr, 133 struct pt_regs *regs) 134 { 135 bust_spinlocks(1); 136 pr_alert("8<--- cut here ---\n"); 137 pr_alert("Unable to handle kernel %s at virtual address %08lx when %s\n", 138 msg, addr, fsr & FSR_LNX_PF ? "execute" : 139 fsr & FSR_WRITE ? "write" : "read"); 140 141 show_pte(KERN_ALERT, mm, addr); 142 die("Oops", regs, fsr); 143 bust_spinlocks(0); 144 make_task_dead(SIGKILL); 145 } 146 147 /* 148 * Oops. The kernel tried to access some page that wasn't present. 149 */ 150 static void 151 __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, 152 struct pt_regs *regs) 153 { 154 const char *msg; 155 /* 156 * Are we prepared to handle this kernel fault? 157 */ 158 if (fixup_exception(regs)) 159 return; 160 161 /* 162 * No handler, we'll have to terminate things with extreme prejudice. 163 */ 164 if (addr < PAGE_SIZE) { 165 msg = "NULL pointer dereference"; 166 } else { 167 if (is_translation_fault(fsr) && 168 kfence_handle_page_fault(addr, is_write_fault(fsr), regs)) 169 return; 170 171 msg = "paging request"; 172 } 173 174 die_kernel_fault(msg, mm, addr, fsr, regs); 175 } 176 177 /* 178 * Something tried to access memory that isn't in our memory map.. 179 * User mode accesses just cause a SIGSEGV 180 */ 181 static void 182 __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig, 183 int code, struct pt_regs *regs) 184 { 185 struct task_struct *tsk = current; 186 187 if (addr > TASK_SIZE) 188 harden_branch_predictor(); 189 190 #ifdef CONFIG_DEBUG_USER 191 if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) || 192 ((user_debug & UDBG_BUS) && (sig == SIGBUS))) { 193 pr_err("8<--- cut here ---\n"); 194 pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", 195 tsk->comm, sig, addr, fsr); 196 show_pte(KERN_ERR, tsk->mm, addr); 197 show_regs(regs); 198 } 199 #endif 200 #ifndef CONFIG_KUSER_HELPERS 201 if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000)) 202 printk_ratelimited(KERN_DEBUG 203 "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n", 204 tsk->comm, addr); 205 #endif 206 207 tsk->thread.address = addr; 208 tsk->thread.error_code = fsr; 209 tsk->thread.trap_no = 14; 210 force_sig_fault(sig, code, (void __user *)addr); 211 } 212 213 void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs) 214 { 215 struct task_struct *tsk = current; 216 struct mm_struct *mm = tsk->active_mm; 217 218 /* 219 * If we are in kernel mode at this point, we 220 * have no context to handle this fault with. 221 */ 222 if (user_mode(regs)) 223 __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs); 224 else 225 __do_kernel_fault(mm, addr, fsr, regs); 226 } 227 228 #ifdef CONFIG_MMU 229 static inline bool is_permission_fault(unsigned int fsr) 230 { 231 int fs = fsr_fs(fsr); 232 #ifdef CONFIG_ARM_LPAE 233 if ((fs & FS_MMU_NOLL_MASK) == FS_PERM_NOLL) 234 return true; 235 #else 236 if (fs == FS_L1_PERM || fs == FS_L2_PERM) 237 return true; 238 #endif 239 return false; 240 } 241 242 #ifdef CONFIG_CPU_TTBR0_PAN 243 static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs) 244 { 245 struct svc_pt_regs *svcregs; 246 247 /* If we are in user mode: permission granted */ 248 if (user_mode(regs)) 249 return true; 250 251 /* uaccess state saved above pt_regs on SVC exception entry */ 252 svcregs = to_svc_pt_regs(regs); 253 254 return !(svcregs->ttbcr & TTBCR_EPD0); 255 } 256 #else 257 static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs) 258 { 259 return true; 260 } 261 #endif 262 263 static int __kprobes 264 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) 265 { 266 struct mm_struct *mm = current->mm; 267 struct vm_area_struct *vma; 268 int sig, code; 269 vm_fault_t fault; 270 unsigned int flags = FAULT_FLAG_DEFAULT; 271 unsigned long vm_flags = VM_ACCESS_FLAGS; 272 273 if (kprobe_page_fault(regs, fsr)) 274 return 0; 275 276 277 /* Enable interrupts if they were enabled in the parent context. */ 278 if (interrupts_enabled(regs)) 279 local_irq_enable(); 280 281 /* 282 * If we're in an interrupt or have no user 283 * context, we must not take the fault.. 284 */ 285 if (faulthandler_disabled() || !mm) 286 goto no_context; 287 288 if (user_mode(regs)) 289 flags |= FAULT_FLAG_USER; 290 291 if (is_write_fault(fsr)) { 292 flags |= FAULT_FLAG_WRITE; 293 vm_flags = VM_WRITE; 294 } 295 296 if (fsr & FSR_LNX_PF) { 297 vm_flags = VM_EXEC; 298 299 if (is_permission_fault(fsr) && !user_mode(regs)) 300 die_kernel_fault("execution of memory", 301 mm, addr, fsr, regs); 302 } 303 304 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 305 306 /* 307 * Privileged access aborts with CONFIG_CPU_TTBR0_PAN enabled are 308 * routed via the translation fault mechanism. Check whether uaccess 309 * is disabled while in kernel mode. 310 */ 311 if (!ttbr0_usermode_access_allowed(regs)) 312 goto no_context; 313 314 if (!(flags & FAULT_FLAG_USER)) 315 goto lock_mmap; 316 317 vma = lock_vma_under_rcu(mm, addr); 318 if (!vma) 319 goto lock_mmap; 320 321 if (!(vma->vm_flags & vm_flags)) { 322 vma_end_read(vma); 323 count_vm_vma_lock_event(VMA_LOCK_SUCCESS); 324 fault = 0; 325 code = SEGV_ACCERR; 326 goto bad_area; 327 } 328 fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); 329 if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) 330 vma_end_read(vma); 331 332 if (!(fault & VM_FAULT_RETRY)) { 333 count_vm_vma_lock_event(VMA_LOCK_SUCCESS); 334 goto done; 335 } 336 count_vm_vma_lock_event(VMA_LOCK_RETRY); 337 if (fault & VM_FAULT_MAJOR) 338 flags |= FAULT_FLAG_TRIED; 339 340 /* Quick path to respond to signals */ 341 if (fault_signal_pending(fault, regs)) { 342 if (!user_mode(regs)) 343 goto no_context; 344 return 0; 345 } 346 lock_mmap: 347 348 retry: 349 vma = lock_mm_and_find_vma(mm, addr, regs); 350 if (unlikely(!vma)) { 351 fault = 0; 352 code = SEGV_MAPERR; 353 goto bad_area; 354 } 355 356 /* 357 * ok, we have a good vm_area for this memory access, check the 358 * permissions on the VMA allow for the fault which occurred. 359 */ 360 if (!(vma->vm_flags & vm_flags)) { 361 mmap_read_unlock(mm); 362 fault = 0; 363 code = SEGV_ACCERR; 364 goto bad_area; 365 } 366 367 fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, regs); 368 369 /* If we need to retry but a fatal signal is pending, handle the 370 * signal first. We do not need to release the mmap_lock because 371 * it would already be released in __lock_page_or_retry in 372 * mm/filemap.c. */ 373 if (fault_signal_pending(fault, regs)) { 374 if (!user_mode(regs)) 375 goto no_context; 376 return 0; 377 } 378 379 /* The fault is fully completed (including releasing mmap lock) */ 380 if (fault & VM_FAULT_COMPLETED) 381 return 0; 382 383 if (!(fault & VM_FAULT_ERROR)) { 384 if (fault & VM_FAULT_RETRY) { 385 flags |= FAULT_FLAG_TRIED; 386 goto retry; 387 } 388 } 389 390 mmap_read_unlock(mm); 391 done: 392 393 /* Handle the "normal" case first */ 394 if (likely(!(fault & VM_FAULT_ERROR))) 395 return 0; 396 397 code = SEGV_MAPERR; 398 bad_area: 399 /* 400 * If we are in kernel mode at this point, we 401 * have no context to handle this fault with. 402 */ 403 if (!user_mode(regs)) 404 goto no_context; 405 406 if (fault & VM_FAULT_OOM) { 407 /* 408 * We ran out of memory, call the OOM killer, and return to 409 * userspace (which will retry the fault, or kill us if we 410 * got oom-killed) 411 */ 412 pagefault_out_of_memory(); 413 return 0; 414 } 415 416 if (fault & VM_FAULT_SIGBUS) { 417 /* 418 * We had some memory, but were unable to 419 * successfully fix up this page fault. 420 */ 421 sig = SIGBUS; 422 code = BUS_ADRERR; 423 } else { 424 /* 425 * Something tried to access memory that 426 * isn't in our memory map.. 427 */ 428 sig = SIGSEGV; 429 } 430 431 __do_user_fault(addr, fsr, sig, code, regs); 432 return 0; 433 434 no_context: 435 __do_kernel_fault(mm, addr, fsr, regs); 436 return 0; 437 } 438 #else /* CONFIG_MMU */ 439 static int 440 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) 441 { 442 return 0; 443 } 444 #endif /* CONFIG_MMU */ 445 446 /* 447 * First Level Translation Fault Handler 448 * 449 * We enter here because the first level page table doesn't contain 450 * a valid entry for the address. 451 * 452 * If the address is in kernel space (>= TASK_SIZE), then we are 453 * probably faulting in the vmalloc() area. 454 * 455 * If the init_task's first level page tables contains the relevant 456 * entry, we copy the it to this task. If not, we send the process 457 * a signal, fixup the exception, or oops the kernel. 458 * 459 * NOTE! We MUST NOT take any locks for this case. We may be in an 460 * interrupt or a critical region, and should only copy the information 461 * from the master page table, nothing more. 462 */ 463 #ifdef CONFIG_MMU 464 static int __kprobes 465 do_translation_fault(unsigned long addr, unsigned int fsr, 466 struct pt_regs *regs) 467 { 468 unsigned int index; 469 pgd_t *pgd, *pgd_k; 470 p4d_t *p4d, *p4d_k; 471 pud_t *pud, *pud_k; 472 pmd_t *pmd, *pmd_k; 473 474 if (addr < TASK_SIZE) 475 return do_page_fault(addr, fsr, regs); 476 477 if (user_mode(regs)) 478 goto bad_area; 479 480 index = pgd_index(addr); 481 482 pgd = cpu_get_pgd() + index; 483 pgd_k = init_mm.pgd + index; 484 485 p4d = p4d_offset(pgd, addr); 486 p4d_k = p4d_offset(pgd_k, addr); 487 488 if (p4d_none(*p4d_k)) 489 goto bad_area; 490 if (!p4d_present(*p4d)) 491 set_p4d(p4d, *p4d_k); 492 493 pud = pud_offset(p4d, addr); 494 pud_k = pud_offset(p4d_k, addr); 495 496 if (pud_none(*pud_k)) 497 goto bad_area; 498 if (!pud_present(*pud)) 499 set_pud(pud, *pud_k); 500 501 pmd = pmd_offset(pud, addr); 502 pmd_k = pmd_offset(pud_k, addr); 503 504 #ifdef CONFIG_ARM_LPAE 505 /* 506 * Only one hardware entry per PMD with LPAE. 507 */ 508 index = 0; 509 #else 510 /* 511 * On ARM one Linux PGD entry contains two hardware entries (see page 512 * tables layout in pgtable.h). We normally guarantee that we always 513 * fill both L1 entries. But create_mapping() doesn't follow the rule. 514 * It can create inidividual L1 entries, so here we have to call 515 * pmd_none() check for the entry really corresponded to address, not 516 * for the first of pair. 517 */ 518 index = (addr >> SECTION_SHIFT) & 1; 519 #endif 520 if (pmd_none(pmd_k[index])) 521 goto bad_area; 522 523 copy_pmd(pmd, pmd_k); 524 return 0; 525 526 bad_area: 527 do_bad_area(addr, fsr, regs); 528 return 0; 529 } 530 #else /* CONFIG_MMU */ 531 static int 532 do_translation_fault(unsigned long addr, unsigned int fsr, 533 struct pt_regs *regs) 534 { 535 return 0; 536 } 537 #endif /* CONFIG_MMU */ 538 539 /* 540 * Some section permission faults need to be handled gracefully. 541 * They can happen due to a __{get,put}_user during an oops. 542 */ 543 #ifndef CONFIG_ARM_LPAE 544 static int 545 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) 546 { 547 do_bad_area(addr, fsr, regs); 548 return 0; 549 } 550 #endif /* CONFIG_ARM_LPAE */ 551 552 /* 553 * This abort handler always returns "fault". 554 */ 555 static int 556 do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs) 557 { 558 return 1; 559 } 560 561 struct fsr_info { 562 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs); 563 int sig; 564 int code; 565 const char *name; 566 }; 567 568 /* FSR definition */ 569 #ifdef CONFIG_ARM_LPAE 570 #include "fsr-3level.c" 571 #else 572 #include "fsr-2level.c" 573 #endif 574 575 void __init 576 hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), 577 int sig, int code, const char *name) 578 { 579 if (nr < 0 || nr >= ARRAY_SIZE(fsr_info)) 580 BUG(); 581 582 fsr_info[nr].fn = fn; 583 fsr_info[nr].sig = sig; 584 fsr_info[nr].code = code; 585 fsr_info[nr].name = name; 586 } 587 588 /* 589 * Dispatch a data abort to the relevant handler. 590 */ 591 asmlinkage void 592 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) 593 { 594 const struct fsr_info *inf = fsr_info + fsr_fs(fsr); 595 596 if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) 597 return; 598 599 pr_alert("8<--- cut here ---\n"); 600 pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n", 601 inf->name, fsr, addr); 602 show_pte(KERN_ALERT, current->mm, addr); 603 604 arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, 605 fsr, 0); 606 } 607 608 void __init 609 hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), 610 int sig, int code, const char *name) 611 { 612 if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info)) 613 BUG(); 614 615 ifsr_info[nr].fn = fn; 616 ifsr_info[nr].sig = sig; 617 ifsr_info[nr].code = code; 618 ifsr_info[nr].name = name; 619 } 620 621 asmlinkage void 622 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) 623 { 624 const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr); 625 626 if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) 627 return; 628 629 pr_alert("8<--- cut here ---\n"); 630 pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", 631 inf->name, ifsr, addr); 632 633 arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, 634 ifsr, 0); 635 } 636 637 /* 638 * Abort handler to be used only during first unmasking of asynchronous aborts 639 * on the boot CPU. This makes sure that the machine will not die if the 640 * firmware/bootloader left an imprecise abort pending for us to trip over. 641 */ 642 static int __init early_abort_handler(unsigned long addr, unsigned int fsr, 643 struct pt_regs *regs) 644 { 645 pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during " 646 "first unmask, this is most likely caused by a " 647 "firmware/bootloader bug.\n", fsr); 648 649 return 0; 650 } 651 652 void __init early_abt_enable(void) 653 { 654 fsr_info[FSR_FS_AEA].fn = early_abort_handler; 655 local_abt_enable(); 656 fsr_info[FSR_FS_AEA].fn = do_bad; 657 } 658 659 #ifndef CONFIG_ARM_LPAE 660 static int __init exceptions_init(void) 661 { 662 if (cpu_architecture() >= CPU_ARCH_ARMv6) { 663 hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR, 664 "I-cache maintenance fault"); 665 } 666 667 if (cpu_architecture() >= CPU_ARCH_ARMv7) { 668 /* 669 * TODO: Access flag faults introduced in ARMv6K. 670 * Runtime check for 'K' extension is needed 671 */ 672 hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR, 673 "section access flag fault"); 674 hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR, 675 "section access flag fault"); 676 } 677 678 return 0; 679 } 680 681 arch_initcall(exceptions_init); 682 #endif 683