1 /* 2 * PowerPC version 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * 5 * Derived from "arch/i386/mm/fault.c" 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * Modified by Cort Dougan and Paul Mackerras. 9 * 10 * Modified for PPC64 by Dave Engebretsen (engebret@ibm.com) 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License 14 * as published by the Free Software Foundation; either version 15 * 2 of the License, or (at your option) any later version. 16 */ 17 18 #include <linux/signal.h> 19 #include <linux/sched.h> 20 #include <linux/sched/task_stack.h> 21 #include <linux/kernel.h> 22 #include <linux/errno.h> 23 #include <linux/string.h> 24 #include <linux/types.h> 25 #include <linux/ptrace.h> 26 #include <linux/mman.h> 27 #include <linux/mm.h> 28 #include <linux/interrupt.h> 29 #include <linux/highmem.h> 30 #include <linux/extable.h> 31 #include <linux/kprobes.h> 32 #include <linux/kdebug.h> 33 #include <linux/perf_event.h> 34 #include <linux/ratelimit.h> 35 #include <linux/context_tracking.h> 36 #include <linux/hugetlb.h> 37 #include <linux/uaccess.h> 38 39 #include <asm/firmware.h> 40 #include <asm/page.h> 41 #include <asm/pgtable.h> 42 #include <asm/mmu.h> 43 #include <asm/mmu_context.h> 44 #include <asm/tlbflush.h> 45 #include <asm/siginfo.h> 46 #include <asm/debug.h> 47 48 static inline bool notify_page_fault(struct pt_regs *regs) 49 { 50 bool ret = false; 51 52 #ifdef CONFIG_KPROBES 53 /* kprobe_running() needs smp_processor_id() */ 54 if (!user_mode(regs)) { 55 preempt_disable(); 56 if (kprobe_running() && kprobe_fault_handler(regs, 11)) 57 ret = true; 58 preempt_enable(); 59 } 60 #endif /* CONFIG_KPROBES */ 61 62 if (unlikely(debugger_fault_handler(regs))) 63 ret = true; 64 65 return ret; 66 } 67 68 /* 69 * Check whether the instruction at regs->nip is a store using 70 * an update addressing form which will update r1. 71 */ 72 static bool store_updates_sp(struct pt_regs *regs) 73 { 74 unsigned int inst; 75 76 if (get_user(inst, (unsigned int __user *)regs->nip)) 77 return false; 78 /* check for 1 in the rA field */ 79 if (((inst >> 16) & 0x1f) != 1) 80 return false; 81 /* check major opcode */ 82 switch (inst >> 26) { 83 case 37: /* stwu */ 84 case 39: /* stbu */ 85 case 45: /* sthu */ 86 case 53: /* stfsu */ 87 case 55: /* stfdu */ 88 return true; 89 case 62: /* std or stdu */ 90 return (inst & 3) == 1; 91 case 31: 92 /* check minor opcode */ 93 switch ((inst >> 1) & 0x3ff) { 94 case 181: /* stdux */ 95 case 183: /* stwux */ 96 case 247: /* stbux */ 97 case 439: /* sthux */ 98 case 695: /* stfsux */ 99 case 759: /* stfdux */ 100 return true; 101 } 102 } 103 return false; 104 } 105 /* 106 * do_page_fault error handling helpers 107 */ 108 109 static int 110 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code) 111 { 112 /* 113 * If we are in kernel mode, bail out with a SEGV, this will 114 * be caught by the assembly which will restore the non-volatile 115 * registers before calling bad_page_fault() 116 */ 117 if (!user_mode(regs)) 118 return SIGSEGV; 119 120 _exception(SIGSEGV, regs, si_code, address); 121 122 return 0; 123 } 124 125 static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long address) 126 { 127 return __bad_area_nosemaphore(regs, address, SEGV_MAPERR); 128 } 129 130 static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code) 131 { 132 struct mm_struct *mm = current->mm; 133 134 /* 135 * Something tried to access memory that isn't in our memory map.. 136 * Fix it, but check if it's kernel or user first.. 137 */ 138 up_read(&mm->mmap_sem); 139 140 return __bad_area_nosemaphore(regs, address, si_code); 141 } 142 143 static noinline int bad_area(struct pt_regs *regs, unsigned long address) 144 { 145 return __bad_area(regs, address, SEGV_MAPERR); 146 } 147 148 static int do_sigbus(struct pt_regs *regs, unsigned long address, 149 unsigned int fault) 150 { 151 siginfo_t info; 152 unsigned int lsb = 0; 153 154 if (!user_mode(regs)) 155 return SIGBUS; 156 157 current->thread.trap_nr = BUS_ADRERR; 158 info.si_signo = SIGBUS; 159 info.si_errno = 0; 160 info.si_code = BUS_ADRERR; 161 info.si_addr = (void __user *)address; 162 #ifdef CONFIG_MEMORY_FAILURE 163 if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { 164 pr_err("MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n", 165 current->comm, current->pid, address); 166 info.si_code = BUS_MCEERR_AR; 167 } 168 169 if (fault & VM_FAULT_HWPOISON_LARGE) 170 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); 171 if (fault & VM_FAULT_HWPOISON) 172 lsb = PAGE_SHIFT; 173 #endif 174 info.si_addr_lsb = lsb; 175 force_sig_info(SIGBUS, &info, current); 176 return 0; 177 } 178 179 static int mm_fault_error(struct pt_regs *regs, unsigned long addr, int fault) 180 { 181 /* 182 * Kernel page fault interrupted by SIGKILL. We have no reason to 183 * continue processing. 184 */ 185 if (fatal_signal_pending(current) && !user_mode(regs)) 186 return SIGKILL; 187 188 /* Out of memory */ 189 if (fault & VM_FAULT_OOM) { 190 /* 191 * We ran out of memory, or some other thing happened to us that 192 * made us unable to handle the page fault gracefully. 193 */ 194 if (!user_mode(regs)) 195 return SIGSEGV; 196 pagefault_out_of_memory(); 197 } else { 198 if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON| 199 VM_FAULT_HWPOISON_LARGE)) 200 return do_sigbus(regs, addr, fault); 201 else if (fault & VM_FAULT_SIGSEGV) 202 return bad_area_nosemaphore(regs, addr); 203 else 204 BUG(); 205 } 206 return 0; 207 } 208 209 /* Is this a bad kernel fault ? */ 210 static bool bad_kernel_fault(bool is_exec, unsigned long error_code, 211 unsigned long address) 212 { 213 if (is_exec && (error_code & (DSISR_NOEXEC_OR_G | DSISR_KEYFAULT))) { 214 printk_ratelimited(KERN_CRIT "kernel tried to execute" 215 " exec-protected page (%lx) -" 216 "exploit attempt? (uid: %d)\n", 217 address, from_kuid(&init_user_ns, 218 current_uid())); 219 } 220 return is_exec || (address >= TASK_SIZE); 221 } 222 223 static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address, 224 struct vm_area_struct *vma, 225 bool store_update_sp) 226 { 227 /* 228 * N.B. The POWER/Open ABI allows programs to access up to 229 * 288 bytes below the stack pointer. 230 * The kernel signal delivery code writes up to about 1.5kB 231 * below the stack pointer (r1) before decrementing it. 232 * The exec code can write slightly over 640kB to the stack 233 * before setting the user r1. Thus we allow the stack to 234 * expand to 1MB without further checks. 235 */ 236 if (address + 0x100000 < vma->vm_end) { 237 /* get user regs even if this fault is in kernel mode */ 238 struct pt_regs *uregs = current->thread.regs; 239 if (uregs == NULL) 240 return true; 241 242 /* 243 * A user-mode access to an address a long way below 244 * the stack pointer is only valid if the instruction 245 * is one which would update the stack pointer to the 246 * address accessed if the instruction completed, 247 * i.e. either stwu rs,n(r1) or stwux rs,r1,rb 248 * (or the byte, halfword, float or double forms). 249 * 250 * If we don't check this then any write to the area 251 * between the last mapped region and the stack will 252 * expand the stack rather than segfaulting. 253 */ 254 if (address + 2048 < uregs->gpr[1] && !store_update_sp) 255 return true; 256 } 257 return false; 258 } 259 260 static bool access_error(bool is_write, bool is_exec, 261 struct vm_area_struct *vma) 262 { 263 /* 264 * Allow execution from readable areas if the MMU does not 265 * provide separate controls over reading and executing. 266 * 267 * Note: That code used to not be enabled for 4xx/BookE. 268 * It is now as I/D cache coherency for these is done at 269 * set_pte_at() time and I see no reason why the test 270 * below wouldn't be valid on those processors. This -may- 271 * break programs compiled with a really old ABI though. 272 */ 273 if (is_exec) { 274 return !(vma->vm_flags & VM_EXEC) && 275 (cpu_has_feature(CPU_FTR_NOEXECUTE) || 276 !(vma->vm_flags & (VM_READ | VM_WRITE))); 277 } 278 279 if (is_write) { 280 if (unlikely(!(vma->vm_flags & VM_WRITE))) 281 return true; 282 return false; 283 } 284 285 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))) 286 return true; 287 288 return false; 289 } 290 291 #ifdef CONFIG_PPC_SMLPAR 292 static inline void cmo_account_page_fault(void) 293 { 294 if (firmware_has_feature(FW_FEATURE_CMO)) { 295 u32 page_ins; 296 297 preempt_disable(); 298 page_ins = be32_to_cpu(get_lppaca()->page_ins); 299 page_ins += 1 << PAGE_FACTOR; 300 get_lppaca()->page_ins = cpu_to_be32(page_ins); 301 preempt_enable(); 302 } 303 } 304 #else 305 static inline void cmo_account_page_fault(void) { } 306 #endif /* CONFIG_PPC_SMLPAR */ 307 308 #ifdef CONFIG_PPC_STD_MMU 309 static void sanity_check_fault(bool is_write, unsigned long error_code) 310 { 311 /* 312 * For hash translation mode, we should never get a 313 * PROTFAULT. Any update to pte to reduce access will result in us 314 * removing the hash page table entry, thus resulting in a DSISR_NOHPTE 315 * fault instead of DSISR_PROTFAULT. 316 * 317 * A pte update to relax the access will not result in a hash page table 318 * entry invalidate and hence can result in DSISR_PROTFAULT. 319 * ptep_set_access_flags() doesn't do a hpte flush. This is why we have 320 * the special !is_write in the below conditional. 321 * 322 * For platforms that doesn't supports coherent icache and do support 323 * per page noexec bit, we do setup things such that we do the 324 * sync between D/I cache via fault. But that is handled via low level 325 * hash fault code (hash_page_do_lazy_icache()) and we should not reach 326 * here in such case. 327 * 328 * For wrong access that can result in PROTFAULT, the above vma->vm_flags 329 * check should handle those and hence we should fall to the bad_area 330 * handling correctly. 331 * 332 * For embedded with per page exec support that doesn't support coherent 333 * icache we do get PROTFAULT and we handle that D/I cache sync in 334 * set_pte_at while taking the noexec/prot fault. Hence this is WARN_ON 335 * is conditional for server MMU. 336 * 337 * For radix, we can get prot fault for autonuma case, because radix 338 * page table will have them marked noaccess for user. 339 */ 340 if (!radix_enabled() && !is_write) 341 WARN_ON_ONCE(error_code & DSISR_PROTFAULT); 342 } 343 #else 344 static void sanity_check_fault(bool is_write, unsigned long error_code) { } 345 #endif /* CONFIG_PPC_STD_MMU */ 346 347 /* 348 * Define the correct "is_write" bit in error_code based 349 * on the processor family 350 */ 351 #if (defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 352 #define page_fault_is_write(__err) ((__err) & ESR_DST) 353 #define page_fault_is_bad(__err) (0) 354 #else 355 #define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE) 356 #if defined(CONFIG_PPC_8xx) 357 #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G) 358 #elif defined(CONFIG_PPC64) 359 #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S) 360 #else 361 #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S) 362 #endif 363 #endif 364 365 /* 366 * For 600- and 800-family processors, the error_code parameter is DSISR 367 * for a data fault, SRR1 for an instruction fault. For 400-family processors 368 * the error_code parameter is ESR for a data fault, 0 for an instruction 369 * fault. 370 * For 64-bit processors, the error_code parameter is 371 * - DSISR for a non-SLB data access fault, 372 * - SRR1 & 0x08000000 for a non-SLB instruction access fault 373 * - 0 any SLB fault. 374 * 375 * The return value is 0 if the fault was handled, or the signal 376 * number if this is a kernel fault that can't be handled here. 377 */ 378 static int __do_page_fault(struct pt_regs *regs, unsigned long address, 379 unsigned long error_code) 380 { 381 struct vm_area_struct * vma; 382 struct mm_struct *mm = current->mm; 383 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 384 int is_exec = TRAP(regs) == 0x400; 385 int is_user = user_mode(regs); 386 int is_write = page_fault_is_write(error_code); 387 int fault, major = 0; 388 bool store_update_sp = false; 389 390 if (notify_page_fault(regs)) 391 return 0; 392 393 if (unlikely(page_fault_is_bad(error_code))) { 394 if (is_user) { 395 _exception(SIGBUS, regs, BUS_OBJERR, address); 396 return 0; 397 } 398 return SIGBUS; 399 } 400 401 /* Additional sanity check(s) */ 402 sanity_check_fault(is_write, error_code); 403 404 /* 405 * The kernel should never take an execute fault nor should it 406 * take a page fault to a kernel address. 407 */ 408 if (unlikely(!is_user && bad_kernel_fault(is_exec, error_code, address))) 409 return SIGSEGV; 410 411 /* 412 * If we're in an interrupt, have no user context or are running 413 * in a region with pagefaults disabled then we must not take the fault 414 */ 415 if (unlikely(faulthandler_disabled() || !mm)) { 416 if (is_user) 417 printk_ratelimited(KERN_ERR "Page fault in user mode" 418 " with faulthandler_disabled()=%d" 419 " mm=%p\n", 420 faulthandler_disabled(), mm); 421 return bad_area_nosemaphore(regs, address); 422 } 423 424 /* We restore the interrupt state now */ 425 if (!arch_irq_disabled_regs(regs)) 426 local_irq_enable(); 427 428 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 429 430 /* 431 * We want to do this outside mmap_sem, because reading code around nip 432 * can result in fault, which will cause a deadlock when called with 433 * mmap_sem held 434 */ 435 if (is_write && is_user) 436 store_update_sp = store_updates_sp(regs); 437 438 if (is_user) 439 flags |= FAULT_FLAG_USER; 440 if (is_write) 441 flags |= FAULT_FLAG_WRITE; 442 if (is_exec) 443 flags |= FAULT_FLAG_INSTRUCTION; 444 445 /* When running in the kernel we expect faults to occur only to 446 * addresses in user space. All other faults represent errors in the 447 * kernel and should generate an OOPS. Unfortunately, in the case of an 448 * erroneous fault occurring in a code path which already holds mmap_sem 449 * we will deadlock attempting to validate the fault against the 450 * address space. Luckily the kernel only validly references user 451 * space from well defined areas of code, which are listed in the 452 * exceptions table. 453 * 454 * As the vast majority of faults will be valid we will only perform 455 * the source reference check when there is a possibility of a deadlock. 456 * Attempt to lock the address space, if we cannot we then validate the 457 * source. If this is invalid we can skip the address space check, 458 * thus avoiding the deadlock. 459 */ 460 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 461 if (!is_user && !search_exception_tables(regs->nip)) 462 return bad_area_nosemaphore(regs, address); 463 464 retry: 465 down_read(&mm->mmap_sem); 466 } else { 467 /* 468 * The above down_read_trylock() might have succeeded in 469 * which case we'll have missed the might_sleep() from 470 * down_read(): 471 */ 472 might_sleep(); 473 } 474 475 vma = find_vma(mm, address); 476 if (unlikely(!vma)) 477 return bad_area(regs, address); 478 if (likely(vma->vm_start <= address)) 479 goto good_area; 480 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) 481 return bad_area(regs, address); 482 483 /* The stack is being expanded, check if it's valid */ 484 if (unlikely(bad_stack_expansion(regs, address, vma, store_update_sp))) 485 return bad_area(regs, address); 486 487 /* Try to expand it */ 488 if (unlikely(expand_stack(vma, address))) 489 return bad_area(regs, address); 490 491 good_area: 492 if (unlikely(access_error(is_write, is_exec, vma))) 493 return bad_area(regs, address); 494 495 /* 496 * If for any reason at all we couldn't handle the fault, 497 * make sure we exit gracefully rather than endlessly redo 498 * the fault. 499 */ 500 fault = handle_mm_fault(vma, address, flags); 501 major |= fault & VM_FAULT_MAJOR; 502 503 /* 504 * Handle the retry right now, the mmap_sem has been released in that 505 * case. 506 */ 507 if (unlikely(fault & VM_FAULT_RETRY)) { 508 /* We retry only once */ 509 if (flags & FAULT_FLAG_ALLOW_RETRY) { 510 /* 511 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 512 * of starvation. 513 */ 514 flags &= ~FAULT_FLAG_ALLOW_RETRY; 515 flags |= FAULT_FLAG_TRIED; 516 if (!fatal_signal_pending(current)) 517 goto retry; 518 } 519 520 /* 521 * User mode? Just return to handle the fatal exception otherwise 522 * return to bad_page_fault 523 */ 524 return is_user ? 0 : SIGBUS; 525 } 526 527 up_read(¤t->mm->mmap_sem); 528 529 if (unlikely(fault & VM_FAULT_ERROR)) 530 return mm_fault_error(regs, address, fault); 531 532 /* 533 * Major/minor page fault accounting. 534 */ 535 if (major) { 536 current->maj_flt++; 537 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); 538 cmo_account_page_fault(); 539 } else { 540 current->min_flt++; 541 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); 542 } 543 return 0; 544 } 545 NOKPROBE_SYMBOL(__do_page_fault); 546 547 int do_page_fault(struct pt_regs *regs, unsigned long address, 548 unsigned long error_code) 549 { 550 enum ctx_state prev_state = exception_enter(); 551 int rc = __do_page_fault(regs, address, error_code); 552 exception_exit(prev_state); 553 return rc; 554 } 555 NOKPROBE_SYMBOL(do_page_fault); 556 557 /* 558 * bad_page_fault is called when we have a bad access from the kernel. 559 * It is called from the DSI and ISI handlers in head.S and from some 560 * of the procedures in traps.c. 561 */ 562 void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) 563 { 564 const struct exception_table_entry *entry; 565 566 /* Are we prepared to handle this fault? */ 567 if ((entry = search_exception_tables(regs->nip)) != NULL) { 568 regs->nip = extable_fixup(entry); 569 return; 570 } 571 572 /* kernel has accessed a bad area */ 573 574 switch (regs->trap) { 575 case 0x300: 576 case 0x380: 577 printk(KERN_ALERT "Unable to handle kernel paging request for " 578 "data at address 0x%08lx\n", regs->dar); 579 break; 580 case 0x400: 581 case 0x480: 582 printk(KERN_ALERT "Unable to handle kernel paging request for " 583 "instruction fetch\n"); 584 break; 585 case 0x600: 586 printk(KERN_ALERT "Unable to handle kernel paging request for " 587 "unaligned access at address 0x%08lx\n", regs->dar); 588 break; 589 default: 590 printk(KERN_ALERT "Unable to handle kernel paging request for " 591 "unknown fault\n"); 592 break; 593 } 594 printk(KERN_ALERT "Faulting instruction address: 0x%08lx\n", 595 regs->nip); 596 597 if (task_stack_end_corrupted(current)) 598 printk(KERN_ALERT "Thread overran stack, or stack corrupted\n"); 599 600 die("Kernel access of bad area", regs, sig); 601 } 602