32 	unsigned long addr = (unsigned long)unsafe_src;  in copy_from_kernel_nofault_allowed()  local
34 	return addr >= TASK_SIZE && ULONG_MAX - addr >= size;  in copy_from_kernel_nofault_allowed()
41 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)  in show_pte()  argument
48 	pgd = pgd_offset(mm, addr);  in show_pte()
49 	printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));  in show_pte()
57 		p4d = p4d_offset(pgd, addr);  in show_pte()
66 		pud = pud_offset(p4d, addr);  in show_pte()
78 		pmd = pmd_offset(pud, addr);  in show_pte()
94 		pte = pte_offset_map(pmd, addr);  in show_pte()
109 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)  in show_pte()  argument
119 			     unsigned long addr, unsigned int fsr,  in die_kernel_fault()  argument
125 		 msg, addr, fsr & FSR_LNX_PF ? "execute" : str_write_read(fsr & FSR_WRITE));  in die_kernel_fault()
127 	show_pte(KERN_ALERT, mm, addr);  in die_kernel_fault()
137 __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,  in __do_kernel_fault()  argument
150 	if (addr < PAGE_SIZE) {  in __do_kernel_fault()
156 		    kfence_handle_page_fault(addr, is_write_fault(fsr), regs))  in __do_kernel_fault()
162 	die_kernel_fault(msg, mm, addr, fsr, regs);  in __do_kernel_fault()
171 __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,  in __do_user_fault()  argument
183 		       tsk->comm, sig, addr, fsr);  in __do_user_fault()
184 		if (likely(addr < TASK_SIZE)) {  in __do_user_fault()
186 			show_pte(KERN_ERR, tsk->mm, addr);  in __do_user_fault()
193 	if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))  in __do_user_fault()
196 				   tsk->comm, addr);  in __do_user_fault()
199 	tsk->thread.address = addr;  in __do_user_fault()
202 	force_sig_fault(sig, code, (void __user *)addr);  in __do_user_fault()
205 void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)  in do_bad_area()  argument
215 		__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);  in do_bad_area()
217 		__do_kernel_fault(mm, addr, fsr, regs);  in do_bad_area()
250 static bool __kprobes __maybe_unused vmalloc_fault(unsigned long addr)  in vmalloc_fault()  argument
258 	index = pgd_index(addr);  in vmalloc_fault()
263 	p4d = p4d_offset(pgd, addr);  in vmalloc_fault()
264 	p4d_k = p4d_offset(pgd_k, addr);  in vmalloc_fault()
271 	pud = pud_offset(p4d, addr);  in vmalloc_fault()
272 	pud_k = pud_offset(p4d_k, addr);  in vmalloc_fault()
279 	pmd = pmd_offset(pud, addr);  in vmalloc_fault()
280 	pmd_k = pmd_offset(pud_k, addr);  in vmalloc_fault()
296 	index = (addr >> SECTION_SHIFT) & 1;  in vmalloc_fault()
307 do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr,  in do_kernel_address_page_fault()  argument
319 		__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);  in do_kernel_address_page_fault()
332 		__do_kernel_fault(mm, addr, fsr, regs);  in do_kernel_address_page_fault()
339 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)  in do_page_fault()  argument
355 	if (addr >= TASK_SIZE)  in do_page_fault()
356 		return do_kernel_address_page_fault(mm, addr, fsr, regs);  in do_page_fault()
382 					 mm, addr, fsr, regs);  in do_page_fault()
385 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);  in do_page_fault()
398 	vma = lock_vma_under_rcu(mm, addr);  in do_page_fault()
409 	fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs);  in do_page_fault()
430 	vma = lock_mm_and_find_vma(mm, addr, regs);  in do_page_fault()
448 	fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, regs);  in do_page_fault()
512 	__do_user_fault(addr, fsr, sig, code, regs);  in do_page_fault()
516 	__do_kernel_fault(mm, addr, fsr, regs);  in do_page_fault()
521 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)  in do_page_fault()  argument
549 do_translation_fault(unsigned long addr, unsigned int fsr,  in do_translation_fault()  argument
552 	if (addr < TASK_SIZE)  in do_translation_fault()
553 		return do_page_fault(addr, fsr, regs);  in do_translation_fault()
555 	if (!user_mode(regs) && vmalloc_fault(addr))  in do_translation_fault()
558 	do_kernel_address_page_fault(current->mm, addr, fsr, regs);  in do_translation_fault()
564 do_translation_fault(unsigned long addr, unsigned int fsr,  in do_translation_fault()  argument
577 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)  in do_sect_fault()  argument
584 	if (addr >= TASK_SIZE && user_mode(regs))  in do_sect_fault()
587 	do_bad_area(addr, fsr, regs);  in do_sect_fault()
597 do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)  in do_bad()  argument
603 	int	(*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
633 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)  in do_DataAbort()  argument
637 	if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))  in do_DataAbort()
645 		inf->name, fsr, addr);  in do_DataAbort()
647 		if (addr < TASK_SIZE) {  in do_DataAbort()
649 			show_pte(KERN_ALERT, current->mm, addr);  in do_DataAbort()
653 		show_pte(KERN_ALERT, current->mm, addr);  in do_DataAbort()
656 	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,  in do_DataAbort()
674 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)  in do_PrefetchAbort()  argument
678 	if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))  in do_PrefetchAbort()
686 		inf->name, ifsr, addr);  in do_PrefetchAbort()
688 	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,  in do_PrefetchAbort()
697 static int __init early_abort_handler(unsigned long addr, unsigned int fsr,  in early_abort_handler()  argument