1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * The out-of-line helpers for get_user(). 4 * 5 * (C) Copyright 1998 Linus Torvalds 6 * (C) Copyright 2005 Andi Kleen 7 * (C) Copyright 2008 Glauber Costa 8 * 9 * These functions have a non-standard call interface 10 * to make them more efficient, especially as they 11 * return an error value in addition to the "real" 12 * return value. 13 */ 14 15/* 16 * __get_user_X 17 * 18 * Inputs: %[r|e]ax contains the address. 19 * 20 * Outputs: %[r|e]ax is error code (0 or -EFAULT) 21 * %[r|e]dx contains zero-extended value 22 * %ecx contains the high half for 32-bit __get_user_8 23 * 24 * 25 * These functions should not modify any other registers, 26 * as they get called from within inline assembly. 27 */ 28 29#include <linux/export.h> 30#include <linux/linkage.h> 31#include <linux/objtool.h> 32#include <asm/page_types.h> 33#include <asm/errno.h> 34#include <asm/asm-offsets.h> 35#include <asm/thread_info.h> 36#include <asm/asm.h> 37#include <asm/smap.h> 38#include <asm/runtime-const.h> 39 40.macro check_range size:req 41.if IS_ENABLED(CONFIG_X86_64) 42 RUNTIME_CONST_PTR USER_PTR_MAX, rdx 43 cmp %rdx, %rax 44 cmova %rdx, %rax 45.else 46 cmp $TASK_SIZE_MAX-\size+1, %eax 47 jae .Lbad_get_user 48 sbb %edx, %edx /* array_index_mask_nospec() */ 49 and %edx, %eax 50.endif 51.endm 52 53.macro UACCESS op src dst 541: \op \src,\dst 55 _ASM_EXTABLE_UA(1b, __get_user_handle_exception) 56.endm 57 58 59 .text 60SYM_FUNC_START(__get_user_1) 61 ANNOTATE_NOENDBR 62 check_range size=1 63 ASM_STAC 64 UACCESS movzbl (%_ASM_AX),%edx 65 xor %eax,%eax 66 ASM_CLAC 67 RET 68SYM_FUNC_END(__get_user_1) 69EXPORT_SYMBOL(__get_user_1) 70 71SYM_FUNC_START(__get_user_2) 72 ANNOTATE_NOENDBR 73 check_range size=2 74 ASM_STAC 75 UACCESS movzwl (%_ASM_AX),%edx 76 xor %eax,%eax 77 ASM_CLAC 78 RET 79SYM_FUNC_END(__get_user_2) 80EXPORT_SYMBOL(__get_user_2) 81 82SYM_FUNC_START(__get_user_4) 83 ANNOTATE_NOENDBR 84 check_range size=4 85 ASM_STAC 86 UACCESS movl (%_ASM_AX),%edx 87 xor %eax,%eax 88 ASM_CLAC 89 RET 90SYM_FUNC_END(__get_user_4) 91EXPORT_SYMBOL(__get_user_4) 92 93SYM_FUNC_START(__get_user_8) 94 ANNOTATE_NOENDBR 95#ifndef CONFIG_X86_64 96 xor %ecx,%ecx 97#endif 98 check_range size=8 99 ASM_STAC 100#ifdef CONFIG_X86_64 101 UACCESS movq (%_ASM_AX),%rdx 102#else 103 UACCESS movl (%_ASM_AX),%edx 104 UACCESS movl 4(%_ASM_AX),%ecx 105#endif 106 xor %eax,%eax 107 ASM_CLAC 108 RET 109SYM_FUNC_END(__get_user_8) 110EXPORT_SYMBOL(__get_user_8) 111 112SYM_CODE_START_LOCAL(__get_user_handle_exception) 113 ASM_CLAC 114.Lbad_get_user: 115 xor %edx,%edx 116 mov $(-EFAULT),%_ASM_AX 117 RET 118SYM_CODE_END(__get_user_handle_exception) 119