1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * __get_user functions. 4 * 5 * (C) Copyright 1998 Linus Torvalds 6 * (C) Copyright 2005 Andi Kleen 7 * (C) Copyright 2008 Glauber Costa 8 * 9 * These functions have a non-standard call interface 10 * to make them more efficient, especially as they 11 * return an error value in addition to the "real" 12 * return value. 13 */ 14 15/* 16 * __get_user_X 17 * 18 * Inputs: %[r|e]ax contains the address. 19 * 20 * Outputs: %[r|e]ax is error code (0 or -EFAULT) 21 * %[r|e]dx contains zero-extended value 22 * %ecx contains the high half for 32-bit __get_user_8 23 * 24 * 25 * These functions should not modify any other registers, 26 * as they get called from within inline assembly. 27 */ 28 29#include <linux/export.h> 30#include <linux/linkage.h> 31#include <asm/page_types.h> 32#include <asm/errno.h> 33#include <asm/asm-offsets.h> 34#include <asm/thread_info.h> 35#include <asm/asm.h> 36#include <asm/smap.h> 37 38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC 39 40.macro check_range size:req 41.if IS_ENABLED(CONFIG_X86_64) 42 movq $0x0123456789abcdef,%rdx 43 1: 44 .pushsection runtime_ptr_USER_PTR_MAX,"a" 45 .long 1b - 8 - . 46 .popsection 47 cmp %rdx, %rax 48 cmova %rdx, %rax 49.else 50 cmp $TASK_SIZE_MAX-\size+1, %eax 51 jae .Lbad_get_user 52 sbb %edx, %edx /* array_index_mask_nospec() */ 53 and %edx, %eax 54.endif 55.endm 56 57.macro UACCESS op src dst 581: \op \src,\dst 59 _ASM_EXTABLE_UA(1b, __get_user_handle_exception) 60.endm 61 62 63 .text 64SYM_FUNC_START(__get_user_1) 65 check_range size=1 66 ASM_STAC 67 UACCESS movzbl (%_ASM_AX),%edx 68 xor %eax,%eax 69 ASM_CLAC 70 RET 71SYM_FUNC_END(__get_user_1) 72EXPORT_SYMBOL(__get_user_1) 73 74SYM_FUNC_START(__get_user_2) 75 check_range size=2 76 ASM_STAC 77 UACCESS movzwl (%_ASM_AX),%edx 78 xor %eax,%eax 79 ASM_CLAC 80 RET 81SYM_FUNC_END(__get_user_2) 82EXPORT_SYMBOL(__get_user_2) 83 84SYM_FUNC_START(__get_user_4) 85 check_range size=4 86 ASM_STAC 87 UACCESS movl (%_ASM_AX),%edx 88 xor %eax,%eax 89 ASM_CLAC 90 RET 91SYM_FUNC_END(__get_user_4) 92EXPORT_SYMBOL(__get_user_4) 93 94SYM_FUNC_START(__get_user_8) 95#ifndef CONFIG_X86_64 96 xor %ecx,%ecx 97#endif 98 check_range size=8 99 ASM_STAC 100#ifdef CONFIG_X86_64 101 UACCESS movq (%_ASM_AX),%rdx 102#else 103 UACCESS movl (%_ASM_AX),%edx 104 UACCESS movl 4(%_ASM_AX),%ecx 105#endif 106 xor %eax,%eax 107 ASM_CLAC 108 RET 109SYM_FUNC_END(__get_user_8) 110EXPORT_SYMBOL(__get_user_8) 111 112/* .. and the same for __get_user, just without the range checks */ 113SYM_FUNC_START(__get_user_nocheck_1) 114 ASM_STAC 115 ASM_BARRIER_NOSPEC 116 UACCESS movzbl (%_ASM_AX),%edx 117 xor %eax,%eax 118 ASM_CLAC 119 RET 120SYM_FUNC_END(__get_user_nocheck_1) 121EXPORT_SYMBOL(__get_user_nocheck_1) 122 123SYM_FUNC_START(__get_user_nocheck_2) 124 ASM_STAC 125 ASM_BARRIER_NOSPEC 126 UACCESS movzwl (%_ASM_AX),%edx 127 xor %eax,%eax 128 ASM_CLAC 129 RET 130SYM_FUNC_END(__get_user_nocheck_2) 131EXPORT_SYMBOL(__get_user_nocheck_2) 132 133SYM_FUNC_START(__get_user_nocheck_4) 134 ASM_STAC 135 ASM_BARRIER_NOSPEC 136 UACCESS movl (%_ASM_AX),%edx 137 xor %eax,%eax 138 ASM_CLAC 139 RET 140SYM_FUNC_END(__get_user_nocheck_4) 141EXPORT_SYMBOL(__get_user_nocheck_4) 142 143SYM_FUNC_START(__get_user_nocheck_8) 144 ASM_STAC 145 ASM_BARRIER_NOSPEC 146#ifdef CONFIG_X86_64 147 UACCESS movq (%_ASM_AX),%rdx 148#else 149 xor %ecx,%ecx 150 UACCESS movl (%_ASM_AX),%edx 151 UACCESS movl 4(%_ASM_AX),%ecx 152#endif 153 xor %eax,%eax 154 ASM_CLAC 155 RET 156SYM_FUNC_END(__get_user_nocheck_8) 157EXPORT_SYMBOL(__get_user_nocheck_8) 158 159 160SYM_CODE_START_LOCAL(__get_user_handle_exception) 161 ASM_CLAC 162.Lbad_get_user: 163 xor %edx,%edx 164 mov $(-EFAULT),%_ASM_AX 165 RET 166SYM_CODE_END(__get_user_handle_exception) 167