1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * __get_user functions. 4 * 5 * (C) Copyright 1998 Linus Torvalds 6 * (C) Copyright 2005 Andi Kleen 7 * (C) Copyright 2008 Glauber Costa 8 * 9 * These functions have a non-standard call interface 10 * to make them more efficient, especially as they 11 * return an error value in addition to the "real" 12 * return value. 13 */ 14 15/* 16 * __get_user_X 17 * 18 * Inputs: %[r|e]ax contains the address. 19 * 20 * Outputs: %[r|e]ax is error code (0 or -EFAULT) 21 * %[r|e]dx contains zero-extended value 22 * %ecx contains the high half for 32-bit __get_user_8 23 * 24 * 25 * These functions should not modify any other registers, 26 * as they get called from within inline assembly. 27 */ 28 29#include <linux/export.h> 30#include <linux/linkage.h> 31#include <linux/objtool.h> 32#include <asm/page_types.h> 33#include <asm/errno.h> 34#include <asm/asm-offsets.h> 35#include <asm/thread_info.h> 36#include <asm/asm.h> 37#include <asm/smap.h> 38 39#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC 40 41.macro check_range size:req 42.if IS_ENABLED(CONFIG_X86_64) 43 movq $0x0123456789abcdef,%rdx 44 1: 45 .pushsection runtime_ptr_USER_PTR_MAX,"a" 46 .long 1b - 8 - . 47 .popsection 48 cmp %rdx, %rax 49 cmova %rdx, %rax 50.else 51 cmp $TASK_SIZE_MAX-\size+1, %eax 52 jae .Lbad_get_user 53 sbb %edx, %edx /* array_index_mask_nospec() */ 54 and %edx, %eax 55.endif 56.endm 57 58.macro UACCESS op src dst 591: \op \src,\dst 60 _ASM_EXTABLE_UA(1b, __get_user_handle_exception) 61.endm 62 63 64 .text 65SYM_FUNC_START(__get_user_1) 66 ANNOTATE_NOENDBR 67 check_range size=1 68 ASM_STAC 69 UACCESS movzbl (%_ASM_AX),%edx 70 xor %eax,%eax 71 ASM_CLAC 72 RET 73SYM_FUNC_END(__get_user_1) 74EXPORT_SYMBOL(__get_user_1) 75 76SYM_FUNC_START(__get_user_2) 77 ANNOTATE_NOENDBR 78 check_range size=2 79 ASM_STAC 80 UACCESS movzwl (%_ASM_AX),%edx 81 xor %eax,%eax 82 ASM_CLAC 83 RET 84SYM_FUNC_END(__get_user_2) 85EXPORT_SYMBOL(__get_user_2) 86 87SYM_FUNC_START(__get_user_4) 88 ANNOTATE_NOENDBR 89 check_range size=4 90 ASM_STAC 91 UACCESS movl (%_ASM_AX),%edx 92 xor %eax,%eax 93 ASM_CLAC 94 RET 95SYM_FUNC_END(__get_user_4) 96EXPORT_SYMBOL(__get_user_4) 97 98SYM_FUNC_START(__get_user_8) 99 ANNOTATE_NOENDBR 100#ifndef CONFIG_X86_64 101 xor %ecx,%ecx 102#endif 103 check_range size=8 104 ASM_STAC 105#ifdef CONFIG_X86_64 106 UACCESS movq (%_ASM_AX),%rdx 107#else 108 UACCESS movl (%_ASM_AX),%edx 109 UACCESS movl 4(%_ASM_AX),%ecx 110#endif 111 xor %eax,%eax 112 ASM_CLAC 113 RET 114SYM_FUNC_END(__get_user_8) 115EXPORT_SYMBOL(__get_user_8) 116 117/* .. and the same for __get_user, just without the range checks */ 118SYM_FUNC_START(__get_user_nocheck_1) 119 ANNOTATE_NOENDBR 120 ASM_STAC 121 ASM_BARRIER_NOSPEC 122 UACCESS movzbl (%_ASM_AX),%edx 123 xor %eax,%eax 124 ASM_CLAC 125 RET 126SYM_FUNC_END(__get_user_nocheck_1) 127EXPORT_SYMBOL(__get_user_nocheck_1) 128 129SYM_FUNC_START(__get_user_nocheck_2) 130 ANNOTATE_NOENDBR 131 ASM_STAC 132 ASM_BARRIER_NOSPEC 133 UACCESS movzwl (%_ASM_AX),%edx 134 xor %eax,%eax 135 ASM_CLAC 136 RET 137SYM_FUNC_END(__get_user_nocheck_2) 138EXPORT_SYMBOL(__get_user_nocheck_2) 139 140SYM_FUNC_START(__get_user_nocheck_4) 141 ANNOTATE_NOENDBR 142 ASM_STAC 143 ASM_BARRIER_NOSPEC 144 UACCESS movl (%_ASM_AX),%edx 145 xor %eax,%eax 146 ASM_CLAC 147 RET 148SYM_FUNC_END(__get_user_nocheck_4) 149EXPORT_SYMBOL(__get_user_nocheck_4) 150 151SYM_FUNC_START(__get_user_nocheck_8) 152 ANNOTATE_NOENDBR 153 ASM_STAC 154 ASM_BARRIER_NOSPEC 155#ifdef CONFIG_X86_64 156 UACCESS movq (%_ASM_AX),%rdx 157#else 158 xor %ecx,%ecx 159 UACCESS movl (%_ASM_AX),%edx 160 UACCESS movl 4(%_ASM_AX),%ecx 161#endif 162 xor %eax,%eax 163 ASM_CLAC 164 RET 165SYM_FUNC_END(__get_user_nocheck_8) 166EXPORT_SYMBOL(__get_user_nocheck_8) 167 168 169SYM_CODE_START_LOCAL(__get_user_handle_exception) 170 ASM_CLAC 171.Lbad_get_user: 172 xor %edx,%edx 173 mov $(-EFAULT),%_ASM_AX 174 RET 175SYM_CODE_END(__get_user_handle_exception) 176