xref: /linux/arch/x86/include/asm/uaccess.h (revision 0526b56cbc3c489642bd6a5fe4b718dea7ef0ee8)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_UACCESS_H
3 #define _ASM_X86_UACCESS_H
4 /*
5  * User space memory access functions
6  */
7 #include <linux/compiler.h>
8 #include <linux/instrumented.h>
9 #include <linux/kasan-checks.h>
10 #include <linux/mm_types.h>
11 #include <linux/string.h>
12 #include <linux/mmap_lock.h>
13 #include <asm/asm.h>
14 #include <asm/page.h>
15 #include <asm/smap.h>
16 #include <asm/extable.h>
17 #include <asm/tlbflush.h>
18 
19 #ifdef CONFIG_X86_32
20 # include <asm/uaccess_32.h>
21 #else
22 # include <asm/uaccess_64.h>
23 #endif
24 
25 #include <asm-generic/access_ok.h>
26 
27 extern int __get_user_1(void);
28 extern int __get_user_2(void);
29 extern int __get_user_4(void);
30 extern int __get_user_8(void);
31 extern int __get_user_nocheck_1(void);
32 extern int __get_user_nocheck_2(void);
33 extern int __get_user_nocheck_4(void);
34 extern int __get_user_nocheck_8(void);
35 extern int __get_user_bad(void);
36 
37 #define __uaccess_begin() stac()
38 #define __uaccess_end()   clac()
39 #define __uaccess_begin_nospec()	\
40 ({					\
41 	stac();				\
42 	barrier_nospec();		\
43 })
44 
45 /*
46  * This is the smallest unsigned integer type that can fit a value
47  * (up to 'long long')
48  */
49 #define __inttype(x) __typeof__(		\
50 	__typefits(x,char,			\
51 	  __typefits(x,short,			\
52 	    __typefits(x,int,			\
53 	      __typefits(x,long,0ULL)))))
54 
55 #define __typefits(x,type,not) \
56 	__builtin_choose_expr(sizeof(x)<=sizeof(type),(unsigned type)0,not)
57 
58 /*
59  * This is used for both get_user() and __get_user() to expand to
60  * the proper special function call that has odd calling conventions
61  * due to returning both a value and an error, and that depends on
62  * the size of the pointer passed in.
63  *
64  * Careful: we have to cast the result to the type of the pointer
65  * for sign reasons.
66  *
67  * The use of _ASM_DX as the register specifier is a bit of a
68  * simplification, as gcc only cares about it as the starting point
69  * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
70  * (%ecx being the next register in gcc's x86 register sequence), and
71  * %rdx on 64 bits.
72  *
73  * Clang/LLVM cares about the size of the register, but still wants
74  * the base register for something that ends up being a pair.
75  */
76 #define do_get_user_call(fn,x,ptr)					\
77 ({									\
78 	int __ret_gu;							\
79 	register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);		\
80 	__chk_user_ptr(ptr);						\
81 	asm volatile("call __" #fn "_%P4"				\
82 		     : "=a" (__ret_gu), "=r" (__val_gu),		\
83 			ASM_CALL_CONSTRAINT				\
84 		     : "0" (ptr), "i" (sizeof(*(ptr))));		\
85 	instrument_get_user(__val_gu);					\
86 	(x) = (__force __typeof__(*(ptr))) __val_gu;			\
87 	__builtin_expect(__ret_gu, 0);					\
88 })
89 
90 /**
91  * get_user - Get a simple variable from user space.
92  * @x:   Variable to store result.
93  * @ptr: Source address, in user space.
94  *
95  * Context: User context only. This function may sleep if pagefaults are
96  *          enabled.
97  *
98  * This macro copies a single simple variable from user space to kernel
99  * space.  It supports simple types like char and int, but not larger
100  * data types like structures or arrays.
101  *
102  * @ptr must have pointer-to-simple-variable type, and the result of
103  * dereferencing @ptr must be assignable to @x without a cast.
104  *
105  * Return: zero on success, or -EFAULT on error.
106  * On error, the variable @x is set to zero.
107  */
108 #define get_user(x,ptr) ({ might_fault(); do_get_user_call(get_user,x,ptr); })
109 
110 /**
111  * __get_user - Get a simple variable from user space, with less checking.
112  * @x:   Variable to store result.
113  * @ptr: Source address, in user space.
114  *
115  * Context: User context only. This function may sleep if pagefaults are
116  *          enabled.
117  *
118  * This macro copies a single simple variable from user space to kernel
119  * space.  It supports simple types like char and int, but not larger
120  * data types like structures or arrays.
121  *
122  * @ptr must have pointer-to-simple-variable type, and the result of
123  * dereferencing @ptr must be assignable to @x without a cast.
124  *
125  * Caller must check the pointer with access_ok() before calling this
126  * function.
127  *
128  * Return: zero on success, or -EFAULT on error.
129  * On error, the variable @x is set to zero.
130  */
131 #define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
132 
133 
134 #ifdef CONFIG_X86_32
135 #define __put_user_goto_u64(x, addr, label)			\
136 	asm_volatile_goto("\n"					\
137 		     "1:	movl %%eax,0(%1)\n"		\
138 		     "2:	movl %%edx,4(%1)\n"		\
139 		     _ASM_EXTABLE_UA(1b, %l2)			\
140 		     _ASM_EXTABLE_UA(2b, %l2)			\
141 		     : : "A" (x), "r" (addr)			\
142 		     : : label)
143 
144 #else
145 #define __put_user_goto_u64(x, ptr, label) \
146 	__put_user_goto(x, ptr, "q", "er", label)
147 #endif
148 
149 extern void __put_user_bad(void);
150 
151 /*
152  * Strange magic calling convention: pointer in %ecx,
153  * value in %eax(:%edx), return value in %ecx. clobbers %rbx
154  */
155 extern void __put_user_1(void);
156 extern void __put_user_2(void);
157 extern void __put_user_4(void);
158 extern void __put_user_8(void);
159 extern void __put_user_nocheck_1(void);
160 extern void __put_user_nocheck_2(void);
161 extern void __put_user_nocheck_4(void);
162 extern void __put_user_nocheck_8(void);
163 
164 /*
165  * ptr must be evaluated and assigned to the temporary __ptr_pu before
166  * the assignment of x to __val_pu, to avoid any function calls
167  * involved in the ptr expression (possibly implicitly generated due
168  * to KASAN) from clobbering %ax.
169  */
170 #define do_put_user_call(fn,x,ptr)					\
171 ({									\
172 	int __ret_pu;							\
173 	void __user *__ptr_pu;						\
174 	register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX);		\
175 	__typeof__(*(ptr)) __x = (x); /* eval x once */			\
176 	__typeof__(ptr) __ptr = (ptr); /* eval ptr once */		\
177 	__chk_user_ptr(__ptr);						\
178 	__ptr_pu = __ptr;						\
179 	__val_pu = __x;							\
180 	asm volatile("call __" #fn "_%P[size]"				\
181 		     : "=c" (__ret_pu),					\
182 			ASM_CALL_CONSTRAINT				\
183 		     : "0" (__ptr_pu),					\
184 		       "r" (__val_pu),					\
185 		       [size] "i" (sizeof(*(ptr)))			\
186 		     :"ebx");						\
187 	instrument_put_user(__x, __ptr, sizeof(*(ptr)));		\
188 	__builtin_expect(__ret_pu, 0);					\
189 })
190 
191 /**
192  * put_user - Write a simple value into user space.
193  * @x:   Value to copy to user space.
194  * @ptr: Destination address, in user space.
195  *
196  * Context: User context only. This function may sleep if pagefaults are
197  *          enabled.
198  *
199  * This macro copies a single simple value from kernel space to user
200  * space.  It supports simple types like char and int, but not larger
201  * data types like structures or arrays.
202  *
203  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
204  * to the result of dereferencing @ptr.
205  *
206  * Return: zero on success, or -EFAULT on error.
207  */
208 #define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
209 
210 /**
211  * __put_user - Write a simple value into user space, with less checking.
212  * @x:   Value to copy to user space.
213  * @ptr: Destination address, in user space.
214  *
215  * Context: User context only. This function may sleep if pagefaults are
216  *          enabled.
217  *
218  * This macro copies a single simple value from kernel space to user
219  * space.  It supports simple types like char and int, but not larger
220  * data types like structures or arrays.
221  *
222  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
223  * to the result of dereferencing @ptr.
224  *
225  * Caller must check the pointer with access_ok() before calling this
226  * function.
227  *
228  * Return: zero on success, or -EFAULT on error.
229  */
230 #define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
231 
232 #define __put_user_size(x, ptr, size, label)				\
233 do {									\
234 	__typeof__(*(ptr)) __x = (x); /* eval x once */			\
235 	__typeof__(ptr) __ptr = (ptr); /* eval ptr once */		\
236 	__chk_user_ptr(__ptr);						\
237 	switch (size) {							\
238 	case 1:								\
239 		__put_user_goto(__x, __ptr, "b", "iq", label);		\
240 		break;							\
241 	case 2:								\
242 		__put_user_goto(__x, __ptr, "w", "ir", label);		\
243 		break;							\
244 	case 4:								\
245 		__put_user_goto(__x, __ptr, "l", "ir", label);		\
246 		break;							\
247 	case 8:								\
248 		__put_user_goto_u64(__x, __ptr, label);			\
249 		break;							\
250 	default:							\
251 		__put_user_bad();					\
252 	}								\
253 	instrument_put_user(__x, __ptr, size);				\
254 } while (0)
255 
256 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
257 
258 #ifdef CONFIG_X86_32
259 #define __get_user_asm_u64(x, ptr, label) do {				\
260 	unsigned int __gu_low, __gu_high;				\
261 	const unsigned int __user *__gu_ptr;				\
262 	__gu_ptr = (const void __user *)(ptr);				\
263 	__get_user_asm(__gu_low, __gu_ptr, "l", "=r", label);		\
264 	__get_user_asm(__gu_high, __gu_ptr+1, "l", "=r", label);	\
265 	(x) = ((unsigned long long)__gu_high << 32) | __gu_low;		\
266 } while (0)
267 #else
268 #define __get_user_asm_u64(x, ptr, label)				\
269 	__get_user_asm(x, ptr, "q", "=r", label)
270 #endif
271 
272 #define __get_user_size(x, ptr, size, label)				\
273 do {									\
274 	__chk_user_ptr(ptr);						\
275 	switch (size) {							\
276 	case 1:	{							\
277 		unsigned char x_u8__;					\
278 		__get_user_asm(x_u8__, ptr, "b", "=q", label);		\
279 		(x) = x_u8__;						\
280 		break;							\
281 	}								\
282 	case 2:								\
283 		__get_user_asm(x, ptr, "w", "=r", label);		\
284 		break;							\
285 	case 4:								\
286 		__get_user_asm(x, ptr, "l", "=r", label);		\
287 		break;							\
288 	case 8:								\
289 		__get_user_asm_u64(x, ptr, label);			\
290 		break;							\
291 	default:							\
292 		(x) = __get_user_bad();					\
293 	}								\
294 	instrument_get_user(x);						\
295 } while (0)
296 
297 #define __get_user_asm(x, addr, itype, ltype, label)			\
298 	asm_volatile_goto("\n"						\
299 		     "1:	mov"itype" %[umem],%[output]\n"		\
300 		     _ASM_EXTABLE_UA(1b, %l2)				\
301 		     : [output] ltype(x)				\
302 		     : [umem] "m" (__m(addr))				\
303 		     : : label)
304 
305 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
306 
307 #ifdef CONFIG_X86_32
308 #define __get_user_asm_u64(x, ptr, retval)				\
309 ({									\
310 	__typeof__(ptr) __ptr = (ptr);					\
311 	asm volatile("\n"						\
312 		     "1:	movl %[lowbits],%%eax\n"		\
313 		     "2:	movl %[highbits],%%edx\n"		\
314 		     "3:\n"						\
315 		     _ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG |	\
316 					   EX_FLAG_CLEAR_AX_DX,		\
317 					   %[errout])			\
318 		     _ASM_EXTABLE_TYPE_REG(2b, 3b, EX_TYPE_EFAULT_REG |	\
319 					   EX_FLAG_CLEAR_AX_DX,		\
320 					   %[errout])			\
321 		     : [errout] "=r" (retval),				\
322 		       [output] "=&A"(x)				\
323 		     : [lowbits] "m" (__m(__ptr)),			\
324 		       [highbits] "m" __m(((u32 __user *)(__ptr)) + 1),	\
325 		       "0" (retval));					\
326 })
327 
328 #else
329 #define __get_user_asm_u64(x, ptr, retval) \
330 	 __get_user_asm(x, ptr, retval, "q")
331 #endif
332 
333 #define __get_user_size(x, ptr, size, retval)				\
334 do {									\
335 	unsigned char x_u8__;						\
336 									\
337 	retval = 0;							\
338 	__chk_user_ptr(ptr);						\
339 	switch (size) {							\
340 	case 1:								\
341 		__get_user_asm(x_u8__, ptr, retval, "b");		\
342 		(x) = x_u8__;						\
343 		break;							\
344 	case 2:								\
345 		__get_user_asm(x, ptr, retval, "w");			\
346 		break;							\
347 	case 4:								\
348 		__get_user_asm(x, ptr, retval, "l");			\
349 		break;							\
350 	case 8:								\
351 		__get_user_asm_u64(x, ptr, retval);			\
352 		break;							\
353 	default:							\
354 		(x) = __get_user_bad();					\
355 	}								\
356 } while (0)
357 
358 #define __get_user_asm(x, addr, err, itype)				\
359 	asm volatile("\n"						\
360 		     "1:	mov"itype" %[umem],%[output]\n"		\
361 		     "2:\n"						\
362 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG | \
363 					   EX_FLAG_CLEAR_AX,		\
364 					   %[errout])			\
365 		     : [errout] "=r" (err),				\
366 		       [output] "=a" (x)				\
367 		     : [umem] "m" (__m(addr)),				\
368 		       "0" (err))
369 
370 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
371 
372 #ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
373 #define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
374 	bool success;							\
375 	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
376 	__typeof__(*(_ptr)) __old = *_old;				\
377 	__typeof__(*(_ptr)) __new = (_new);				\
378 	asm_volatile_goto("\n"						\
379 		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
380 		     _ASM_EXTABLE_UA(1b, %l[label])			\
381 		     : CC_OUT(z) (success),				\
382 		       [ptr] "+m" (*_ptr),				\
383 		       [old] "+a" (__old)				\
384 		     : [new] ltype (__new)				\
385 		     : "memory"						\
386 		     : label);						\
387 	if (unlikely(!success))						\
388 		*_old = __old;						\
389 	likely(success);					})
390 
391 #ifdef CONFIG_X86_32
392 #define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label)	({	\
393 	bool success;							\
394 	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
395 	__typeof__(*(_ptr)) __old = *_old;				\
396 	__typeof__(*(_ptr)) __new = (_new);				\
397 	asm_volatile_goto("\n"						\
398 		     "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"		\
399 		     _ASM_EXTABLE_UA(1b, %l[label])			\
400 		     : CC_OUT(z) (success),				\
401 		       "+A" (__old),					\
402 		       [ptr] "+m" (*_ptr)				\
403 		     : "b" ((u32)__new),				\
404 		       "c" ((u32)((u64)__new >> 32))			\
405 		     : "memory"						\
406 		     : label);						\
407 	if (unlikely(!success))						\
408 		*_old = __old;						\
409 	likely(success);					})
410 #endif // CONFIG_X86_32
411 #else  // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
412 #define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
413 	int __err = 0;							\
414 	bool success;							\
415 	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
416 	__typeof__(*(_ptr)) __old = *_old;				\
417 	__typeof__(*(_ptr)) __new = (_new);				\
418 	asm volatile("\n"						\
419 		     "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
420 		     CC_SET(z)						\
421 		     "2:\n"						\
422 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
423 					   %[errout])			\
424 		     : CC_OUT(z) (success),				\
425 		       [errout] "+r" (__err),				\
426 		       [ptr] "+m" (*_ptr),				\
427 		       [old] "+a" (__old)				\
428 		     : [new] ltype (__new)				\
429 		     : "memory");					\
430 	if (unlikely(__err))						\
431 		goto label;						\
432 	if (unlikely(!success))						\
433 		*_old = __old;						\
434 	likely(success);					})
435 
436 #ifdef CONFIG_X86_32
437 /*
438  * Unlike the normal CMPXCHG, use output GPR for both success/fail and error.
439  * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
440  * hardcoded by CMPXCHG8B, leaving only ESI and EDI.  If the compiler uses
441  * both ESI and EDI for the memory operand, compilation will fail if the error
442  * is an input+output as there will be no register available for input.
443  */
444 #define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label)	({	\
445 	int __result;							\
446 	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
447 	__typeof__(*(_ptr)) __old = *_old;				\
448 	__typeof__(*(_ptr)) __new = (_new);				\
449 	asm volatile("\n"						\
450 		     "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"		\
451 		     "mov $0, %[result]\n\t"				\
452 		     "setz %b[result]\n"				\
453 		     "2:\n"						\
454 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
455 					   %[result])			\
456 		     : [result] "=q" (__result),			\
457 		       "+A" (__old),					\
458 		       [ptr] "+m" (*_ptr)				\
459 		     : "b" ((u32)__new),				\
460 		       "c" ((u32)((u64)__new >> 32))			\
461 		     : "memory", "cc");					\
462 	if (unlikely(__result < 0))					\
463 		goto label;						\
464 	if (unlikely(!__result))					\
465 		*_old = __old;						\
466 	likely(__result);					})
467 #endif // CONFIG_X86_32
468 #endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
469 
470 /* FIXME: this hack is definitely wrong -AK */
471 struct __large_struct { unsigned long buf[100]; };
472 #define __m(x) (*(struct __large_struct __user *)(x))
473 
474 /*
475  * Tell gcc we read from memory instead of writing: this is because
476  * we do not write to any memory gcc knows about, so there are no
477  * aliasing issues.
478  */
479 #define __put_user_goto(x, addr, itype, ltype, label)			\
480 	asm_volatile_goto("\n"						\
481 		"1:	mov"itype" %0,%1\n"				\
482 		_ASM_EXTABLE_UA(1b, %l2)				\
483 		: : ltype(x), "m" (__m(addr))				\
484 		: : label)
485 
486 extern unsigned long
487 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
488 extern __must_check long
489 strncpy_from_user(char *dst, const char __user *src, long count);
490 
491 extern __must_check long strnlen_user(const char __user *str, long n);
492 
493 #ifdef CONFIG_ARCH_HAS_COPY_MC
494 unsigned long __must_check
495 copy_mc_to_kernel(void *to, const void *from, unsigned len);
496 #define copy_mc_to_kernel copy_mc_to_kernel
497 
498 unsigned long __must_check
499 copy_mc_to_user(void *to, const void *from, unsigned len);
500 #endif
501 
502 /*
503  * movsl can be slow when source and dest are not both 8-byte aligned
504  */
505 #ifdef CONFIG_X86_INTEL_USERCOPY
506 extern struct movsl_mask {
507 	int mask;
508 } ____cacheline_aligned_in_smp movsl_mask;
509 #endif
510 
511 #define ARCH_HAS_NOCACHE_UACCESS 1
512 
513 /*
514  * The "unsafe" user accesses aren't really "unsafe", but the naming
515  * is a big fat warning: you have to not only do the access_ok()
516  * checking before using them, but you have to surround them with the
517  * user_access_begin/end() pair.
518  */
519 static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
520 {
521 	if (unlikely(!access_ok(ptr,len)))
522 		return 0;
523 	__uaccess_begin_nospec();
524 	return 1;
525 }
526 #define user_access_begin(a,b)	user_access_begin(a,b)
527 #define user_access_end()	__uaccess_end()
528 
529 #define user_access_save()	smap_save()
530 #define user_access_restore(x)	smap_restore(x)
531 
532 #define unsafe_put_user(x, ptr, label)	\
533 	__put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
534 
535 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
536 #define unsafe_get_user(x, ptr, err_label)					\
537 do {										\
538 	__inttype(*(ptr)) __gu_val;						\
539 	__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label);		\
540 	(x) = (__force __typeof__(*(ptr)))__gu_val;				\
541 } while (0)
542 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
543 #define unsafe_get_user(x, ptr, err_label)					\
544 do {										\
545 	int __gu_err;								\
546 	__inttype(*(ptr)) __gu_val;						\
547 	__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err);		\
548 	(x) = (__force __typeof__(*(ptr)))__gu_val;				\
549 	if (unlikely(__gu_err)) goto err_label;					\
550 } while (0)
551 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
552 
553 extern void __try_cmpxchg_user_wrong_size(void);
554 
555 #ifndef CONFIG_X86_32
556 #define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)		\
557 	__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
558 #endif
559 
560 /*
561  * Force the pointer to u<size> to match the size expected by the asm helper.
562  * clang/LLVM compiles all cases and only discards the unused paths after
563  * processing errors, which breaks i386 if the pointer is an 8-byte value.
564  */
565 #define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({			\
566 	bool __ret;								\
567 	__chk_user_ptr(_ptr);							\
568 	switch (sizeof(*(_ptr))) {						\
569 	case 1:	__ret = __try_cmpxchg_user_asm("b", "q",			\
570 					       (__force u8 *)(_ptr), (_oldp),	\
571 					       (_nval), _label);		\
572 		break;								\
573 	case 2:	__ret = __try_cmpxchg_user_asm("w", "r",			\
574 					       (__force u16 *)(_ptr), (_oldp),	\
575 					       (_nval), _label);		\
576 		break;								\
577 	case 4:	__ret = __try_cmpxchg_user_asm("l", "r",			\
578 					       (__force u32 *)(_ptr), (_oldp),	\
579 					       (_nval), _label);		\
580 		break;								\
581 	case 8:	__ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
582 						 (_nval), _label);		\
583 		break;								\
584 	default: __try_cmpxchg_user_wrong_size();				\
585 	}									\
586 	__ret;						})
587 
588 /* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
589 #define __try_cmpxchg_user(_ptr, _oldp, _nval, _label)	({		\
590 	int __ret = -EFAULT;						\
591 	__uaccess_begin_nospec();					\
592 	__ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label);	\
593 _label:									\
594 	__uaccess_end();						\
595 	__ret;								\
596 							})
597 
598 /*
599  * We want the unsafe accessors to always be inlined and use
600  * the error labels - thus the macro games.
601  */
602 #define unsafe_copy_loop(dst, src, len, type, label)				\
603 	while (len >= sizeof(type)) {						\
604 		unsafe_put_user(*(type *)(src),(type __user *)(dst),label);	\
605 		dst += sizeof(type);						\
606 		src += sizeof(type);						\
607 		len -= sizeof(type);						\
608 	}
609 
610 #define unsafe_copy_to_user(_dst,_src,_len,label)			\
611 do {									\
612 	char __user *__ucu_dst = (_dst);				\
613 	const char *__ucu_src = (_src);					\
614 	size_t __ucu_len = (_len);					\
615 	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u64, label);	\
616 	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u32, label);	\
617 	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u16, label);	\
618 	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label);	\
619 } while (0)
620 
621 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
622 #define __get_kernel_nofault(dst, src, type, err_label)			\
623 	__get_user_size(*((type *)(dst)), (__force type __user *)(src),	\
624 			sizeof(type), err_label)
625 #else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
626 #define __get_kernel_nofault(dst, src, type, err_label)			\
627 do {									\
628 	int __kr_err;							\
629 									\
630 	__get_user_size(*((type *)(dst)), (__force type __user *)(src),	\
631 			sizeof(type), __kr_err);			\
632 	if (unlikely(__kr_err))						\
633 		goto err_label;						\
634 } while (0)
635 #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
636 
637 #define __put_kernel_nofault(dst, src, type, err_label)			\
638 	__put_user_size(*((type *)(src)), (__force type __user *)(dst),	\
639 			sizeof(type), err_label)
640 
641 #endif /* _ASM_X86_UACCESS_H */
642 
643