xref: /linux/arch/s390/include/asm/uaccess.h (revision 2d7eaeb05ae265bd64270376e6a2c0d80244771f) 
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  *  S390 version
4  *    Copyright IBM Corp. 1999, 2000
5  *    Author(s): Hartmut Penner (hp@de.ibm.com),
6  *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
7  *
8  *  Derived from "include/asm-i386/uaccess.h"
9  */
10 #ifndef __S390_UACCESS_H
11 #define __S390_UACCESS_H
12 
13 /*
14  * User space memory access functions
15  */
16 #include <linux/pgtable.h>
17 #include <asm/asm-extable.h>
18 #include <asm/processor.h>
19 #include <asm/extable.h>
20 #include <asm/facility.h>
21 #include <asm-generic/access_ok.h>
22 #include <asm/asce.h>
23 #include <linux/instrumented.h>
24 
25 void debug_user_asce(int exit);
26 
27 #ifdef CONFIG_KMSAN
28 #define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory
29 #else
30 #define uaccess_kmsan_or_inline __always_inline
31 #endif
32 
33 #define INLINE_COPY_USER
34 
35 static uaccess_kmsan_or_inline __must_check unsigned long
36 raw_copy_from_user(void *to, const void __user *from, unsigned long size)
37 {
38 	unsigned long osize;
39 	int cc;
40 
41 	while (1) {
42 		osize = size;
43 		asm_inline volatile(
44 			"	lhi	%%r0,%[spec]\n"
45 			"0:	mvcos	%[to],%[from],%[size]\n"
46 			CC_IPM(cc)
47 			EX_TABLE_INSN_UA_MVCOS_FROM(0b, 0b)
48 			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
49 			: [spec] "I" (0x81), [from] "Q" (*(const char __user *)from)
50 			: CC_CLOBBER_LIST("memory", "0"));
51 		if (__builtin_constant_p(osize) && osize <= 4096)
52 			return osize - size;
53 		if (likely(CC_TRANSFORM(cc) == 0))
54 			return osize - size;
55 		size -= 4096;
56 		to += 4096;
57 		from += 4096;
58 	}
59 }
60 
61 static uaccess_kmsan_or_inline __must_check unsigned long
62 raw_copy_to_user(void __user *to, const void *from, unsigned long size)
63 {
64 	unsigned long osize;
65 	int cc;
66 
67 	while (1) {
68 		osize = size;
69 		asm_inline volatile(
70 			"	llilh	%%r0,%[spec]\n"
71 			"0:	mvcos	%[to],%[from],%[size]\n"
72 			CC_IPM(cc)
73 			EX_TABLE_INSN_UA_MVCOS_TO(0b, 0b)
74 			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
75 			: [spec] "I" (0x81), [from] "Q" (*(const char *)from)
76 			: CC_CLOBBER_LIST("memory", "0"));
77 		if (__builtin_constant_p(osize) && osize <= 4096)
78 			return osize - size;
79 		if (likely(CC_TRANSFORM(cc) == 0))
80 			return osize - size;
81 		size -= 4096;
82 		to += 4096;
83 		from += 4096;
84 	}
85 }
86 
87 unsigned long __must_check
88 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key);
89 
90 static __always_inline unsigned long __must_check
91 copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key)
92 {
93 	if (check_copy_size(to, n, false))
94 		n = _copy_from_user_key(to, from, n, key);
95 	return n;
96 }
97 
98 unsigned long __must_check
99 _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key);
100 
101 static __always_inline unsigned long __must_check
102 copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key)
103 {
104 	if (check_copy_size(from, n, true))
105 		n = _copy_to_user_key(to, from, n, key);
106 	return n;
107 }
108 
109 int __noreturn __put_user_bad(void);
110 
111 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
112 
113 #define DEFINE_PUT_USER_NOINSTR(type)					\
114 static uaccess_kmsan_or_inline int					\
115 __put_user_##type##_noinstr(unsigned type __user *to,			\
116 			    unsigned type *from,			\
117 			    unsigned long size)				\
118 {									\
119 	asm goto(							\
120 		"	llilh	%%r0,%[spec]\n"				\
121 		"0:	mvcos	%[to],%[from],%[size]\n"		\
122 		EX_TABLE_INSN(0b, %l[Efault])				\
123 		: [to] "+Q" (*to)					\
124 		: [size] "d" (size), [from] "Q" (*from),		\
125 		  [spec] "I" (0x81)					\
126 		: "cc", "0"						\
127 		: Efault						\
128 		);							\
129 	return 0;							\
130 Efault:									\
131 	return -EFAULT;							\
132 }
133 
134 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
135 
136 #define DEFINE_PUT_USER_NOINSTR(type)					\
137 static uaccess_kmsan_or_inline int					\
138 __put_user_##type##_noinstr(unsigned type __user *to,			\
139 			    unsigned type *from,			\
140 			    unsigned long size)				\
141 {									\
142 	int rc;								\
143 									\
144 	asm_inline volatile(						\
145 		"	llilh	%%r0,%[spec]\n"				\
146 		"0:	mvcos	%[to],%[from],%[size]\n"		\
147 		"	lhi	%[rc],0\n"				\
148 		"1:\n"							\
149 		EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc])			\
150 		: [rc] "=d" (rc), [to] "+Q" (*to)			\
151 		: [size] "d" (size), [from] "Q" (*from),		\
152 		  [spec] "I" (0x81)					\
153 		: "cc", "0");						\
154 	return rc;							\
155 }
156 
157 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
158 
159 DEFINE_PUT_USER_NOINSTR(char);
160 DEFINE_PUT_USER_NOINSTR(short);
161 DEFINE_PUT_USER_NOINSTR(int);
162 DEFINE_PUT_USER_NOINSTR(long);
163 
164 #define DEFINE_PUT_USER(type)						\
165 static __always_inline int						\
166 __put_user_##type(unsigned type __user *to, unsigned type *from,	\
167 		  unsigned long size)					\
168 {									\
169 	int rc;								\
170 									\
171 	rc = __put_user_##type##_noinstr(to, from, size);		\
172 	instrument_put_user(*from, to, size);				\
173 	return rc;							\
174 }
175 
176 DEFINE_PUT_USER(char);
177 DEFINE_PUT_USER(short);
178 DEFINE_PUT_USER(int);
179 DEFINE_PUT_USER(long);
180 
181 #define __put_user(x, ptr)						\
182 ({									\
183 	__typeof__(*(ptr)) __x = (x);					\
184 	int __prc;							\
185 									\
186 	__chk_user_ptr(ptr);						\
187 	switch (sizeof(*(ptr))) {					\
188 	case 1:								\
189 		__prc = __put_user_char((unsigned char __user *)(ptr),	\
190 					(unsigned char *)&__x,		\
191 					sizeof(*(ptr)));		\
192 		break;							\
193 	case 2:								\
194 		__prc = __put_user_short((unsigned short __user *)(ptr),\
195 					 (unsigned short *)&__x,	\
196 					 sizeof(*(ptr)));		\
197 		break;							\
198 	case 4:								\
199 		__prc = __put_user_int((unsigned int __user *)(ptr),	\
200 				       (unsigned int *)&__x,		\
201 				       sizeof(*(ptr)));			\
202 		break;							\
203 	case 8:								\
204 		__prc = __put_user_long((unsigned long __user *)(ptr),	\
205 					(unsigned long *)&__x,		\
206 					sizeof(*(ptr)));		\
207 		break;							\
208 	default:							\
209 		__prc = __put_user_bad();				\
210 		break;							\
211 	}								\
212 	__builtin_expect(__prc, 0);					\
213 })
214 
215 #define put_user(x, ptr)						\
216 ({									\
217 	might_fault();							\
218 	__put_user(x, ptr);						\
219 })
220 
221 int __noreturn __get_user_bad(void);
222 
223 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
224 
225 #define DEFINE_GET_USER_NOINSTR(type)					\
226 static uaccess_kmsan_or_inline int					\
227 __get_user_##type##_noinstr(unsigned type *to,				\
228 			    const unsigned type __user *from,		\
229 			    unsigned long size)				\
230 {									\
231 	asm goto(							\
232 		"	lhi	%%r0,%[spec]\n"				\
233 		"0:	mvcos	%[to],%[from],%[size]\n"		\
234 		EX_TABLE_INSN(0b, %l[Efault])				\
235 		: [to] "=Q" (*to)					\
236 		: [size] "d" (size), [from] "Q" (*from),		\
237 		  [spec] "I" (0x81)					\
238 		: "cc", "0"						\
239 		: Efault						\
240 		);							\
241 	return 0;							\
242 Efault:									\
243 	*to = 0;							\
244 	return -EFAULT;							\
245 }
246 
247 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
248 
249 #define DEFINE_GET_USER_NOINSTR(type)					\
250 static uaccess_kmsan_or_inline int					\
251 __get_user_##type##_noinstr(unsigned type *to,				\
252 			    const unsigned type __user *from,		\
253 			    unsigned long size)				\
254 {									\
255 	int rc;								\
256 									\
257 	asm_inline volatile(						\
258 		"	lhi	%%r0,%[spec]\n"				\
259 		"0:	mvcos	%[to],%[from],%[size]\n"		\
260 		"	lhi	%[rc],0\n"				\
261 		"1:\n"							\
262 		EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc])			\
263 		: [rc] "=d" (rc), [to] "=Q" (*to)			\
264 		: [size] "d" (size), [from] "Q" (*from),		\
265 		  [spec] "I" (0x81)					\
266 		: "cc", "0");						\
267 	if (likely(!rc))						\
268 		return 0;						\
269 	*to = 0;							\
270 	return rc;							\
271 }
272 
273 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
274 
275 DEFINE_GET_USER_NOINSTR(char);
276 DEFINE_GET_USER_NOINSTR(short);
277 DEFINE_GET_USER_NOINSTR(int);
278 DEFINE_GET_USER_NOINSTR(long);
279 
280 #define DEFINE_GET_USER(type)						\
281 static __always_inline int						\
282 __get_user_##type(unsigned type *to, const unsigned type __user *from,	\
283 		  unsigned long size)					\
284 {									\
285 	int rc;								\
286 									\
287 	rc = __get_user_##type##_noinstr(to, from, size);		\
288 	instrument_get_user(*to);					\
289 	return rc;							\
290 }
291 
292 DEFINE_GET_USER(char);
293 DEFINE_GET_USER(short);
294 DEFINE_GET_USER(int);
295 DEFINE_GET_USER(long);
296 
297 #define __get_user(x, ptr)						\
298 ({									\
299 	const __user void *____guptr = (ptr);				\
300 	int __grc;							\
301 									\
302 	__chk_user_ptr(ptr);						\
303 	switch (sizeof(*(ptr))) {					\
304 	case 1: {							\
305 		const unsigned char __user *__guptr = ____guptr;	\
306 		unsigned char __x;					\
307 									\
308 		__grc = __get_user_char(&__x, __guptr, sizeof(*(ptr)));	\
309 		(x) = *(__force __typeof__(*(ptr)) *)&__x;		\
310 		break;							\
311 	};								\
312 	case 2: {							\
313 		const unsigned short __user *__guptr = ____guptr;	\
314 		unsigned short __x;					\
315 									\
316 		__grc = __get_user_short(&__x, __guptr, sizeof(*(ptr)));\
317 		(x) = *(__force __typeof__(*(ptr)) *)&__x;		\
318 		break;							\
319 	};								\
320 	case 4: {							\
321 		const unsigned int __user *__guptr = ____guptr;		\
322 		unsigned int __x;					\
323 									\
324 		__grc = __get_user_int(&__x, __guptr, sizeof(*(ptr)));	\
325 		(x) = *(__force __typeof__(*(ptr)) *)&__x;		\
326 		break;							\
327 	};								\
328 	case 8: {							\
329 		const unsigned long __user *__guptr = ____guptr;	\
330 		unsigned long __x;					\
331 									\
332 		__grc = __get_user_long(&__x, __guptr, sizeof(*(ptr)));	\
333 		(x) = *(__force __typeof__(*(ptr)) *)&__x;		\
334 		break;							\
335 	};								\
336 	default:							\
337 		__grc = __get_user_bad();				\
338 		break;							\
339 	}								\
340 	__builtin_expect(__grc, 0);					\
341 })
342 
343 #define get_user(x, ptr)						\
344 ({									\
345 	might_fault();							\
346 	__get_user(x, ptr);						\
347 })
348 
349 /*
350  * Copy a null terminated string from userspace.
351  */
352 long __must_check strncpy_from_user(char *dst, const char __user *src, long count);
353 
354 long __must_check strnlen_user(const char __user *src, long count);
355 
356 static uaccess_kmsan_or_inline __must_check unsigned long
357 __clear_user(void __user *to, unsigned long size)
358 {
359 	unsigned long osize;
360 	int cc;
361 
362 	while (1) {
363 		osize = size;
364 		asm_inline volatile(
365 			"	llilh	%%r0,%[spec]\n"
366 			"0:	mvcos	%[to],%[from],%[size]\n"
367 			CC_IPM(cc)
368 			EX_TABLE_INSN_UA_MVCOS_TO(0b, 0b)
369 			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
370 			: [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page)
371 			: CC_CLOBBER_LIST("memory", "0"));
372 		if (__builtin_constant_p(osize) && osize <= 4096)
373 			return osize - size;
374 		if (CC_TRANSFORM(cc) == 0)
375 			return osize - size;
376 		size -= 4096;
377 		to += 4096;
378 	}
379 }
380 
381 static __always_inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
382 {
383 	might_fault();
384 	return __clear_user(to, n);
385 }
386 
387 void *__s390_kernel_write(void *dst, const void *src, size_t size);
388 
389 static inline void *s390_kernel_write(void *dst, const void *src, size_t size)
390 {
391 	if (__is_defined(__DECOMPRESSOR))
392 		return memcpy(dst, src, size);
393 	return __s390_kernel_write(dst, src, size);
394 }
395 
396 void __noreturn __mvc_kernel_nofault_bad(void);
397 
398 #if defined(CONFIG_CC_HAS_ASM_GOTO_OUTPUT) && defined(CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS)
399 
400 #define __mvc_kernel_nofault(dst, src, type, err_label)			\
401 do {									\
402 	switch (sizeof(type)) {						\
403 	case 1:								\
404 	case 2:								\
405 	case 4:								\
406 	case 8:								\
407 		asm goto(						\
408 			"0:	mvc	%O[_dst](%[_len],%R[_dst]),%[_src]\n" \
409 			EX_TABLE_INSN(0b, %l[err_label])		\
410 			: [_dst] "=Q" (*(type *)dst)			\
411 			: [_src] "Q" (*(type *)(src)),			\
412 			  [_len] "I" (sizeof(type))			\
413 			:						\
414 			: err_label);					\
415 		break;							\
416 	default:							\
417 		__mvc_kernel_nofault_bad();				\
418 		break;							\
419 	}								\
420 } while (0)
421 
422 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT) && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
423 
424 #define __mvc_kernel_nofault(dst, src, type, err_label)			\
425 do {									\
426 	type *(__dst) = (type *)(dst);					\
427 	int __rc;							\
428 									\
429 	switch (sizeof(type)) {						\
430 	case 1:								\
431 	case 2:								\
432 	case 4:								\
433 	case 8:								\
434 		asm_inline volatile(					\
435 			"0:	mvc	0(%[_len],%[_dst]),%[_src]\n"	\
436 			"	lhi	%[_rc],0\n"			\
437 			"1:\n"						\
438 			EX_TABLE_INSN_UA_FAULT(0b, 1b, %[_rc])		\
439 			: [_rc] "=d" (__rc),				\
440 			  "=m" (*__dst)					\
441 			: [_src] "Q" (*(type *)(src)),			\
442 			[_dst] "a" (__dst),				\
443 			[_len] "I" (sizeof(type)));			\
444 		if (__rc)						\
445 			goto err_label;					\
446 		break;							\
447 	default:							\
448 		__mvc_kernel_nofault_bad();				\
449 		break;							\
450 	}								\
451 } while (0)
452 
453 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
454 
455 #define arch_get_kernel_nofault __mvc_kernel_nofault
456 #define arch_put_kernel_nofault __mvc_kernel_nofault
457 
458 int __cmpxchg_key1(void *address, unsigned char *uval, unsigned char old,
459 		   unsigned char new, unsigned long key);
460 int __cmpxchg_key2(void *address, unsigned short *uval, unsigned short old,
461 		   unsigned short new, unsigned long key);
462 int __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old,
463 		   unsigned int new, unsigned long key);
464 int __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old,
465 		   unsigned long new, unsigned long key);
466 int __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old,
467 		    __uint128_t new, unsigned long key);
468 
469 #endif /* __S390_UACCESS_H */
470