1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Standard user space access functions based on mvcp/mvcs and doing
4 * interesting things in the secondary space mode.
5 *
6 * Copyright IBM Corp. 2006,2014
7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
8 * Gerald Schaefer (gerald.schaefer@de.ibm.com)
9 */
10
11 #include <linux/kprobes.h>
12 #include <linux/uaccess.h>
13 #include <linux/export.h>
14 #include <linux/mm.h>
15 #include <asm/asm-extable.h>
16 #include <asm/ctlreg.h>
17 #include <asm/skey.h>
18
19 #ifdef CONFIG_DEBUG_ENTRY
debug_user_asce(int exit)20 void debug_user_asce(int exit)
21 {
22 struct lowcore *lc = get_lowcore();
23 struct ctlreg cr1, cr7;
24
25 local_ctl_store(1, &cr1);
26 local_ctl_store(7, &cr7);
27 if (cr1.val == lc->user_asce.val && cr7.val == lc->user_asce.val)
28 return;
29 panic("incorrect ASCE on kernel %s\n"
30 "cr1: %016lx cr7: %016lx\n"
31 "kernel: %016lx user: %016lx\n",
32 exit ? "exit" : "entry", cr1.val, cr7.val,
33 lc->kernel_asce.val, lc->user_asce.val);
34 }
35 #endif /*CONFIG_DEBUG_ENTRY */
36
37 union oac {
38 unsigned int val;
39 struct {
40 struct {
41 unsigned short key : 4;
42 unsigned short : 4;
43 unsigned short as : 2;
44 unsigned short : 4;
45 unsigned short k : 1;
46 unsigned short a : 1;
47 } oac1;
48 struct {
49 unsigned short key : 4;
50 unsigned short : 4;
51 unsigned short as : 2;
52 unsigned short : 4;
53 unsigned short k : 1;
54 unsigned short a : 1;
55 } oac2;
56 };
57 };
58
59 static uaccess_kmsan_or_inline __must_check unsigned long
raw_copy_from_user_key(void * to,const void __user * from,unsigned long size,unsigned long key)60 raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
61 {
62 unsigned long osize;
63 union oac spec = {
64 .oac2.key = key,
65 .oac2.as = PSW_BITS_AS_SECONDARY,
66 .oac2.k = 1,
67 .oac2.a = 1,
68 };
69 int cc;
70
71 while (1) {
72 osize = size;
73 asm_inline volatile(
74 " lr %%r0,%[spec]\n"
75 "0: mvcos %[to],%[from],%[size]\n"
76 "1: nopr %%r7\n"
77 CC_IPM(cc)
78 EX_TABLE_UA_MVCOS_FROM(0b, 0b)
79 EX_TABLE_UA_MVCOS_FROM(1b, 0b)
80 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
81 : [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
82 : CC_CLOBBER_LIST("memory", "0"));
83 if (CC_TRANSFORM(cc) == 0)
84 return osize - size;
85 size -= 4096;
86 to += 4096;
87 from += 4096;
88 }
89 }
90
_copy_from_user_key(void * to,const void __user * from,unsigned long n,unsigned long key)91 unsigned long _copy_from_user_key(void *to, const void __user *from,
92 unsigned long n, unsigned long key)
93 {
94 unsigned long res = n;
95
96 might_fault();
97 if (!should_fail_usercopy()) {
98 instrument_copy_from_user_before(to, from, n);
99 res = raw_copy_from_user_key(to, from, n, key);
100 instrument_copy_from_user_after(to, from, n, res);
101 }
102 if (unlikely(res))
103 memset(to + (n - res), 0, res);
104 return res;
105 }
106 EXPORT_SYMBOL(_copy_from_user_key);
107
108 static uaccess_kmsan_or_inline __must_check unsigned long
raw_copy_to_user_key(void __user * to,const void * from,unsigned long size,unsigned long key)109 raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
110 {
111 unsigned long osize;
112 union oac spec = {
113 .oac1.key = key,
114 .oac1.as = PSW_BITS_AS_SECONDARY,
115 .oac1.k = 1,
116 .oac1.a = 1,
117 };
118 int cc;
119
120 while (1) {
121 osize = size;
122 asm_inline volatile(
123 " lr %%r0,%[spec]\n"
124 "0: mvcos %[to],%[from],%[size]\n"
125 "1: nopr %%r7\n"
126 CC_IPM(cc)
127 EX_TABLE_UA_MVCOS_TO(0b, 0b)
128 EX_TABLE_UA_MVCOS_TO(1b, 0b)
129 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
130 : [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
131 : CC_CLOBBER_LIST("memory", "0"));
132 if (CC_TRANSFORM(cc) == 0)
133 return osize - size;
134 size -= 4096;
135 to += 4096;
136 from += 4096;
137 }
138 }
139
_copy_to_user_key(void __user * to,const void * from,unsigned long n,unsigned long key)140 unsigned long _copy_to_user_key(void __user *to, const void *from,
141 unsigned long n, unsigned long key)
142 {
143 might_fault();
144 if (should_fail_usercopy())
145 return n;
146 instrument_copy_to_user(to, from, n);
147 return raw_copy_to_user_key(to, from, n, key);
148 }
149 EXPORT_SYMBOL(_copy_to_user_key);
150
151 #define CMPXCHG_USER_KEY_MAX_LOOPS 128
152
__cmpxchg_user_key_small(unsigned long address,unsigned int * uval,unsigned int old,unsigned int new,unsigned int mask,unsigned long key)153 static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsigned int *uval,
154 unsigned int old, unsigned int new,
155 unsigned int mask, unsigned long key)
156 {
157 unsigned long count;
158 unsigned int prev;
159 bool sacf_flag;
160 int rc = 0;
161
162 skey_regions_initialize();
163 sacf_flag = enable_sacf_uaccess();
164 asm_inline volatile(
165 "20: spka 0(%[key])\n"
166 " sacf 256\n"
167 " llill %[count],%[max_loops]\n"
168 "0: l %[prev],%[address]\n"
169 "1: nr %[prev],%[mask]\n"
170 " xilf %[mask],0xffffffff\n"
171 " or %[new],%[prev]\n"
172 " or %[prev],%[tmp]\n"
173 "2: lr %[tmp],%[prev]\n"
174 "3: cs %[prev],%[new],%[address]\n"
175 "4: jnl 5f\n"
176 " xr %[tmp],%[prev]\n"
177 " xr %[new],%[tmp]\n"
178 " nr %[tmp],%[mask]\n"
179 " jnz 5f\n"
180 " brct %[count],2b\n"
181 "5: sacf 768\n"
182 " spka %[default_key]\n"
183 "21:\n"
184 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
185 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
186 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
187 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
188 SKEY_REGION(20b, 21b)
189 : [rc] "+&d" (rc),
190 [prev] "=&d" (prev),
191 [address] "+Q" (*(int *)address),
192 [tmp] "+&d" (old),
193 [new] "+&d" (new),
194 [mask] "+&d" (mask),
195 [count] "=a" (count)
196 : [key] "%[count]" (key << 4),
197 [default_key] "J" (PAGE_DEFAULT_KEY),
198 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
199 : "memory", "cc");
200 disable_sacf_uaccess(sacf_flag);
201 *uval = prev;
202 if (!count)
203 rc = -EAGAIN;
204 return rc;
205 }
206
__cmpxchg_user_key1(unsigned long address,unsigned char * uval,unsigned char old,unsigned char new,unsigned long key)207 int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
208 unsigned char old, unsigned char new, unsigned long key)
209 {
210 unsigned int prev, shift, mask, _old, _new;
211 int rc;
212
213 shift = (3 ^ (address & 3)) << 3;
214 address ^= address & 3;
215 _old = (unsigned int)old << shift;
216 _new = (unsigned int)new << shift;
217 mask = ~(0xff << shift);
218 rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key);
219 *uval = prev >> shift;
220 return rc;
221 }
222 EXPORT_SYMBOL(__cmpxchg_user_key1);
223
__cmpxchg_user_key2(unsigned long address,unsigned short * uval,unsigned short old,unsigned short new,unsigned long key)224 int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
225 unsigned short old, unsigned short new, unsigned long key)
226 {
227 unsigned int prev, shift, mask, _old, _new;
228 int rc;
229
230 shift = (2 ^ (address & 2)) << 3;
231 address ^= address & 2;
232 _old = (unsigned int)old << shift;
233 _new = (unsigned int)new << shift;
234 mask = ~(0xffff << shift);
235 rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key);
236 *uval = prev >> shift;
237 return rc;
238 }
239 EXPORT_SYMBOL(__cmpxchg_user_key2);
240
__cmpxchg_user_key4(unsigned long address,unsigned int * uval,unsigned int old,unsigned int new,unsigned long key)241 int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval,
242 unsigned int old, unsigned int new, unsigned long key)
243 {
244 unsigned int prev = old;
245 bool sacf_flag;
246 int rc = 0;
247
248 skey_regions_initialize();
249 sacf_flag = enable_sacf_uaccess();
250 asm_inline volatile(
251 "20: spka 0(%[key])\n"
252 " sacf 256\n"
253 "0: cs %[prev],%[new],%[address]\n"
254 "1: sacf 768\n"
255 " spka %[default_key]\n"
256 "21:\n"
257 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
258 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
259 SKEY_REGION(20b, 21b)
260 : [rc] "+&d" (rc),
261 [prev] "+&d" (prev),
262 [address] "+Q" (*(int *)address)
263 : [new] "d" (new),
264 [key] "a" (key << 4),
265 [default_key] "J" (PAGE_DEFAULT_KEY)
266 : "memory", "cc");
267 disable_sacf_uaccess(sacf_flag);
268 *uval = prev;
269 return rc;
270 }
271 EXPORT_SYMBOL(__cmpxchg_user_key4);
272
__cmpxchg_user_key8(unsigned long address,unsigned long * uval,unsigned long old,unsigned long new,unsigned long key)273 int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval,
274 unsigned long old, unsigned long new, unsigned long key)
275 {
276 unsigned long prev = old;
277 bool sacf_flag;
278 int rc = 0;
279
280 skey_regions_initialize();
281 sacf_flag = enable_sacf_uaccess();
282 asm_inline volatile(
283 "20: spka 0(%[key])\n"
284 " sacf 256\n"
285 "0: csg %[prev],%[new],%[address]\n"
286 "1: sacf 768\n"
287 " spka %[default_key]\n"
288 "21:\n"
289 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
290 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
291 SKEY_REGION(20b, 21b)
292 : [rc] "+&d" (rc),
293 [prev] "+&d" (prev),
294 [address] "+QS" (*(long *)address)
295 : [new] "d" (new),
296 [key] "a" (key << 4),
297 [default_key] "J" (PAGE_DEFAULT_KEY)
298 : "memory", "cc");
299 disable_sacf_uaccess(sacf_flag);
300 *uval = prev;
301 return rc;
302 }
303 EXPORT_SYMBOL(__cmpxchg_user_key8);
304
__cmpxchg_user_key16(unsigned long address,__uint128_t * uval,__uint128_t old,__uint128_t new,unsigned long key)305 int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval,
306 __uint128_t old, __uint128_t new, unsigned long key)
307 {
308 __uint128_t prev = old;
309 bool sacf_flag;
310 int rc = 0;
311
312 skey_regions_initialize();
313 sacf_flag = enable_sacf_uaccess();
314 asm_inline volatile(
315 "20: spka 0(%[key])\n"
316 " sacf 256\n"
317 "0: cdsg %[prev],%[new],%[address]\n"
318 "1: sacf 768\n"
319 " spka %[default_key]\n"
320 "21:\n"
321 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
322 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
323 SKEY_REGION(20b, 21b)
324 : [rc] "+&d" (rc),
325 [prev] "+&d" (prev),
326 [address] "+QS" (*(__int128_t *)address)
327 : [new] "d" (new),
328 [key] "a" (key << 4),
329 [default_key] "J" (PAGE_DEFAULT_KEY)
330 : "memory", "cc");
331 disable_sacf_uaccess(sacf_flag);
332 *uval = prev;
333 return rc;
334 }
335 EXPORT_SYMBOL(__cmpxchg_user_key16);
336