1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Standard user space access functions based on mvcp/mvcs and doing 4 * interesting things in the secondary space mode. 5 * 6 * Copyright IBM Corp. 2006,2014 7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 8 * Gerald Schaefer (gerald.schaefer@de.ibm.com) 9 */ 10 11 #include <linux/kprobes.h> 12 #include <linux/uaccess.h> 13 #include <linux/export.h> 14 #include <linux/mm.h> 15 #include <asm/asm-extable.h> 16 #include <asm/ctlreg.h> 17 #include <asm/skey.h> 18 19 #ifdef CONFIG_DEBUG_ENTRY 20 void debug_user_asce(int exit) 21 { 22 struct lowcore *lc = get_lowcore(); 23 struct ctlreg cr1, cr7; 24 25 local_ctl_store(1, &cr1); 26 local_ctl_store(7, &cr7); 27 if (cr1.val == lc->user_asce.val && cr7.val == lc->user_asce.val) 28 return; 29 panic("incorrect ASCE on kernel %s\n" 30 "cr1: %016lx cr7: %016lx\n" 31 "kernel: %016lx user: %016lx\n", 32 exit ? "exit" : "entry", cr1.val, cr7.val, 33 lc->kernel_asce.val, lc->user_asce.val); 34 } 35 #endif /*CONFIG_DEBUG_ENTRY */ 36 37 union oac { 38 unsigned int val; 39 struct { 40 struct { 41 unsigned short key : 4; 42 unsigned short : 4; 43 unsigned short as : 2; 44 unsigned short : 4; 45 unsigned short k : 1; 46 unsigned short a : 1; 47 } oac1; 48 struct { 49 unsigned short key : 4; 50 unsigned short : 4; 51 unsigned short as : 2; 52 unsigned short : 4; 53 unsigned short k : 1; 54 unsigned short a : 1; 55 } oac2; 56 }; 57 }; 58 59 static uaccess_kmsan_or_inline __must_check unsigned long 60 raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key) 61 { 62 unsigned long osize; 63 union oac spec = { 64 .oac2.key = key, 65 .oac2.as = PSW_BITS_AS_SECONDARY, 66 .oac2.k = 1, 67 .oac2.a = 1, 68 }; 69 int cc; 70 71 while (1) { 72 osize = size; 73 asm_inline volatile( 74 " lr %%r0,%[spec]\n" 75 "0: mvcos %[to],%[from],%[size]\n" 76 "1: nopr %%r7\n" 77 CC_IPM(cc) 78 EX_TABLE_UA_MVCOS_FROM(0b, 0b) 79 EX_TABLE_UA_MVCOS_FROM(1b, 0b) 80 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to) 81 : [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from) 82 : CC_CLOBBER_LIST("memory", "0")); 83 if (CC_TRANSFORM(cc) == 0) 84 return osize - size; 85 size -= 4096; 86 to += 4096; 87 from += 4096; 88 } 89 } 90 91 unsigned long _copy_from_user_key(void *to, const void __user *from, 92 unsigned long n, unsigned long key) 93 { 94 unsigned long res = n; 95 96 might_fault(); 97 if (!should_fail_usercopy()) { 98 instrument_copy_from_user_before(to, from, n); 99 res = raw_copy_from_user_key(to, from, n, key); 100 instrument_copy_from_user_after(to, from, n, res); 101 } 102 if (unlikely(res)) 103 memset(to + (n - res), 0, res); 104 return res; 105 } 106 EXPORT_SYMBOL(_copy_from_user_key); 107 108 static uaccess_kmsan_or_inline __must_check unsigned long 109 raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key) 110 { 111 unsigned long osize; 112 union oac spec = { 113 .oac1.key = key, 114 .oac1.as = PSW_BITS_AS_SECONDARY, 115 .oac1.k = 1, 116 .oac1.a = 1, 117 }; 118 int cc; 119 120 while (1) { 121 osize = size; 122 asm_inline volatile( 123 " lr %%r0,%[spec]\n" 124 "0: mvcos %[to],%[from],%[size]\n" 125 "1: nopr %%r7\n" 126 CC_IPM(cc) 127 EX_TABLE_UA_MVCOS_TO(0b, 0b) 128 EX_TABLE_UA_MVCOS_TO(1b, 0b) 129 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) 130 : [spec] "d" (spec.val), [from] "Q" (*(const char *)from) 131 : CC_CLOBBER_LIST("memory", "0")); 132 if (CC_TRANSFORM(cc) == 0) 133 return osize - size; 134 size -= 4096; 135 to += 4096; 136 from += 4096; 137 } 138 } 139 140 unsigned long _copy_to_user_key(void __user *to, const void *from, 141 unsigned long n, unsigned long key) 142 { 143 might_fault(); 144 if (should_fail_usercopy()) 145 return n; 146 instrument_copy_to_user(to, from, n); 147 return raw_copy_to_user_key(to, from, n, key); 148 } 149 EXPORT_SYMBOL(_copy_to_user_key); 150 151 #define CMPXCHG_USER_KEY_MAX_LOOPS 128 152 153 static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsigned int *uval, 154 unsigned int old, unsigned int new, 155 unsigned int mask, unsigned long key) 156 { 157 unsigned long count; 158 unsigned int prev; 159 bool sacf_flag; 160 int rc = 0; 161 162 skey_regions_initialize(); 163 sacf_flag = enable_sacf_uaccess(); 164 asm_inline volatile( 165 "20: spka 0(%[key])\n" 166 " sacf 256\n" 167 " llill %[count],%[max_loops]\n" 168 "0: l %[prev],%[address]\n" 169 "1: nr %[prev],%[mask]\n" 170 " xilf %[mask],0xffffffff\n" 171 " or %[new],%[prev]\n" 172 " or %[prev],%[tmp]\n" 173 "2: lr %[tmp],%[prev]\n" 174 "3: cs %[prev],%[new],%[address]\n" 175 "4: jnl 5f\n" 176 " xr %[tmp],%[prev]\n" 177 " xr %[new],%[tmp]\n" 178 " nr %[tmp],%[mask]\n" 179 " jnz 5f\n" 180 " brct %[count],2b\n" 181 "5: sacf 768\n" 182 " spka %[default_key]\n" 183 "21:\n" 184 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) 185 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) 186 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) 187 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) 188 SKEY_REGION(20b, 21b) 189 : [rc] "+&d" (rc), 190 [prev] "=&d" (prev), 191 [address] "+Q" (*(int *)address), 192 [tmp] "+&d" (old), 193 [new] "+&d" (new), 194 [mask] "+&d" (mask), 195 [count] "=a" (count) 196 : [key] "%[count]" (key << 4), 197 [default_key] "J" (PAGE_DEFAULT_KEY), 198 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) 199 : "memory", "cc"); 200 disable_sacf_uaccess(sacf_flag); 201 *uval = prev; 202 if (!count) 203 rc = -EAGAIN; 204 return rc; 205 } 206 207 int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval, 208 unsigned char old, unsigned char new, unsigned long key) 209 { 210 unsigned int prev, shift, mask, _old, _new; 211 int rc; 212 213 shift = (3 ^ (address & 3)) << 3; 214 address ^= address & 3; 215 _old = (unsigned int)old << shift; 216 _new = (unsigned int)new << shift; 217 mask = ~(0xff << shift); 218 rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key); 219 *uval = prev >> shift; 220 return rc; 221 } 222 EXPORT_SYMBOL(__cmpxchg_user_key1); 223 224 int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval, 225 unsigned short old, unsigned short new, unsigned long key) 226 { 227 unsigned int prev, shift, mask, _old, _new; 228 int rc; 229 230 shift = (2 ^ (address & 2)) << 3; 231 address ^= address & 2; 232 _old = (unsigned int)old << shift; 233 _new = (unsigned int)new << shift; 234 mask = ~(0xffff << shift); 235 rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key); 236 *uval = prev >> shift; 237 return rc; 238 } 239 EXPORT_SYMBOL(__cmpxchg_user_key2); 240 241 int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval, 242 unsigned int old, unsigned int new, unsigned long key) 243 { 244 unsigned int prev = old; 245 bool sacf_flag; 246 int rc = 0; 247 248 skey_regions_initialize(); 249 sacf_flag = enable_sacf_uaccess(); 250 asm_inline volatile( 251 "20: spka 0(%[key])\n" 252 " sacf 256\n" 253 "0: cs %[prev],%[new],%[address]\n" 254 "1: sacf 768\n" 255 " spka %[default_key]\n" 256 "21:\n" 257 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 258 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 259 SKEY_REGION(20b, 21b) 260 : [rc] "+&d" (rc), 261 [prev] "+&d" (prev), 262 [address] "+Q" (*(int *)address) 263 : [new] "d" (new), 264 [key] "a" (key << 4), 265 [default_key] "J" (PAGE_DEFAULT_KEY) 266 : "memory", "cc"); 267 disable_sacf_uaccess(sacf_flag); 268 *uval = prev; 269 return rc; 270 } 271 EXPORT_SYMBOL(__cmpxchg_user_key4); 272 273 int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval, 274 unsigned long old, unsigned long new, unsigned long key) 275 { 276 unsigned long prev = old; 277 bool sacf_flag; 278 int rc = 0; 279 280 skey_regions_initialize(); 281 sacf_flag = enable_sacf_uaccess(); 282 asm_inline volatile( 283 "20: spka 0(%[key])\n" 284 " sacf 256\n" 285 "0: csg %[prev],%[new],%[address]\n" 286 "1: sacf 768\n" 287 " spka %[default_key]\n" 288 "21:\n" 289 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 290 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 291 SKEY_REGION(20b, 21b) 292 : [rc] "+&d" (rc), 293 [prev] "+&d" (prev), 294 [address] "+QS" (*(long *)address) 295 : [new] "d" (new), 296 [key] "a" (key << 4), 297 [default_key] "J" (PAGE_DEFAULT_KEY) 298 : "memory", "cc"); 299 disable_sacf_uaccess(sacf_flag); 300 *uval = prev; 301 return rc; 302 } 303 EXPORT_SYMBOL(__cmpxchg_user_key8); 304 305 int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval, 306 __uint128_t old, __uint128_t new, unsigned long key) 307 { 308 __uint128_t prev = old; 309 bool sacf_flag; 310 int rc = 0; 311 312 skey_regions_initialize(); 313 sacf_flag = enable_sacf_uaccess(); 314 asm_inline volatile( 315 "20: spka 0(%[key])\n" 316 " sacf 256\n" 317 "0: cdsg %[prev],%[new],%[address]\n" 318 "1: sacf 768\n" 319 " spka %[default_key]\n" 320 "21:\n" 321 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) 322 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) 323 SKEY_REGION(20b, 21b) 324 : [rc] "+&d" (rc), 325 [prev] "+&d" (prev), 326 [address] "+QS" (*(__int128_t *)address) 327 : [new] "d" (new), 328 [key] "a" (key << 4), 329 [default_key] "J" (PAGE_DEFAULT_KEY) 330 : "memory", "cc"); 331 disable_sacf_uaccess(sacf_flag); 332 *uval = prev; 333 return rc; 334 } 335 EXPORT_SYMBOL(__cmpxchg_user_key16); 336