1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Standard user space access functions based on mvcp/mvcs and doing 4 * interesting things in the secondary space mode. 5 * 6 * Copyright IBM Corp. 2006,2014 7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 8 * Gerald Schaefer (gerald.schaefer@de.ibm.com) 9 */ 10 11 #include <linux/kprobes.h> 12 #include <linux/uaccess.h> 13 #include <linux/export.h> 14 #include <linux/mm.h> 15 #include <asm/asm-extable.h> 16 #include <asm/ctlreg.h> 17 #include <asm/skey.h> 18 19 #ifdef CONFIG_DEBUG_ENTRY 20 void debug_user_asce(int exit) 21 { 22 struct lowcore *lc = get_lowcore(); 23 struct ctlreg cr1, cr7; 24 25 local_ctl_store(1, &cr1); 26 local_ctl_store(7, &cr7); 27 if (cr1.val == lc->user_asce.val && cr7.val == lc->user_asce.val) 28 return; 29 panic("incorrect ASCE on kernel %s\n" 30 "cr1: %016lx cr7: %016lx\n" 31 "kernel: %016lx user: %016lx\n", 32 exit ? "exit" : "entry", cr1.val, cr7.val, 33 lc->kernel_asce.val, lc->user_asce.val); 34 } 35 #endif /*CONFIG_DEBUG_ENTRY */ 36 37 #define CMPXCHG_USER_KEY_MAX_LOOPS 128 38 39 static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval, 40 unsigned int old, unsigned int new, 41 unsigned int mask, unsigned long key) 42 { 43 unsigned long count; 44 unsigned int prev; 45 int rc = 0; 46 47 skey_regions_initialize(); 48 asm_inline volatile( 49 "20: spka 0(%[key])\n" 50 " llill %[count],%[max_loops]\n" 51 "0: l %[prev],%[address]\n" 52 "1: nr %[prev],%[mask]\n" 53 " xilf %[mask],0xffffffff\n" 54 " or %[new],%[prev]\n" 55 " or %[prev],%[tmp]\n" 56 "2: lr %[tmp],%[prev]\n" 57 "3: cs %[prev],%[new],%[address]\n" 58 "4: jnl 5f\n" 59 " xr %[tmp],%[prev]\n" 60 " xr %[new],%[tmp]\n" 61 " nr %[tmp],%[mask]\n" 62 " jnz 5f\n" 63 " brct %[count],2b\n" 64 "5: spka %[default_key]\n" 65 "21:\n" 66 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) 67 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) 68 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) 69 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) 70 SKEY_REGION(20b, 21b) 71 : [rc] "+&d" (rc), 72 [prev] "=&d" (prev), 73 [address] "+Q" (*(int *)address), 74 [tmp] "+&d" (old), 75 [new] "+&d" (new), 76 [mask] "+&d" (mask), 77 [count] "=a" (count) 78 : [key] "%[count]" (key << 4), 79 [default_key] "J" (PAGE_DEFAULT_KEY), 80 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) 81 : "memory", "cc"); 82 *uval = prev; 83 if (!count) 84 rc = -EAGAIN; 85 return rc; 86 } 87 88 int __kprobes __cmpxchg_key1(void *addr, unsigned char *uval, unsigned char old, 89 unsigned char new, unsigned long key) 90 { 91 unsigned long address = (unsigned long)addr; 92 unsigned int prev, shift, mask, _old, _new; 93 int rc; 94 95 shift = (3 ^ (address & 3)) << 3; 96 address ^= address & 3; 97 _old = (unsigned int)old << shift; 98 _new = (unsigned int)new << shift; 99 mask = ~(0xff << shift); 100 rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key); 101 *uval = prev >> shift; 102 return rc; 103 } 104 EXPORT_SYMBOL(__cmpxchg_key1); 105 106 int __kprobes __cmpxchg_key2(void *addr, unsigned short *uval, unsigned short old, 107 unsigned short new, unsigned long key) 108 { 109 unsigned long address = (unsigned long)addr; 110 unsigned int prev, shift, mask, _old, _new; 111 int rc; 112 113 shift = (2 ^ (address & 2)) << 3; 114 address ^= address & 2; 115 _old = (unsigned int)old << shift; 116 _new = (unsigned int)new << shift; 117 mask = ~(0xffff << shift); 118 rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key); 119 *uval = prev >> shift; 120 return rc; 121 } 122 EXPORT_SYMBOL(__cmpxchg_key2); 123 124 int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old, 125 unsigned int new, unsigned long key) 126 { 127 unsigned int prev = old; 128 int rc = 0; 129 130 skey_regions_initialize(); 131 asm_inline volatile( 132 "20: spka 0(%[key])\n" 133 "0: cs %[prev],%[new],%[address]\n" 134 "1: spka %[default_key]\n" 135 "21:\n" 136 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 137 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 138 SKEY_REGION(20b, 21b) 139 : [rc] "+&d" (rc), 140 [prev] "+&d" (prev), 141 [address] "+Q" (*(int *)address) 142 : [new] "d" (new), 143 [key] "a" (key << 4), 144 [default_key] "J" (PAGE_DEFAULT_KEY) 145 : "memory", "cc"); 146 *uval = prev; 147 return rc; 148 } 149 EXPORT_SYMBOL(__cmpxchg_key4); 150 151 int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old, 152 unsigned long new, unsigned long key) 153 { 154 unsigned long prev = old; 155 int rc = 0; 156 157 skey_regions_initialize(); 158 asm_inline volatile( 159 "20: spka 0(%[key])\n" 160 "0: csg %[prev],%[new],%[address]\n" 161 "1: spka %[default_key]\n" 162 "21:\n" 163 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 164 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 165 SKEY_REGION(20b, 21b) 166 : [rc] "+&d" (rc), 167 [prev] "+&d" (prev), 168 [address] "+QS" (*(long *)address) 169 : [new] "d" (new), 170 [key] "a" (key << 4), 171 [default_key] "J" (PAGE_DEFAULT_KEY) 172 : "memory", "cc"); 173 *uval = prev; 174 return rc; 175 } 176 EXPORT_SYMBOL(__cmpxchg_key8); 177 178 int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, 179 __uint128_t new, unsigned long key) 180 { 181 __uint128_t prev = old; 182 int rc = 0; 183 184 skey_regions_initialize(); 185 asm_inline volatile( 186 "20: spka 0(%[key])\n" 187 "0: cdsg %[prev],%[new],%[address]\n" 188 "1: spka %[default_key]\n" 189 "21:\n" 190 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) 191 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) 192 SKEY_REGION(20b, 21b) 193 : [rc] "+&d" (rc), 194 [prev] "+&d" (prev), 195 [address] "+QS" (*(__int128_t *)address) 196 : [new] "d" (new), 197 [key] "a" (key << 4), 198 [default_key] "J" (PAGE_DEFAULT_KEY) 199 : "memory", "cc"); 200 *uval = prev; 201 return rc; 202 } 203 EXPORT_SYMBOL(__cmpxchg_key16); 204