1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Standard user space access functions based on mvcp/mvcs and doing 4 * interesting things in the secondary space mode. 5 * 6 * Copyright IBM Corp. 2006,2014 7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 8 * Gerald Schaefer (gerald.schaefer@de.ibm.com) 9 */ 10 11 #include <linux/kprobes.h> 12 #include <linux/uaccess.h> 13 #include <linux/export.h> 14 #include <linux/mm.h> 15 #include <asm/asm-extable.h> 16 #include <asm/ctlreg.h> 17 #include <asm/skey.h> 18 19 #ifdef CONFIG_DEBUG_ENTRY 20 void debug_user_asce(int exit) 21 { 22 struct lowcore *lc = get_lowcore(); 23 struct ctlreg cr1, cr7; 24 25 local_ctl_store(1, &cr1); 26 local_ctl_store(7, &cr7); 27 if (cr1.val == lc->user_asce.val && cr7.val == lc->user_asce.val) 28 return; 29 panic("incorrect ASCE on kernel %s\n" 30 "cr1: %016lx cr7: %016lx\n" 31 "kernel: %016lx user: %016lx\n", 32 exit ? "exit" : "entry", cr1.val, cr7.val, 33 lc->kernel_asce.val, lc->user_asce.val); 34 } 35 #endif /*CONFIG_DEBUG_ENTRY */ 36 37 #define CMPXCHG_USER_KEY_MAX_LOOPS 128 38 39 static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval, 40 unsigned int old, unsigned int new, 41 unsigned int mask, unsigned long key) 42 { 43 unsigned long count; 44 unsigned int prev; 45 int rc = 0; 46 47 skey_regions_initialize(); 48 asm_inline volatile( 49 "20: spka 0(%[key])\n" 50 " llill %[count],%[max_loops]\n" 51 "0: l %[prev],%[address]\n" 52 " nr %[prev],%[mask]\n" 53 " xilf %[mask],0xffffffff\n" 54 " or %[new],%[prev]\n" 55 " or %[prev],%[tmp]\n" 56 "1: lr %[tmp],%[prev]\n" 57 "2: cs %[prev],%[new],%[address]\n" 58 " jnl 3f\n" 59 " xr %[tmp],%[prev]\n" 60 " xr %[new],%[tmp]\n" 61 " nr %[tmp],%[mask]\n" 62 " jnz 3f\n" 63 " brct %[count],1b\n" 64 "3: spka %[default_key]\n" 65 "21:\n" 66 EX_TABLE_INSN_UA_LOAD_REG(0b, 3b, %[rc], %[prev]) 67 EX_TABLE_INSN_UA_LOAD_REG(2b, 3b, %[rc], %[prev]) 68 SKEY_REGION(20b, 21b) 69 : [rc] "+&d" (rc), 70 [prev] "=&d" (prev), 71 [address] "+Q" (*(int *)address), 72 [tmp] "+&d" (old), 73 [new] "+&d" (new), 74 [mask] "+&d" (mask), 75 [count] "=a" (count) 76 : [key] "%[count]" (key << 4), 77 [default_key] "J" (PAGE_DEFAULT_KEY), 78 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) 79 : "memory", "cc"); 80 *uval = prev; 81 if (!count) 82 rc = -EAGAIN; 83 return rc; 84 } 85 86 int __kprobes __cmpxchg_key1(void *addr, unsigned char *uval, unsigned char old, 87 unsigned char new, unsigned long key) 88 { 89 unsigned long address = (unsigned long)addr; 90 unsigned int prev, shift, mask, _old, _new; 91 int rc; 92 93 shift = (3 ^ (address & 3)) << 3; 94 address ^= address & 3; 95 _old = (unsigned int)old << shift; 96 _new = (unsigned int)new << shift; 97 mask = ~(0xff << shift); 98 rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key); 99 *uval = prev >> shift; 100 return rc; 101 } 102 EXPORT_SYMBOL(__cmpxchg_key1); 103 104 int __kprobes __cmpxchg_key2(void *addr, unsigned short *uval, unsigned short old, 105 unsigned short new, unsigned long key) 106 { 107 unsigned long address = (unsigned long)addr; 108 unsigned int prev, shift, mask, _old, _new; 109 int rc; 110 111 shift = (2 ^ (address & 2)) << 3; 112 address ^= address & 2; 113 _old = (unsigned int)old << shift; 114 _new = (unsigned int)new << shift; 115 mask = ~(0xffff << shift); 116 rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key); 117 *uval = prev >> shift; 118 return rc; 119 } 120 EXPORT_SYMBOL(__cmpxchg_key2); 121 122 int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old, 123 unsigned int new, unsigned long key) 124 { 125 unsigned int prev = old; 126 int rc = 0; 127 128 skey_regions_initialize(); 129 asm_inline volatile( 130 "20: spka 0(%[key])\n" 131 "0: cs %[prev],%[new],%[address]\n" 132 "1: spka %[default_key]\n" 133 "21:\n" 134 EX_TABLE_INSN_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 135 SKEY_REGION(20b, 21b) 136 : [rc] "+&d" (rc), 137 [prev] "+&d" (prev), 138 [address] "+Q" (*(int *)address) 139 : [new] "d" (new), 140 [key] "a" (key << 4), 141 [default_key] "J" (PAGE_DEFAULT_KEY) 142 : "memory", "cc"); 143 *uval = prev; 144 return rc; 145 } 146 EXPORT_SYMBOL(__cmpxchg_key4); 147 148 int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old, 149 unsigned long new, unsigned long key) 150 { 151 unsigned long prev = old; 152 int rc = 0; 153 154 skey_regions_initialize(); 155 asm_inline volatile( 156 "20: spka 0(%[key])\n" 157 "0: csg %[prev],%[new],%[address]\n" 158 "1: spka %[default_key]\n" 159 "21:\n" 160 EX_TABLE_INSN_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 161 SKEY_REGION(20b, 21b) 162 : [rc] "+&d" (rc), 163 [prev] "+&d" (prev), 164 [address] "+QS" (*(long *)address) 165 : [new] "d" (new), 166 [key] "a" (key << 4), 167 [default_key] "J" (PAGE_DEFAULT_KEY) 168 : "memory", "cc"); 169 *uval = prev; 170 return rc; 171 } 172 EXPORT_SYMBOL(__cmpxchg_key8); 173 174 int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, 175 __uint128_t new, unsigned long key) 176 { 177 __uint128_t prev = old; 178 int rc = 0; 179 180 skey_regions_initialize(); 181 asm_inline volatile( 182 "20: spka 0(%[key])\n" 183 "0: cdsg %[prev],%[new],%[address]\n" 184 "1: spka %[default_key]\n" 185 "21:\n" 186 EX_TABLE_INSN_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) 187 SKEY_REGION(20b, 21b) 188 : [rc] "+&d" (rc), 189 [prev] "+&d" (prev), 190 [address] "+QS" (*(__int128_t *)address) 191 : [new] "d" (new), 192 [key] "a" (key << 4), 193 [default_key] "J" (PAGE_DEFAULT_KEY) 194 : "memory", "cc"); 195 *uval = prev; 196 return rc; 197 } 198 EXPORT_SYMBOL(__cmpxchg_key16); 199