1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_RISCV_RUNTIME_CONST_H 3 #define _ASM_RISCV_RUNTIME_CONST_H 4 5 #ifdef MODULE 6 #error "Cannot use runtime-const infrastructure from modules" 7 #endif 8 9 #include <asm/asm.h> 10 #include <asm/alternative.h> 11 #include <asm/cacheflush.h> 12 #include <asm/insn-def.h> 13 #include <linux/memory.h> 14 #include <asm/text-patching.h> 15 16 #include <linux/uaccess.h> 17 18 #define RUNTIME_MAGIC __ASM_STR(0x89ABCDEF) 19 20 #ifdef CONFIG_32BIT 21 #define runtime_const_ptr(sym) \ 22 ({ \ 23 typeof(sym) __ret; \ 24 asm_inline(".option push\n\t" \ 25 ".option norvc\n\t" \ 26 ".option norelax\n\t" \ 27 "1:\t" \ 28 "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ 29 "addi %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ 30 ".option pop\n\t" \ 31 ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ 32 ".long 1b - .\n\t" \ 33 ".popsection" \ 34 : [__ret] "=r" (__ret)); \ 35 __ret; \ 36 }) 37 #else 38 /* 39 * Loading 64-bit constants into a register from immediates is a non-trivial 40 * task on riscv64. To get it somewhat performant, load 32 bits into two 41 * different registers and then combine the results. 42 * 43 * If the processor supports the Zbkb extension, we can combine the final 44 * "slli,slli,srli,add" into the single "pack" instruction. If the processor 45 * doesn't support Zbkb but does support the Zbb extension, we can 46 * combine the final "slli,srli,add" into one instruction "add.uw". 47 */ 48 #define RISCV_RUNTIME_CONST_64_PREAMBLE \ 49 ".option push\n\t" \ 50 ".option norvc\n\t" \ 51 ".option norelax\n\t" \ 52 "1:\t" \ 53 "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ 54 "lui %[__tmp], %%hi(" RUNTIME_MAGIC ")\n\t" \ 55 "addiw %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ 56 "addiw %[__tmp],%[__tmp], %%lo(" RUNTIME_MAGIC ")\n\t" \ 57 58 #define RISCV_RUNTIME_CONST_64_BASE \ 59 "slli %[__tmp],%[__tmp],32\n\t" \ 60 "slli %[__ret],%[__ret],32\n\t" \ 61 "srli %[__ret],%[__ret],32\n\t" \ 62 "add %[__ret],%[__ret],%[__tmp]\n\t" \ 63 64 #define RISCV_RUNTIME_CONST_64_ZBA \ 65 ".option push\n\t" \ 66 ".option arch,+zba\n\t" \ 67 ".option norvc\n\t" \ 68 "slli %[__tmp],%[__tmp],32\n\t" \ 69 "add.uw %[__ret],%[__ret],%[__tmp]\n\t" \ 70 "nop\n\t" \ 71 "nop\n\t" \ 72 ".option pop\n\t" \ 73 74 #define RISCV_RUNTIME_CONST_64_ZBKB \ 75 ".option push\n\t" \ 76 ".option arch,+zbkb\n\t" \ 77 ".option norvc\n\t" \ 78 "pack %[__ret],%[__ret],%[__tmp]\n\t" \ 79 "nop\n\t" \ 80 "nop\n\t" \ 81 "nop\n\t" \ 82 ".option pop\n\t" \ 83 84 #define RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ 85 ".option pop\n\t" \ 86 ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ 87 ".long 1b - .\n\t" \ 88 ".popsection" \ 89 90 #if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) \ 91 && defined(CONFIG_RISCV_ISA_ZBKB) 92 #define runtime_const_ptr(sym) \ 93 ({ \ 94 typeof(sym) __ret, __tmp; \ 95 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ 96 ALTERNATIVE_2( \ 97 RISCV_RUNTIME_CONST_64_BASE, \ 98 RISCV_RUNTIME_CONST_64_ZBA, \ 99 0, RISCV_ISA_EXT_ZBA, 1, \ 100 RISCV_RUNTIME_CONST_64_ZBKB, \ 101 0, RISCV_ISA_EXT_ZBKB, 1 \ 102 ) \ 103 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ 104 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ 105 __ret; \ 106 }) 107 #elif defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) 108 #define runtime_const_ptr(sym) \ 109 ({ \ 110 typeof(sym) __ret, __tmp; \ 111 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ 112 ALTERNATIVE( \ 113 RISCV_RUNTIME_CONST_64_BASE, \ 114 RISCV_RUNTIME_CONST_64_ZBA, \ 115 0, RISCV_ISA_EXT_ZBA, 1 \ 116 ) \ 117 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ 118 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ 119 __ret; \ 120 }) 121 #elif defined(CONFIG_RISCV_ISA_ZBKB) 122 #define runtime_const_ptr(sym) \ 123 ({ \ 124 typeof(sym) __ret, __tmp; \ 125 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ 126 ALTERNATIVE( \ 127 RISCV_RUNTIME_CONST_64_BASE, \ 128 RISCV_RUNTIME_CONST_64_ZBKB, \ 129 0, RISCV_ISA_EXT_ZBKB, 1 \ 130 ) \ 131 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ 132 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ 133 __ret; \ 134 }) 135 #else 136 #define runtime_const_ptr(sym) \ 137 ({ \ 138 typeof(sym) __ret, __tmp; \ 139 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ 140 RISCV_RUNTIME_CONST_64_BASE \ 141 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ 142 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ 143 __ret; \ 144 }) 145 #endif 146 #endif 147 148 #define runtime_const_shift_right_32(val, sym) \ 149 ({ \ 150 u32 __ret; \ 151 asm_inline(".option push\n\t" \ 152 ".option norvc\n\t" \ 153 "1:\t" \ 154 SRLI " %[__ret],%[__val],12\n\t" \ 155 ".option pop\n\t" \ 156 ".pushsection runtime_shift_" #sym ",\"a\"\n\t" \ 157 ".long 1b - .\n\t" \ 158 ".popsection" \ 159 : [__ret] "=r" (__ret) \ 160 : [__val] "r" (val)); \ 161 __ret; \ 162 }) 163 164 #define runtime_const_mask_32(val, sym) \ 165 ({ \ 166 u32 __ret; \ 167 asm_inline(".option push\n\t" \ 168 ".option norvc\n\t" \ 169 "1:\t" \ 170 SLLI " %[__ret],%[__val],12\n\t" \ 171 SRLI " %[__ret],%[__ret],12\n\t" \ 172 ".option pop\n\t" \ 173 ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \ 174 ".long 1b - .\n\t" \ 175 ".popsection" \ 176 : [__ret] "=r" (__ret) \ 177 : [__val] "r" (val)); \ 178 __ret; \ 179 }) 180 181 #define runtime_const_init(type, sym) do { \ 182 extern s32 __start_runtime_##type##_##sym[]; \ 183 extern s32 __stop_runtime_##type##_##sym[]; \ 184 \ 185 runtime_const_fixup(__runtime_fixup_##type, \ 186 (unsigned long)(sym), \ 187 __start_runtime_##type##_##sym, \ 188 __stop_runtime_##type##_##sym); \ 189 } while (0) 190 191 static inline void __runtime_fixup_caches(void *where, unsigned int insns) 192 { 193 /* On riscv there are currently only cache-wide flushes so va is ignored. */ 194 __always_unused uintptr_t va = (uintptr_t)where; 195 196 flush_icache_range(va, va + 4 * insns); 197 } 198 199 /* 200 * The 32-bit immediate is stored in a lui+addi pairing. 201 * lui holds the upper 20 bits of the immediate in the first 20 bits of the instruction. 202 * addi holds the lower 12 bits of the immediate in the first 12 bits of the instruction. 203 */ 204 static inline void __runtime_fixup_32(__le16 *lui_parcel, __le16 *addi_parcel, unsigned int val) 205 { 206 unsigned int lower_immediate, upper_immediate; 207 u32 lui_insn, addi_insn, addi_insn_mask; 208 __le32 lui_res, addi_res; 209 210 /* Mask out upper 12 bit of addi */ 211 addi_insn_mask = 0x000fffff; 212 213 lui_insn = (u32)le16_to_cpu(lui_parcel[0]) | (u32)le16_to_cpu(lui_parcel[1]) << 16; 214 addi_insn = (u32)le16_to_cpu(addi_parcel[0]) | (u32)le16_to_cpu(addi_parcel[1]) << 16; 215 216 lower_immediate = sign_extend32(val, 11); 217 upper_immediate = (val - lower_immediate); 218 219 if (upper_immediate & 0xfffff000) { 220 /* replace upper 20 bits of lui with upper immediate */ 221 lui_insn &= 0x00000fff; 222 lui_insn |= upper_immediate & 0xfffff000; 223 } else { 224 /* replace lui with nop if immediate is small enough to fit in addi */ 225 lui_insn = RISCV_INSN_NOP4; 226 /* 227 * lui is being skipped, so do a load instead of an add. A load 228 * is performed by adding with the x0 register. Setting rs to 229 * zero with the following mask will accomplish this goal. 230 */ 231 addi_insn_mask &= 0x07fff; 232 } 233 234 if (lower_immediate & 0x00000fff || lui_insn == RISCV_INSN_NOP4) { 235 /* replace upper 12 bits of addi with lower 12 bits of val */ 236 addi_insn &= addi_insn_mask; 237 addi_insn |= (lower_immediate & 0x00000fff) << 20; 238 } else { 239 /* replace addi with nop if lower_immediate is empty */ 240 addi_insn = RISCV_INSN_NOP4; 241 } 242 243 addi_res = cpu_to_le32(addi_insn); 244 lui_res = cpu_to_le32(lui_insn); 245 mutex_lock(&text_mutex); 246 patch_insn_write(addi_parcel, &addi_res, sizeof(addi_res)); 247 patch_insn_write(lui_parcel, &lui_res, sizeof(lui_res)); 248 mutex_unlock(&text_mutex); 249 } 250 251 static inline void __runtime_fixup_ptr(void *where, unsigned long val) 252 { 253 #ifdef CONFIG_32BIT 254 __runtime_fixup_32(where, where + 4, val); 255 __runtime_fixup_caches(where, 2); 256 #else 257 __runtime_fixup_32(where, where + 8, val); 258 __runtime_fixup_32(where + 4, where + 12, val >> 32); 259 __runtime_fixup_caches(where, 4); 260 #endif 261 } 262 263 /* 264 * Replace the least significant 5 bits of the srli/srliw immediate that is 265 * located at bits 20-24 266 */ 267 static inline void __runtime_fixup_shift(void *where, unsigned long val) 268 { 269 __le16 *parcel = where; 270 __le32 res; 271 u32 insn; 272 273 insn = (u32)le16_to_cpu(parcel[0]) | (u32)le16_to_cpu(parcel[1]) << 16; 274 275 insn &= 0xfe0fffff; 276 insn |= (val & 0b11111) << 20; 277 278 res = cpu_to_le32(insn); 279 mutex_lock(&text_mutex); 280 patch_text_nosync(where, &res, sizeof(insn)); 281 mutex_unlock(&text_mutex); 282 } 283 284 static inline void __runtime_fixup_mask(void *where, unsigned long val) 285 { 286 unsigned int width = (val) ? __fls(val) + 1 : 0; 287 288 /* 289 * XXX: Current implementation only supports patching masks of 290 * form GENMASK(width, 0) (width >= 0) using a SRLI + SLLI 291 * sequence instead of LUI + ADDI + AND sequence to improve 292 * performance, density, and covers all the current use-cases. 293 * 294 * When the need arises to support any generic mask, and this 295 * BUG_ON() is tripped, consider using a: 296 * 297 * lui %[__ret], #imm16 298 * addi %[__ret], #imm16 299 * 300 * sequence to load the 32bit const mask, and perform a logical 301 * and outside the asm block before returning the result. Fixup 302 * can simply reuse the existing __runtime_fixup_32() to patch 303 * the LUI + ADDI sequence. 304 */ 305 BUG_ON(!val || width > 31 || (GENMASK(width - 1, 0) != val)); 306 307 __runtime_fixup_shift(where, 32 - width); 308 __runtime_fixup_shift(where + 4, 32 - width); 309 } 310 311 static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), 312 unsigned long val, s32 *start, s32 *end) 313 { 314 while (start < end) { 315 fn(*start + (void *)start, val); 316 start++; 317 } 318 } 319 320 #endif /* _ASM_RISCV_RUNTIME_CONST_H */ 321