1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_RUNTIME_CONST_H 3 #define _ASM_RUNTIME_CONST_H 4 5 #ifdef MODULE 6 #error "Cannot use runtime-const infrastructure from modules" 7 #endif 8 9 #include <asm/cacheflush.h> 10 #include <asm/text-patching.h> 11 12 /* Sigh. You can still run arm64 in BE mode */ 13 #include <asm/byteorder.h> 14 15 #define runtime_const_ptr(sym) ({ \ 16 typeof(sym) __ret; \ 17 asm_inline("1:\t" \ 18 "movz %0, #0xcdef\n\t" \ 19 "movk %0, #0x89ab, lsl #16\n\t" \ 20 "movk %0, #0x4567, lsl #32\n\t" \ 21 "movk %0, #0x0123, lsl #48\n\t" \ 22 ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ 23 ".long 1b - .\n\t" \ 24 ".popsection" \ 25 :"=r" (__ret)); \ 26 __ret; }) 27 28 #define runtime_const_shift_right_32(val, sym) ({ \ 29 unsigned long __ret; \ 30 asm_inline("1:\t" \ 31 "lsr %w0,%w1,#12\n\t" \ 32 ".pushsection runtime_shift_" #sym ",\"a\"\n\t" \ 33 ".long 1b - .\n\t" \ 34 ".popsection" \ 35 :"=r" (__ret) \ 36 :"r" (0u+(val))); \ 37 __ret; }) 38 39 #define runtime_const_mask_32(val, sym) ({ \ 40 unsigned long __ret; \ 41 asm_inline("1:\t" \ 42 "ubfx %w0, %w1, #0, #32\n\t" \ 43 ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \ 44 ".long 1b - .\n\t" \ 45 ".popsection" \ 46 :"=r" (__ret) \ 47 :"r" (0u+(val))); \ 48 __ret; }) 49 50 #define runtime_const_init(type, sym) do { \ 51 extern s32 __start_runtime_##type##_##sym[]; \ 52 extern s32 __stop_runtime_##type##_##sym[]; \ 53 runtime_const_fixup(__runtime_fixup_##type, \ 54 (unsigned long)(sym), \ 55 __start_runtime_##type##_##sym, \ 56 __stop_runtime_##type##_##sym); \ 57 } while (0) 58 59 /* 16-bit immediate for wide move (movz and movk) in bits 5..20 */ 60 static inline void __runtime_fixup_16(__le32 *p, unsigned int val) 61 { 62 u32 insn = le32_to_cpu(*p); 63 insn &= 0xffe0001f; 64 insn |= (val & 0xffff) << 5; 65 aarch64_insn_patch_text_nosync(p, insn); 66 } 67 68 static inline void __runtime_fixup_ptr(void *where, unsigned long val) 69 { 70 __le32 *p = where; 71 __runtime_fixup_16(p, val); 72 __runtime_fixup_16(p+1, val >> 16); 73 __runtime_fixup_16(p+2, val >> 32); 74 __runtime_fixup_16(p+3, val >> 48); 75 } 76 77 /* Immediate value is 6 bits starting at bit #16 */ 78 static inline void __runtime_fixup_shift(void *where, unsigned long val) 79 { 80 __le32 *p = where; 81 u32 insn = le32_to_cpu(*p); 82 insn &= 0xffc0ffff; 83 insn |= (val & 63) << 16; 84 aarch64_insn_patch_text_nosync(p, insn); 85 } 86 87 static inline void __runtime_fixup_mask(void *where, unsigned long val) 88 { 89 unsigned int width = (val) ? __fls(val) + 1 : 0; 90 __le32 *p = where; 91 u32 insn; 92 93 /* 94 * XXX: Current implementation only supports patching masks of 95 * form GENMASK(n, 0) (n >= 0) using a single UBFX instruction 96 * to improve performance, density, and covers all the current 97 * use-cases. 98 * 99 * When the need arises to support any generic mask, and this 100 * BUG_ON() is tripped, consider using a: 101 * 102 * movz %w0, #imm16 103 * movk %w0, #imm16, lsl #16 104 * 105 * sequence to load the 32bit const mask, and perform a logical 106 * and outside the asm block before returning the result. Fixup 107 * can simply reuse the existing __runtime_fixup_16() to patch 108 * the individual mov instructions. 109 */ 110 BUG_ON(!val || width > 32 || (GENMASK(width - 1, 0) != val)); 111 112 /* 113 * The width of the mask is encoded as (width - 1) in imms 114 * which is 6 bits starting at bit #10. 115 */ 116 insn = le32_to_cpu(*p); 117 insn &= 0xffff03ff; 118 insn |= ((width - 1) & 0x1f) << 10; 119 aarch64_insn_patch_text_nosync(p, insn); 120 } 121 122 static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), 123 unsigned long val, s32 *start, s32 *end) 124 { 125 while (start < end) { 126 fn(*start + (void *)start, val); 127 start++; 128 } 129 } 130 131 #endif 132