1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/mm.h> 8 #include <linux/memory.h> 9 #include <linux/string.h> 10 #include <linux/uaccess.h> 11 #include <linux/stop_machine.h> 12 #include <asm/kprobes.h> 13 #include <asm/cacheflush.h> 14 #include <asm/fixmap.h> 15 #include <asm/ftrace.h> 16 #include <asm/text-patching.h> 17 #include <asm/sections.h> 18 19 struct patch_insn { 20 void *addr; 21 u32 *insns; 22 size_t len; 23 atomic_t cpu_count; 24 }; 25 26 int riscv_patch_in_stop_machine = false; 27 28 #ifdef CONFIG_MMU 29 30 static inline bool is_kernel_exittext(uintptr_t addr) 31 { 32 return system_state < SYSTEM_RUNNING && 33 addr >= (uintptr_t)__exittext_begin && 34 addr < (uintptr_t)__exittext_end; 35 } 36 37 /* 38 * The fix_to_virt(, idx) needs a const value (not a dynamic variable of 39 * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". 40 * So use '__always_inline' and 'const unsigned int fixmap' here. 41 */ 42 static __always_inline void *patch_map(void *addr, const unsigned int fixmap) 43 { 44 uintptr_t uintaddr = (uintptr_t) addr; 45 phys_addr_t phys; 46 47 if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) { 48 if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 49 return addr; 50 phys = __pa_symbol(addr); 51 } else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) { 52 struct page *page = vmalloc_to_page(addr); 53 54 BUG_ON(!page); 55 phys = page_to_phys(page) + offset_in_page(addr); 56 } else { 57 return addr; 58 } 59 60 return (void *)set_fixmap_offset(fixmap, phys); 61 } 62 63 static void patch_unmap(int fixmap) 64 { 65 clear_fixmap(fixmap); 66 } 67 NOKPROBE_SYMBOL(patch_unmap); 68 69 static int __patch_insn_set(void *addr, u8 c, size_t len) 70 { 71 bool across_pages = (offset_in_page(addr) + len) > PAGE_SIZE; 72 void *waddr = addr; 73 74 /* 75 * Only two pages can be mapped at a time for writing. 76 */ 77 if (len + offset_in_page(addr) > 2 * PAGE_SIZE) 78 return -EINVAL; 79 /* 80 * Before reaching here, it was expected to lock the text_mutex 81 * already, so we don't need to give another lock here and could 82 * ensure that it was safe between each cores. 83 */ 84 lockdep_assert_held(&text_mutex); 85 86 preempt_disable(); 87 88 if (across_pages) 89 patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1); 90 91 waddr = patch_map(addr, FIX_TEXT_POKE0); 92 93 memset(waddr, c, len); 94 95 /* 96 * We could have just patched a function that is about to be 97 * called so make sure we don't execute partially patched 98 * instructions by flushing the icache as soon as possible. 99 */ 100 local_flush_icache_range((unsigned long)waddr, 101 (unsigned long)waddr + len); 102 103 patch_unmap(FIX_TEXT_POKE0); 104 105 if (across_pages) 106 patch_unmap(FIX_TEXT_POKE1); 107 108 preempt_enable(); 109 110 return 0; 111 } 112 NOKPROBE_SYMBOL(__patch_insn_set); 113 114 static int __patch_insn_write(void *addr, const void *insn, size_t len) 115 { 116 bool across_pages = (offset_in_page(addr) + len) > PAGE_SIZE; 117 void *waddr = addr; 118 int ret; 119 120 /* 121 * Only two pages can be mapped at a time for writing. 122 */ 123 if (len + offset_in_page(addr) > 2 * PAGE_SIZE) 124 return -EINVAL; 125 126 /* 127 * Before reaching here, it was expected to lock the text_mutex 128 * already, so we don't need to give another lock here and could 129 * ensure that it was safe between each cores. 130 * 131 * We're currently using stop_machine() for ftrace & kprobes, and while 132 * that ensures text_mutex is held before installing the mappings it 133 * does not ensure text_mutex is held by the calling thread. That's 134 * safe but triggers a lockdep failure, so just elide it for that 135 * specific case. 136 */ 137 if (!riscv_patch_in_stop_machine) 138 lockdep_assert_held(&text_mutex); 139 140 preempt_disable(); 141 142 if (across_pages) 143 patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1); 144 145 waddr = patch_map(addr, FIX_TEXT_POKE0); 146 147 ret = copy_to_kernel_nofault(waddr, insn, len); 148 149 /* 150 * We could have just patched a function that is about to be 151 * called so make sure we don't execute partially patched 152 * instructions by flushing the icache as soon as possible. 153 */ 154 local_flush_icache_range((unsigned long)waddr, 155 (unsigned long)waddr + len); 156 157 patch_unmap(FIX_TEXT_POKE0); 158 159 if (across_pages) 160 patch_unmap(FIX_TEXT_POKE1); 161 162 preempt_enable(); 163 164 return ret; 165 } 166 NOKPROBE_SYMBOL(__patch_insn_write); 167 #else 168 static int __patch_insn_set(void *addr, u8 c, size_t len) 169 { 170 memset(addr, c, len); 171 172 return 0; 173 } 174 NOKPROBE_SYMBOL(__patch_insn_set); 175 176 static int __patch_insn_write(void *addr, const void *insn, size_t len) 177 { 178 return copy_to_kernel_nofault(addr, insn, len); 179 } 180 NOKPROBE_SYMBOL(__patch_insn_write); 181 #endif /* CONFIG_MMU */ 182 183 static int patch_insn_set(void *addr, u8 c, size_t len) 184 { 185 size_t size; 186 int ret; 187 188 /* 189 * __patch_insn_set() can only work on 2 pages at a time so call it in a 190 * loop with len <= 2 * PAGE_SIZE. 191 */ 192 while (len) { 193 size = min(len, PAGE_SIZE * 2 - offset_in_page(addr)); 194 ret = __patch_insn_set(addr, c, size); 195 if (ret) 196 return ret; 197 198 addr += size; 199 len -= size; 200 } 201 202 return 0; 203 } 204 NOKPROBE_SYMBOL(patch_insn_set); 205 206 int patch_text_set_nosync(void *addr, u8 c, size_t len) 207 { 208 int ret; 209 210 ret = patch_insn_set(addr, c, len); 211 if (!ret) 212 flush_icache_range((uintptr_t)addr, (uintptr_t)addr + len); 213 214 return ret; 215 } 216 NOKPROBE_SYMBOL(patch_text_set_nosync); 217 218 int patch_insn_write(void *addr, const void *insn, size_t len) 219 { 220 size_t size; 221 int ret; 222 223 /* 224 * Copy the instructions to the destination address, two pages at a time 225 * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE. 226 */ 227 while (len) { 228 size = min(len, PAGE_SIZE * 2 - offset_in_page(addr)); 229 ret = __patch_insn_write(addr, insn, size); 230 if (ret) 231 return ret; 232 233 addr += size; 234 insn += size; 235 len -= size; 236 } 237 238 return 0; 239 } 240 NOKPROBE_SYMBOL(patch_insn_write); 241 242 int patch_text_nosync(void *addr, const void *insns, size_t len) 243 { 244 int ret; 245 246 ret = patch_insn_write(addr, insns, len); 247 if (!ret) 248 flush_icache_range((uintptr_t)addr, (uintptr_t)addr + len); 249 250 return ret; 251 } 252 NOKPROBE_SYMBOL(patch_text_nosync); 253 254 static int patch_text_cb(void *data) 255 { 256 struct patch_insn *patch = data; 257 int ret = 0; 258 259 if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { 260 ret = patch_insn_write(patch->addr, patch->insns, patch->len); 261 /* 262 * Make sure the patching store is effective *before* we 263 * increment the counter which releases all waiting CPUs 264 * by using the release variant of atomic increment. The 265 * release pairs with the call to local_flush_icache_all() 266 * on the waiting CPU. 267 */ 268 atomic_inc_return_release(&patch->cpu_count); 269 } else { 270 while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 271 cpu_relax(); 272 273 local_flush_icache_all(); 274 } 275 276 return ret; 277 } 278 NOKPROBE_SYMBOL(patch_text_cb); 279 280 int patch_text(void *addr, u32 *insns, size_t len) 281 { 282 int ret; 283 struct patch_insn patch = { 284 .addr = addr, 285 .insns = insns, 286 .len = len, 287 .cpu_count = ATOMIC_INIT(0), 288 }; 289 290 /* 291 * kprobes takes text_mutex, before calling patch_text(), but as we call 292 * calls stop_machine(), the lockdep assertion in patch_insn_write() 293 * gets confused by the context in which the lock is taken. 294 * Instead, ensure the lock is held before calling stop_machine(), and 295 * set riscv_patch_in_stop_machine to skip the check in 296 * patch_insn_write(). 297 */ 298 lockdep_assert_held(&text_mutex); 299 riscv_patch_in_stop_machine = true; 300 ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask); 301 riscv_patch_in_stop_machine = false; 302 return ret; 303 } 304 NOKPROBE_SYMBOL(patch_text); 305