1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #define pr_fmt(fmt) "callthunks: " fmt 4 5 #include <linux/debugfs.h> 6 #include <linux/kallsyms.h> 7 #include <linux/memory.h> 8 #include <linux/moduleloader.h> 9 #include <linux/static_call.h> 10 11 #include <asm/alternative.h> 12 #include <asm/asm-offsets.h> 13 #include <asm/cpu.h> 14 #include <asm/ftrace.h> 15 #include <asm/insn.h> 16 #include <asm/kexec.h> 17 #include <asm/nospec-branch.h> 18 #include <asm/paravirt.h> 19 #include <asm/sections.h> 20 #include <asm/switch_to.h> 21 #include <asm/sync_core.h> 22 #include <asm/text-patching.h> 23 #include <asm/xen/hypercall.h> 24 25 static int __initdata_or_module debug_callthunks; 26 27 #define MAX_PATCH_LEN (255-1) 28 29 #define prdbg(fmt, args...) \ 30 do { \ 31 if (debug_callthunks) \ 32 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 33 } while(0) 34 35 static int __init debug_thunks(char *str) 36 { 37 debug_callthunks = 1; 38 return 1; 39 } 40 __setup("debug-callthunks", debug_thunks); 41 42 #ifdef CONFIG_CALL_THUNKS_DEBUG 43 DEFINE_PER_CPU(u64, __x86_call_count); 44 DEFINE_PER_CPU(u64, __x86_ret_count); 45 DEFINE_PER_CPU(u64, __x86_stuffs_count); 46 DEFINE_PER_CPU(u64, __x86_ctxsw_count); 47 EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count); 48 EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count); 49 #endif 50 51 extern s32 __call_sites[], __call_sites_end[]; 52 53 struct core_text { 54 unsigned long base; 55 unsigned long end; 56 const char *name; 57 }; 58 59 static bool thunks_initialized __ro_after_init; 60 61 static const struct core_text builtin_coretext = { 62 .base = (unsigned long)_text, 63 .end = (unsigned long)_etext, 64 .name = "builtin", 65 }; 66 67 asm ( 68 ".pushsection .rodata \n" 69 ".global skl_call_thunk_template \n" 70 "skl_call_thunk_template: \n" 71 __stringify(INCREMENT_CALL_DEPTH)" \n" 72 ".global skl_call_thunk_tail \n" 73 "skl_call_thunk_tail: \n" 74 ".popsection \n" 75 ); 76 77 extern u8 skl_call_thunk_template[]; 78 extern u8 skl_call_thunk_tail[]; 79 80 #define SKL_TMPL_SIZE \ 81 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) 82 83 extern void error_entry(void); 84 extern void xen_error_entry(void); 85 extern void paranoid_entry(void); 86 87 static inline bool within_coretext(const struct core_text *ct, void *addr) 88 { 89 unsigned long p = (unsigned long)addr; 90 91 return ct->base <= p && p < ct->end; 92 } 93 94 static inline bool within_module_coretext(void *addr) 95 { 96 bool ret = false; 97 98 #ifdef CONFIG_MODULES 99 struct module *mod; 100 101 preempt_disable(); 102 mod = __module_address((unsigned long)addr); 103 if (mod && within_module_core((unsigned long)addr, mod)) 104 ret = true; 105 preempt_enable(); 106 #endif 107 return ret; 108 } 109 110 static bool is_coretext(const struct core_text *ct, void *addr) 111 { 112 if (ct && within_coretext(ct, addr)) 113 return true; 114 if (within_coretext(&builtin_coretext, addr)) 115 return true; 116 return within_module_coretext(addr); 117 } 118 119 static bool skip_addr(void *dest) 120 { 121 if (dest == error_entry) 122 return true; 123 if (dest == paranoid_entry) 124 return true; 125 if (dest == xen_error_entry) 126 return true; 127 /* Does FILL_RSB... */ 128 if (dest == __switch_to_asm) 129 return true; 130 /* Accounts directly */ 131 if (dest == ret_from_fork) 132 return true; 133 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT) 134 if (dest == soft_restart_cpu) 135 return true; 136 #endif 137 #ifdef CONFIG_FUNCTION_TRACER 138 if (dest == __fentry__) 139 return true; 140 #endif 141 #ifdef CONFIG_KEXEC_CORE 142 if (dest >= (void *)relocate_kernel && 143 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 144 return true; 145 #endif 146 #ifdef CONFIG_XEN 147 if (dest >= (void *)hypercall_page && 148 dest < (void*)hypercall_page + PAGE_SIZE) 149 return true; 150 #endif 151 return false; 152 } 153 154 static __init_or_module void *call_get_dest(void *addr) 155 { 156 struct insn insn; 157 void *dest; 158 int ret; 159 160 ret = insn_decode_kernel(&insn, addr); 161 if (ret) 162 return ERR_PTR(ret); 163 164 /* Patched out call? */ 165 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 166 return NULL; 167 168 dest = addr + insn.length + insn.immediate.value; 169 if (skip_addr(dest)) 170 return NULL; 171 return dest; 172 } 173 174 static const u8 nops[] = { 175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 178 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 179 }; 180 181 static void *patch_dest(void *dest, bool direct) 182 { 183 unsigned int tsize = SKL_TMPL_SIZE; 184 u8 insn_buff[MAX_PATCH_LEN]; 185 u8 *pad = dest - tsize; 186 187 memcpy(insn_buff, skl_call_thunk_template, tsize); 188 apply_relocation(insn_buff, tsize, pad, 189 skl_call_thunk_template, tsize); 190 191 /* Already patched? */ 192 if (!bcmp(pad, insn_buff, tsize)) 193 return pad; 194 195 /* Ensure there are nops */ 196 if (bcmp(pad, nops, tsize)) { 197 pr_warn_once("Invalid padding area for %pS\n", dest); 198 return NULL; 199 } 200 201 if (direct) 202 memcpy(pad, insn_buff, tsize); 203 else 204 text_poke_copy_locked(pad, insn_buff, tsize, true); 205 return pad; 206 } 207 208 static __init_or_module void patch_call(void *addr, const struct core_text *ct) 209 { 210 void *pad, *dest; 211 u8 bytes[8]; 212 213 if (!within_coretext(ct, addr)) 214 return; 215 216 dest = call_get_dest(addr); 217 if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 218 return; 219 220 if (!is_coretext(ct, dest)) 221 return; 222 223 pad = patch_dest(dest, within_coretext(ct, dest)); 224 if (!pad) 225 return; 226 227 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 228 dest, dest, pad); 229 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 230 text_poke_early(addr, bytes, CALL_INSN_SIZE); 231 } 232 233 static __init_or_module void 234 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 235 { 236 s32 *s; 237 238 for (s = start; s < end; s++) 239 patch_call((void *)s + *s, ct); 240 } 241 242 static __init_or_module void 243 patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end, 244 const struct core_text *ct) 245 { 246 struct alt_instr *a; 247 248 for (a = start; a < end; a++) 249 patch_call((void *)&a->instr_offset + a->instr_offset, ct); 250 } 251 252 static __init_or_module void 253 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 254 { 255 prdbg("Patching call sites %s\n", ct->name); 256 patch_call_sites(cs->call_start, cs->call_end, ct); 257 patch_alt_call_sites(cs->alt_start, cs->alt_end, ct); 258 prdbg("Patching call sites done%s\n", ct->name); 259 } 260 261 void __init callthunks_patch_builtin_calls(void) 262 { 263 struct callthunk_sites cs = { 264 .call_start = __call_sites, 265 .call_end = __call_sites_end, 266 .alt_start = __alt_instructions, 267 .alt_end = __alt_instructions_end 268 }; 269 270 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 271 return; 272 273 pr_info("Setting up call depth tracking\n"); 274 mutex_lock(&text_mutex); 275 callthunks_setup(&cs, &builtin_coretext); 276 thunks_initialized = true; 277 mutex_unlock(&text_mutex); 278 } 279 280 void *callthunks_translate_call_dest(void *dest) 281 { 282 void *target; 283 284 lockdep_assert_held(&text_mutex); 285 286 if (!thunks_initialized || skip_addr(dest)) 287 return dest; 288 289 if (!is_coretext(NULL, dest)) 290 return dest; 291 292 target = patch_dest(dest, false); 293 return target ? : dest; 294 } 295 296 #ifdef CONFIG_BPF_JIT 297 static bool is_callthunk(void *addr) 298 { 299 unsigned int tmpl_size = SKL_TMPL_SIZE; 300 u8 insn_buff[MAX_PATCH_LEN]; 301 unsigned long dest; 302 u8 *pad; 303 304 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT); 305 if (!thunks_initialized || skip_addr((void *)dest)) 306 return false; 307 308 pad = (void *)(dest - tmpl_size); 309 310 memcpy(insn_buff, skl_call_thunk_template, tmpl_size); 311 apply_relocation(insn_buff, tmpl_size, pad, 312 skl_call_thunk_template, tmpl_size); 313 314 return !bcmp(pad, insn_buff, tmpl_size); 315 } 316 317 int x86_call_depth_emit_accounting(u8 **pprog, void *func) 318 { 319 unsigned int tmpl_size = SKL_TMPL_SIZE; 320 u8 insn_buff[MAX_PATCH_LEN]; 321 322 if (!thunks_initialized) 323 return 0; 324 325 /* Is function call target a thunk? */ 326 if (func && is_callthunk(func)) 327 return 0; 328 329 memcpy(insn_buff, skl_call_thunk_template, tmpl_size); 330 apply_relocation(insn_buff, tmpl_size, *pprog, 331 skl_call_thunk_template, tmpl_size); 332 333 memcpy(*pprog, insn_buff, tmpl_size); 334 *pprog += tmpl_size; 335 return tmpl_size; 336 } 337 #endif 338 339 #ifdef CONFIG_MODULES 340 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 341 struct module *mod) 342 { 343 struct core_text ct = { 344 .base = (unsigned long)mod->mem[MOD_TEXT].base, 345 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size, 346 .name = mod->name, 347 }; 348 349 if (!thunks_initialized) 350 return; 351 352 mutex_lock(&text_mutex); 353 callthunks_setup(cs, &ct); 354 mutex_unlock(&text_mutex); 355 } 356 #endif /* CONFIG_MODULES */ 357 358 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS) 359 static int callthunks_debug_show(struct seq_file *m, void *p) 360 { 361 unsigned long cpu = (unsigned long)m->private; 362 363 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,", 364 per_cpu(__x86_call_count, cpu), 365 per_cpu(__x86_ret_count, cpu), 366 per_cpu(__x86_stuffs_count, cpu), 367 per_cpu(__x86_ctxsw_count, cpu)); 368 return 0; 369 } 370 371 static int callthunks_debug_open(struct inode *inode, struct file *file) 372 { 373 return single_open(file, callthunks_debug_show, inode->i_private); 374 } 375 376 static const struct file_operations dfs_ops = { 377 .open = callthunks_debug_open, 378 .read = seq_read, 379 .llseek = seq_lseek, 380 .release = single_release, 381 }; 382 383 static int __init callthunks_debugfs_init(void) 384 { 385 struct dentry *dir; 386 unsigned long cpu; 387 388 dir = debugfs_create_dir("callthunks", NULL); 389 for_each_possible_cpu(cpu) { 390 void *arg = (void *)cpu; 391 char name [10]; 392 393 sprintf(name, "cpu%lu", cpu); 394 debugfs_create_file(name, 0644, dir, arg, &dfs_ops); 395 } 396 return 0; 397 } 398 __initcall(callthunks_debugfs_init); 399 #endif 400