1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #define pr_fmt(fmt) "callthunks: " fmt 4 5 #include <linux/debugfs.h> 6 #include <linux/kallsyms.h> 7 #include <linux/memory.h> 8 #include <linux/moduleloader.h> 9 #include <linux/static_call.h> 10 11 #include <asm/alternative.h> 12 #include <asm/asm-offsets.h> 13 #include <asm/cpu.h> 14 #include <asm/ftrace.h> 15 #include <asm/insn.h> 16 #include <asm/kexec.h> 17 #include <asm/nospec-branch.h> 18 #include <asm/sections.h> 19 #include <asm/switch_to.h> 20 #include <asm/sync_core.h> 21 #include <asm/text-patching.h> 22 #include <asm/xen/hypercall.h> 23 24 static int __initdata_or_module debug_callthunks; 25 26 #define MAX_PATCH_LEN (255-1) 27 28 #define prdbg(fmt, args...) \ 29 do { \ 30 if (debug_callthunks) \ 31 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 32 } while(0) 33 34 static int __init debug_thunks(char *str) 35 { 36 debug_callthunks = 1; 37 return 1; 38 } 39 __setup("debug-callthunks", debug_thunks); 40 41 #ifdef CONFIG_CALL_THUNKS_DEBUG 42 DEFINE_PER_CPU(u64, __x86_call_count); 43 DEFINE_PER_CPU(u64, __x86_ret_count); 44 DEFINE_PER_CPU(u64, __x86_stuffs_count); 45 DEFINE_PER_CPU(u64, __x86_ctxsw_count); 46 EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count); 47 EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count); 48 #endif 49 50 extern s32 __call_sites[], __call_sites_end[]; 51 52 struct core_text { 53 unsigned long base; 54 unsigned long end; 55 const char *name; 56 }; 57 58 static bool thunks_initialized __ro_after_init; 59 60 static const struct core_text builtin_coretext = { 61 .base = (unsigned long)_text, 62 .end = (unsigned long)_etext, 63 .name = "builtin", 64 }; 65 66 asm ( 67 ".pushsection .rodata \n" 68 ".global skl_call_thunk_template \n" 69 "skl_call_thunk_template: \n" 70 __stringify(INCREMENT_CALL_DEPTH)" \n" 71 ".global skl_call_thunk_tail \n" 72 "skl_call_thunk_tail: \n" 73 ".popsection \n" 74 ); 75 76 extern u8 skl_call_thunk_template[]; 77 extern u8 skl_call_thunk_tail[]; 78 79 #define SKL_TMPL_SIZE \ 80 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) 81 82 extern void error_entry(void); 83 extern void xen_error_entry(void); 84 extern void paranoid_entry(void); 85 86 static inline bool within_coretext(const struct core_text *ct, void *addr) 87 { 88 unsigned long p = (unsigned long)addr; 89 90 return ct->base <= p && p < ct->end; 91 } 92 93 static inline bool within_module_coretext(void *addr) 94 { 95 bool ret = false; 96 97 #ifdef CONFIG_MODULES 98 struct module *mod; 99 100 guard(rcu)(); 101 mod = __module_address((unsigned long)addr); 102 if (mod && within_module_core((unsigned long)addr, mod)) 103 ret = true; 104 #endif 105 return ret; 106 } 107 108 static bool is_coretext(const struct core_text *ct, void *addr) 109 { 110 if (ct && within_coretext(ct, addr)) 111 return true; 112 if (within_coretext(&builtin_coretext, addr)) 113 return true; 114 return within_module_coretext(addr); 115 } 116 117 static bool skip_addr(void *dest) 118 { 119 if (dest == error_entry) 120 return true; 121 if (dest == paranoid_entry) 122 return true; 123 if (dest == xen_error_entry) 124 return true; 125 /* Does FILL_RSB... */ 126 if (dest == __switch_to_asm) 127 return true; 128 /* Accounts directly */ 129 if (dest == ret_from_fork) 130 return true; 131 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT) 132 if (dest == soft_restart_cpu) 133 return true; 134 #endif 135 #ifdef CONFIG_FUNCTION_TRACER 136 if (dest == __fentry__) 137 return true; 138 #endif 139 #ifdef CONFIG_KEXEC_CORE 140 # ifdef CONFIG_X86_64 141 if (dest >= (void *)__relocate_kernel_start && 142 dest < (void *)__relocate_kernel_end) 143 return true; 144 # else 145 if (dest >= (void *)relocate_kernel && 146 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 147 return true; 148 # endif 149 #endif 150 return false; 151 } 152 153 static __init_or_module void *call_get_dest(void *addr) 154 { 155 struct insn insn; 156 void *dest; 157 int ret; 158 159 ret = insn_decode_kernel(&insn, addr); 160 if (ret) 161 return ERR_PTR(ret); 162 163 /* Patched out call? */ 164 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 165 return NULL; 166 167 dest = addr + insn.length + insn.immediate.value; 168 if (skip_addr(dest)) 169 return NULL; 170 return dest; 171 } 172 173 static const u8 nops[] = { 174 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 178 }; 179 180 static void *patch_dest(void *dest, bool direct) 181 { 182 unsigned int tsize = SKL_TMPL_SIZE; 183 u8 insn_buff[MAX_PATCH_LEN]; 184 u8 *pad = dest - tsize; 185 186 memcpy(insn_buff, skl_call_thunk_template, tsize); 187 text_poke_apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize); 188 189 /* Already patched? */ 190 if (!bcmp(pad, insn_buff, tsize)) 191 return pad; 192 193 /* Ensure there are nops */ 194 if (bcmp(pad, nops, tsize)) { 195 pr_warn_once("Invalid padding area for %pS\n", dest); 196 return NULL; 197 } 198 199 if (direct) 200 memcpy(pad, insn_buff, tsize); 201 else 202 text_poke_copy_locked(pad, insn_buff, tsize, true); 203 return pad; 204 } 205 206 static __init_or_module void patch_call(void *addr, const struct core_text *ct) 207 { 208 void *pad, *dest; 209 u8 bytes[8]; 210 211 if (!within_coretext(ct, addr)) 212 return; 213 214 dest = call_get_dest(addr); 215 if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 216 return; 217 218 if (!is_coretext(ct, dest)) 219 return; 220 221 pad = patch_dest(dest, within_coretext(ct, dest)); 222 if (!pad) 223 return; 224 225 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 226 dest, dest, pad); 227 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 228 text_poke_early(addr, bytes, CALL_INSN_SIZE); 229 } 230 231 static __init_or_module void 232 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 233 { 234 s32 *s; 235 236 for (s = start; s < end; s++) 237 patch_call((void *)s + *s, ct); 238 } 239 240 static __init_or_module void 241 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 242 { 243 prdbg("Patching call sites %s\n", ct->name); 244 patch_call_sites(cs->call_start, cs->call_end, ct); 245 prdbg("Patching call sites done%s\n", ct->name); 246 } 247 248 void __init callthunks_patch_builtin_calls(void) 249 { 250 struct callthunk_sites cs = { 251 .call_start = __call_sites, 252 .call_end = __call_sites_end, 253 }; 254 255 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 256 return; 257 258 pr_info("Setting up call depth tracking\n"); 259 mutex_lock(&text_mutex); 260 callthunks_setup(&cs, &builtin_coretext); 261 thunks_initialized = true; 262 mutex_unlock(&text_mutex); 263 } 264 265 void *callthunks_translate_call_dest(void *dest) 266 { 267 void *target; 268 269 lockdep_assert_held(&text_mutex); 270 271 if (!thunks_initialized || skip_addr(dest)) 272 return dest; 273 274 if (!is_coretext(NULL, dest)) 275 return dest; 276 277 target = patch_dest(dest, false); 278 return target ? : dest; 279 } 280 281 #ifdef CONFIG_BPF_JIT 282 static bool is_callthunk(void *addr) 283 { 284 unsigned int tmpl_size = SKL_TMPL_SIZE; 285 u8 insn_buff[MAX_PATCH_LEN]; 286 unsigned long dest; 287 u8 *pad; 288 289 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT); 290 if (!thunks_initialized || skip_addr((void *)dest)) 291 return false; 292 293 pad = (void *)(dest - tmpl_size); 294 295 memcpy(insn_buff, skl_call_thunk_template, tmpl_size); 296 text_poke_apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size); 297 298 return !bcmp(pad, insn_buff, tmpl_size); 299 } 300 301 int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip) 302 { 303 unsigned int tmpl_size = SKL_TMPL_SIZE; 304 u8 insn_buff[MAX_PATCH_LEN]; 305 306 if (!thunks_initialized) 307 return 0; 308 309 /* Is function call target a thunk? */ 310 if (func && is_callthunk(func)) 311 return 0; 312 313 memcpy(insn_buff, skl_call_thunk_template, tmpl_size); 314 text_poke_apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size); 315 316 memcpy(*pprog, insn_buff, tmpl_size); 317 *pprog += tmpl_size; 318 return tmpl_size; 319 } 320 #endif 321 322 #ifdef CONFIG_MODULES 323 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 324 struct module *mod) 325 { 326 struct core_text ct = { 327 .base = (unsigned long)mod->mem[MOD_TEXT].base, 328 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size, 329 .name = mod->name, 330 }; 331 332 if (!thunks_initialized) 333 return; 334 335 mutex_lock(&text_mutex); 336 callthunks_setup(cs, &ct); 337 mutex_unlock(&text_mutex); 338 } 339 #endif /* CONFIG_MODULES */ 340 341 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS) 342 static int callthunks_debug_show(struct seq_file *m, void *p) 343 { 344 unsigned long cpu = (unsigned long)m->private; 345 346 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,", 347 per_cpu(__x86_call_count, cpu), 348 per_cpu(__x86_ret_count, cpu), 349 per_cpu(__x86_stuffs_count, cpu), 350 per_cpu(__x86_ctxsw_count, cpu)); 351 return 0; 352 } 353 354 static int callthunks_debug_open(struct inode *inode, struct file *file) 355 { 356 return single_open(file, callthunks_debug_show, inode->i_private); 357 } 358 359 static const struct file_operations dfs_ops = { 360 .open = callthunks_debug_open, 361 .read = seq_read, 362 .llseek = seq_lseek, 363 .release = single_release, 364 }; 365 366 static int __init callthunks_debugfs_init(void) 367 { 368 struct dentry *dir; 369 unsigned long cpu; 370 371 dir = debugfs_create_dir("callthunks", NULL); 372 for_each_possible_cpu(cpu) { 373 void *arg = (void *)cpu; 374 char name [10]; 375 376 sprintf(name, "cpu%lu", cpu); 377 debugfs_create_file(name, 0644, dir, arg, &dfs_ops); 378 } 379 return 0; 380 } 381 __initcall(callthunks_debugfs_init); 382 #endif 383