1 /* 2 * Code for replacing ftrace calls with jumps. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * 6 * Thanks goes to Ingo Molnar, for suggesting the idea. 7 * Mathieu Desnoyers, for suggesting postponing the modifications. 8 * Arjan van de Ven, for keeping me straight, and explaining to me 9 * the dangers of modifying code on the run. 10 */ 11 12 #include <linux/spinlock.h> 13 #include <linux/hardirq.h> 14 #include <linux/uaccess.h> 15 #include <linux/ftrace.h> 16 #include <linux/percpu.h> 17 #include <linux/init.h> 18 #include <linux/list.h> 19 20 #include <asm/ftrace.h> 21 #include <asm/nops.h> 22 23 24 /* Long is fine, even if it is only 4 bytes ;-) */ 25 static unsigned long *ftrace_nop; 26 27 union ftrace_code_union { 28 char code[MCOUNT_INSN_SIZE]; 29 struct { 30 char e8; 31 int offset; 32 } __attribute__((packed)); 33 }; 34 35 36 static int notrace ftrace_calc_offset(long ip, long addr) 37 { 38 return (int)(addr - ip); 39 } 40 41 notrace unsigned char *ftrace_nop_replace(void) 42 { 43 return (char *)ftrace_nop; 44 } 45 46 notrace unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) 47 { 48 static union ftrace_code_union calc; 49 50 calc.e8 = 0xe8; 51 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr); 52 53 /* 54 * No locking needed, this must be called via kstop_machine 55 * which in essence is like running on a uniprocessor machine. 56 */ 57 return calc.code; 58 } 59 60 notrace int 61 ftrace_modify_code(unsigned long ip, unsigned char *old_code, 62 unsigned char *new_code) 63 { 64 unsigned char replaced[MCOUNT_INSN_SIZE]; 65 66 /* 67 * Note: Due to modules and __init, code can 68 * disappear and change, we need to protect against faulting 69 * as well as code changing. 70 * 71 * No real locking needed, this code is run through 72 * kstop_machine, or before SMP starts. 73 */ 74 if (__copy_from_user_inatomic(replaced, (char __user *)ip, MCOUNT_INSN_SIZE)) 75 return 1; 76 77 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0) 78 return 2; 79 80 WARN_ON_ONCE(__copy_to_user_inatomic((char __user *)ip, new_code, 81 MCOUNT_INSN_SIZE)); 82 83 sync_core(); 84 85 return 0; 86 } 87 88 notrace int ftrace_update_ftrace_func(ftrace_func_t func) 89 { 90 unsigned long ip = (unsigned long)(&ftrace_call); 91 unsigned char old[MCOUNT_INSN_SIZE], *new; 92 int ret; 93 94 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE); 95 new = ftrace_call_replace(ip, (unsigned long)func); 96 ret = ftrace_modify_code(ip, old, new); 97 98 return ret; 99 } 100 101 notrace int ftrace_mcount_set(unsigned long *data) 102 { 103 /* mcount is initialized as a nop */ 104 *data = 0; 105 return 0; 106 } 107 108 int __init ftrace_dyn_arch_init(void *data) 109 { 110 extern const unsigned char ftrace_test_p6nop[]; 111 extern const unsigned char ftrace_test_nop5[]; 112 extern const unsigned char ftrace_test_jmp[]; 113 int faulted = 0; 114 115 /* 116 * There is no good nop for all x86 archs. 117 * We will default to using the P6_NOP5, but first we 118 * will test to make sure that the nop will actually 119 * work on this CPU. If it faults, we will then 120 * go to a lesser efficient 5 byte nop. If that fails 121 * we then just use a jmp as our nop. This isn't the most 122 * efficient nop, but we can not use a multi part nop 123 * since we would then risk being preempted in the middle 124 * of that nop, and if we enabled tracing then, it might 125 * cause a system crash. 126 * 127 * TODO: check the cpuid to determine the best nop. 128 */ 129 asm volatile ( 130 "jmp ftrace_test_jmp\n" 131 /* This code needs to stay around */ 132 ".section .text, \"ax\"\n" 133 "ftrace_test_jmp:" 134 "jmp ftrace_test_p6nop\n" 135 "nop\n" 136 "nop\n" 137 "nop\n" /* 2 byte jmp + 3 bytes */ 138 "ftrace_test_p6nop:" 139 P6_NOP5 140 "jmp 1f\n" 141 "ftrace_test_nop5:" 142 ".byte 0x66,0x66,0x66,0x66,0x90\n" 143 "jmp 1f\n" 144 ".previous\n" 145 "1:" 146 ".section .fixup, \"ax\"\n" 147 "2: movl $1, %0\n" 148 " jmp ftrace_test_nop5\n" 149 "3: movl $2, %0\n" 150 " jmp 1b\n" 151 ".previous\n" 152 _ASM_EXTABLE(ftrace_test_p6nop, 2b) 153 _ASM_EXTABLE(ftrace_test_nop5, 3b) 154 : "=r"(faulted) : "0" (faulted)); 155 156 switch (faulted) { 157 case 0: 158 pr_info("ftrace: converting mcount calls to 0f 1f 44 00 00\n"); 159 ftrace_nop = (unsigned long *)ftrace_test_p6nop; 160 break; 161 case 1: 162 pr_info("ftrace: converting mcount calls to 66 66 66 66 90\n"); 163 ftrace_nop = (unsigned long *)ftrace_test_nop5; 164 break; 165 case 2: 166 pr_info("ftrace: converting mcount calls to jmp . + 5\n"); 167 ftrace_nop = (unsigned long *)ftrace_test_jmp; 168 break; 169 } 170 171 /* The return code is retured via data */ 172 *(unsigned long *)data = 0; 173 174 return 0; 175 } 176