1 /* 2 * jump label support 3 * 4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 5 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> 6 * 7 */ 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/list.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 #include <linux/err.h> 15 #include <linux/jump_label.h> 16 17 #ifdef HAVE_JUMP_LABEL 18 19 /* mutex to protect coming/going of the the jump_label table */ 20 static DEFINE_MUTEX(jump_label_mutex); 21 22 void jump_label_lock(void) 23 { 24 mutex_lock(&jump_label_mutex); 25 } 26 27 void jump_label_unlock(void) 28 { 29 mutex_unlock(&jump_label_mutex); 30 } 31 32 bool jump_label_enabled(struct jump_label_key *key) 33 { 34 return !!atomic_read(&key->enabled); 35 } 36 37 static int jump_label_cmp(const void *a, const void *b) 38 { 39 const struct jump_entry *jea = a; 40 const struct jump_entry *jeb = b; 41 42 if (jea->key < jeb->key) 43 return -1; 44 45 if (jea->key > jeb->key) 46 return 1; 47 48 return 0; 49 } 50 51 static void 52 jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) 53 { 54 unsigned long size; 55 56 size = (((unsigned long)stop - (unsigned long)start) 57 / sizeof(struct jump_entry)); 58 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 59 } 60 61 static void jump_label_update(struct jump_label_key *key, int enable); 62 63 void jump_label_inc(struct jump_label_key *key) 64 { 65 if (atomic_inc_not_zero(&key->enabled)) 66 return; 67 68 jump_label_lock(); 69 if (atomic_add_return(1, &key->enabled) == 1) 70 jump_label_update(key, JUMP_LABEL_ENABLE); 71 jump_label_unlock(); 72 } 73 74 void jump_label_dec(struct jump_label_key *key) 75 { 76 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) 77 return; 78 79 jump_label_update(key, JUMP_LABEL_DISABLE); 80 jump_label_unlock(); 81 } 82 83 static int addr_conflict(struct jump_entry *entry, void *start, void *end) 84 { 85 if (entry->code <= (unsigned long)end && 86 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 87 return 1; 88 89 return 0; 90 } 91 92 static int __jump_label_text_reserved(struct jump_entry *iter_start, 93 struct jump_entry *iter_stop, void *start, void *end) 94 { 95 struct jump_entry *iter; 96 97 iter = iter_start; 98 while (iter < iter_stop) { 99 if (addr_conflict(iter, start, end)) 100 return 1; 101 iter++; 102 } 103 104 return 0; 105 } 106 107 static void __jump_label_update(struct jump_label_key *key, 108 struct jump_entry *entry, int enable) 109 { 110 for (; entry->key == (jump_label_t)(unsigned long)key; entry++) { 111 /* 112 * entry->code set to 0 invalidates module init text sections 113 * kernel_text_address() verifies we are not in core kernel 114 * init code, see jump_label_invalidate_module_init(). 115 */ 116 if (entry->code && kernel_text_address(entry->code)) 117 arch_jump_label_transform(entry, enable); 118 } 119 } 120 121 /* 122 * Not all archs need this. 123 */ 124 void __weak arch_jump_label_text_poke_early(jump_label_t addr) 125 { 126 } 127 128 static __init int jump_label_init(void) 129 { 130 struct jump_entry *iter_start = __start___jump_table; 131 struct jump_entry *iter_stop = __stop___jump_table; 132 struct jump_label_key *key = NULL; 133 struct jump_entry *iter; 134 135 jump_label_lock(); 136 jump_label_sort_entries(iter_start, iter_stop); 137 138 for (iter = iter_start; iter < iter_stop; iter++) { 139 arch_jump_label_text_poke_early(iter->code); 140 if (iter->key == (jump_label_t)(unsigned long)key) 141 continue; 142 143 key = (struct jump_label_key *)(unsigned long)iter->key; 144 atomic_set(&key->enabled, 0); 145 key->entries = iter; 146 #ifdef CONFIG_MODULES 147 key->next = NULL; 148 #endif 149 } 150 jump_label_unlock(); 151 152 return 0; 153 } 154 early_initcall(jump_label_init); 155 156 #ifdef CONFIG_MODULES 157 158 struct jump_label_mod { 159 struct jump_label_mod *next; 160 struct jump_entry *entries; 161 struct module *mod; 162 }; 163 164 static int __jump_label_mod_text_reserved(void *start, void *end) 165 { 166 struct module *mod; 167 168 mod = __module_text_address((unsigned long)start); 169 if (!mod) 170 return 0; 171 172 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); 173 174 return __jump_label_text_reserved(mod->jump_entries, 175 mod->jump_entries + mod->num_jump_entries, 176 start, end); 177 } 178 179 static void __jump_label_mod_update(struct jump_label_key *key, int enable) 180 { 181 struct jump_label_mod *mod = key->next; 182 183 while (mod) { 184 __jump_label_update(key, mod->entries, enable); 185 mod = mod->next; 186 } 187 } 188 189 /*** 190 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() 191 * @mod: module to patch 192 * 193 * Allow for run-time selection of the optimal nops. Before the module 194 * loads patch these with arch_get_jump_label_nop(), which is specified by 195 * the arch specific jump label code. 196 */ 197 void jump_label_apply_nops(struct module *mod) 198 { 199 struct jump_entry *iter_start = mod->jump_entries; 200 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 201 struct jump_entry *iter; 202 203 /* if the module doesn't have jump label entries, just return */ 204 if (iter_start == iter_stop) 205 return; 206 207 for (iter = iter_start; iter < iter_stop; iter++) 208 arch_jump_label_text_poke_early(iter->code); 209 } 210 211 static int jump_label_add_module(struct module *mod) 212 { 213 struct jump_entry *iter_start = mod->jump_entries; 214 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 215 struct jump_entry *iter; 216 struct jump_label_key *key = NULL; 217 struct jump_label_mod *jlm; 218 219 /* if the module doesn't have jump label entries, just return */ 220 if (iter_start == iter_stop) 221 return 0; 222 223 jump_label_sort_entries(iter_start, iter_stop); 224 225 for (iter = iter_start; iter < iter_stop; iter++) { 226 if (iter->key == (jump_label_t)(unsigned long)key) 227 continue; 228 229 key = (struct jump_label_key *)(unsigned long)iter->key; 230 231 if (__module_address(iter->key) == mod) { 232 atomic_set(&key->enabled, 0); 233 key->entries = iter; 234 key->next = NULL; 235 continue; 236 } 237 238 jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); 239 if (!jlm) 240 return -ENOMEM; 241 242 jlm->mod = mod; 243 jlm->entries = iter; 244 jlm->next = key->next; 245 key->next = jlm; 246 247 if (jump_label_enabled(key)) 248 __jump_label_update(key, iter, JUMP_LABEL_ENABLE); 249 } 250 251 return 0; 252 } 253 254 static void jump_label_del_module(struct module *mod) 255 { 256 struct jump_entry *iter_start = mod->jump_entries; 257 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 258 struct jump_entry *iter; 259 struct jump_label_key *key = NULL; 260 struct jump_label_mod *jlm, **prev; 261 262 for (iter = iter_start; iter < iter_stop; iter++) { 263 if (iter->key == (jump_label_t)(unsigned long)key) 264 continue; 265 266 key = (struct jump_label_key *)(unsigned long)iter->key; 267 268 if (__module_address(iter->key) == mod) 269 continue; 270 271 prev = &key->next; 272 jlm = key->next; 273 274 while (jlm && jlm->mod != mod) { 275 prev = &jlm->next; 276 jlm = jlm->next; 277 } 278 279 if (jlm) { 280 *prev = jlm->next; 281 kfree(jlm); 282 } 283 } 284 } 285 286 static void jump_label_invalidate_module_init(struct module *mod) 287 { 288 struct jump_entry *iter_start = mod->jump_entries; 289 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 290 struct jump_entry *iter; 291 292 for (iter = iter_start; iter < iter_stop; iter++) { 293 if (within_module_init(iter->code, mod)) 294 iter->code = 0; 295 } 296 } 297 298 static int 299 jump_label_module_notify(struct notifier_block *self, unsigned long val, 300 void *data) 301 { 302 struct module *mod = data; 303 int ret = 0; 304 305 switch (val) { 306 case MODULE_STATE_COMING: 307 jump_label_lock(); 308 ret = jump_label_add_module(mod); 309 if (ret) 310 jump_label_del_module(mod); 311 jump_label_unlock(); 312 break; 313 case MODULE_STATE_GOING: 314 jump_label_lock(); 315 jump_label_del_module(mod); 316 jump_label_unlock(); 317 break; 318 case MODULE_STATE_LIVE: 319 jump_label_lock(); 320 jump_label_invalidate_module_init(mod); 321 jump_label_unlock(); 322 break; 323 } 324 325 return notifier_from_errno(ret); 326 } 327 328 struct notifier_block jump_label_module_nb = { 329 .notifier_call = jump_label_module_notify, 330 .priority = 1, /* higher than tracepoints */ 331 }; 332 333 static __init int jump_label_init_module(void) 334 { 335 return register_module_notifier(&jump_label_module_nb); 336 } 337 early_initcall(jump_label_init_module); 338 339 #endif /* CONFIG_MODULES */ 340 341 /*** 342 * jump_label_text_reserved - check if addr range is reserved 343 * @start: start text addr 344 * @end: end text addr 345 * 346 * checks if the text addr located between @start and @end 347 * overlaps with any of the jump label patch addresses. Code 348 * that wants to modify kernel text should first verify that 349 * it does not overlap with any of the jump label addresses. 350 * Caller must hold jump_label_mutex. 351 * 352 * returns 1 if there is an overlap, 0 otherwise 353 */ 354 int jump_label_text_reserved(void *start, void *end) 355 { 356 int ret = __jump_label_text_reserved(__start___jump_table, 357 __stop___jump_table, start, end); 358 359 if (ret) 360 return ret; 361 362 #ifdef CONFIG_MODULES 363 ret = __jump_label_mod_text_reserved(start, end); 364 #endif 365 return ret; 366 } 367 368 static void jump_label_update(struct jump_label_key *key, int enable) 369 { 370 struct jump_entry *entry = key->entries; 371 372 /* if there are no users, entry can be NULL */ 373 if (entry) 374 __jump_label_update(key, entry, enable); 375 376 #ifdef CONFIG_MODULES 377 __jump_label_mod_update(key, enable); 378 #endif 379 } 380 381 #endif 382