1 /* 2 * jump label support 3 * 4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 5 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> 6 * 7 */ 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/list.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 #include <linux/err.h> 15 #include <linux/jump_label.h> 16 17 #ifdef HAVE_JUMP_LABEL 18 19 /* mutex to protect coming/going of the the jump_label table */ 20 static DEFINE_MUTEX(jump_label_mutex); 21 22 void jump_label_lock(void) 23 { 24 mutex_lock(&jump_label_mutex); 25 } 26 27 void jump_label_unlock(void) 28 { 29 mutex_unlock(&jump_label_mutex); 30 } 31 32 bool jump_label_enabled(struct jump_label_key *key) 33 { 34 return !!atomic_read(&key->enabled); 35 } 36 37 static int jump_label_cmp(const void *a, const void *b) 38 { 39 const struct jump_entry *jea = a; 40 const struct jump_entry *jeb = b; 41 42 if (jea->key < jeb->key) 43 return -1; 44 45 if (jea->key > jeb->key) 46 return 1; 47 48 return 0; 49 } 50 51 static void 52 jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) 53 { 54 unsigned long size; 55 56 size = (((unsigned long)stop - (unsigned long)start) 57 / sizeof(struct jump_entry)); 58 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 59 } 60 61 static void jump_label_update(struct jump_label_key *key, int enable); 62 63 void jump_label_inc(struct jump_label_key *key) 64 { 65 if (atomic_inc_not_zero(&key->enabled)) 66 return; 67 68 jump_label_lock(); 69 if (atomic_add_return(1, &key->enabled) == 1) 70 jump_label_update(key, JUMP_LABEL_ENABLE); 71 jump_label_unlock(); 72 } 73 74 void jump_label_dec(struct jump_label_key *key) 75 { 76 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) 77 return; 78 79 jump_label_update(key, JUMP_LABEL_DISABLE); 80 jump_label_unlock(); 81 } 82 83 static int addr_conflict(struct jump_entry *entry, void *start, void *end) 84 { 85 if (entry->code <= (unsigned long)end && 86 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 87 return 1; 88 89 return 0; 90 } 91 92 static int __jump_label_text_reserved(struct jump_entry *iter_start, 93 struct jump_entry *iter_stop, void *start, void *end) 94 { 95 struct jump_entry *iter; 96 97 iter = iter_start; 98 while (iter < iter_stop) { 99 if (addr_conflict(iter, start, end)) 100 return 1; 101 iter++; 102 } 103 104 return 0; 105 } 106 107 static void __jump_label_update(struct jump_label_key *key, 108 struct jump_entry *entry, 109 struct jump_entry *stop, int enable) 110 { 111 for (; (entry < stop) && 112 (entry->key == (jump_label_t)(unsigned long)key); 113 entry++) { 114 /* 115 * entry->code set to 0 invalidates module init text sections 116 * kernel_text_address() verifies we are not in core kernel 117 * init code, see jump_label_invalidate_module_init(). 118 */ 119 if (entry->code && kernel_text_address(entry->code)) 120 arch_jump_label_transform(entry, enable); 121 } 122 } 123 124 /* 125 * Not all archs need this. 126 */ 127 void __weak arch_jump_label_text_poke_early(jump_label_t addr) 128 { 129 } 130 131 static __init int jump_label_init(void) 132 { 133 struct jump_entry *iter_start = __start___jump_table; 134 struct jump_entry *iter_stop = __stop___jump_table; 135 struct jump_label_key *key = NULL; 136 struct jump_entry *iter; 137 138 jump_label_lock(); 139 jump_label_sort_entries(iter_start, iter_stop); 140 141 for (iter = iter_start; iter < iter_stop; iter++) { 142 arch_jump_label_text_poke_early(iter->code); 143 if (iter->key == (jump_label_t)(unsigned long)key) 144 continue; 145 146 key = (struct jump_label_key *)(unsigned long)iter->key; 147 atomic_set(&key->enabled, 0); 148 key->entries = iter; 149 #ifdef CONFIG_MODULES 150 key->next = NULL; 151 #endif 152 } 153 jump_label_unlock(); 154 155 return 0; 156 } 157 early_initcall(jump_label_init); 158 159 #ifdef CONFIG_MODULES 160 161 struct jump_label_mod { 162 struct jump_label_mod *next; 163 struct jump_entry *entries; 164 struct module *mod; 165 }; 166 167 static int __jump_label_mod_text_reserved(void *start, void *end) 168 { 169 struct module *mod; 170 171 mod = __module_text_address((unsigned long)start); 172 if (!mod) 173 return 0; 174 175 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); 176 177 return __jump_label_text_reserved(mod->jump_entries, 178 mod->jump_entries + mod->num_jump_entries, 179 start, end); 180 } 181 182 static void __jump_label_mod_update(struct jump_label_key *key, int enable) 183 { 184 struct jump_label_mod *mod = key->next; 185 186 while (mod) { 187 struct module *m = mod->mod; 188 189 __jump_label_update(key, mod->entries, 190 m->jump_entries + m->num_jump_entries, 191 enable); 192 mod = mod->next; 193 } 194 } 195 196 /*** 197 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() 198 * @mod: module to patch 199 * 200 * Allow for run-time selection of the optimal nops. Before the module 201 * loads patch these with arch_get_jump_label_nop(), which is specified by 202 * the arch specific jump label code. 203 */ 204 void jump_label_apply_nops(struct module *mod) 205 { 206 struct jump_entry *iter_start = mod->jump_entries; 207 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 208 struct jump_entry *iter; 209 210 /* if the module doesn't have jump label entries, just return */ 211 if (iter_start == iter_stop) 212 return; 213 214 for (iter = iter_start; iter < iter_stop; iter++) 215 arch_jump_label_text_poke_early(iter->code); 216 } 217 218 static int jump_label_add_module(struct module *mod) 219 { 220 struct jump_entry *iter_start = mod->jump_entries; 221 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 222 struct jump_entry *iter; 223 struct jump_label_key *key = NULL; 224 struct jump_label_mod *jlm; 225 226 /* if the module doesn't have jump label entries, just return */ 227 if (iter_start == iter_stop) 228 return 0; 229 230 jump_label_sort_entries(iter_start, iter_stop); 231 232 for (iter = iter_start; iter < iter_stop; iter++) { 233 if (iter->key == (jump_label_t)(unsigned long)key) 234 continue; 235 236 key = (struct jump_label_key *)(unsigned long)iter->key; 237 238 if (__module_address(iter->key) == mod) { 239 atomic_set(&key->enabled, 0); 240 key->entries = iter; 241 key->next = NULL; 242 continue; 243 } 244 245 jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); 246 if (!jlm) 247 return -ENOMEM; 248 249 jlm->mod = mod; 250 jlm->entries = iter; 251 jlm->next = key->next; 252 key->next = jlm; 253 254 if (jump_label_enabled(key)) 255 __jump_label_update(key, iter, iter_stop, 256 JUMP_LABEL_ENABLE); 257 } 258 259 return 0; 260 } 261 262 static void jump_label_del_module(struct module *mod) 263 { 264 struct jump_entry *iter_start = mod->jump_entries; 265 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 266 struct jump_entry *iter; 267 struct jump_label_key *key = NULL; 268 struct jump_label_mod *jlm, **prev; 269 270 for (iter = iter_start; iter < iter_stop; iter++) { 271 if (iter->key == (jump_label_t)(unsigned long)key) 272 continue; 273 274 key = (struct jump_label_key *)(unsigned long)iter->key; 275 276 if (__module_address(iter->key) == mod) 277 continue; 278 279 prev = &key->next; 280 jlm = key->next; 281 282 while (jlm && jlm->mod != mod) { 283 prev = &jlm->next; 284 jlm = jlm->next; 285 } 286 287 if (jlm) { 288 *prev = jlm->next; 289 kfree(jlm); 290 } 291 } 292 } 293 294 static void jump_label_invalidate_module_init(struct module *mod) 295 { 296 struct jump_entry *iter_start = mod->jump_entries; 297 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 298 struct jump_entry *iter; 299 300 for (iter = iter_start; iter < iter_stop; iter++) { 301 if (within_module_init(iter->code, mod)) 302 iter->code = 0; 303 } 304 } 305 306 static int 307 jump_label_module_notify(struct notifier_block *self, unsigned long val, 308 void *data) 309 { 310 struct module *mod = data; 311 int ret = 0; 312 313 switch (val) { 314 case MODULE_STATE_COMING: 315 jump_label_lock(); 316 ret = jump_label_add_module(mod); 317 if (ret) 318 jump_label_del_module(mod); 319 jump_label_unlock(); 320 break; 321 case MODULE_STATE_GOING: 322 jump_label_lock(); 323 jump_label_del_module(mod); 324 jump_label_unlock(); 325 break; 326 case MODULE_STATE_LIVE: 327 jump_label_lock(); 328 jump_label_invalidate_module_init(mod); 329 jump_label_unlock(); 330 break; 331 } 332 333 return notifier_from_errno(ret); 334 } 335 336 struct notifier_block jump_label_module_nb = { 337 .notifier_call = jump_label_module_notify, 338 .priority = 1, /* higher than tracepoints */ 339 }; 340 341 static __init int jump_label_init_module(void) 342 { 343 return register_module_notifier(&jump_label_module_nb); 344 } 345 early_initcall(jump_label_init_module); 346 347 #endif /* CONFIG_MODULES */ 348 349 /*** 350 * jump_label_text_reserved - check if addr range is reserved 351 * @start: start text addr 352 * @end: end text addr 353 * 354 * checks if the text addr located between @start and @end 355 * overlaps with any of the jump label patch addresses. Code 356 * that wants to modify kernel text should first verify that 357 * it does not overlap with any of the jump label addresses. 358 * Caller must hold jump_label_mutex. 359 * 360 * returns 1 if there is an overlap, 0 otherwise 361 */ 362 int jump_label_text_reserved(void *start, void *end) 363 { 364 int ret = __jump_label_text_reserved(__start___jump_table, 365 __stop___jump_table, start, end); 366 367 if (ret) 368 return ret; 369 370 #ifdef CONFIG_MODULES 371 ret = __jump_label_mod_text_reserved(start, end); 372 #endif 373 return ret; 374 } 375 376 static void jump_label_update(struct jump_label_key *key, int enable) 377 { 378 struct jump_entry *entry = key->entries, *stop = __stop___jump_table; 379 380 #ifdef CONFIG_MODULES 381 struct module *mod = __module_address((jump_label_t)key); 382 383 __jump_label_mod_update(key, enable); 384 385 if (mod) 386 stop = mod->jump_entries + mod->num_jump_entries; 387 #endif 388 /* if there are no users, entry can be NULL */ 389 if (entry) 390 __jump_label_update(key, entry, stop, enable); 391 } 392 393 #endif 394