1 /* 2 * jump label support 3 * 4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 5 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> 6 * 7 */ 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/list.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 #include <linux/err.h> 15 #include <linux/jump_label.h> 16 17 #ifdef HAVE_JUMP_LABEL 18 19 /* mutex to protect coming/going of the the jump_label table */ 20 static DEFINE_MUTEX(jump_label_mutex); 21 22 void jump_label_lock(void) 23 { 24 mutex_lock(&jump_label_mutex); 25 } 26 27 void jump_label_unlock(void) 28 { 29 mutex_unlock(&jump_label_mutex); 30 } 31 32 bool jump_label_enabled(struct jump_label_key *key) 33 { 34 return !!atomic_read(&key->enabled); 35 } 36 37 static int jump_label_cmp(const void *a, const void *b) 38 { 39 const struct jump_entry *jea = a; 40 const struct jump_entry *jeb = b; 41 42 if (jea->key < jeb->key) 43 return -1; 44 45 if (jea->key > jeb->key) 46 return 1; 47 48 return 0; 49 } 50 51 static void 52 jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) 53 { 54 unsigned long size; 55 56 size = (((unsigned long)stop - (unsigned long)start) 57 / sizeof(struct jump_entry)); 58 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 59 } 60 61 static void jump_label_update(struct jump_label_key *key, int enable); 62 63 void jump_label_inc(struct jump_label_key *key) 64 { 65 if (atomic_inc_not_zero(&key->enabled)) 66 return; 67 68 jump_label_lock(); 69 if (atomic_read(&key->enabled) == 0) 70 jump_label_update(key, JUMP_LABEL_ENABLE); 71 atomic_inc(&key->enabled); 72 jump_label_unlock(); 73 } 74 75 void jump_label_dec(struct jump_label_key *key) 76 { 77 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) 78 return; 79 80 jump_label_update(key, JUMP_LABEL_DISABLE); 81 jump_label_unlock(); 82 } 83 84 static int addr_conflict(struct jump_entry *entry, void *start, void *end) 85 { 86 if (entry->code <= (unsigned long)end && 87 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 88 return 1; 89 90 return 0; 91 } 92 93 static int __jump_label_text_reserved(struct jump_entry *iter_start, 94 struct jump_entry *iter_stop, void *start, void *end) 95 { 96 struct jump_entry *iter; 97 98 iter = iter_start; 99 while (iter < iter_stop) { 100 if (addr_conflict(iter, start, end)) 101 return 1; 102 iter++; 103 } 104 105 return 0; 106 } 107 108 /* 109 * Update code which is definitely not currently executing. 110 * Architectures which need heavyweight synchronization to modify 111 * running code can override this to make the non-live update case 112 * cheaper. 113 */ 114 void __weak arch_jump_label_transform_static(struct jump_entry *entry, 115 enum jump_label_type type) 116 { 117 arch_jump_label_transform(entry, type); 118 } 119 120 static void __jump_label_update(struct jump_label_key *key, 121 struct jump_entry *entry, 122 struct jump_entry *stop, int enable) 123 { 124 for (; (entry < stop) && 125 (entry->key == (jump_label_t)(unsigned long)key); 126 entry++) { 127 /* 128 * entry->code set to 0 invalidates module init text sections 129 * kernel_text_address() verifies we are not in core kernel 130 * init code, see jump_label_invalidate_module_init(). 131 */ 132 if (entry->code && kernel_text_address(entry->code)) 133 arch_jump_label_transform(entry, enable); 134 } 135 } 136 137 void __init jump_label_init(void) 138 { 139 struct jump_entry *iter_start = __start___jump_table; 140 struct jump_entry *iter_stop = __stop___jump_table; 141 struct jump_label_key *key = NULL; 142 struct jump_entry *iter; 143 144 jump_label_lock(); 145 jump_label_sort_entries(iter_start, iter_stop); 146 147 for (iter = iter_start; iter < iter_stop; iter++) { 148 struct jump_label_key *iterk; 149 150 iterk = (struct jump_label_key *)(unsigned long)iter->key; 151 arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? 152 JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); 153 if (iterk == key) 154 continue; 155 156 key = iterk; 157 key->entries = iter; 158 #ifdef CONFIG_MODULES 159 key->next = NULL; 160 #endif 161 } 162 jump_label_unlock(); 163 } 164 165 #ifdef CONFIG_MODULES 166 167 struct jump_label_mod { 168 struct jump_label_mod *next; 169 struct jump_entry *entries; 170 struct module *mod; 171 }; 172 173 static int __jump_label_mod_text_reserved(void *start, void *end) 174 { 175 struct module *mod; 176 177 mod = __module_text_address((unsigned long)start); 178 if (!mod) 179 return 0; 180 181 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); 182 183 return __jump_label_text_reserved(mod->jump_entries, 184 mod->jump_entries + mod->num_jump_entries, 185 start, end); 186 } 187 188 static void __jump_label_mod_update(struct jump_label_key *key, int enable) 189 { 190 struct jump_label_mod *mod = key->next; 191 192 while (mod) { 193 struct module *m = mod->mod; 194 195 __jump_label_update(key, mod->entries, 196 m->jump_entries + m->num_jump_entries, 197 enable); 198 mod = mod->next; 199 } 200 } 201 202 /*** 203 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() 204 * @mod: module to patch 205 * 206 * Allow for run-time selection of the optimal nops. Before the module 207 * loads patch these with arch_get_jump_label_nop(), which is specified by 208 * the arch specific jump label code. 209 */ 210 void jump_label_apply_nops(struct module *mod) 211 { 212 struct jump_entry *iter_start = mod->jump_entries; 213 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 214 struct jump_entry *iter; 215 216 /* if the module doesn't have jump label entries, just return */ 217 if (iter_start == iter_stop) 218 return; 219 220 for (iter = iter_start; iter < iter_stop; iter++) 221 arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); 222 } 223 224 static int jump_label_add_module(struct module *mod) 225 { 226 struct jump_entry *iter_start = mod->jump_entries; 227 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 228 struct jump_entry *iter; 229 struct jump_label_key *key = NULL; 230 struct jump_label_mod *jlm; 231 232 /* if the module doesn't have jump label entries, just return */ 233 if (iter_start == iter_stop) 234 return 0; 235 236 jump_label_sort_entries(iter_start, iter_stop); 237 238 for (iter = iter_start; iter < iter_stop; iter++) { 239 if (iter->key == (jump_label_t)(unsigned long)key) 240 continue; 241 242 key = (struct jump_label_key *)(unsigned long)iter->key; 243 244 if (__module_address(iter->key) == mod) { 245 atomic_set(&key->enabled, 0); 246 key->entries = iter; 247 key->next = NULL; 248 continue; 249 } 250 251 jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); 252 if (!jlm) 253 return -ENOMEM; 254 255 jlm->mod = mod; 256 jlm->entries = iter; 257 jlm->next = key->next; 258 key->next = jlm; 259 260 if (jump_label_enabled(key)) 261 __jump_label_update(key, iter, iter_stop, 262 JUMP_LABEL_ENABLE); 263 } 264 265 return 0; 266 } 267 268 static void jump_label_del_module(struct module *mod) 269 { 270 struct jump_entry *iter_start = mod->jump_entries; 271 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 272 struct jump_entry *iter; 273 struct jump_label_key *key = NULL; 274 struct jump_label_mod *jlm, **prev; 275 276 for (iter = iter_start; iter < iter_stop; iter++) { 277 if (iter->key == (jump_label_t)(unsigned long)key) 278 continue; 279 280 key = (struct jump_label_key *)(unsigned long)iter->key; 281 282 if (__module_address(iter->key) == mod) 283 continue; 284 285 prev = &key->next; 286 jlm = key->next; 287 288 while (jlm && jlm->mod != mod) { 289 prev = &jlm->next; 290 jlm = jlm->next; 291 } 292 293 if (jlm) { 294 *prev = jlm->next; 295 kfree(jlm); 296 } 297 } 298 } 299 300 static void jump_label_invalidate_module_init(struct module *mod) 301 { 302 struct jump_entry *iter_start = mod->jump_entries; 303 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 304 struct jump_entry *iter; 305 306 for (iter = iter_start; iter < iter_stop; iter++) { 307 if (within_module_init(iter->code, mod)) 308 iter->code = 0; 309 } 310 } 311 312 static int 313 jump_label_module_notify(struct notifier_block *self, unsigned long val, 314 void *data) 315 { 316 struct module *mod = data; 317 int ret = 0; 318 319 switch (val) { 320 case MODULE_STATE_COMING: 321 jump_label_lock(); 322 ret = jump_label_add_module(mod); 323 if (ret) 324 jump_label_del_module(mod); 325 jump_label_unlock(); 326 break; 327 case MODULE_STATE_GOING: 328 jump_label_lock(); 329 jump_label_del_module(mod); 330 jump_label_unlock(); 331 break; 332 case MODULE_STATE_LIVE: 333 jump_label_lock(); 334 jump_label_invalidate_module_init(mod); 335 jump_label_unlock(); 336 break; 337 } 338 339 return notifier_from_errno(ret); 340 } 341 342 struct notifier_block jump_label_module_nb = { 343 .notifier_call = jump_label_module_notify, 344 .priority = 1, /* higher than tracepoints */ 345 }; 346 347 static __init int jump_label_init_module(void) 348 { 349 return register_module_notifier(&jump_label_module_nb); 350 } 351 early_initcall(jump_label_init_module); 352 353 #endif /* CONFIG_MODULES */ 354 355 /*** 356 * jump_label_text_reserved - check if addr range is reserved 357 * @start: start text addr 358 * @end: end text addr 359 * 360 * checks if the text addr located between @start and @end 361 * overlaps with any of the jump label patch addresses. Code 362 * that wants to modify kernel text should first verify that 363 * it does not overlap with any of the jump label addresses. 364 * Caller must hold jump_label_mutex. 365 * 366 * returns 1 if there is an overlap, 0 otherwise 367 */ 368 int jump_label_text_reserved(void *start, void *end) 369 { 370 int ret = __jump_label_text_reserved(__start___jump_table, 371 __stop___jump_table, start, end); 372 373 if (ret) 374 return ret; 375 376 #ifdef CONFIG_MODULES 377 ret = __jump_label_mod_text_reserved(start, end); 378 #endif 379 return ret; 380 } 381 382 static void jump_label_update(struct jump_label_key *key, int enable) 383 { 384 struct jump_entry *entry = key->entries, *stop = __stop___jump_table; 385 386 #ifdef CONFIG_MODULES 387 struct module *mod = __module_address((jump_label_t)key); 388 389 __jump_label_mod_update(key, enable); 390 391 if (mod) 392 stop = mod->jump_entries + mod->num_jump_entries; 393 #endif 394 /* if there are no users, entry can be NULL */ 395 if (entry) 396 __jump_label_update(key, entry, stop, enable); 397 } 398 399 #endif 400