1 /* 2 * jump label support 3 * 4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 5 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> 6 * 7 */ 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/list.h> 12 #include <linux/slab.h> 13 #include <linux/sort.h> 14 #include <linux/err.h> 15 #include <linux/jump_label.h> 16 17 #ifdef HAVE_JUMP_LABEL 18 19 /* mutex to protect coming/going of the the jump_label table */ 20 static DEFINE_MUTEX(jump_label_mutex); 21 22 void jump_label_lock(void) 23 { 24 mutex_lock(&jump_label_mutex); 25 } 26 27 void jump_label_unlock(void) 28 { 29 mutex_unlock(&jump_label_mutex); 30 } 31 32 bool jump_label_enabled(struct jump_label_key *key) 33 { 34 return !!atomic_read(&key->enabled); 35 } 36 37 static int jump_label_cmp(const void *a, const void *b) 38 { 39 const struct jump_entry *jea = a; 40 const struct jump_entry *jeb = b; 41 42 if (jea->key < jeb->key) 43 return -1; 44 45 if (jea->key > jeb->key) 46 return 1; 47 48 return 0; 49 } 50 51 static void 52 jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) 53 { 54 unsigned long size; 55 56 size = (((unsigned long)stop - (unsigned long)start) 57 / sizeof(struct jump_entry)); 58 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 59 } 60 61 static void jump_label_update(struct jump_label_key *key, int enable); 62 63 void jump_label_inc(struct jump_label_key *key) 64 { 65 if (atomic_inc_not_zero(&key->enabled)) 66 return; 67 68 jump_label_lock(); 69 if (atomic_add_return(1, &key->enabled) == 1) 70 jump_label_update(key, JUMP_LABEL_ENABLE); 71 jump_label_unlock(); 72 } 73 74 void jump_label_dec(struct jump_label_key *key) 75 { 76 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) 77 return; 78 79 jump_label_update(key, JUMP_LABEL_DISABLE); 80 jump_label_unlock(); 81 } 82 83 static int addr_conflict(struct jump_entry *entry, void *start, void *end) 84 { 85 if (entry->code <= (unsigned long)end && 86 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 87 return 1; 88 89 return 0; 90 } 91 92 static int __jump_label_text_reserved(struct jump_entry *iter_start, 93 struct jump_entry *iter_stop, void *start, void *end) 94 { 95 struct jump_entry *iter; 96 97 iter = iter_start; 98 while (iter < iter_stop) { 99 if (addr_conflict(iter, start, end)) 100 return 1; 101 iter++; 102 } 103 104 return 0; 105 } 106 107 /* 108 * Update code which is definitely not currently executing. 109 * Architectures which need heavyweight synchronization to modify 110 * running code can override this to make the non-live update case 111 * cheaper. 112 */ 113 void __weak arch_jump_label_transform_static(struct jump_entry *entry, 114 enum jump_label_type type) 115 { 116 arch_jump_label_transform(entry, type); 117 } 118 119 static void __jump_label_update(struct jump_label_key *key, 120 struct jump_entry *entry, 121 struct jump_entry *stop, int enable) 122 { 123 for (; (entry < stop) && 124 (entry->key == (jump_label_t)(unsigned long)key); 125 entry++) { 126 /* 127 * entry->code set to 0 invalidates module init text sections 128 * kernel_text_address() verifies we are not in core kernel 129 * init code, see jump_label_invalidate_module_init(). 130 */ 131 if (entry->code && kernel_text_address(entry->code)) 132 arch_jump_label_transform(entry, enable); 133 } 134 } 135 136 void __init jump_label_init(void) 137 { 138 struct jump_entry *iter_start = __start___jump_table; 139 struct jump_entry *iter_stop = __stop___jump_table; 140 struct jump_label_key *key = NULL; 141 struct jump_entry *iter; 142 143 jump_label_lock(); 144 jump_label_sort_entries(iter_start, iter_stop); 145 146 for (iter = iter_start; iter < iter_stop; iter++) { 147 struct jump_label_key *iterk; 148 149 iterk = (struct jump_label_key *)(unsigned long)iter->key; 150 arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? 151 JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); 152 if (iterk == key) 153 continue; 154 155 key = iterk; 156 key->entries = iter; 157 #ifdef CONFIG_MODULES 158 key->next = NULL; 159 #endif 160 } 161 jump_label_unlock(); 162 } 163 164 #ifdef CONFIG_MODULES 165 166 struct jump_label_mod { 167 struct jump_label_mod *next; 168 struct jump_entry *entries; 169 struct module *mod; 170 }; 171 172 static int __jump_label_mod_text_reserved(void *start, void *end) 173 { 174 struct module *mod; 175 176 mod = __module_text_address((unsigned long)start); 177 if (!mod) 178 return 0; 179 180 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); 181 182 return __jump_label_text_reserved(mod->jump_entries, 183 mod->jump_entries + mod->num_jump_entries, 184 start, end); 185 } 186 187 static void __jump_label_mod_update(struct jump_label_key *key, int enable) 188 { 189 struct jump_label_mod *mod = key->next; 190 191 while (mod) { 192 struct module *m = mod->mod; 193 194 __jump_label_update(key, mod->entries, 195 m->jump_entries + m->num_jump_entries, 196 enable); 197 mod = mod->next; 198 } 199 } 200 201 /*** 202 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() 203 * @mod: module to patch 204 * 205 * Allow for run-time selection of the optimal nops. Before the module 206 * loads patch these with arch_get_jump_label_nop(), which is specified by 207 * the arch specific jump label code. 208 */ 209 void jump_label_apply_nops(struct module *mod) 210 { 211 struct jump_entry *iter_start = mod->jump_entries; 212 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 213 struct jump_entry *iter; 214 215 /* if the module doesn't have jump label entries, just return */ 216 if (iter_start == iter_stop) 217 return; 218 219 for (iter = iter_start; iter < iter_stop; iter++) 220 arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); 221 } 222 223 static int jump_label_add_module(struct module *mod) 224 { 225 struct jump_entry *iter_start = mod->jump_entries; 226 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 227 struct jump_entry *iter; 228 struct jump_label_key *key = NULL; 229 struct jump_label_mod *jlm; 230 231 /* if the module doesn't have jump label entries, just return */ 232 if (iter_start == iter_stop) 233 return 0; 234 235 jump_label_sort_entries(iter_start, iter_stop); 236 237 for (iter = iter_start; iter < iter_stop; iter++) { 238 if (iter->key == (jump_label_t)(unsigned long)key) 239 continue; 240 241 key = (struct jump_label_key *)(unsigned long)iter->key; 242 243 if (__module_address(iter->key) == mod) { 244 atomic_set(&key->enabled, 0); 245 key->entries = iter; 246 key->next = NULL; 247 continue; 248 } 249 250 jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); 251 if (!jlm) 252 return -ENOMEM; 253 254 jlm->mod = mod; 255 jlm->entries = iter; 256 jlm->next = key->next; 257 key->next = jlm; 258 259 if (jump_label_enabled(key)) 260 __jump_label_update(key, iter, iter_stop, 261 JUMP_LABEL_ENABLE); 262 } 263 264 return 0; 265 } 266 267 static void jump_label_del_module(struct module *mod) 268 { 269 struct jump_entry *iter_start = mod->jump_entries; 270 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 271 struct jump_entry *iter; 272 struct jump_label_key *key = NULL; 273 struct jump_label_mod *jlm, **prev; 274 275 for (iter = iter_start; iter < iter_stop; iter++) { 276 if (iter->key == (jump_label_t)(unsigned long)key) 277 continue; 278 279 key = (struct jump_label_key *)(unsigned long)iter->key; 280 281 if (__module_address(iter->key) == mod) 282 continue; 283 284 prev = &key->next; 285 jlm = key->next; 286 287 while (jlm && jlm->mod != mod) { 288 prev = &jlm->next; 289 jlm = jlm->next; 290 } 291 292 if (jlm) { 293 *prev = jlm->next; 294 kfree(jlm); 295 } 296 } 297 } 298 299 static void jump_label_invalidate_module_init(struct module *mod) 300 { 301 struct jump_entry *iter_start = mod->jump_entries; 302 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 303 struct jump_entry *iter; 304 305 for (iter = iter_start; iter < iter_stop; iter++) { 306 if (within_module_init(iter->code, mod)) 307 iter->code = 0; 308 } 309 } 310 311 static int 312 jump_label_module_notify(struct notifier_block *self, unsigned long val, 313 void *data) 314 { 315 struct module *mod = data; 316 int ret = 0; 317 318 switch (val) { 319 case MODULE_STATE_COMING: 320 jump_label_lock(); 321 ret = jump_label_add_module(mod); 322 if (ret) 323 jump_label_del_module(mod); 324 jump_label_unlock(); 325 break; 326 case MODULE_STATE_GOING: 327 jump_label_lock(); 328 jump_label_del_module(mod); 329 jump_label_unlock(); 330 break; 331 case MODULE_STATE_LIVE: 332 jump_label_lock(); 333 jump_label_invalidate_module_init(mod); 334 jump_label_unlock(); 335 break; 336 } 337 338 return notifier_from_errno(ret); 339 } 340 341 struct notifier_block jump_label_module_nb = { 342 .notifier_call = jump_label_module_notify, 343 .priority = 1, /* higher than tracepoints */ 344 }; 345 346 static __init int jump_label_init_module(void) 347 { 348 return register_module_notifier(&jump_label_module_nb); 349 } 350 early_initcall(jump_label_init_module); 351 352 #endif /* CONFIG_MODULES */ 353 354 /*** 355 * jump_label_text_reserved - check if addr range is reserved 356 * @start: start text addr 357 * @end: end text addr 358 * 359 * checks if the text addr located between @start and @end 360 * overlaps with any of the jump label patch addresses. Code 361 * that wants to modify kernel text should first verify that 362 * it does not overlap with any of the jump label addresses. 363 * Caller must hold jump_label_mutex. 364 * 365 * returns 1 if there is an overlap, 0 otherwise 366 */ 367 int jump_label_text_reserved(void *start, void *end) 368 { 369 int ret = __jump_label_text_reserved(__start___jump_table, 370 __stop___jump_table, start, end); 371 372 if (ret) 373 return ret; 374 375 #ifdef CONFIG_MODULES 376 ret = __jump_label_mod_text_reserved(start, end); 377 #endif 378 return ret; 379 } 380 381 static void jump_label_update(struct jump_label_key *key, int enable) 382 { 383 struct jump_entry *entry = key->entries, *stop = __stop___jump_table; 384 385 #ifdef CONFIG_MODULES 386 struct module *mod = __module_address((jump_label_t)key); 387 388 __jump_label_mod_update(key, enable); 389 390 if (mod) 391 stop = mod->jump_entries + mod->num_jump_entries; 392 #endif 393 /* if there are no users, entry can be NULL */ 394 if (entry) 395 __jump_label_update(key, entry, stop, enable); 396 } 397 398 #endif 399