1 /* 2 * jump label support 3 * 4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 5 * 6 */ 7 #include <linux/jump_label.h> 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/list.h> 12 #include <linux/jhash.h> 13 #include <linux/slab.h> 14 #include <linux/sort.h> 15 #include <linux/err.h> 16 17 #ifdef HAVE_JUMP_LABEL 18 19 #define JUMP_LABEL_HASH_BITS 6 20 #define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS) 21 static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE]; 22 23 /* mutex to protect coming/going of the the jump_label table */ 24 static DEFINE_MUTEX(jump_label_mutex); 25 26 struct jump_label_entry { 27 struct hlist_node hlist; 28 struct jump_entry *table; 29 int nr_entries; 30 /* hang modules off here */ 31 struct hlist_head modules; 32 unsigned long key; 33 }; 34 35 struct jump_label_module_entry { 36 struct hlist_node hlist; 37 struct jump_entry *table; 38 int nr_entries; 39 struct module *mod; 40 }; 41 42 static int jump_label_cmp(const void *a, const void *b) 43 { 44 const struct jump_entry *jea = a; 45 const struct jump_entry *jeb = b; 46 47 if (jea->key < jeb->key) 48 return -1; 49 50 if (jea->key > jeb->key) 51 return 1; 52 53 return 0; 54 } 55 56 static void 57 sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) 58 { 59 unsigned long size; 60 61 size = (((unsigned long)stop - (unsigned long)start) 62 / sizeof(struct jump_entry)); 63 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 64 } 65 66 static struct jump_label_entry *get_jump_label_entry(jump_label_t key) 67 { 68 struct hlist_head *head; 69 struct hlist_node *node; 70 struct jump_label_entry *e; 71 u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0); 72 73 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; 74 hlist_for_each_entry(e, node, head, hlist) { 75 if (key == e->key) 76 return e; 77 } 78 return NULL; 79 } 80 81 static struct jump_label_entry * 82 add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table) 83 { 84 struct hlist_head *head; 85 struct jump_label_entry *e; 86 u32 hash; 87 88 e = get_jump_label_entry(key); 89 if (e) 90 return ERR_PTR(-EEXIST); 91 92 e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL); 93 if (!e) 94 return ERR_PTR(-ENOMEM); 95 96 hash = jhash((void *)&key, sizeof(jump_label_t), 0); 97 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; 98 e->key = key; 99 e->table = table; 100 e->nr_entries = nr_entries; 101 INIT_HLIST_HEAD(&(e->modules)); 102 hlist_add_head(&e->hlist, head); 103 return e; 104 } 105 106 static int 107 build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop) 108 { 109 struct jump_entry *iter, *iter_begin; 110 struct jump_label_entry *entry; 111 int count; 112 113 sort_jump_label_entries(start, stop); 114 iter = start; 115 while (iter < stop) { 116 entry = get_jump_label_entry(iter->key); 117 if (!entry) { 118 iter_begin = iter; 119 count = 0; 120 while ((iter < stop) && 121 (iter->key == iter_begin->key)) { 122 iter++; 123 count++; 124 } 125 entry = add_jump_label_entry(iter_begin->key, 126 count, iter_begin); 127 if (IS_ERR(entry)) 128 return PTR_ERR(entry); 129 } else { 130 WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n"); 131 return -1; 132 } 133 } 134 return 0; 135 } 136 137 /*** 138 * jump_label_update - update jump label text 139 * @key - key value associated with a a jump label 140 * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE 141 * 142 * Will enable/disable the jump for jump label @key, depending on the 143 * value of @type. 144 * 145 */ 146 147 void jump_label_update(unsigned long key, enum jump_label_type type) 148 { 149 struct jump_entry *iter; 150 struct jump_label_entry *entry; 151 struct hlist_node *module_node; 152 struct jump_label_module_entry *e_module; 153 int count; 154 155 mutex_lock(&jump_label_mutex); 156 entry = get_jump_label_entry((jump_label_t)key); 157 if (entry) { 158 count = entry->nr_entries; 159 iter = entry->table; 160 while (count--) { 161 if (kernel_text_address(iter->code)) 162 arch_jump_label_transform(iter, type); 163 iter++; 164 } 165 /* eanble/disable jump labels in modules */ 166 hlist_for_each_entry(e_module, module_node, &(entry->modules), 167 hlist) { 168 count = e_module->nr_entries; 169 iter = e_module->table; 170 while (count--) { 171 if (kernel_text_address(iter->code)) 172 arch_jump_label_transform(iter, type); 173 iter++; 174 } 175 } 176 } 177 mutex_unlock(&jump_label_mutex); 178 } 179 180 static int addr_conflict(struct jump_entry *entry, void *start, void *end) 181 { 182 if (entry->code <= (unsigned long)end && 183 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 184 return 1; 185 186 return 0; 187 } 188 189 #ifdef CONFIG_MODULES 190 191 static int module_conflict(void *start, void *end) 192 { 193 struct hlist_head *head; 194 struct hlist_node *node, *node_next, *module_node, *module_node_next; 195 struct jump_label_entry *e; 196 struct jump_label_module_entry *e_module; 197 struct jump_entry *iter; 198 int i, count; 199 int conflict = 0; 200 201 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { 202 head = &jump_label_table[i]; 203 hlist_for_each_entry_safe(e, node, node_next, head, hlist) { 204 hlist_for_each_entry_safe(e_module, module_node, 205 module_node_next, 206 &(e->modules), hlist) { 207 count = e_module->nr_entries; 208 iter = e_module->table; 209 while (count--) { 210 if (addr_conflict(iter, start, end)) { 211 conflict = 1; 212 goto out; 213 } 214 iter++; 215 } 216 } 217 } 218 } 219 out: 220 return conflict; 221 } 222 223 #endif 224 225 /*** 226 * jump_label_text_reserved - check if addr range is reserved 227 * @start: start text addr 228 * @end: end text addr 229 * 230 * checks if the text addr located between @start and @end 231 * overlaps with any of the jump label patch addresses. Code 232 * that wants to modify kernel text should first verify that 233 * it does not overlap with any of the jump label addresses. 234 * 235 * returns 1 if there is an overlap, 0 otherwise 236 */ 237 int jump_label_text_reserved(void *start, void *end) 238 { 239 struct jump_entry *iter; 240 struct jump_entry *iter_start = __start___jump_table; 241 struct jump_entry *iter_stop = __start___jump_table; 242 int conflict = 0; 243 244 mutex_lock(&jump_label_mutex); 245 iter = iter_start; 246 while (iter < iter_stop) { 247 if (addr_conflict(iter, start, end)) { 248 conflict = 1; 249 goto out; 250 } 251 iter++; 252 } 253 254 /* now check modules */ 255 #ifdef CONFIG_MODULES 256 conflict = module_conflict(start, end); 257 #endif 258 out: 259 mutex_unlock(&jump_label_mutex); 260 return conflict; 261 } 262 263 static __init int init_jump_label(void) 264 { 265 int ret; 266 struct jump_entry *iter_start = __start___jump_table; 267 struct jump_entry *iter_stop = __stop___jump_table; 268 struct jump_entry *iter; 269 270 mutex_lock(&jump_label_mutex); 271 ret = build_jump_label_hashtable(__start___jump_table, 272 __stop___jump_table); 273 iter = iter_start; 274 while (iter < iter_stop) { 275 arch_jump_label_text_poke_early(iter->code); 276 iter++; 277 } 278 mutex_unlock(&jump_label_mutex); 279 return ret; 280 } 281 early_initcall(init_jump_label); 282 283 #ifdef CONFIG_MODULES 284 285 static struct jump_label_module_entry * 286 add_jump_label_module_entry(struct jump_label_entry *entry, 287 struct jump_entry *iter_begin, 288 int count, struct module *mod) 289 { 290 struct jump_label_module_entry *e; 291 292 e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL); 293 if (!e) 294 return ERR_PTR(-ENOMEM); 295 e->mod = mod; 296 e->nr_entries = count; 297 e->table = iter_begin; 298 hlist_add_head(&e->hlist, &entry->modules); 299 return e; 300 } 301 302 static int add_jump_label_module(struct module *mod) 303 { 304 struct jump_entry *iter, *iter_begin; 305 struct jump_label_entry *entry; 306 struct jump_label_module_entry *module_entry; 307 int count; 308 309 /* if the module doesn't have jump label entries, just return */ 310 if (!mod->num_jump_entries) 311 return 0; 312 313 sort_jump_label_entries(mod->jump_entries, 314 mod->jump_entries + mod->num_jump_entries); 315 iter = mod->jump_entries; 316 while (iter < mod->jump_entries + mod->num_jump_entries) { 317 entry = get_jump_label_entry(iter->key); 318 iter_begin = iter; 319 count = 0; 320 while ((iter < mod->jump_entries + mod->num_jump_entries) && 321 (iter->key == iter_begin->key)) { 322 iter++; 323 count++; 324 } 325 if (!entry) { 326 entry = add_jump_label_entry(iter_begin->key, 0, NULL); 327 if (IS_ERR(entry)) 328 return PTR_ERR(entry); 329 } 330 module_entry = add_jump_label_module_entry(entry, iter_begin, 331 count, mod); 332 if (IS_ERR(module_entry)) 333 return PTR_ERR(module_entry); 334 } 335 return 0; 336 } 337 338 static void remove_jump_label_module(struct module *mod) 339 { 340 struct hlist_head *head; 341 struct hlist_node *node, *node_next, *module_node, *module_node_next; 342 struct jump_label_entry *e; 343 struct jump_label_module_entry *e_module; 344 int i; 345 346 /* if the module doesn't have jump label entries, just return */ 347 if (!mod->num_jump_entries) 348 return; 349 350 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { 351 head = &jump_label_table[i]; 352 hlist_for_each_entry_safe(e, node, node_next, head, hlist) { 353 hlist_for_each_entry_safe(e_module, module_node, 354 module_node_next, 355 &(e->modules), hlist) { 356 if (e_module->mod == mod) { 357 hlist_del(&e_module->hlist); 358 kfree(e_module); 359 } 360 } 361 if (hlist_empty(&e->modules) && (e->nr_entries == 0)) { 362 hlist_del(&e->hlist); 363 kfree(e); 364 } 365 } 366 } 367 } 368 369 static int 370 jump_label_module_notify(struct notifier_block *self, unsigned long val, 371 void *data) 372 { 373 struct module *mod = data; 374 int ret = 0; 375 376 switch (val) { 377 case MODULE_STATE_COMING: 378 mutex_lock(&jump_label_mutex); 379 ret = add_jump_label_module(mod); 380 if (ret) 381 remove_jump_label_module(mod); 382 mutex_unlock(&jump_label_mutex); 383 break; 384 case MODULE_STATE_GOING: 385 mutex_lock(&jump_label_mutex); 386 remove_jump_label_module(mod); 387 mutex_unlock(&jump_label_mutex); 388 break; 389 } 390 return ret; 391 } 392 393 /*** 394 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() 395 * @mod: module to patch 396 * 397 * Allow for run-time selection of the optimal nops. Before the module 398 * loads patch these with arch_get_jump_label_nop(), which is specified by 399 * the arch specific jump label code. 400 */ 401 void jump_label_apply_nops(struct module *mod) 402 { 403 struct jump_entry *iter; 404 405 /* if the module doesn't have jump label entries, just return */ 406 if (!mod->num_jump_entries) 407 return; 408 409 iter = mod->jump_entries; 410 while (iter < mod->jump_entries + mod->num_jump_entries) { 411 arch_jump_label_text_poke_early(iter->code); 412 iter++; 413 } 414 } 415 416 struct notifier_block jump_label_module_nb = { 417 .notifier_call = jump_label_module_notify, 418 .priority = 0, 419 }; 420 421 static __init int init_jump_label_module(void) 422 { 423 return register_module_notifier(&jump_label_module_nb); 424 } 425 early_initcall(init_jump_label_module); 426 427 #endif /* CONFIG_MODULES */ 428 429 #endif 430