Lines Matching +full:patch +full:- +full:address
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * core.c - Kernel Live Patching Core
25 #include "patch.h"
31 * accesses to klp-related variables and structures must have mutex protection,
34 * - klp_ftrace_handler()
35 * - klp_update_patch_state()
36 * - __klp_sched_try_switch()
51 return obj->name; in klp_is_module()
54 /* sets obj->mod if object is not vmlinux and module is found */
68 mod = find_module(obj->name); in klp_find_object_module()
71 * Note that the patch might still be needed before klp_module_going() in klp_find_object_module()
73 * until mod->exit() finishes. This is especially important for in klp_find_object_module()
76 if (mod && mod->klp_alive) in klp_find_object_module()
77 obj->mod = mod; in klp_find_object_module()
95 if ((strcmp(old_func->old_name, func->old_name) == 0) && in klp_find_func()
96 ((old_func->old_sympos == func->old_sympos) || in klp_find_func()
97 (old_func->old_sympos == 0 && func->old_sympos == 1) || in klp_find_func()
98 (old_func->old_sympos == 1 && func->old_sympos == 0))) { in klp_find_func()
106 static struct klp_object *klp_find_object(struct klp_patch *patch, in klp_find_object() argument
111 klp_for_each_object(patch, obj) { in klp_find_object()
114 strcmp(old_obj->name, obj->name) == 0) { in klp_find_object()
136 args->addr = addr; in klp_match_callback()
137 args->count++; in klp_match_callback()
141 * or the position is not defined for a non-unique symbol. in klp_match_callback()
143 if ((args->pos && (args->count == args->pos)) || in klp_match_callback()
144 (!args->pos && (args->count > 1))) in klp_match_callback()
154 if (strcmp(args->name, name)) in klp_find_callback()
176 * Ensure an address was found. If sympos is 0, ensure symbol is unique; in klp_find_object_symbol()
193 return -EINVAL; in klp_find_object_symbol()
211 * call are hard-coded and correspond to MODULE_NAME_LEN and in klp_resolve_symbols()
221 relas = (Elf_Rela *) relasec->sh_addr; in klp_resolve_symbols()
223 for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) { in klp_resolve_symbols()
225 if (sym->st_shndx != SHN_LIVEPATCH) { in klp_resolve_symbols()
227 strtab + sym->st_name, symndx, i); in klp_resolve_symbols()
228 return -EINVAL; in klp_resolve_symbols()
232 cnt = sscanf(strtab + sym->st_name, in klp_resolve_symbols()
237 strtab + sym->st_name); in klp_resolve_symbols()
238 return -EINVAL; in klp_resolve_symbols()
244 * Prevent module-specific KLP rela sections from referencing in klp_resolve_symbols()
250 …pr_err("invalid access to vmlinux symbol '%s' from module-specific livepatch relocation section\n", in klp_resolve_symbols()
252 return -EINVAL; in klp_resolve_symbols()
261 sym->st_value = addr; in klp_resolve_symbols()
276 * At a high-level, there are two types of klp relocation sections: those which
281 * write vmlinux-specific klp relocations (.klp.rela.vmlinux.* sections).
288 * 2) When a to-be-patched module loads -- or is already loaded when a
289 * corresponding klp module loads -- klp code calls this function to write
290 * module-specific klp relocations (.klp.rela.{module}.* sections). These
292 * reference symbols which live in the to-be-patched module or one of its
295 * the to-be-patched module to be loaded and patched sometime *after* the
312 cnt = sscanf(shstrtab + sec->sh_name, KLP_RELOC_SEC_PREFIX "%55[^.]", in klp_write_section_relocs()
316 shstrtab + sec->sh_name); in klp_write_section_relocs()
317 return -EINVAL; in klp_write_section_relocs()
349 * /sys/kernel/livepatch/<patch>
350 * /sys/kernel/livepatch/<patch>/enabled
351 * /sys/kernel/livepatch/<patch>/transition
352 * /sys/kernel/livepatch/<patch>/force
353 * /sys/kernel/livepatch/<patch>/replace
354 * /sys/kernel/livepatch/<patch>/stack_order
355 * /sys/kernel/livepatch/<patch>/<object>
356 * /sys/kernel/livepatch/<patch>/<object>/patched
357 * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
359 static int __klp_disable_patch(struct klp_patch *patch);
364 struct klp_patch *patch; in enabled_store() local
372 patch = container_of(kobj, struct klp_patch, kobj); in enabled_store()
376 if (patch->enabled == enabled) { in enabled_store()
378 ret = -EINVAL; in enabled_store()
387 * Do not allow to re-enable a disabled patch. in enabled_store()
389 if (patch == klp_transition_patch) in enabled_store()
392 ret = __klp_disable_patch(patch); in enabled_store()
394 ret = -EINVAL; in enabled_store()
407 struct klp_patch *patch; in enabled_show() local
409 patch = container_of(kobj, struct klp_patch, kobj); in enabled_show()
410 return sysfs_emit(buf, "%d\n", patch->enabled); in enabled_show()
416 struct klp_patch *patch; in transition_show() local
418 patch = container_of(kobj, struct klp_patch, kobj); in transition_show()
419 return sysfs_emit(buf, "%d\n", patch == klp_transition_patch); in transition_show()
425 struct klp_patch *patch; in force_store() local
438 patch = container_of(kobj, struct klp_patch, kobj); in force_store()
439 if (patch != klp_transition_patch) { in force_store()
441 return -EINVAL; in force_store()
454 struct klp_patch *patch; in replace_show() local
456 patch = container_of(kobj, struct klp_patch, kobj); in replace_show()
457 return sysfs_emit(buf, "%d\n", patch->replace); in replace_show()
463 struct klp_patch *patch, *this_patch; in stack_order_show() local
470 klp_for_each_patch(patch) { in stack_order_show()
472 if (patch == this_patch) in stack_order_show()
502 return sysfs_emit(buf, "%d\n", obj->patched); in patched_show()
514 kfree(obj->name); in klp_free_object_dynamic()
520 static void klp_init_object_early(struct klp_patch *patch,
524 struct klp_patch *patch) in klp_alloc_object_dynamic() argument
533 obj->name = kstrdup(name, GFP_KERNEL); in klp_alloc_object_dynamic()
534 if (!obj->name) { in klp_alloc_object_dynamic()
540 klp_init_object_early(patch, obj); in klp_alloc_object_dynamic()
541 obj->dynamic = true; in klp_alloc_object_dynamic()
548 kfree(func->old_name); in klp_free_func_nop()
561 if (old_func->old_name) { in klp_alloc_func_nop()
562 func->old_name = kstrdup(old_func->old_name, GFP_KERNEL); in klp_alloc_func_nop()
563 if (!func->old_name) { in klp_alloc_func_nop()
571 * func->new_func is same as func->old_func. These addresses are in klp_alloc_func_nop()
574 func->old_sympos = old_func->old_sympos; in klp_alloc_func_nop()
575 func->nop = true; in klp_alloc_func_nop()
580 static int klp_add_object_nops(struct klp_patch *patch, in klp_add_object_nops() argument
586 obj = klp_find_object(patch, old_obj); in klp_add_object_nops()
589 obj = klp_alloc_object_dynamic(old_obj->name, patch); in klp_add_object_nops()
591 return -ENOMEM; in klp_add_object_nops()
601 return -ENOMEM; in klp_add_object_nops()
615 static int klp_add_nops(struct klp_patch *patch) in klp_add_nops() argument
624 err = klp_add_object_nops(patch, old_obj); in klp_add_nops()
635 struct klp_patch *patch; in klp_kobj_release_patch() local
637 patch = container_of(kobj, struct klp_patch, kobj); in klp_kobj_release_patch()
638 complete(&patch->finish); in klp_kobj_release_patch()
653 if (obj->dynamic) in klp_kobj_release_object()
669 if (func->nop) in klp_kobj_release_func()
683 if (nops_only && !func->nop) in __klp_free_funcs()
686 list_del(&func->node); in __klp_free_funcs()
687 kobject_put(&func->kobj); in __klp_free_funcs()
696 obj->mod = NULL; in klp_free_object_loaded()
699 func->old_func = NULL; in klp_free_object_loaded()
701 if (func->nop) in klp_free_object_loaded()
702 func->new_func = NULL; in klp_free_object_loaded()
706 static void __klp_free_objects(struct klp_patch *patch, bool nops_only) in __klp_free_objects() argument
710 klp_for_each_object_safe(patch, obj, tmp_obj) { in __klp_free_objects()
713 if (nops_only && !obj->dynamic) in __klp_free_objects()
716 list_del(&obj->node); in __klp_free_objects()
717 kobject_put(&obj->kobj); in __klp_free_objects()
721 static void klp_free_objects(struct klp_patch *patch) in klp_free_objects() argument
723 __klp_free_objects(patch, false); in klp_free_objects()
726 static void klp_free_objects_dynamic(struct klp_patch *patch) in klp_free_objects_dynamic() argument
728 __klp_free_objects(patch, true); in klp_free_objects_dynamic()
738 static void klp_free_patch_start(struct klp_patch *patch) in klp_free_patch_start() argument
740 if (!list_empty(&patch->list)) in klp_free_patch_start()
741 list_del(&patch->list); in klp_free_patch_start()
743 klp_free_objects(patch); in klp_free_patch_start()
751 * the last function accessing the livepatch structures when the patch
754 static void klp_free_patch_finish(struct klp_patch *patch) in klp_free_patch_finish() argument
759 * this is called when the patch gets disabled and it in klp_free_patch_finish()
762 kobject_put(&patch->kobj); in klp_free_patch_finish()
763 wait_for_completion(&patch->finish); in klp_free_patch_finish()
766 if (!patch->forced) in klp_free_patch_finish()
767 module_put(patch->mod); in klp_free_patch_finish()
771 * The livepatch might be freed from sysfs interface created by the patch.
777 struct klp_patch *patch = in klp_free_patch_work_fn() local
780 klp_free_patch_finish(patch); in klp_free_patch_work_fn()
783 void klp_free_patch_async(struct klp_patch *patch) in klp_free_patch_async() argument
785 klp_free_patch_start(patch); in klp_free_patch_async()
786 schedule_work(&patch->free_work); in klp_free_patch_async()
802 if (!func->old_name) in klp_init_func()
803 return -EINVAL; in klp_init_func()
806 * NOPs get the address later. The patched module must be loaded, in klp_init_func()
809 if (!func->new_func && !func->nop) in klp_init_func()
810 return -EINVAL; in klp_init_func()
812 if (strlen(func->old_name) >= KSYM_NAME_LEN) in klp_init_func()
813 return -EINVAL; in klp_init_func()
815 INIT_LIST_HEAD(&func->stack_node); in klp_init_func()
816 func->patched = false; in klp_init_func()
817 func->transition = false; in klp_init_func()
824 return kobject_add(&func->kobj, &obj->kobj, "%s,%lu", in klp_init_func()
825 func->old_name, in klp_init_func()
826 func->old_sympos ? func->old_sympos : 1); in klp_init_func()
829 static int klp_write_object_relocs(struct klp_patch *patch, in klp_write_object_relocs() argument
834 struct klp_modinfo *info = patch->mod->klp_info; in klp_write_object_relocs()
836 for (i = 1; i < info->hdr.e_shnum; i++) { in klp_write_object_relocs()
837 Elf_Shdr *sec = info->sechdrs + i; in klp_write_object_relocs()
839 if (!(sec->sh_flags & SHF_RELA_LIVEPATCH)) in klp_write_object_relocs()
842 ret = klp_write_section_relocs(patch->mod, info->sechdrs, in klp_write_object_relocs()
843 info->secstrings, in klp_write_object_relocs()
844 patch->mod->core_kallsyms.strtab, in klp_write_object_relocs()
845 info->symndx, i, obj->name, apply); in klp_write_object_relocs()
853 static int klp_apply_object_relocs(struct klp_patch *patch, in klp_apply_object_relocs() argument
856 return klp_write_object_relocs(patch, obj, true); in klp_apply_object_relocs()
859 static void klp_clear_object_relocs(struct klp_patch *patch, in klp_clear_object_relocs() argument
862 klp_write_object_relocs(patch, obj, false); in klp_clear_object_relocs()
866 static int klp_init_object_loaded(struct klp_patch *patch, in klp_init_object_loaded() argument
874 * Only write module-specific relocations here in klp_init_object_loaded()
875 * (.klp.rela.{module}.*). vmlinux-specific relocations were in klp_init_object_loaded()
879 ret = klp_apply_object_relocs(patch, obj); in klp_init_object_loaded()
885 ret = klp_find_object_symbol(obj->name, func->old_name, in klp_init_object_loaded()
886 func->old_sympos, in klp_init_object_loaded()
887 (unsigned long *)&func->old_func); in klp_init_object_loaded()
891 ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, in klp_init_object_loaded()
892 &func->old_size, NULL); in klp_init_object_loaded()
895 func->old_name); in klp_init_object_loaded()
896 return -ENOENT; in klp_init_object_loaded()
899 if (func->nop) in klp_init_object_loaded()
900 func->new_func = func->old_func; in klp_init_object_loaded()
902 ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, in klp_init_object_loaded()
903 &func->new_size, NULL); in klp_init_object_loaded()
906 func->old_name); in klp_init_object_loaded()
907 return -ENOENT; in klp_init_object_loaded()
914 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) in klp_init_object() argument
920 if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN) in klp_init_object()
921 return -EINVAL; in klp_init_object()
923 obj->patched = false; in klp_init_object()
924 obj->mod = NULL; in klp_init_object()
928 name = klp_is_module(obj) ? obj->name : "vmlinux"; in klp_init_object()
929 ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name); in klp_init_object()
940 ret = klp_init_object_loaded(patch, obj); in klp_init_object()
948 kobject_init(&func->kobj, &klp_ktype_func); in klp_init_func_early()
949 list_add_tail(&func->node, &obj->func_list); in klp_init_func_early()
952 static void klp_init_object_early(struct klp_patch *patch, in klp_init_object_early() argument
955 INIT_LIST_HEAD(&obj->func_list); in klp_init_object_early()
956 kobject_init(&obj->kobj, &klp_ktype_object); in klp_init_object_early()
957 list_add_tail(&obj->node, &patch->obj_list); in klp_init_object_early()
960 static void klp_init_patch_early(struct klp_patch *patch) in klp_init_patch_early() argument
965 INIT_LIST_HEAD(&patch->list); in klp_init_patch_early()
966 INIT_LIST_HEAD(&patch->obj_list); in klp_init_patch_early()
967 kobject_init(&patch->kobj, &klp_ktype_patch); in klp_init_patch_early()
968 patch->enabled = false; in klp_init_patch_early()
969 patch->forced = false; in klp_init_patch_early()
970 INIT_WORK(&patch->free_work, klp_free_patch_work_fn); in klp_init_patch_early()
971 init_completion(&patch->finish); in klp_init_patch_early()
973 klp_for_each_object_static(patch, obj) { in klp_init_patch_early()
974 klp_init_object_early(patch, obj); in klp_init_patch_early()
982 static int klp_init_patch(struct klp_patch *patch) in klp_init_patch() argument
987 ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name); in klp_init_patch()
991 if (patch->replace) { in klp_init_patch()
992 ret = klp_add_nops(patch); in klp_init_patch()
997 klp_for_each_object(patch, obj) { in klp_init_patch()
998 ret = klp_init_object(patch, obj); in klp_init_patch()
1003 list_add_tail(&patch->list, &klp_patches); in klp_init_patch()
1008 static int __klp_disable_patch(struct klp_patch *patch) in __klp_disable_patch() argument
1012 if (WARN_ON(!patch->enabled)) in __klp_disable_patch()
1013 return -EINVAL; in __klp_disable_patch()
1016 return -EBUSY; in __klp_disable_patch()
1018 klp_init_transition(patch, KLP_TRANSITION_UNPATCHED); in __klp_disable_patch()
1020 klp_for_each_object(patch, obj) in __klp_disable_patch()
1021 if (obj->patched) in __klp_disable_patch()
1025 * Enforce the order of the func->transition writes in in __klp_disable_patch()
1029 * this ensures the handler sees that func->transition is set. in __klp_disable_patch()
1034 patch->enabled = false; in __klp_disable_patch()
1040 static int __klp_enable_patch(struct klp_patch *patch) in __klp_enable_patch() argument
1046 return -EBUSY; in __klp_enable_patch()
1048 if (WARN_ON(patch->enabled)) in __klp_enable_patch()
1049 return -EINVAL; in __klp_enable_patch()
1051 pr_notice("enabling patch '%s'\n", patch->mod->name); in __klp_enable_patch()
1053 klp_init_transition(patch, KLP_TRANSITION_PATCHED); in __klp_enable_patch()
1056 * Enforce the order of the func->transition writes in in __klp_enable_patch()
1057 * klp_init_transition() and the ops->func_stack writes in in __klp_enable_patch()
1059 * func->transition updates before the handler is registered and the in __klp_enable_patch()
1064 klp_for_each_object(patch, obj) { in __klp_enable_patch()
1070 pr_warn("pre-patch callback failed for object '%s'\n", in __klp_enable_patch()
1071 klp_is_module(obj) ? obj->name : "vmlinux"); in __klp_enable_patch()
1077 pr_warn("failed to patch object '%s'\n", in __klp_enable_patch()
1078 klp_is_module(obj) ? obj->name : "vmlinux"); in __klp_enable_patch()
1084 patch->enabled = true; in __klp_enable_patch()
1089 pr_warn("failed to enable patch '%s'\n", patch->mod->name); in __klp_enable_patch()
1096 * klp_enable_patch() - enable the livepatch
1097 * @patch: patch to be enabled
1099 * Initializes the data structure associated with the patch, creates the sysfs
1108 int klp_enable_patch(struct klp_patch *patch) in klp_enable_patch() argument
1113 if (!patch || !patch->mod || !patch->objs) in klp_enable_patch()
1114 return -EINVAL; in klp_enable_patch()
1116 klp_for_each_object_static(patch, obj) { in klp_enable_patch()
1117 if (!obj->funcs) in klp_enable_patch()
1118 return -EINVAL; in klp_enable_patch()
1122 if (!is_livepatch_module(patch->mod)) { in klp_enable_patch()
1124 patch->mod->name); in klp_enable_patch()
1125 return -EINVAL; in klp_enable_patch()
1129 return -ENODEV; in klp_enable_patch()
1138 if (!klp_is_patch_compatible(patch)) { in klp_enable_patch()
1139 pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", in klp_enable_patch()
1140 patch->mod->name); in klp_enable_patch()
1142 return -EINVAL; in klp_enable_patch()
1145 if (!try_module_get(patch->mod)) { in klp_enable_patch()
1147 return -ENODEV; in klp_enable_patch()
1150 klp_init_patch_early(patch); in klp_enable_patch()
1152 ret = klp_init_patch(patch); in klp_enable_patch()
1156 ret = __klp_enable_patch(patch); in klp_enable_patch()
1165 klp_free_patch_start(patch); in klp_enable_patch()
1169 klp_free_patch_finish(patch); in klp_enable_patch()
1185 * klp_ftrace_handler() might still see the replaced patch on the stack.
1188 * this is handled transparently by patch->module_put.
1198 old_patch->enabled = false; in klp_unpatch_replaced_patches()
1210 * will see a valid ops->func_stack entry thanks to RCU.
1213 * in ops->func_stack. Therefore unregister_ftrace_function() is called.
1233 struct klp_patch *patch; in klp_cleanup_module_patches_limited() local
1236 klp_for_each_patch(patch) { in klp_cleanup_module_patches_limited()
1237 if (patch == limit) in klp_cleanup_module_patches_limited()
1240 klp_for_each_object(patch, obj) { in klp_cleanup_module_patches_limited()
1241 if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) in klp_cleanup_module_patches_limited()
1244 if (patch != klp_transition_patch) in klp_cleanup_module_patches_limited()
1247 pr_notice("reverting patch '%s' on unloading module '%s'\n", in klp_cleanup_module_patches_limited()
1248 patch->mod->name, obj->mod->name); in klp_cleanup_module_patches_limited()
1252 klp_clear_object_relocs(patch, obj); in klp_cleanup_module_patches_limited()
1262 struct klp_patch *patch; in klp_module_coming() local
1265 if (WARN_ON(mod->state != MODULE_STATE_COMING)) in klp_module_coming()
1266 return -EINVAL; in klp_module_coming()
1268 if (!strcmp(mod->name, "vmlinux")) { in klp_module_coming()
1270 return -EINVAL; in klp_module_coming()
1277 * get patched by a new patch. in klp_module_coming()
1279 mod->klp_alive = true; in klp_module_coming()
1281 klp_for_each_patch(patch) { in klp_module_coming()
1282 klp_for_each_object(patch, obj) { in klp_module_coming()
1283 if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) in klp_module_coming()
1286 obj->mod = mod; in klp_module_coming()
1288 ret = klp_init_object_loaded(patch, obj); in klp_module_coming()
1290 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n", in klp_module_coming()
1291 patch->mod->name, obj->mod->name, ret); in klp_module_coming()
1295 pr_notice("applying patch '%s' to loading module '%s'\n", in klp_module_coming()
1296 patch->mod->name, obj->mod->name); in klp_module_coming()
1300 pr_warn("pre-patch callback failed for object '%s'\n", in klp_module_coming()
1301 obj->name); in klp_module_coming()
1307 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n", in klp_module_coming()
1308 patch->mod->name, obj->mod->name, ret); in klp_module_coming()
1314 if (patch != klp_transition_patch) in klp_module_coming()
1327 * If a patch is unsuccessfully applied, return in klp_module_coming()
1330 pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n", in klp_module_coming()
1331 patch->mod->name, obj->mod->name, obj->mod->name); in klp_module_coming()
1332 mod->klp_alive = false; in klp_module_coming()
1333 obj->mod = NULL; in klp_module_coming()
1334 klp_cleanup_module_patches_limited(mod, patch); in klp_module_coming()
1342 if (WARN_ON(mod->state != MODULE_STATE_GOING && in klp_module_going()
1343 mod->state != MODULE_STATE_COMING)) in klp_module_going()
1350 * get patched by a new patch. in klp_module_going()
1352 mod->klp_alive = false; in klp_module_going()
1362 struct klp_modinfo *info = mod->klp_info; in klp_find_section_by_name()
1364 for (int i = 1; i < info->hdr.e_shnum; i++) { in klp_find_section_by_name()
1365 Elf_Shdr *shdr = &info->sechdrs[i]; in klp_find_section_by_name()
1367 if (!strcmp(info->secstrings + shdr->sh_name, name)) { in klp_find_section_by_name()
1368 *sec_size = shdr->sh_size; in klp_find_section_by_name()
1369 return (void *)shdr->sh_addr; in klp_find_section_by_name()
1382 return -ENOMEM; in klp_init()