hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- There are some codes that can be used in the solution without ftrace, such as module initialization and symbol relocation and so on, so include them into CONFIG_LIVEPATCH. Also note that CONFIG_LIVEPATCH_WO_FTRACE is still not implemented in this patch. Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/livepatch.h | 31 +++++++++------ kernel/livepatch/Makefile | 5 ++- kernel/livepatch/core.c | 80 +++++++++++++++++++++++++++++++++++++-- kernel/livepatch/core.h | 4 ++ 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 2c6ce24b641a..f635bd6b2c6f 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -72,11 +72,16 @@ struct klp_func { struct list_head node; struct list_head stack_node; unsigned long old_size, new_size; +#ifdef CONFIG_LIVEPATCH_FTRACE bool nop; +#endif bool patched; +#ifdef CONFIG_LIVEPATCH_FTRACE bool transition; +#endif }; +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_object; /** @@ -100,6 +105,7 @@ struct klp_callbacks { void (*post_unpatch)(struct klp_object *obj); bool post_unpatch_enabled; }; +#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_object - kernel object structure for live patching @@ -118,17 +124,22 @@ struct klp_object { /* external */ const char *name; struct klp_func *funcs; +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_callbacks callbacks; +#endif /* internal */ struct kobject kobj; struct list_head func_list; struct list_head node; struct module *mod; +#ifdef CONFIG_LIVEPATCH_FTRACE bool dynamic; +#endif bool patched; }; +#ifdef CONFIG_LIVEPATCH_FTRACE /** * struct klp_state - state of the system modified by the livepatch * @id: system state identifier (non-zero) @@ -140,6 +151,7 @@ struct klp_state { unsigned int version; void *data; }; +#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_patch - patch structure for live patching @@ -159,15 +171,19 @@ struct klp_patch { /* external */ struct module *mod; struct klp_object *objs; +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_state *states; bool replace; +#endif /* internal */ struct list_head list; struct kobject kobj; struct list_head obj_list; bool enabled; +#ifdef CONFIG_LIVEPATCH_FTRACE bool forced; +#endif struct work_struct free_work; struct completion finish; }; @@ -231,11 +247,6 @@ void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor); struct klp_state *klp_get_state(struct klp_patch *patch, unsigned long id); struct klp_state *klp_get_prev_state(unsigned long id); -int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, - const char *shstrtab, const char *strtab, - unsigned int symindex, unsigned int secindex, - const char *objname); - #else /* !CONFIG_LIVEPATCH_FTRACE */ static inline int klp_module_coming(struct module *mod) { return 0; } @@ -245,16 +256,12 @@ static inline void klp_update_patch_state(struct task_struct *task) {} static inline void klp_copy_process(struct task_struct *child) {} static inline bool klp_have_reliable_stack(void) { return true; } -static inline +#endif /* CONFIG_LIVEPATCH_FTRACE */ + int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, const char *shstrtab, const char *strtab, unsigned int symindex, unsigned int secindex, - const char *objname) -{ - return 0; -} - -#endif /* CONFIG_LIVEPATCH_FTRACE */ + const char *objname); #else /* !CONFIG_LIVEPATCH */ diff --git a/kernel/livepatch/Makefile b/kernel/livepatch/Makefile index 18f9cb687576..facf512b237a 100644 --- a/kernel/livepatch/Makefile +++ b/kernel/livepatch/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_LIVEPATCH_FTRACE) += livepatch.o +obj-$(CONFIG_LIVEPATCH) += livepatch.o -livepatch-objs := core.o patch.o shadow.o state.o transition.o +obj-$(CONFIG_LIVEPATCH_FTRACE) += patch.o shadow.o state.o transition.o +livepatch-objs := core.o diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index ecbc9b6aba3a..28705113b75c 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -22,9 +22,11 @@ #include <linux/rcupdate.h> #include <asm/cacheflush.h> #include "core.h" +#ifdef CONFIG_LIVEPATCH_FTRACE #include "patch.h" #include "state.h" #include "transition.h" +#endif /* CONFIG_LIVEPATCH_FTRACE */ /* * klp_mutex is a coarse lock which serializes access to klp data. All @@ -84,6 +86,7 @@ static bool klp_initialized(void) return !!klp_root_kobj; } +#ifdef CONFIG_LIVEPATCH_FTRACE static struct klp_func *klp_find_func(struct klp_object *obj, struct klp_func *old_func) { @@ -117,6 +120,7 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, return NULL; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ struct klp_find_arg { const char *name; @@ -260,6 +264,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, return 0; } +#ifdef CONFIG_LIVEPATCH_FTRACE void __weak clear_relocate_add(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -267,6 +272,7 @@ void __weak clear_relocate_add(Elf_Shdr *sechdrs, struct module *me) { } +#endif /* * At a high-level, there are two types of klp relocation sections: those which @@ -325,7 +331,9 @@ static int klp_write_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, return apply_relocate_add(sechdrs, strtab, symndx, secndx, pmod); } +#ifdef CONFIG_LIVEPATCH_FTRACE clear_relocate_add(sechdrs, strtab, symndx, secndx, pmod); +#endif return 0; } @@ -350,6 +358,8 @@ int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, * /sys/kernel/livepatch/<patch>/<object>/patched * /sys/kernel/livepatch/<patch>/<object>/<function,sympos> */ +#ifdef CONFIG_LIVEPATCH_FTRACE + static int __klp_disable_patch(struct klp_patch *patch); static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, @@ -395,6 +405,21 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ + +static int __klp_enable_patch(struct klp_patch *patch) +{ + return -EINVAL; +} + +static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + return -EINVAL; +} + +#endif /* CONFIG_LIVEPATCH_FTRACE */ + static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -404,6 +429,7 @@ static ssize_t enabled_show(struct kobject *kobj, return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled); } +#ifdef CONFIG_LIVEPATCH_FTRACE static ssize_t transition_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -442,18 +468,24 @@ static ssize_t force_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled); +#ifdef CONFIG_LIVEPATCH_FTRACE static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition); static struct kobj_attribute force_kobj_attr = __ATTR_WO(force); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static struct attribute *klp_patch_attrs[] = { &enabled_kobj_attr.attr, +#ifdef CONFIG_LIVEPATCH_FTRACE &transition_kobj_attr.attr, &force_kobj_attr.attr, +#endif /* CONFIG_LIVEPATCH_FTRACE */ NULL }; ATTRIBUTE_GROUPS(klp_patch); +#ifdef CONFIG_LIVEPATCH_FTRACE static ssize_t patched_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -475,12 +507,14 @@ static void klp_free_object_dynamic(struct klp_object *obj) kfree(obj->name); kfree(obj); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void klp_init_func_early(struct klp_object *obj, struct klp_func *func); static void klp_init_object_early(struct klp_patch *patch, struct klp_object *obj); +#ifdef CONFIG_LIVEPATCH_FTRACE static struct klp_object *klp_alloc_object_dynamic(const char *name, struct klp_patch *patch) { @@ -587,6 +621,7 @@ static int klp_add_nops(struct klp_patch *patch) return 0; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void klp_kobj_release_patch(struct kobject *kobj) { @@ -604,28 +639,34 @@ static const struct kobj_type klp_ktype_patch = { static void klp_kobj_release_object(struct kobject *kobj) { +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_object *obj; obj = container_of(kobj, struct klp_object, kobj); if (obj->dynamic) klp_free_object_dynamic(obj); +#endif } static const struct kobj_type klp_ktype_object = { .release = klp_kobj_release_object, .sysfs_ops = &kobj_sysfs_ops, +#ifdef CONFIG_LIVEPATCH_FTRACE .default_groups = klp_object_groups, +#endif }; static void klp_kobj_release_func(struct kobject *kobj) { +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_func *func; func = container_of(kobj, struct klp_func, kobj); if (func->nop) klp_free_func_nop(func); +#endif } static const struct kobj_type klp_ktype_func = { @@ -638,14 +679,17 @@ static void __klp_free_funcs(struct klp_object *obj, bool nops_only) struct klp_func *func, *tmp_func; klp_for_each_func_safe(obj, func, tmp_func) { +#ifdef CONFIG_LIVEPATCH_FTRACE if (nops_only && !func->nop) continue; +#endif list_del(&func->node); kobject_put(&func->kobj); } } +#ifdef CONFIG_LIVEPATCH_FTRACE /* Clean up when a patched object is unloaded */ static void klp_free_object_loaded(struct klp_object *obj) { @@ -660,6 +704,7 @@ static void klp_free_object_loaded(struct klp_object *obj) func->new_func = NULL; } } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void __klp_free_objects(struct klp_patch *patch, bool nops_only) { @@ -667,10 +712,10 @@ static void __klp_free_objects(struct klp_patch *patch, bool nops_only) klp_for_each_object_safe(patch, obj, tmp_obj) { __klp_free_funcs(obj, nops_only); - +#ifdef CONFIG_LIVEPATCH_FTRACE if (nops_only && !obj->dynamic) continue; - +#endif list_del(&obj->node); kobject_put(&obj->kobj); } @@ -681,10 +726,12 @@ static void klp_free_objects(struct klp_patch *patch) __klp_free_objects(patch, false); } +#ifdef CONFIG_LIVEPATCH_FTRACE static void klp_free_objects_dynamic(struct klp_patch *patch) { __klp_free_objects(patch, true); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ /* * This function implements the free operations that can be called safely @@ -720,9 +767,13 @@ static void klp_free_patch_finish(struct klp_patch *patch) kobject_put(&patch->kobj); wait_for_completion(&patch->finish); +#ifdef CONFIG_LIVEPATCH_FTRACE /* Put the module after the last access to struct klp_patch. */ if (!patch->forced) module_put(patch->mod); +#else + module_put(patch->mod); +#endif /* CONFIG_LIVEPATCH_FTRACE */ } /* @@ -738,6 +789,7 @@ static void klp_free_patch_work_fn(struct work_struct *work) klp_free_patch_finish(patch); } +#ifdef CONFIG_LIVEPATCH_FTRACE void klp_free_patch_async(struct klp_patch *patch) { klp_free_patch_start(patch); @@ -754,25 +806,33 @@ void klp_free_replaced_patches_async(struct klp_patch *new_patch) klp_free_patch_async(old_patch); } } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static int klp_init_func(struct klp_object *obj, struct klp_func *func) { if (!func->old_name) return -EINVAL; +#ifdef CONFIG_LIVEPATCH_FTRACE /* * NOPs get the address later. The patched module must be loaded, * see klp_init_object_loaded(). */ if (!func->new_func && !func->nop) return -EINVAL; +#else /* !CONFIG_LIVEPATCH_FTRACE */ + if (!func->new_func) + return -EINVAL; +#endif /* CONFIG_LIVEPATCH_FTRACE */ if (strlen(func->old_name) >= KSYM_NAME_LEN) return -EINVAL; INIT_LIST_HEAD(&func->stack_node); func->patched = false; +#ifdef CONFIG_LIVEPATCH_FTRACE func->transition = false; +#endif /* The format for the sysfs directory is <function,sympos> where sympos * is the nth occurrence of this symbol in kallsyms for the patched @@ -814,11 +874,13 @@ static int klp_apply_object_relocs(struct klp_patch *patch, return klp_write_object_relocs(patch, obj, true); } +#ifdef CONFIG_LIVEPATCH_FTRACE static void klp_clear_object_relocs(struct klp_patch *patch, struct klp_object *obj) { klp_write_object_relocs(patch, obj, false); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ /* parts of the initialization that is done only when the object is loaded */ static int klp_init_object_loaded(struct klp_patch *patch, @@ -853,10 +915,10 @@ static int klp_init_object_loaded(struct klp_patch *patch, func->old_name); return -ENOENT; } - +#ifdef CONFIG_LIVEPATCH_FTRACE if (func->nop) func->new_func = func->old_func; - +#endif ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, &func->new_size, NULL); if (!ret) { @@ -924,7 +986,9 @@ static void klp_init_patch_early(struct klp_patch *patch) INIT_LIST_HEAD(&patch->obj_list); kobject_init(&patch->kobj, &klp_ktype_patch); patch->enabled = false; +#ifdef CONFIG_LIVEPATCH_FTRACE patch->forced = false; +#endif INIT_WORK(&patch->free_work, klp_free_patch_work_fn); init_completion(&patch->finish); @@ -946,11 +1010,13 @@ static int klp_init_patch(struct klp_patch *patch) if (ret) return ret; +#ifdef CONFIG_LIVEPATCH_FTRACE if (patch->replace) { ret = klp_add_nops(patch); if (ret) return ret; } +#endif klp_for_each_object(patch, obj) { ret = klp_init_object(patch, obj); @@ -963,6 +1029,7 @@ static int klp_init_patch(struct klp_patch *patch) return 0; } +#ifdef CONFIG_LIVEPATCH_FTRACE static int __klp_disable_patch(struct klp_patch *patch) { struct klp_object *obj; @@ -1049,6 +1116,7 @@ static int __klp_enable_patch(struct klp_patch *patch) klp_cancel_transition(); return ret; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * klp_enable_patch() - enable the livepatch @@ -1093,12 +1161,14 @@ int klp_enable_patch(struct klp_patch *patch) mutex_lock(&klp_mutex); +#ifdef CONFIG_LIVEPATCH_FTRACE if (!klp_is_patch_compatible(patch)) { pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", patch->mod->name); mutex_unlock(&klp_mutex); return -EINVAL; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ if (!try_module_get(patch->mod)) { mutex_unlock(&klp_mutex); @@ -1130,6 +1200,7 @@ int klp_enable_patch(struct klp_patch *patch) } EXPORT_SYMBOL_GPL(klp_enable_patch); +#ifdef CONFIG_LIVEPATCH_FTRACE /* * This function unpatches objects from the replaced livepatches. * @@ -1313,6 +1384,7 @@ void klp_module_going(struct module *mod) mutex_unlock(&klp_mutex); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static int __init klp_init(void) { diff --git a/kernel/livepatch/core.h b/kernel/livepatch/core.h index 38209c7361b6..20f916239125 100644 --- a/kernel/livepatch/core.h +++ b/kernel/livepatch/core.h @@ -13,16 +13,19 @@ extern struct list_head klp_patches; #define klp_for_each_patch(patch) \ list_for_each_entry(patch, &klp_patches, list) +#ifdef CONFIG_LIVEPATCH_FTRACE void klp_free_patch_async(struct klp_patch *patch); void klp_free_replaced_patches_async(struct klp_patch *new_patch); void klp_unpatch_replaced_patches(struct klp_patch *new_patch); void klp_discard_nops(struct klp_patch *new_patch); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static inline bool klp_is_object_loaded(struct klp_object *obj) { return !obj->name || obj->mod; } +#ifdef CONFIG_LIVEPATCH_FTRACE static inline int klp_pre_patch_callback(struct klp_object *obj) { int ret = 0; @@ -55,5 +58,6 @@ static inline void klp_post_unpatch_callback(struct klp_object *obj) obj->callbacks.post_unpatch_enabled = false; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ #endif /* _LIVEPATCH_CORE_H */ -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- support livepatch without ftrace for x86_64 supported now: livepatch relocation when init_patch after load_module; instruction patched when enable; activeness function check; enforcing the patch stacking principle; x86_64 use variable length instruction, so there's no need to consider extra implementation for long jumps. Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Li Bin <huawei.libin@huawei.com> Signed-off-by: Wang ShaoBo <bobo.shaobowang@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/x86/configs/openeuler_defconfig | 4 +- arch/x86/include/asm/livepatch.h | 33 + arch/x86/kernel/Makefile | 1 + arch/x86/kernel/livepatch.c | 289 ++++++++ include/linux/livepatch.h | 89 ++- kernel/livepatch/Kconfig | 21 +- kernel/livepatch/core.c | 1013 +++++++++++++++++++++++++- 7 files changed, 1405 insertions(+), 45 deletions(-) create mode 100644 arch/x86/include/asm/livepatch.h create mode 100644 arch/x86/kernel/livepatch.c diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 465fea71fc97..84a402352c06 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -502,9 +502,9 @@ CONFIG_LEGACY_VSYSCALL_XONLY=y # CONFIG_CMDLINE_BOOL is not set CONFIG_MODIFY_LDT_SYSCALL=y # CONFIG_STRICT_SIGALTSTACK_SIZE is not set -CONFIG_HAVE_LIVEPATCH_FTRACE=y +CONFIG_HAVE_LIVEPATCH_WO_FTRACE=y CONFIG_LIVEPATCH=y -CONFIG_LIVEPATCH_FTRACE=y +CONFIG_LIVEPATCH_WO_FTRACE=y # end of Processor type and features CONFIG_FUNCTION_PADDING_CFI=11 diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h new file mode 100644 index 000000000000..359c06ad1cc2 --- /dev/null +++ b/arch/x86/include/asm/livepatch.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * livepatch.h - x86-specific Kernel Live Patching Core + * + * Copyright (C) 2023 Huawei. + */ + +#ifndef _ASM_X86_LIVEPATCH_H +#define _ASM_X86_LIVEPATCH_H + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + +#define JMP_E9_INSN_SIZE 5 +struct arch_klp_data { + unsigned char old_insns[JMP_E9_INSN_SIZE]; +}; + +#define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) + +struct klp_func; + +#define klp_smp_isb() +int arch_klp_patch_func(struct klp_func *func); +void arch_klp_unpatch_func(struct klp_func *func); +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func); +bool arch_check_jump_insn(unsigned long func_addr); +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); +void arch_klp_code_modify_prepare(void); +void arch_klp_code_modify_post_process(void); + +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + +#endif /* _ASM_X86_LIVEPATCH_H */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 3269a0e23d3a..6fafa4560c6d 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -90,6 +90,7 @@ obj-$(CONFIG_SMP) += setup_percpu.o obj-$(CONFIG_X86_MPPARSE) += mpparse.o obj-y += apic/ obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o +obj-$(CONFIG_LIVEPATCH_WO_FTRACE) += livepatch.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace_$(BITS).o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o diff --git a/arch/x86/kernel/livepatch.c b/arch/x86/kernel/livepatch.c new file mode 100644 index 000000000000..a877ffdf96ac --- /dev/null +++ b/arch/x86/kernel/livepatch.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * livepatch.c - x86-specific Kernel Live Patching Core + * + * Copyright (C) 2023 Huawei Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/kallsyms.h> +#include <linux/livepatch.h> +#include <linux/stacktrace.h> +#include <linux/memory.h> +#include <asm/text-patching.h> +#include <asm/stacktrace.h> +#include <asm/set_memory.h> +#include <asm/insn.h> + +#include <linux/slab.h> +#include <asm/nops.h> +#include <asm/sections.h> +#include <linux/kprobes.h> + +/* + * The instruction set on x86 is CISC. + * The instructions of call in same segment are 11101000(direct), + * 11111111(register indirect) and 11111111(memory indirect). + * The instructions of call in other segment are 10011010(direct), + * 11111111(indirect). + */ +static bool is_jump_insn(u8 *insn) +{ + if ((insn[0] == 0xE8) || (insn[0] == 0x9a)) + return true; + else if ((insn[0] == 0xFF) && ((insn[1] & 0x30) == 0x10)) + return true; + return false; +} + +bool arch_check_jump_insn(unsigned long func_addr) +{ + int len = JMP_E9_INSN_SIZE; + struct insn insn; + u8 *addr = (u8 *)func_addr; + + do { + if (is_jump_insn(addr)) + return true; + insn_init(&insn, addr, MAX_INSN_SIZE, 1); + insn_get_length(&insn); + if (!insn.length || !insn_complete(&insn)) + return true; + len -= insn.length; + addr += insn.length; + } while (len > 0); + + return false; +} + +static void klp_print_stack_trace(void *trace_ptr, int trace_len) +{ + int i; +#ifdef CONFIG_ARCH_STACKWALK + unsigned long *trace = trace_ptr; +#else + struct stack_trace *trace = trace_ptr; +#endif + + pr_err("Call Trace:\n"); +#ifdef CONFIG_ARCH_STACKWALK + for (i = 0; i < trace_len; i++) { + pr_err("[<%pK>] %pS\n", + (void *)trace[i], + (void *)trace[i]); + } +#else + for (i = 0; i < trace->nr_entries; i++) { + pr_err("[<%pK>] %pS\n", + (void *)trace->entries[i], + (void *)trace->entries[i]); + } +#endif + +} + +#ifdef MAX_STACK_ENTRIES +#undef MAX_STACK_ENTRIES +#endif +#define MAX_STACK_ENTRIES 100 + +static int klp_check_stack(void *trace_ptr, int trace_len, + bool (*fn)(void *, int *, unsigned long), void *data) +{ +#ifdef CONFIG_ARCH_STACKWALK + unsigned long *trace = trace_ptr; +#else + struct stack_trace *trace = trace_ptr; +#endif + unsigned long address; + int i, ret; + +#ifdef CONFIG_ARCH_STACKWALK + for (i = 0; i < trace_len; i++) { + address = trace[i]; +#else + for (i = 0; i < trace->nr_entries; i++) { + address = trace->entries[i]; +#endif + if (!fn(data, &ret, address)) { +#ifdef CONFIG_ARCH_STACKWALK + klp_print_stack_trace(trace_ptr, trace_len); +#else + klp_print_stack_trace(trace_ptr, 0); +#endif + return ret; + } + } + + return 0; +} + +static int check_task_calltrace(struct task_struct *t, + bool (*fn)(void *, int *, unsigned long), + void *data) +{ + int ret = 0; + static unsigned long trace_entries[MAX_STACK_ENTRIES]; +#ifdef CONFIG_ARCH_STACKWALK + int trace_len; +#else + struct stack_trace trace; +#endif + +#ifdef CONFIG_ARCH_STACKWALK + ret = stack_trace_save_tsk_reliable(t, trace_entries, MAX_STACK_ENTRIES); + if (ret < 0) { + pr_err("%s:%d has an unreliable stack, ret=%d\n", + t->comm, t->pid, ret); + return ret; + } + trace_len = ret; + ret = klp_check_stack(trace_entries, trace_len, fn, data); +#else + trace.skip = 0; + trace.nr_entries = 0; + trace.max_entries = MAX_STACK_ENTRIES; + trace.entries = trace_entries; + ret = save_stack_trace_tsk_reliable(t, &trace); + if (ret) { + pr_err("%s: %s:%d has an unreliable stack, ret=%d\n", + __func__, t->comm, t->pid, ret); + return ret; + } + ret = klp_check_stack(&trace, 0, fn, data); +#endif + if (ret) { + pr_err("%s:%d check stack failed, ret=%d\n", + t->comm, t->pid, ret); + return ret; + } + return 0; +} + +static int do_check_calltrace(bool (*fn)(void *, int *, unsigned long), void *data) +{ + int ret = 0; + struct task_struct *g, *t; + unsigned int cpu; + + for_each_process_thread(g, t) { + if (klp_is_migration_thread(t->comm)) + continue; + + ret = check_task_calltrace(t, fn, data); + if (ret) + return ret; + } + for_each_online_cpu(cpu) { + ret = check_task_calltrace(idle_task(cpu), fn, data); + if (ret) + return ret; + } + return 0; +} + +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data) +{ + return do_check_calltrace(check_func, data); +} + +void arch_klp_code_modify_prepare(void) + __acquires(&text_mutex) +{ + mutex_lock(&text_mutex); +} + +void arch_klp_code_modify_post_process(void) + __releases(&text_mutex) +{ + text_poke_sync(); + mutex_unlock(&text_mutex); +} + +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + long ret; + + /* Prevent text modification */ + mutex_lock(&text_mutex); + ret = copy_from_kernel_nofault(arch_data->old_insns, + old_func, JMP_E9_INSN_SIZE); + mutex_unlock(&text_mutex); + + return ret; +} + +static void klp_patch_text(void *dst, const void *src, int len) +{ + if (len <= 1) + return; + /* skip breakpoint at first */ + text_poke(dst + 1, src + 1, len - 1); + /* + * Avoid compile optimization, make sure that instructions + * except first breakpoint has been patched. + */ + barrier(); + /* update jmp opcode */ + text_poke(dst, src, 1); +} + +static void *klp_jmp_code(unsigned long ip, unsigned long addr) +{ + return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr); +} + +int arch_klp_patch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + unsigned long ip, new_addr; + unsigned char *new; + + func_node = func->func_node; + ip = (unsigned long)func->old_func; + list_add_rcu(&func->stack_node, &func_node->func_stack); + new_addr = (unsigned long)func->new_func; + /* replace the text with the new text */ + new = (unsigned char *)klp_jmp_code(ip, new_addr); + klp_patch_text((void *)ip, (const void *)new, JMP_E9_INSN_SIZE); + return 0; +} + +void arch_klp_unpatch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + struct klp_func *next_func; + unsigned long ip, new_addr; + void *new; + + func_node = func->func_node; + ip = (unsigned long)func_node->old_func; + list_del_rcu(&func->stack_node); + if (list_empty(&func_node->func_stack)) { + new = func_node->arch_data.old_insns; + } else { + next_func = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + + new_addr = (unsigned long)next_func->new_func; + new = klp_jmp_code(ip, new_addr); + } + + /* replace the text with the new text */ + klp_patch_text((void *)ip, (const void *)new, JMP_E9_INSN_SIZE); +} diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index f635bd6b2c6f..b5ef9c2d8718 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -17,11 +17,17 @@ #if IS_ENABLED(CONFIG_LIVEPATCH) +#include <asm/livepatch.h> + /* task patch states */ #define KLP_UNDEFINED -1 #define KLP_UNPATCHED 0 #define KLP_PATCHED 1 +#define KLP_NORMAL_FORCE 0 +#define KLP_ENFORCEMENT 1 +#define KLP_STACK_OPTIMIZE 2 + /** * struct klp_func - function structure for live patching * @old_name: name of the function to be patched @@ -65,6 +71,7 @@ struct klp_func { * in kallsyms for the given object is used. */ unsigned long old_sympos; + int force; /* Only used in the solution without ftrace */ /* internal */ void *old_func; @@ -72,16 +79,14 @@ struct klp_func { struct list_head node; struct list_head stack_node; unsigned long old_size, new_size; -#ifdef CONFIG_LIVEPATCH_FTRACE - bool nop; -#endif + bool nop; /* Not used in the solution without ftrace */ bool patched; #ifdef CONFIG_LIVEPATCH_FTRACE bool transition; #endif + void *func_node; /* Only used in the solution without ftrace */ }; -#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_object; /** @@ -105,7 +110,6 @@ struct klp_callbacks { void (*post_unpatch)(struct klp_object *obj); bool post_unpatch_enabled; }; -#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_object - kernel object structure for live patching @@ -124,22 +128,17 @@ struct klp_object { /* external */ const char *name; struct klp_func *funcs; -#ifdef CONFIG_LIVEPATCH_FTRACE - struct klp_callbacks callbacks; -#endif + struct klp_callbacks callbacks; /* Not used in the solution without ftrace */ /* internal */ struct kobject kobj; struct list_head func_list; struct list_head node; struct module *mod; -#ifdef CONFIG_LIVEPATCH_FTRACE - bool dynamic; -#endif + bool dynamic; /* Not used in the solution without ftrace */ bool patched; }; -#ifdef CONFIG_LIVEPATCH_FTRACE /** * struct klp_state - state of the system modified by the livepatch * @id: system state identifier (non-zero) @@ -151,7 +150,6 @@ struct klp_state { unsigned int version; void *data; }; -#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_patch - patch structure for live patching @@ -171,20 +169,16 @@ struct klp_patch { /* external */ struct module *mod; struct klp_object *objs; -#ifdef CONFIG_LIVEPATCH_FTRACE - struct klp_state *states; - bool replace; -#endif + struct klp_state *states; /* Not used in the solution without ftrace */ + bool replace; /* Not used in the solution without ftrace */ /* internal */ struct list_head list; struct kobject kobj; struct list_head obj_list; bool enabled; -#ifdef CONFIG_LIVEPATCH_FTRACE - bool forced; -#endif - struct work_struct free_work; + bool forced; /* Not used in the solution without ftrace */ + struct work_struct free_work; /* Not used in the solution without ftrace */ struct completion finish; }; @@ -208,9 +202,9 @@ struct klp_patch { #define klp_for_each_func(obj, func) \ list_for_each_entry(func, &obj->func_list, node) +#ifdef CONFIG_LIVEPATCH_FTRACE int klp_enable_patch(struct klp_patch *); -#ifdef CONFIG_LIVEPATCH_FTRACE /* Called from the module loader during module coming/going states */ int klp_module_coming(struct module *mod); void klp_module_going(struct module *mod); @@ -249,6 +243,57 @@ struct klp_state *klp_get_prev_state(unsigned long id); #else /* !CONFIG_LIVEPATCH_FTRACE */ +struct klp_func_node { + struct list_head node; + struct list_head func_stack; + void *old_func; + struct arch_klp_data arch_data; +}; + +static inline +int klp_compare_address(unsigned long pc, unsigned long func_addr, + const char *func_name, unsigned long check_size) +{ + if (pc >= func_addr && pc < func_addr + check_size) { + pr_warn("func %s is in use!\n", func_name); + /* Return -EAGAIN for next retry */ + return -EAGAIN; + } + return 0; +} + +typedef int (*klp_add_func_t)(struct list_head *func_list, + unsigned long func_addr, unsigned long func_size, + const char *func_name, int force); + +struct walk_stackframe_args { + void *data; + int ret; + bool (*check_func)(void *data, int *ret, unsigned long pc); +}; + +#ifndef klp_smp_isb +#define klp_smp_isb() +#endif + +#define KLP_MIGRATION_NAME_PREFIX "migration/" +static inline bool klp_is_migration_thread(const char *task_name) +{ + /* + * current on other CPU + * we call this in stop_machine, so the current + * of each CPUs is migration, just compare the + * task_comm here, because we can't get the + * cpu_curr(task_cpu(t))). This assumes that no + * other thread will pretend to be a stopper via + * task_comm. + */ + return !strncmp(task_name, KLP_MIGRATION_NAME_PREFIX, + sizeof(KLP_MIGRATION_NAME_PREFIX) - 1); +} + +int klp_register_patch(struct klp_patch *patch); +int klp_unregister_patch(struct klp_patch *patch); static inline int klp_module_coming(struct module *mod) { return 0; } static inline void klp_module_going(struct module *mod) {} static inline bool klp_patch_pending(struct task_struct *task) { return false; } diff --git a/kernel/livepatch/Kconfig b/kernel/livepatch/Kconfig index 8ffe3b9ca1f9..d67a3e59a372 100644 --- a/kernel/livepatch/Kconfig +++ b/kernel/livepatch/Kconfig @@ -13,11 +13,12 @@ menu "Enable Livepatch" config LIVEPATCH bool "Kernel Live Patching" - depends on (HAVE_LIVEPATCH_FTRACE && (DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS)) || HAVE_LIVEPATCH_WO_FTRACE + depends on (HAVE_LIVEPATCH_FTRACE && (DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS)) || (HAVE_LIVEPATCH_WO_FTRACE && DEBUG_INFO) depends on MODULES depends on SYSFS depends on KALLSYMS_ALL depends on !TRIM_UNUSED_KSYMS + default n help Say Y here if you want to support kernel live patching. This option has no runtime impact until a kernel "patch" @@ -37,6 +38,7 @@ choice config LIVEPATCH_FTRACE bool "based on ftrace" depends on HAVE_LIVEPATCH_FTRACE + depends on DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS help Supports kernel live patching based on ftrace. This is the original implementation of kernel live @@ -46,6 +48,8 @@ config LIVEPATCH_FTRACE config LIVEPATCH_WO_FTRACE bool "without ftrace" depends on HAVE_LIVEPATCH_WO_FTRACE + depends on DEBUG_INFO + select LIVEPATCH_STOP_MACHINE_CONSISTENCY help Supports kernel live patching without ftrace. This solution will patch the first few instructions @@ -57,4 +61,19 @@ config LIVEPATCH_WO_FTRACE endchoice +config LIVEPATCH_STOP_MACHINE_CONSISTENCY + bool "Stop machine consistency" + depends on LIVEPATCH_WO_FTRACE + help + Use stop machine consistency model + stop-machine consistency and kpatch's stack + trace checking. + +config LIVEPATCH_STACK + bool "Enforcing the patch stacking principle" + depends on LIVEPATCH_WO_FTRACE + default y + help + Say N here if you want to remove the patch stacking principle. + endmenu diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 28705113b75c..72db3e79c303 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -4,6 +4,7 @@ * * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> * Copyright (C) 2014 SUSE + * Copyright (C) 2023 Huawei Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -26,6 +27,10 @@ #include "patch.h" #include "state.h" #include "transition.h" +#else /* !CONFIG_LIVEPATCH_FTRACE */ +#include <linux/proc_fs.h> +#include <linux/seq_file.h> +#include <linux/stop_machine.h> #endif /* CONFIG_LIVEPATCH_FTRACE */ /* @@ -53,6 +58,7 @@ static bool klp_is_module(struct klp_object *obj) return obj->name; } +#ifdef CONFIG_LIVEPATCH_FTRACE /* sets obj->mod if object is not vmlinux and module is found */ static void klp_find_object_module(struct klp_object *obj) { @@ -80,6 +86,9 @@ static void klp_find_object_module(struct klp_object *obj) rcu_read_unlock_sched(); } +#else /* !CONFIG_LIVEPATCH_FTRACE */ +static int klp_find_object_module(struct klp_object *obj); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static bool klp_initialized(void) { @@ -407,16 +416,8 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, #else /* !CONFIG_LIVEPATCH_FTRACE */ -static int __klp_enable_patch(struct klp_patch *patch) -{ - return -EINVAL; -} - static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t count) -{ - return -EINVAL; -} + const char *buf, size_t count); #endif /* CONFIG_LIVEPATCH_FTRACE */ @@ -711,6 +712,12 @@ static void __klp_free_objects(struct klp_patch *patch, bool nops_only) struct klp_object *obj, *tmp_obj; klp_for_each_object_safe(patch, obj, tmp_obj) { +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + if (klp_is_module(obj) && obj->mod) { + module_put(obj->mod); + obj->mod = NULL; + } +#endif __klp_free_funcs(obj, nops_only); #ifdef CONFIG_LIVEPATCH_FTRACE if (nops_only && !obj->dynamic) @@ -748,6 +755,7 @@ static void klp_free_patch_start(struct klp_patch *patch) klp_free_objects(patch); } +#ifdef CONFIG_LIVEPATCH_FTRACE /* * This function implements the free part that must be called outside * klp_mutex. @@ -767,13 +775,9 @@ static void klp_free_patch_finish(struct klp_patch *patch) kobject_put(&patch->kobj); wait_for_completion(&patch->finish); -#ifdef CONFIG_LIVEPATCH_FTRACE /* Put the module after the last access to struct klp_patch. */ if (!patch->forced) module_put(patch->mod); -#else - module_put(patch->mod); -#endif /* CONFIG_LIVEPATCH_FTRACE */ } /* @@ -789,7 +793,6 @@ static void klp_free_patch_work_fn(struct work_struct *work) klp_free_patch_finish(patch); } -#ifdef CONFIG_LIVEPATCH_FTRACE void klp_free_patch_async(struct klp_patch *patch) { klp_free_patch_start(patch); @@ -910,11 +913,25 @@ static int klp_init_object_loaded(struct klp_patch *patch, ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, &func->old_size, NULL); +#ifdef CONFIG_LIVEPATCH_FTRACE if (!ret) { pr_err("kallsyms size lookup failed for '%s'\n", func->old_name); return -ENOENT; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ + if (!ret || ((long)func->old_size < 0)) { + pr_err("kallsyms size lookup failed for '%s'\n", + func->old_name); + return -ENOENT; + } + if (func->old_size < KLP_MAX_REPLACE_SIZE) { + pr_err("%s size less than limit (%lu < %zu)\n", func->old_name, + func->old_size, KLP_MAX_REPLACE_SIZE); + return -EINVAL; + } +#endif /* CONFIG_LIVEPATCH_FTRACE */ + #ifdef CONFIG_LIVEPATCH_FTRACE if (func->nop) func->new_func = func->old_func; @@ -931,6 +948,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, return 0; } +#ifdef CONFIG_LIVEPATCH_FTRACE static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; @@ -961,12 +979,18 @@ static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) return ret; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ +static int klp_init_object(struct klp_patch *patch, struct klp_object *obj); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void klp_init_func_early(struct klp_object *obj, struct klp_func *func) { kobject_init(&func->kobj, &klp_ktype_func); list_add_tail(&func->node, &obj->func_list); +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + func->func_node = NULL; +#endif } static void klp_init_object_early(struct klp_patch *patch, @@ -975,6 +999,9 @@ static void klp_init_object_early(struct klp_patch *patch, INIT_LIST_HEAD(&obj->func_list); kobject_init(&obj->kobj, &klp_ktype_object); list_add_tail(&obj->node, &patch->obj_list); +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + obj->mod = NULL; +#endif } static void klp_init_patch_early(struct klp_patch *patch) @@ -988,8 +1015,8 @@ static void klp_init_patch_early(struct klp_patch *patch) patch->enabled = false; #ifdef CONFIG_LIVEPATCH_FTRACE patch->forced = false; -#endif INIT_WORK(&patch->free_work, klp_free_patch_work_fn); +#endif init_completion(&patch->finish); klp_for_each_object_static(patch, obj) { @@ -1116,7 +1143,6 @@ static int __klp_enable_patch(struct klp_patch *patch) klp_cancel_transition(); return ret; } -#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * klp_enable_patch() - enable the livepatch @@ -1161,14 +1187,12 @@ int klp_enable_patch(struct klp_patch *patch) mutex_lock(&klp_mutex); -#ifdef CONFIG_LIVEPATCH_FTRACE if (!klp_is_patch_compatible(patch)) { pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", patch->mod->name); mutex_unlock(&klp_mutex); return -EINVAL; } -#endif /* CONFIG_LIVEPATCH_FTRACE */ if (!try_module_get(patch->mod)) { mutex_unlock(&klp_mutex); @@ -1200,7 +1224,6 @@ int klp_enable_patch(struct klp_patch *patch) } EXPORT_SYMBOL_GPL(klp_enable_patch); -#ifdef CONFIG_LIVEPATCH_FTRACE /* * This function unpatches objects from the replaced livepatches. * @@ -1384,7 +1407,6 @@ void klp_module_going(struct module *mod) mutex_unlock(&klp_mutex); } -#endif /* CONFIG_LIVEPATCH_FTRACE */ static int __init klp_init(void) { @@ -1395,4 +1417,955 @@ static int __init klp_init(void) return 0; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ + +struct patch_data { + struct klp_patch *patch; + atomic_t cpu_count; +}; + +static bool klp_is_patch_registered(struct klp_patch *patch) +{ + struct klp_patch *mypatch; + + list_for_each_entry(mypatch, &klp_patches, list) + if (mypatch == patch) + return true; + + return false; +} + +static int state_show(struct seq_file *m, void *v) +{ + struct klp_patch *patch; + char *state; + int index = 0; + + seq_printf(m, "%-5s\t%-26s\t%-8s\n", "Index", "Patch", "State"); + seq_puts(m, "-----------------------------------------------\n"); + mutex_lock(&klp_mutex); + list_for_each_entry(patch, &klp_patches, list) { + if (patch->enabled) + state = "enabled"; + else + state = "disabled"; + + seq_printf(m, "%-5d\t%-26s\t%-8s\n", ++index, + patch->mod->name, state); + } + mutex_unlock(&klp_mutex); + seq_puts(m, "-----------------------------------------------\n"); + + return 0; +} + +static int klp_state_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, state_show, NULL); +} + +static const struct proc_ops proc_klpstate_operations = { + .proc_open = klp_state_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, +}; + +static int klp_find_object_module(struct klp_object *obj) +{ + struct module *mod; + + if (!klp_is_module(obj)) + return 0; + + rcu_read_lock_sched(); + /* + * We do not want to block removal of patched modules and therefore + * we do not take a reference here. The patches are removed by + * klp_module_going() instead. + */ + mod = find_module(obj->name); + if (!mod) { + pr_err("module '%s' not loaded\n", obj->name); + rcu_read_unlock_sched(); + return -ENOPKG; /* the deponds module is not loaded */ + } + + if (mod->state == MODULE_STATE_COMING || !try_module_get(mod)) { + rcu_read_unlock_sched(); + return -EINVAL; + } + + obj->mod = mod; + + rcu_read_unlock_sched(); + return 0; +} + +static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) +{ + struct klp_func *func; + int ret; + const char *name; + + if (klp_is_module(obj) && strnlen(obj->name, MODULE_NAME_LEN) >= MODULE_NAME_LEN) { + pr_err("obj name is too long\n"); + return -EINVAL; + } + klp_for_each_func(obj, func) { + if (!func->old_name) { + pr_err("old name is invalid\n"); + return -EINVAL; + } + /* + * NOPs get the address later. The patched module must be loaded, + * see klp_init_object_loaded(). + */ + if (!func->new_func && !func->nop) { + pr_err("new_func is invalid\n"); + return -EINVAL; + } + if (strlen(func->old_name) >= KSYM_NAME_LEN) { + pr_err("function old name is too long\n"); + return -EINVAL; + } + } + + obj->patched = false; + obj->mod = NULL; + + ret = klp_find_object_module(obj); + if (ret) + return ret; + + name = klp_is_module(obj) ? obj->name : "vmlinux"; + ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name); + if (ret) + goto out; + + /* + * For livepatch without ftrace, we need to modify the first N + * instructions of the to-be-patched func. So should check if the + * func length enough to allow this modification. + * + * We add check hook in klp_init_func and will using the old_size + * internally, so the klp_init_object_loaded should called first + * to fill the klp_func struct. + */ + if (klp_is_object_loaded(obj)) { + ret = klp_init_object_loaded(patch, obj); + if (ret) + goto out; + } + + klp_for_each_func(obj, func) { + ret = klp_init_func(obj, func); + if (ret) + goto out; + } + + return 0; + +out: + if (klp_is_module(obj)) { + module_put(obj->mod); + obj->mod = NULL; + } + return ret; +} + +int __weak arch_klp_check_calltrace(bool (*fn)(void *, int *, unsigned long), void *data) +{ + return -EINVAL; +} + +bool __weak arch_check_jump_insn(unsigned long func_addr) +{ + return true; +} + +int __weak arch_klp_check_activeness_func(struct klp_func *func, int enable, + klp_add_func_t add_func, + struct list_head *func_list) +{ + int ret; + unsigned long func_addr = 0; + unsigned long func_size; + struct klp_func_node *func_node = NULL; + unsigned long old_func = (unsigned long)func->old_func; + + func_node = func->func_node; + /* Check func address in stack */ + if (enable) { + if (func->patched || func->force == KLP_ENFORCEMENT) + return 0; + /* + * When enable, checking the currently active functions. + */ + if (list_empty(&func_node->func_stack)) { + /* + * Not patched on this function [the origin one] + */ + func_addr = old_func; + func_size = func->old_size; + } else { + /* + * Previously patched function [the active one] + */ + struct klp_func *prev; + + prev = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + func_addr = (unsigned long)prev->new_func; + func_size = prev->new_size; + } + /* + * When preemption is disabled and the replacement area + * does not contain a jump instruction, the migration + * thread is scheduled to run stop machine only after the + * execution of instructions to be replaced is complete. + */ + if (IS_ENABLED(CONFIG_PREEMPTION) || + (func->force == KLP_NORMAL_FORCE) || + arch_check_jump_insn(func_addr)) { + ret = add_func(func_list, func_addr, func_size, + func->old_name, func->force); + if (ret) + return ret; + if (func_addr != old_func) { + ret = add_func(func_list, old_func, KLP_MAX_REPLACE_SIZE, + func->old_name, func->force); + if (ret) + return ret; + } + } + } else { +#ifdef CONFIG_PREEMPTION + /* + * No scheduling point in the replacement instructions. Therefore, + * when preemption is not enabled, atomic execution is performed + * and these instructions will not appear on the stack. + */ + if (list_is_singular(&func_node->func_stack)) { + func_addr = old_func; + func_size = func->old_size; + } else { + struct klp_func *prev; + + prev = list_first_or_null_rcu( + &func_node->func_stack, + struct klp_func, stack_node); + func_addr = (unsigned long)prev->new_func; + func_size = prev->new_size; + } + ret = add_func(func_list, func_addr, + func_size, func->old_name, 0); + if (ret) + return ret; + if (func_addr != old_func) { + ret = add_func(func_list, old_func, KLP_MAX_REPLACE_SIZE, + func->old_name, 0); + if (ret) + return ret; + } +#endif + + func_addr = (unsigned long)func->new_func; + func_size = func->new_size; + ret = add_func(func_list, func_addr, + func_size, func->old_name, 0); + if (ret) + return ret; + } + return 0; +} + +static inline unsigned long klp_size_to_check(unsigned long func_size, + int force) +{ + unsigned long size = func_size; + + if (force == KLP_STACK_OPTIMIZE && size > KLP_MAX_REPLACE_SIZE) + size = KLP_MAX_REPLACE_SIZE; + return size; +} + +struct actv_func { + struct list_head list; + unsigned long func_addr; + unsigned long func_size; + const char *func_name; + int force; +}; + +static bool check_func_list(void *data, int *ret, unsigned long pc) +{ + struct list_head *func_list = (struct list_head *)data; + struct actv_func *func = NULL; + + list_for_each_entry(func, func_list, list) { + *ret = klp_compare_address(pc, func->func_addr, func->func_name, + klp_size_to_check(func->func_size, func->force)); + if (*ret) + return false; + } + return true; +} + +static int add_func_to_list(struct list_head *func_list, unsigned long func_addr, + unsigned long func_size, const char *func_name, + int force) +{ + struct actv_func *func = kzalloc(sizeof(struct actv_func), GFP_ATOMIC); + + if (!func) + return -ENOMEM; + func->func_addr = func_addr; + func->func_size = func_size; + func->func_name = func_name; + func->force = force; + list_add_tail(&func->list, func_list); + return 0; +} + +static void free_func_list(struct list_head *func_list) +{ + struct actv_func *func = NULL; + struct actv_func *tmp = NULL; + + list_for_each_entry_safe(func, tmp, func_list, list) { + list_del(&func->list); + kfree(func); + } +} + +static int klp_check_activeness_func(struct klp_patch *patch, int enable, + struct list_head *func_list) +{ + int ret; + struct klp_object *obj = NULL; + struct klp_func *func = NULL; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + ret = arch_klp_check_activeness_func(func, enable, + add_func_to_list, + func_list); + if (ret) + return ret; + } + } + return 0; +} + +static int klp_check_calltrace(struct klp_patch *patch, int enable) +{ + int ret = 0; + LIST_HEAD(func_list); + + ret = klp_check_activeness_func(patch, enable, &func_list); + if (ret) { + pr_err("collect active functions failed, ret=%d\n", ret); + goto out; + } + + if (list_empty(&func_list)) + goto out; + + ret = arch_klp_check_calltrace(check_func_list, (void *)&func_list); + +out: + free_func_list(&func_list); + return ret; +} + +static LIST_HEAD(klp_func_list); + +/* + * The caller must ensure that the klp_mutex lock is held or is in the rcu read + * critical area. + */ +static struct klp_func_node *klp_find_func_node(const void *old_func) +{ + struct klp_func_node *func_node; + + list_for_each_entry_rcu(func_node, &klp_func_list, node, + lockdep_is_held(&klp_mutex)) { + if (func_node->old_func == old_func) + return func_node; + } + + return NULL; +} + +static void klp_add_func_node(struct klp_func_node *func_node) +{ + list_add_rcu(&func_node->node, &klp_func_list); +} + +static void klp_del_func_node(struct klp_func_node *func_node) +{ + list_del_rcu(&func_node->node); +} + +void __weak *arch_klp_mem_alloc(size_t size) +{ + return kzalloc(size, GFP_ATOMIC); +} + +void __weak arch_klp_mem_free(void *mem) +{ + kfree(mem); +} + +void __weak arch_klp_code_modify_prepare(void) +{ +} + +void __weak arch_klp_code_modify_post_process(void) +{ +} + +long __weak arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + return -EINVAL; +} + +static struct klp_func_node *func_node_alloc(struct klp_func *func) +{ + long ret; + struct klp_func_node *func_node = NULL; + + func_node = klp_find_func_node(func->old_func); + if (func_node) /* The old_func has ever been patched */ + return func_node; + func_node = arch_klp_mem_alloc(sizeof(struct klp_func_node)); + if (func_node) { + INIT_LIST_HEAD(&func_node->func_stack); + func_node->old_func = func->old_func; + /* + * Module which contains 'old_func' would not be removed because + * it's reference count has been held during registration. + * But it's not in stop_machine context here, 'old_func' should + * not be modified as saving old code. + */ + ret = arch_klp_save_old_code(&func_node->arch_data, func->old_func); + if (ret) { + arch_klp_mem_free(func_node); + pr_err("save old code failed, ret=%ld\n", ret); + return NULL; + } + klp_add_func_node(func_node); + } + return func_node; +} + +static void func_node_free(struct klp_func *func) +{ + struct klp_func_node *func_node; + + func_node = func->func_node; + if (func_node) { + func->func_node = NULL; + if (list_empty(&func_node->func_stack)) { + klp_del_func_node(func_node); + synchronize_rcu(); + arch_klp_mem_free(func_node); + } + } +} + +static void klp_mem_recycle(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + func_node_free(func); + } + } +} + +static int klp_mem_prepare(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + func->func_node = func_node_alloc(func); + if (func->func_node == NULL) { + klp_mem_recycle(patch); + pr_err("alloc func_node failed\n"); + return -ENOMEM; + } + } + } + return 0; +} + +void __weak arch_klp_unpatch_func(struct klp_func *func) +{ +} + +int __weak arch_klp_patch_func(struct klp_func *func) +{ + return -EINVAL; +} + +static void klp_unpatch_func(struct klp_func *func) +{ + if (WARN_ON(!func->patched)) + return; + if (WARN_ON(!func->old_func)) + return; + if (WARN_ON(!func->func_node)) + return; + + arch_klp_unpatch_func(func); + + func->patched = false; +} + +static inline int klp_patch_func(struct klp_func *func) +{ + int ret = 0; + + if (func->patched) + return 0; + if (WARN_ON(!func->old_func)) + return -EINVAL; + if (WARN_ON(!func->func_node)) + return -EINVAL; + + ret = arch_klp_patch_func(func); + if (!ret) + func->patched = true; + + return ret; +} + +static void klp_unpatch_object(struct klp_object *obj) +{ + struct klp_func *func; + + klp_for_each_func(obj, func) { + if (func->patched) + klp_unpatch_func(func); + } + obj->patched = false; +} + +static int klp_patch_object(struct klp_object *obj) +{ + struct klp_func *func; + int ret; + + if (obj->patched) + return 0; + + klp_for_each_func(obj, func) { + ret = klp_patch_func(func); + if (ret) { + klp_unpatch_object(obj); + return ret; + } + } + obj->patched = true; + + return 0; +} + +static void klp_unpatch_objects(struct klp_patch *patch) +{ + struct klp_object *obj; + + klp_for_each_object(patch, obj) + if (obj->patched) + klp_unpatch_object(obj); +} + +static int klp_stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus) +{ + int ret; + + /* + * Cpu hotplug locking is a "percpu" rw semaphore, however write + * lock and read lock on it are globally mutual exclusive, that is + * cpus_write_lock() on one cpu can block all cpus_read_lock() + * on other cpus, vice versa. + * + * Since cpu hotplug take the cpus_write_lock() before text_mutex, + * here take cpus_read_lock() before text_mutex to avoid deadlock. + */ + cpus_read_lock(); + arch_klp_code_modify_prepare(); + ret = stop_machine_cpuslocked(fn, data, cpus); + arch_klp_code_modify_post_process(); + cpus_read_unlock(); + return ret; +} + +static int disable_patch(struct klp_patch *patch) +{ + pr_notice("disabling patch '%s'\n", patch->mod->name); + + klp_unpatch_objects(patch); + patch->enabled = false; + module_put(patch->mod); + return 0; +} + +static int klp_try_disable_patch(void *data) +{ + int ret = 0; + struct patch_data *pd = (struct patch_data *)data; + + if (atomic_inc_return(&pd->cpu_count) == 1) { + struct klp_patch *patch = pd->patch; + + ret = klp_check_calltrace(patch, 0); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + ret = disable_patch(patch); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + atomic_inc(&pd->cpu_count); + } else { + while (atomic_read(&pd->cpu_count) <= num_online_cpus()) + cpu_relax(); + + klp_smp_isb(); + } + + return ret; +} + +static int __klp_disable_patch(struct klp_patch *patch) +{ + int ret; + struct patch_data patch_data = { + .patch = patch, + .cpu_count = ATOMIC_INIT(0), + }; + + if (WARN_ON(!patch->enabled)) + return -EINVAL; + +#ifdef CONFIG_LIVEPATCH_STACK + /* enforce stacking: only the last enabled patch can be disabled */ + if (!list_is_last(&patch->list, &klp_patches) && + list_next_entry(patch, list)->enabled) { + pr_err("only the last enabled patch can be disabled\n"); + return -EBUSY; + } +#endif + + ret = klp_stop_machine(klp_try_disable_patch, &patch_data, cpu_online_mask); + if (ret) + return ret; + + klp_mem_recycle(patch); + return 0; +} + +/* + * This function is called from stop_machine() context. + */ +static int enable_patch(struct klp_patch *patch) +{ + struct klp_object *obj; + int ret; + + pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n"); + add_taint(TAINT_LIVEPATCH, LOCKDEP_STILL_OK); + + if (!patch->enabled) { + if (!try_module_get(patch->mod)) + return -ENODEV; + + patch->enabled = true; + + pr_notice("enabling patch '%s'\n", patch->mod->name); + } + + klp_for_each_object(patch, obj) { + if (!klp_is_object_loaded(obj)) + continue; + + ret = klp_patch_object(obj); + if (ret) { + pr_warn("failed to patch object '%s'\n", + klp_is_module(obj) ? obj->name : "vmlinux"); + goto disable; + } + } + + return 0; + +disable: + disable_patch(patch); + return ret; +} + +static int klp_try_enable_patch(void *data) +{ + int ret = 0; + struct patch_data *pd = (struct patch_data *)data; + + if (atomic_inc_return(&pd->cpu_count) == 1) { + struct klp_patch *patch = pd->patch; + + ret = klp_check_calltrace(patch, 1); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + ret = enable_patch(patch); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + atomic_inc(&pd->cpu_count); + } else { + while (atomic_read(&pd->cpu_count) <= num_online_cpus()) + cpu_relax(); + + klp_smp_isb(); + } + + return ret; +} + +static int __klp_enable_patch(struct klp_patch *patch) +{ + int ret; + struct patch_data patch_data = { + .patch = patch, + .cpu_count = ATOMIC_INIT(0), + }; + + if (WARN_ON(patch->enabled)) + return -EINVAL; + +#ifdef CONFIG_LIVEPATCH_STACK + /* enforce stacking: only the first disabled patch can be enabled */ + if (patch->list.prev != &klp_patches && + !list_prev_entry(patch, list)->enabled) { + pr_err("only the first disabled patch can be enabled\n"); + return -EBUSY; + } +#endif + + ret = klp_mem_prepare(patch); + if (ret) + return ret; + + ret = klp_stop_machine(klp_try_enable_patch, &patch_data, cpu_online_mask); + if (ret) + goto err_out; + +#ifndef CONFIG_LIVEPATCH_STACK + /* move the enabled patch to the list tail */ + list_del(&patch->list); + list_add_tail(&patch->list, &klp_patches); +#endif + + return 0; + +err_out: + klp_mem_recycle(patch); + return ret; +} + + +static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct klp_patch *patch; + int ret; + bool enabled; + + ret = kstrtobool(buf, &enabled); + if (ret) + return ret; + + patch = container_of(kobj, struct klp_patch, kobj); + + mutex_lock(&klp_mutex); + + if (!klp_is_patch_registered(patch)) { + /* + * Module with the patch could either disappear meanwhile or is + * not properly initialized yet. + */ + ret = -EINVAL; + goto out; + } + + if (patch->enabled == enabled) { + /* already in requested state */ + ret = -EINVAL; + goto out; + } + + if (enabled) + ret = __klp_enable_patch(patch); + else + ret = __klp_disable_patch(patch); + +out: + mutex_unlock(&klp_mutex); + + if (ret) + return ret; + return count; +} + +/** + * klp_register_patch() - registers a patch + * @patch: Patch to be registered + * + * Initializes the data structure associated with the patch and + * creates the sysfs interface. + * + * Return: 0 on success, otherwise error + */ +int klp_register_patch(struct klp_patch *patch) +{ + int ret; + struct klp_object *obj; + + if (!patch) { + pr_err("patch invalid\n"); + return -EINVAL; + } + if (!patch->mod) { + pr_err("patch->mod invalid\n"); + return -EINVAL; + } + if (!patch->objs) { + pr_err("patch->objs invalid\n"); + return -EINVAL; + } + + klp_for_each_object_static(patch, obj) { + if (!obj->funcs) { + pr_err("obj->funcs invalid\n"); + return -EINVAL; + } + } + + if (!is_livepatch_module(patch->mod)) { + pr_err("module %s is not marked as a livepatch module\n", + patch->mod->name); + return -EINVAL; + } + + if (!klp_initialized()) { + pr_err("kernel live patch not available\n"); + return -ENODEV; + } + + mutex_lock(&klp_mutex); + + if (klp_is_patch_registered(patch)) { + mutex_unlock(&klp_mutex); + return -EINVAL; + } + + klp_init_patch_early(patch); + + ret = klp_init_patch(patch); + if (ret) + goto err; + + mutex_unlock(&klp_mutex); + + return 0; + +err: + klp_free_patch_start(patch); + + mutex_unlock(&klp_mutex); + + kobject_put(&patch->kobj); + wait_for_completion(&patch->finish); + + return ret; +} +EXPORT_SYMBOL_GPL(klp_register_patch); + +/** + * klp_unregister_patch() - unregisters a patch + * @patch: Disabled patch to be unregistered + * + * Frees the data structures and removes the sysfs interface. + * + * Return: 0 on success, otherwise error + */ +int klp_unregister_patch(struct klp_patch *patch) +{ + int ret = 0; + + mutex_lock(&klp_mutex); + + if (!klp_is_patch_registered(patch)) { + ret = -EINVAL; + goto out; + } + + if (patch->enabled) { + ret = -EBUSY; + goto out; + } + + klp_free_patch_start(patch); + + mutex_unlock(&klp_mutex); + + kobject_put(&patch->kobj); + wait_for_completion(&patch->finish); + + return 0; +out: + mutex_unlock(&klp_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(klp_unregister_patch); + +static int __init klp_init(void) +{ + struct proc_dir_entry *root_klp_dir, *res; + + root_klp_dir = proc_mkdir("livepatch", NULL); + if (!root_klp_dir) + goto error_out; + + res = proc_create("livepatch/state", 0, NULL, + &proc_klpstate_operations); + if (!res) + goto error_remove; + + klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj); + if (!klp_root_kobj) + goto error_remove_state; + + return 0; + +error_remove_state: + remove_proc_entry("livepatch/state", NULL); +error_remove: + remove_proc_entry("livepatch", NULL); +error_out: + return -ENOMEM; +} + +#endif /* CONFIG_LIVEPATCH_FTRACE */ + module_init(klp_init); -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- livepatch wo_ftrace and kprobe are in conflict, because kprobe may modify the instructions anywhere in the function. So it's dangerous to patched/unpatched an function when there are some kprobes registered on it. Restrict these situation. we should hold kprobe_mutex in klp_check_patch_kprobed, but it's static and can't export, so protect klp_check_patch_probe in stop_machine to avoid registing kprobes when patching. we do nothing for (un)register kprobes on the (old) function which has been patched. because there are sone engineers need this. certainly, it will not lead to hangs, but not recommended. Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Wang ShaoBo <bobo.shaobowang@huawei.com> Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- kernel/livepatch/Kconfig | 11 ++++++++++ kernel/livepatch/core.c | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/kernel/livepatch/Kconfig b/kernel/livepatch/Kconfig index d67a3e59a372..ad15685dfd53 100644 --- a/kernel/livepatch/Kconfig +++ b/kernel/livepatch/Kconfig @@ -76,4 +76,15 @@ config LIVEPATCH_STACK help Say N here if you want to remove the patch stacking principle. +config LIVEPATCH_RESTRICT_KPROBE + bool "Enforing check livepatch and kprobe restrict" + depends on LIVEPATCH_WO_FTRACE + depends on KPROBES + default y + help + Livepatch without ftrace and kprobe are conflicting. + We should not patch for the functions where registered with kprobe, + and vice versa. + Say Y here if you want to check those. + endmenu diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index fef672d7b4b6..0e745f489cee 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -31,6 +31,9 @@ #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/stop_machine.h> +#ifdef CONFIG_LIVEPATCH_RESTRICT_KPROBE +#include <linux/kprobes.h> +#endif /* CONFIG_LIVEPATCH_RESTRICT_KPROBE */ #endif /* CONFIG_LIVEPATCH_FTRACE */ /* @@ -1908,6 +1911,40 @@ static int klp_mem_prepare(struct klp_patch *patch) return 0; } +#ifdef CONFIG_LIVEPATCH_RESTRICT_KPROBE +/* + * Check whether a function has been registered with kprobes before patched. + * We can't patched this function util we unregistered the kprobes. + */ +static struct kprobe *klp_check_patch_kprobed(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + struct kprobe *kp; + int i; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + for (i = 0; i < func->old_size; i++) { + kp = get_kprobe(func->old_func + i); + if (kp) { + pr_err("func %s has been probed, (un)patch failed\n", + func->old_name); + return kp; + } + } + } + } + + return NULL; +} +#else +static inline struct kprobe *klp_check_patch_kprobed(struct klp_patch *patch) +{ + return NULL; +} +#endif /* CONFIG_LIVEPATCH_RESTRICT_KPROBE */ + void __weak arch_klp_unpatch_func(struct klp_func *func) { } @@ -2028,6 +2065,11 @@ static int klp_try_disable_patch(void *data) if (atomic_inc_return(&pd->cpu_count) == 1) { struct klp_patch *patch = pd->patch; + if (klp_check_patch_kprobed(patch)) { + atomic_inc(&pd->cpu_count); + return -EINVAL; + } + ret = klp_check_calltrace(patch, 0); if (ret) { atomic_inc(&pd->cpu_count); @@ -2124,6 +2166,11 @@ static int klp_try_enable_patch(void *data) if (atomic_inc_return(&pd->cpu_count) == 1) { struct klp_patch *patch = pd->patch; + if (klp_check_patch_kprobed(patch)) { + atomic_inc(&pd->cpu_count); + return -EINVAL; + } + ret = klp_check_calltrace(patch, 1); if (ret) { atomic_inc(&pd->cpu_count); -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- samples/Kconfig | 2 +- samples/livepatch/Makefile | 2 ++ samples/livepatch/livepatch-sample.c | 36 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/samples/Kconfig b/samples/Kconfig index db611889bd11..b0ddf5f36738 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -130,7 +130,7 @@ config SAMPLE_RPMSG_CLIENT config SAMPLE_LIVEPATCH tristate "Build live patching samples -- loadable modules only" - depends on LIVEPATCH_FTRACE && m + depends on LIVEPATCH && m help Build sample live patch demonstrations. diff --git a/samples/livepatch/Makefile b/samples/livepatch/Makefile index 9f853eeb6140..1e384d50c73f 100644 --- a/samples/livepatch/Makefile +++ b/samples/livepatch/Makefile @@ -1,8 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-sample.o +ifeq ($(CONFIG_LIVEPATCH_FTRACE), y) obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-shadow-mod.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-shadow-fix1.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-shadow-fix2.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-callbacks-demo.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-callbacks-mod.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-callbacks-busymod.o +endif diff --git a/samples/livepatch/livepatch-sample.c b/samples/livepatch/livepatch-sample.c index cd76d7ebe598..6649e7fb9149 100644 --- a/samples/livepatch/livepatch-sample.c +++ b/samples/livepatch/livepatch-sample.c @@ -30,6 +30,31 @@ */ #include <linux/seq_file.h> + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +void load_hook(void) +{ + pr_info("load_hook\n"); +} + +void unload_hook(void) +{ + pr_info("unload_hook\n"); +} + +static struct klp_hook hooks_load[] = { + { + .hook = load_hook + }, { } +}; + +static struct klp_hook hooks_unload[] = { + { + .hook = unload_hook + }, { } +}; +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + static int livepatch_cmdline_proc_show(struct seq_file *m, void *v) { seq_printf(m, "%s\n", "this has been live patched"); @@ -47,6 +72,10 @@ static struct klp_object objs[] = { { /* name being NULL means vmlinux */ .funcs = funcs, +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + .hooks_load = hooks_load, + .hooks_unload = hooks_unload, +#endif }, { } }; @@ -57,11 +86,18 @@ static struct klp_patch patch = { static int livepatch_init(void) { +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + return klp_register_patch(&patch); +#else return klp_enable_patch(&patch); +#endif } static void livepatch_exit(void) { +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + WARN_ON(klp_unregister_patch(&patch)); +#endif } module_init(livepatch_init); -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- Livepatch allows users to call hook functions to perform some customized operations when insmod the .ko. During the test, it is found that in the ARM64 architecture, if accesses the global variables defined by other ko in hook functions, a crash occurs. Since relocation is performed during the livepatch insertion, instructions in the icache should be invalid. If the instructions in the icache are directly obtained, incorrect addresses may be obtained, caseing crash. Therefore, flush the icache before calling the hook functions. Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/moduleloader.h | 2 ++ kernel/livepatch/core.c | 1 + kernel/module/main.c | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index 068362e08747..176fef9870b9 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -121,6 +121,8 @@ void module_arch_cleanup(struct module *mod); /* Any cleanup before freeing mod->module_init */ void module_arch_freeing_init(struct module *mod); +void flush_module_icache(const struct module *mod); + #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ !defined(CONFIG_KASAN_VMALLOC) #include <linux/kasan.h> diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 8d323994284e..900270647a62 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -1060,6 +1060,7 @@ static int klp_init_patch(struct klp_patch *patch) } #ifdef CONFIG_LIVEPATCH_WO_FTRACE + flush_module_icache(patch->mod); set_mod_klp_rel_state(patch->mod, MODULE_KLP_REL_DONE); ret = jump_label_register(patch->mod); if (ret) { diff --git a/kernel/module/main.c b/kernel/module/main.c index d5765c6cd188..4f1592035229 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2317,7 +2317,7 @@ static int check_export_symbol_versions(struct module *mod) return 0; } -static void flush_module_icache(const struct module *mod) +void flush_module_icache(const struct module *mod) { /* * Flush the instruction cache, since we've played with text. -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- static call and static key allow user to modify instructions on call site, relate configs are: CONFIG_HAVE_STATIC_CALL_INLINE for static call, CONFIG_JUMP_LABEL for static key. When they exist in first several instruction of an old function, and livepatch could also modify there, then confliction happened. To avoid the confliction, we don't allow a livepatch module of this case to be inserted. Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- kernel/livepatch/core.c | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 0df2fc0e6c76..ab0ac54c7ef4 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -424,6 +424,7 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count); static inline int klp_load_hook(struct klp_object *obj); static inline int klp_unload_hook(struct klp_object *obj); +static int check_address_conflict(struct klp_patch *patch); #endif /* CONFIG_LIVEPATCH_FTRACE */ @@ -1080,6 +1081,11 @@ static int klp_init_patch(struct klp_patch *patch) pr_err("register static call failed, ret=%d\n", ret); return ret; } + + ret = check_address_conflict(patch); + if (ret) + return ret; + klp_for_each_object(patch, obj) klp_load_hook(obj); #endif @@ -1468,6 +1474,43 @@ static bool klp_is_patch_registered(struct klp_patch *patch) return false; } +static int check_address_conflict(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + int ret; + void *start; + void *end; + + /* + * Locks seem required as comment of jump_label_text_reserved() said: + * Caller must hold jump_label_mutex. + * But looking into implementation of jump_label_text_reserved() and + * static_call_text_reserved(), call sites of every jump_label or static_call + * are checked, and they won't be changed after corresponding module inserted, + * so no need to take jump_label_lock and static_call_lock here. + */ + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + start = func->old_func; + end = start + KLP_MAX_REPLACE_SIZE - 1; + ret = jump_label_text_reserved(start, end); + if (ret) { + pr_err("'%s' has static key in first %zu bytes, ret=%d\n", + func->old_name, KLP_MAX_REPLACE_SIZE, ret); + return -EINVAL; + } + ret = static_call_text_reserved(start, end); + if (ret) { + pr_err("'%s' has static call in first %zu bytes, ret=%d\n", + func->old_name, KLP_MAX_REPLACE_SIZE, ret); + return -EINVAL; + } + } + } + return 0; +} + static int state_show(struct seq_file *m, void *v) { struct klp_patch *patch; -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- After commit d556e1be3332 ("livepatch: Remove module_disable_ro() usage") and commit 0d9fbf78fefb ("module: Remove module_disable_ro()") and commit e6eff4376e28 ("module: Make module_enable_ro() static again"), the module_disable_ro is removed and module_enable_ro is make static. It's ok for x86/ppc platform because the livepatch module relocation is done by text poke func which internally modify the text addr by remap to high virtaddr which has write permission. However for arm/arm64 platform, it's apply_relocate[_add] still directly modify the text code so we should change the module text permission before relocation. Otherwise it will lead to following problem: Unable to handle kernel write to read-only memory at virtual address ffff800008a95288 Mem abort info: ESR = 0x9600004f EC = 0x25: DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x0000004f CM = 0, WnR = 1 swapper pgtable: 4k pages, 48-bit VAs, pgdp=000000004133c000 [ffff800008a95288] pgd=00000000bdfff003, p4d=00000000bdfff003, pud=00000000bdffe003, pmd=0000000080ce7003, pte=0040000080d5d783 Internal error: Oops: 9600004f [#1] PREEMPT SMP Modules linked in: livepatch_testmod_drv(OK+) testmod_drv(O) CPU: 0 PID: 139 Comm: insmod Tainted: G O K 5.10.0-01131-gf6b4602e09b2-dirty #35 Hardware name: linux,dummy-virt (DT) pstate: 80000005 (Nzcv daif -PAN -UAO -TCO BTYPE=--) pc : reloc_insn_imm+0x54/0x78 lr : reloc_insn_imm+0x50/0x78 sp : ffff800011cf3910 ... Call trace: reloc_insn_imm+0x54/0x78 apply_relocate_add+0x464/0x680 klp_apply_section_relocs+0x11c/0x148 klp_enable_patch+0x338/0x998 patch_init+0x338/0x1000 [livepatch_testmod_drv] do_one_initcall+0x60/0x1d8 do_init_module+0x58/0x1e0 load_module+0x1fb4/0x2688 __do_sys_finit_module+0xc0/0x128 __arm64_sys_finit_module+0x20/0x30 do_el0_svc+0x84/0x1b0 el0_svc+0x14/0x20 el0_sync_handler+0x90/0xc8 el0_sync+0x158/0x180 Code: 2a0503e0 9ad42a73 97d6a499 91000673 (b90002a0) ---[ end trace 67dd2ef1203ed335 ]--- Though the permission change is not necessary to x86/ppc platform, consider that the jump_label_register api may modify the text code either, we just put the change handle here instead of putting it in arch-specific relocate. Besides, the jump_label_module_nb callback called in jump_label_register also maybe need motify the module code either, it sort and swap the jump entries if necessary. So just disable ro before jump_label handling and restore it back. Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> [Introduce klp_module_enable_ro() and klp_module_disable_ro() to solve conflicts and compile issues, and only implement them on arm and arm64] Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/livepatch.h | 2 ++ kernel/livepatch/core.c | 27 ++++++++++++++++++++++++++- kernel/module/strict_rwx.c | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 864e75727200..fc5fac56cc58 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -310,6 +310,8 @@ static inline bool klp_patch_pending(struct task_struct *task) { return false; } static inline void klp_update_patch_state(struct task_struct *task) {} static inline void klp_copy_process(struct task_struct *child) {} static inline bool klp_have_reliable_stack(void) { return true; } +extern void module_enable_ro(const struct module *mod, bool after_init); +extern void module_disable_ro(const struct module *mod); #endif /* CONFIG_LIVEPATCH_FTRACE */ diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index ab0ac54c7ef4..053e66ad6e7c 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -418,6 +418,9 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } +static inline void klp_module_enable_ro(const struct module *mod, bool after_init) {} +static inline void klp_module_disable_ro(const struct module *mod) {} + #else /* !CONFIG_LIVEPATCH_FTRACE */ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, @@ -426,6 +429,20 @@ static inline int klp_load_hook(struct klp_object *obj); static inline int klp_unload_hook(struct klp_object *obj); static int check_address_conflict(struct klp_patch *patch); +static void klp_module_enable_ro(const struct module *mod, bool after_init) +{ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) + module_enable_ro(mod, after_init); +#endif +} + +static void klp_module_disable_ro(const struct module *mod) +{ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) + module_disable_ro(mod); +#endif +} + #endif /* CONFIG_LIVEPATCH_FTRACE */ static ssize_t enabled_show(struct kobject *kobj, @@ -899,6 +916,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, struct klp_func *func; int ret; + klp_module_disable_ro(patch->mod); if (klp_is_module(obj)) { /* * Only write module-specific relocations here @@ -907,9 +925,12 @@ static int klp_init_object_loaded(struct klp_patch *patch, * itself. */ ret = klp_apply_object_relocs(patch, obj); - if (ret) + if (ret) { + klp_module_enable_ro(patch->mod, true); return ret; + } } + klp_module_enable_ro(patch->mod, true); klp_for_each_func(obj, func) { ret = klp_find_object_symbol(obj->name, func->old_name, @@ -1064,8 +1085,10 @@ static int klp_init_patch(struct klp_patch *patch) #ifdef CONFIG_LIVEPATCH_WO_FTRACE flush_module_icache(patch->mod); set_mod_klp_rel_state(patch->mod, MODULE_KLP_REL_DONE); + klp_module_disable_ro(patch->mod); ret = jump_label_register(patch->mod); if (ret) { + klp_module_enable_ro(patch->mod, true); pr_err("register jump label failed, ret=%d\n", ret); return ret; } @@ -1078,9 +1101,11 @@ static int klp_init_patch(struct klp_patch *patch) * do_init_module * fail_free_freeinit: <-- notify GOING here */ + klp_module_enable_ro(patch->mod, true); pr_err("register static call failed, ret=%d\n", ret); return ret; } + klp_module_enable_ro(patch->mod, true); ret = check_address_conflict(patch); if (ret) diff --git a/kernel/module/strict_rwx.c b/kernel/module/strict_rwx.c index a2b656b4e3d2..397e18f50517 100644 --- a/kernel/module/strict_rwx.c +++ b/kernel/module/strict_rwx.c @@ -32,6 +32,23 @@ void module_enable_x(const struct module *mod) module_set_memory(mod, type, set_memory_x); } +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +void module_disable_ro(const struct module *mod) +{ + if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) + return; +#ifdef CONFIG_STRICT_MODULE_RWX + if (!rodata_enabled) + return; +#endif + + module_set_memory(mod, MOD_TEXT, set_memory_rw); + module_set_memory(mod, MOD_INIT_TEXT, set_memory_rw); + module_set_memory(mod, MOD_RODATA, set_memory_rw); + module_set_memory(mod, MOD_INIT_RODATA, set_memory_rw); +} +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + void module_enable_ro(const struct module *mod, bool after_init) { if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA --------------------------- We are planning to add livepatch without ftrace support for arm in the next commit. However after commit 425595a7fc20 ("livepatch: reuse module loader code to write relocations") merged, the klp relocations is done by apply_relocate function. The mod->arch.{core,init}.plt pointers were problematic for livepatch because they pointed within temporary section headers (provided by the module loader via info->sechdrs) that would be freed after module load. Here we take same modification based on commit c8ebf64eab74 ("arm64/module: use plt section indices for relocations") to solve. Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/arm/include/asm/module.h | 4 +++- arch/arm/kernel/ftrace.c | 4 ++-- arch/arm/kernel/module-plts.c | 22 ++++++++++++++-------- arch/arm/kernel/module.c | 4 ++-- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/arch/arm/include/asm/module.h b/arch/arm/include/asm/module.h index 07c51a34f77d..761305646831 100644 --- a/arch/arm/include/asm/module.h +++ b/arch/arm/include/asm/module.h @@ -21,6 +21,7 @@ struct plt_entries { struct mod_plt_sec { struct elf32_shdr *plt; struct plt_entries *plt_ent; + int plt_shndx; int plt_count; }; @@ -36,7 +37,8 @@ struct mod_arch_specific { }; struct module; -u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val); +u32 get_module_plt(struct module *mod, Elf32_Shdr *sechdrs, + unsigned long loc, Elf32_Addr val); #ifdef CONFIG_ARM_MODULE_PLTS bool in_module_plt(unsigned long loc); #else diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index a0b6d1e3812f..d99874cc9b2e 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -158,7 +158,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) new = ftrace_call_replace(ip, aaddr, !mod); #ifdef CONFIG_ARM_MODULE_PLTS if (!new && mod) { - aaddr = get_module_plt(mod, ip, aaddr); + aaddr = get_module_plt(mod, NULL, ip, aaddr); new = ftrace_call_replace(ip, aaddr, true); } #endif @@ -204,7 +204,7 @@ int ftrace_make_nop(struct module *mod, !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod); #ifdef CONFIG_ARM_MODULE_PLTS if (!old && mod) { - aaddr = get_module_plt(mod, ip, aaddr); + aaddr = get_module_plt(mod, NULL, ip, aaddr); old = ftrace_call_replace(ip, aaddr, true); } #endif diff --git a/arch/arm/kernel/module-plts.c b/arch/arm/kernel/module-plts.c index da2ee8d6ef1a..630388dbbf25 100644 --- a/arch/arm/kernel/module-plts.c +++ b/arch/arm/kernel/module-plts.c @@ -43,16 +43,19 @@ static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) memcpy(plt->lit, fixed_plts, sizeof(fixed_plts)); } -u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) +u32 get_module_plt(struct module *mod, Elf32_Shdr *sechdrs, + unsigned long loc, Elf32_Addr val) { struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ? &mod->arch.core : &mod->arch.init; + Elf32_Shdr *plt_shdr = sechdrs ? &sechdrs[pltsec->plt_shndx] : + pltsec->plt; struct plt_entries *plt; int idx; /* cache the address, ELF header is available only during module load */ if (!pltsec->plt_ent) - pltsec->plt_ent = (struct plt_entries *)pltsec->plt->sh_addr; + pltsec->plt_ent = (struct plt_entries *)plt_shdr->sh_addr; plt = pltsec->plt_ent; prealloc_fixed(pltsec, plt); @@ -80,7 +83,7 @@ u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) } pltsec->plt_count++; - BUG_ON(pltsec->plt_count * PLT_ENT_SIZE > pltsec->plt->sh_size); + BUG_ON(pltsec->plt_count * PLT_ENT_SIZE > plt_shdr->sh_size); if (!idx) /* Populate a new set of entries */ @@ -213,21 +216,24 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, unsigned long init_plts = ARRAY_SIZE(fixed_plts); Elf32_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum; Elf32_Sym *syms = NULL; + int i = 0; /* * To store the PLTs, we expand the .text section for core module code * and for initialization code. */ - for (s = sechdrs; s < sechdrs_end; ++s) { - if (strcmp(".plt", secstrings + s->sh_name) == 0) + for (s = sechdrs; s < sechdrs_end; ++s, ++i) { + if (strcmp(".plt", secstrings + s->sh_name) == 0) { mod->arch.core.plt = s; - else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) + mod->arch.core.plt_shndx = i; + } else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) { mod->arch.init.plt = s; - else if (s->sh_type == SHT_SYMTAB) + mod->arch.init.plt_shndx = i; + } else if (s->sh_type == SHT_SYMTAB) syms = (Elf32_Sym *)s->sh_addr; } - if (!mod->arch.core.plt || !mod->arch.init.plt) { + if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) { pr_err("%s: module PLT section(s) missing\n", mod->name); return -ENOEXEC; } diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index e74d84f58b77..6c310d49f71c 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -182,7 +182,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && (offset <= (s32)0xfe000000 || offset >= (s32)0x02000000)) - offset = get_module_plt(module, loc, + offset = get_module_plt(module, sechdrs, loc, offset + loc + 8) - loc - 8; @@ -353,7 +353,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && (offset <= (s32)0xff000000 || offset >= (s32)0x01000000)) - offset = get_module_plt(module, loc, + offset = get_module_plt(module, sechdrs, loc, offset + loc + 4) - loc - 4; -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA ---------------------------------------- support livepatch without ftrace for ARM supported now: livepatch relocation when init_patch after load_module; instruction patched when enable; activeness function check; enforcing the patch stacking principle; Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Wang ShaoBo <bobo.shaobowang@huawei.com> Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/arm/Kconfig | 3 + arch/arm/include/asm/livepatch.h | 51 ++++++ arch/arm/kernel/Makefile | 1 + arch/arm/kernel/livepatch.c | 256 +++++++++++++++++++++++++++++++ 4 files changed, 311 insertions(+) create mode 100644 arch/arm/include/asm/livepatch.h create mode 100644 arch/arm/kernel/livepatch.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 9557808e8937..e093c9ba408a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -139,6 +139,7 @@ config ARM select RTC_LIB select SPARSE_IRQ if !(ARCH_FOOTBRIDGE || ARCH_RPC) select SYS_SUPPORTS_APM_EMULATION + select HAVE_LIVEPATCH_WO_FTRACE select THREAD_INFO_IN_TASK select TIMER_OF if OF select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS @@ -1811,3 +1812,5 @@ config ARCH_HIBERNATION_POSSIBLE endmenu source "arch/arm/Kconfig.assembler" + +source "kernel/livepatch/Kconfig" diff --git a/arch/arm/include/asm/livepatch.h b/arch/arm/include/asm/livepatch.h new file mode 100644 index 000000000000..892231ca55ce --- /dev/null +++ b/arch/arm/include/asm/livepatch.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * livepatch.h - arm-specific Kernel Live Patching Core + * + * Copyright (C) 2018 Huawei Technologies Co., Ltd. + * Copyright (C) 2023 Huawei Technologies Co., Ltd. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ASM_ARM_LIVEPATCH_H +#define _ASM_ARM_LIVEPATCH_H + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + +#ifdef CONFIG_ARM_MODULE_PLTS +#define LJMP_INSN_SIZE 3 +#else +#define LJMP_INSN_SIZE 1 +#endif /* CONFIG_ARM_MODULE_PLTS */ + +struct arch_klp_data { + u32 old_insns[LJMP_INSN_SIZE]; +}; + +#define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) + +struct klp_func; + +/* kernel livepatch instruction barrier */ +#define klp_smp_isb() isb() +int arch_klp_patch_func(struct klp_func *func); +void arch_klp_unpatch_func(struct klp_func *func); +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func); +bool arch_check_jump_insn(unsigned long func_addr); +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); + +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + +#endif /* _ASM_ARM_LIVEPATCH_H */ diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index d53f56d6f840..3d1f8ff10a86 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -58,6 +58,7 @@ obj-$(CONFIG_ARM_ARCH_TIMER) += arch_timer.o obj-$(CONFIG_FUNCTION_TRACER) += entry-ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o patch.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o patch.o +obj-$(CONFIG_LIVEPATCH_WO_FTRACE) += livepatch.o insn.o patch.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o # Main staffs in KPROBES are in arch/arm/probes/ . diff --git a/arch/arm/kernel/livepatch.c b/arch/arm/kernel/livepatch.c new file mode 100644 index 000000000000..193cfcd0aca5 --- /dev/null +++ b/arch/arm/kernel/livepatch.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * livepatch.c - arm-specific Kernel Live Patching Core + * + * Copyright (C) 2018 Huawei Technologies Co., Ltd. + * Copyright (C) 2023 Huawei Technologies Co., Ltd. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/uaccess.h> +#include <linux/livepatch.h> +#include <linux/sched/debug.h> +#include <asm/livepatch.h> +#include <asm/stacktrace.h> +#include <asm/cacheflush.h> +#include <linux/slab.h> +#include <linux/ptrace.h> +#include <asm/traps.h> +#include <asm/insn.h> +#include <asm/patch.h> + +#ifdef ARM_INSN_SIZE +#error "ARM_INSN_SIZE have been redefined, please check" +#else +#define ARM_INSN_SIZE 4 +#endif + +#define CHECK_JUMP_RANGE LJMP_INSN_SIZE + +/* + * The instruction set on arm is A32. + * The instruction of BL is xxxx1011xxxxxxxxxxxxxxxxxxxxxxxx, and first four + * bits could not be 1111. + * The instruction of BLX(immediate) is 1111101xxxxxxxxxxxxxxxxxxxxxxxxx. + * The instruction of BLX(register) is xxxx00010010xxxxxxxxxxxx0011xxxx, and + * first four bits could not be 1111. + */ +static bool is_jump_insn(u32 insn) +{ + if (((insn & 0x0f000000) == 0x0b000000) && + ((insn & 0xf0000000) != 0xf0000000)) + return true; + if ((insn & 0xfe000000) == 0xfa000000) + return true; + if (((insn & 0x0ff000f0) == 0x01200030) && + ((insn & 0xf0000000) != 0xf0000000)) + return true; + return false; +} + +bool arch_check_jump_insn(unsigned long func_addr) +{ + unsigned long i; + u32 *insn = (u32 *)func_addr; + + for (i = 0; i < CHECK_JUMP_RANGE; i++) { + if (is_jump_insn(*insn)) + return true; + insn++; + } + return false; +} + +static bool klp_check_jump_func(void *ws_args, unsigned long pc) +{ + struct walk_stackframe_args *args = ws_args; + + return args->check_func(args->data, &args->ret, pc); +} + +static int check_task_calltrace(struct task_struct *t, + struct walk_stackframe_args *args, + bool (*fn)(void *, unsigned long)) +{ + struct stackframe frame; + + if (t == current) { + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_stack_pointer; + frame.lr = (unsigned long)__builtin_return_address(0); + frame.pc = (unsigned long)check_task_calltrace; + } else { + frame.fp = thread_saved_fp(t); + frame.sp = thread_saved_sp(t); + frame.lr = 0; /* recovered from the stack */ + frame.pc = thread_saved_pc(t); + } + walk_stackframe(&frame, fn, args); + if (args->ret) { + pr_info("PID: %d Comm: %.20s\n", t->pid, t->comm); + show_stack(t, NULL, KERN_INFO); + return args->ret; + } + return 0; +} + +static int do_check_calltrace(struct walk_stackframe_args *args, + bool (*fn)(void *, unsigned long)) +{ + int ret; + struct task_struct *g, *t; + unsigned int cpu; + + for_each_process_thread(g, t) { + if (klp_is_migration_thread(t->comm)) + continue; + ret = check_task_calltrace(t, args, fn); + if (ret) + return ret; + } + for_each_online_cpu(cpu) { + ret = check_task_calltrace(idle_task(cpu), args, fn); + if (ret) + return ret; + } + return 0; +} + +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data) +{ + struct walk_stackframe_args args = { + .data = data, + .ret = 0, + .check_func = check_func, + }; + + return do_check_calltrace(&args, klp_check_jump_func); +} + +static inline bool offset_in_range(unsigned long pc, unsigned long addr, + long range) +{ + long offset = addr - pc; + + return (offset >= -range && offset < range); +} + +long arm_insn_read(void *addr, u32 *insnp) +{ + long ret; + u32 val; + + ret = copy_from_kernel_nofault(&val, addr, ARM_INSN_SIZE); + if (!ret) + *insnp = le32_to_cpu(val); + + return ret; +} + +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + long ret; + int i; + + for (i = 0; i < LJMP_INSN_SIZE; i++) { + ret = arm_insn_read((u32 *)old_func + i, &arch_data->old_insns[i]); + if (ret) + break; + } + return ret; +} + +static void klp_patch_text(u32 *dst, const u32 *src, int len) +{ + int i; + + if (len <= 0) + return; + /* skip breakpoint at first */ + for (i = 1; i < len; i++) + __patch_text(dst + i, src[i]); + /* + * Avoid compile optimization, make sure that instructions + * except first breakpoint has been patched. + */ + barrier(); + __patch_text(dst, src[0]); +} + +static int do_patch(unsigned long pc, unsigned long new_addr) +{ + u32 insns[LJMP_INSN_SIZE]; + + if (!offset_in_range(pc, new_addr, SZ_32M)) { +#ifdef CONFIG_ARM_MODULE_PLTS + /* + * [0] LDR PC, [PC+8] + * [4] nop + * [8] new_addr_to_jump + */ + insns[0] = __opcode_to_mem_arm(0xe59ff000); + insns[1] = __opcode_to_mem_arm(0xe320f000); + insns[2] = new_addr; + + klp_patch_text((u32 *)pc, insns, LJMP_INSN_SIZE); +#else + /* + * When offset from 'new_addr' to 'pc' is out of SZ_32M range but + * CONFIG_ARM_MODULE_PLTS not enabled, we should stop patching. + */ + pr_err("new address out of range\n"); + return -EFAULT; +#endif + } else { + insns[0] = arm_gen_branch(pc, new_addr); + klp_patch_text((u32 *)pc, insns, 1); + } + return 0; +} + +int arch_klp_patch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + int ret; + + func_node = func->func_node; + list_add_rcu(&func->stack_node, &func_node->func_stack); + ret = do_patch((unsigned long)func->old_func, (unsigned long)func->new_func); + if (ret) + list_del_rcu(&func->stack_node); + return ret; +} + +void arch_klp_unpatch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + struct klp_func *next_func; + unsigned long pc; + + func_node = func->func_node; + pc = (unsigned long)func_node->old_func; + list_del_rcu(&func->stack_node); + if (list_empty(&func_node->func_stack)) { + klp_patch_text((u32 *)pc, func_node->arch_data.old_insns, LJMP_INSN_SIZE); + } else { + next_func = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + + do_patch(pc, (unsigned long)next_func->new_func); + } +} -- 2.25.1