hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- There are some codes that can be used in the solution without ftrace, such as module initialization and symbol relocation and so on, so include them into CONFIG_LIVEPATCH. Also note that CONFIG_LIVEPATCH_WO_FTRACE is still not implemented in this patch. Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/livepatch.h | 31 +++++++++------ kernel/livepatch/Makefile | 5 ++- kernel/livepatch/core.c | 80 +++++++++++++++++++++++++++++++++++++-- kernel/livepatch/core.h | 4 ++ 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 2c6ce24b641a..f635bd6b2c6f 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -72,11 +72,16 @@ struct klp_func { struct list_head node; struct list_head stack_node; unsigned long old_size, new_size; +#ifdef CONFIG_LIVEPATCH_FTRACE bool nop; +#endif bool patched; +#ifdef CONFIG_LIVEPATCH_FTRACE bool transition; +#endif }; +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_object; /** @@ -100,6 +105,7 @@ struct klp_callbacks { void (*post_unpatch)(struct klp_object *obj); bool post_unpatch_enabled; }; +#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_object - kernel object structure for live patching @@ -118,17 +124,22 @@ struct klp_object { /* external */ const char *name; struct klp_func *funcs; +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_callbacks callbacks; +#endif /* internal */ struct kobject kobj; struct list_head func_list; struct list_head node; struct module *mod; +#ifdef CONFIG_LIVEPATCH_FTRACE bool dynamic; +#endif bool patched; }; +#ifdef CONFIG_LIVEPATCH_FTRACE /** * struct klp_state - state of the system modified by the livepatch * @id: system state identifier (non-zero) @@ -140,6 +151,7 @@ struct klp_state { unsigned int version; void *data; }; +#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_patch - patch structure for live patching @@ -159,15 +171,19 @@ struct klp_patch { /* external */ struct module *mod; struct klp_object *objs; +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_state *states; bool replace; +#endif /* internal */ struct list_head list; struct kobject kobj; struct list_head obj_list; bool enabled; +#ifdef CONFIG_LIVEPATCH_FTRACE bool forced; +#endif struct work_struct free_work; struct completion finish; }; @@ -231,11 +247,6 @@ void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor); struct klp_state *klp_get_state(struct klp_patch *patch, unsigned long id); struct klp_state *klp_get_prev_state(unsigned long id); -int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, - const char *shstrtab, const char *strtab, - unsigned int symindex, unsigned int secindex, - const char *objname); - #else /* !CONFIG_LIVEPATCH_FTRACE */ static inline int klp_module_coming(struct module *mod) { return 0; } @@ -245,16 +256,12 @@ static inline void klp_update_patch_state(struct task_struct *task) {} static inline void klp_copy_process(struct task_struct *child) {} static inline bool klp_have_reliable_stack(void) { return true; } -static inline +#endif /* CONFIG_LIVEPATCH_FTRACE */ + int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, const char *shstrtab, const char *strtab, unsigned int symindex, unsigned int secindex, - const char *objname) -{ - return 0; -} - -#endif /* CONFIG_LIVEPATCH_FTRACE */ + const char *objname); #else /* !CONFIG_LIVEPATCH */ diff --git a/kernel/livepatch/Makefile b/kernel/livepatch/Makefile index 18f9cb687576..facf512b237a 100644 --- a/kernel/livepatch/Makefile +++ b/kernel/livepatch/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_LIVEPATCH_FTRACE) += livepatch.o +obj-$(CONFIG_LIVEPATCH) += livepatch.o -livepatch-objs := core.o patch.o shadow.o state.o transition.o +obj-$(CONFIG_LIVEPATCH_FTRACE) += patch.o shadow.o state.o transition.o +livepatch-objs := core.o diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index ecbc9b6aba3a..28705113b75c 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -22,9 +22,11 @@ #include <linux/rcupdate.h> #include <asm/cacheflush.h> #include "core.h" +#ifdef CONFIG_LIVEPATCH_FTRACE #include "patch.h" #include "state.h" #include "transition.h" +#endif /* CONFIG_LIVEPATCH_FTRACE */ /* * klp_mutex is a coarse lock which serializes access to klp data. All @@ -84,6 +86,7 @@ static bool klp_initialized(void) return !!klp_root_kobj; } +#ifdef CONFIG_LIVEPATCH_FTRACE static struct klp_func *klp_find_func(struct klp_object *obj, struct klp_func *old_func) { @@ -117,6 +120,7 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, return NULL; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ struct klp_find_arg { const char *name; @@ -260,6 +264,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, return 0; } +#ifdef CONFIG_LIVEPATCH_FTRACE void __weak clear_relocate_add(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -267,6 +272,7 @@ void __weak clear_relocate_add(Elf_Shdr *sechdrs, struct module *me) { } +#endif /* * At a high-level, there are two types of klp relocation sections: those which @@ -325,7 +331,9 @@ static int klp_write_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, return apply_relocate_add(sechdrs, strtab, symndx, secndx, pmod); } +#ifdef CONFIG_LIVEPATCH_FTRACE clear_relocate_add(sechdrs, strtab, symndx, secndx, pmod); +#endif return 0; } @@ -350,6 +358,8 @@ int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, * /sys/kernel/livepatch/<patch>/<object>/patched * /sys/kernel/livepatch/<patch>/<object>/<function,sympos> */ +#ifdef CONFIG_LIVEPATCH_FTRACE + static int __klp_disable_patch(struct klp_patch *patch); static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, @@ -395,6 +405,21 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ + +static int __klp_enable_patch(struct klp_patch *patch) +{ + return -EINVAL; +} + +static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + return -EINVAL; +} + +#endif /* CONFIG_LIVEPATCH_FTRACE */ + static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -404,6 +429,7 @@ static ssize_t enabled_show(struct kobject *kobj, return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled); } +#ifdef CONFIG_LIVEPATCH_FTRACE static ssize_t transition_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -442,18 +468,24 @@ static ssize_t force_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled); +#ifdef CONFIG_LIVEPATCH_FTRACE static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition); static struct kobj_attribute force_kobj_attr = __ATTR_WO(force); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static struct attribute *klp_patch_attrs[] = { &enabled_kobj_attr.attr, +#ifdef CONFIG_LIVEPATCH_FTRACE &transition_kobj_attr.attr, &force_kobj_attr.attr, +#endif /* CONFIG_LIVEPATCH_FTRACE */ NULL }; ATTRIBUTE_GROUPS(klp_patch); +#ifdef CONFIG_LIVEPATCH_FTRACE static ssize_t patched_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -475,12 +507,14 @@ static void klp_free_object_dynamic(struct klp_object *obj) kfree(obj->name); kfree(obj); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void klp_init_func_early(struct klp_object *obj, struct klp_func *func); static void klp_init_object_early(struct klp_patch *patch, struct klp_object *obj); +#ifdef CONFIG_LIVEPATCH_FTRACE static struct klp_object *klp_alloc_object_dynamic(const char *name, struct klp_patch *patch) { @@ -587,6 +621,7 @@ static int klp_add_nops(struct klp_patch *patch) return 0; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void klp_kobj_release_patch(struct kobject *kobj) { @@ -604,28 +639,34 @@ static const struct kobj_type klp_ktype_patch = { static void klp_kobj_release_object(struct kobject *kobj) { +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_object *obj; obj = container_of(kobj, struct klp_object, kobj); if (obj->dynamic) klp_free_object_dynamic(obj); +#endif } static const struct kobj_type klp_ktype_object = { .release = klp_kobj_release_object, .sysfs_ops = &kobj_sysfs_ops, +#ifdef CONFIG_LIVEPATCH_FTRACE .default_groups = klp_object_groups, +#endif }; static void klp_kobj_release_func(struct kobject *kobj) { +#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_func *func; func = container_of(kobj, struct klp_func, kobj); if (func->nop) klp_free_func_nop(func); +#endif } static const struct kobj_type klp_ktype_func = { @@ -638,14 +679,17 @@ static void __klp_free_funcs(struct klp_object *obj, bool nops_only) struct klp_func *func, *tmp_func; klp_for_each_func_safe(obj, func, tmp_func) { +#ifdef CONFIG_LIVEPATCH_FTRACE if (nops_only && !func->nop) continue; +#endif list_del(&func->node); kobject_put(&func->kobj); } } +#ifdef CONFIG_LIVEPATCH_FTRACE /* Clean up when a patched object is unloaded */ static void klp_free_object_loaded(struct klp_object *obj) { @@ -660,6 +704,7 @@ static void klp_free_object_loaded(struct klp_object *obj) func->new_func = NULL; } } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void __klp_free_objects(struct klp_patch *patch, bool nops_only) { @@ -667,10 +712,10 @@ static void __klp_free_objects(struct klp_patch *patch, bool nops_only) klp_for_each_object_safe(patch, obj, tmp_obj) { __klp_free_funcs(obj, nops_only); - +#ifdef CONFIG_LIVEPATCH_FTRACE if (nops_only && !obj->dynamic) continue; - +#endif list_del(&obj->node); kobject_put(&obj->kobj); } @@ -681,10 +726,12 @@ static void klp_free_objects(struct klp_patch *patch) __klp_free_objects(patch, false); } +#ifdef CONFIG_LIVEPATCH_FTRACE static void klp_free_objects_dynamic(struct klp_patch *patch) { __klp_free_objects(patch, true); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ /* * This function implements the free operations that can be called safely @@ -720,9 +767,13 @@ static void klp_free_patch_finish(struct klp_patch *patch) kobject_put(&patch->kobj); wait_for_completion(&patch->finish); +#ifdef CONFIG_LIVEPATCH_FTRACE /* Put the module after the last access to struct klp_patch. */ if (!patch->forced) module_put(patch->mod); +#else + module_put(patch->mod); +#endif /* CONFIG_LIVEPATCH_FTRACE */ } /* @@ -738,6 +789,7 @@ static void klp_free_patch_work_fn(struct work_struct *work) klp_free_patch_finish(patch); } +#ifdef CONFIG_LIVEPATCH_FTRACE void klp_free_patch_async(struct klp_patch *patch) { klp_free_patch_start(patch); @@ -754,25 +806,33 @@ void klp_free_replaced_patches_async(struct klp_patch *new_patch) klp_free_patch_async(old_patch); } } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static int klp_init_func(struct klp_object *obj, struct klp_func *func) { if (!func->old_name) return -EINVAL; +#ifdef CONFIG_LIVEPATCH_FTRACE /* * NOPs get the address later. The patched module must be loaded, * see klp_init_object_loaded(). */ if (!func->new_func && !func->nop) return -EINVAL; +#else /* !CONFIG_LIVEPATCH_FTRACE */ + if (!func->new_func) + return -EINVAL; +#endif /* CONFIG_LIVEPATCH_FTRACE */ if (strlen(func->old_name) >= KSYM_NAME_LEN) return -EINVAL; INIT_LIST_HEAD(&func->stack_node); func->patched = false; +#ifdef CONFIG_LIVEPATCH_FTRACE func->transition = false; +#endif /* The format for the sysfs directory is <function,sympos> where sympos * is the nth occurrence of this symbol in kallsyms for the patched @@ -814,11 +874,13 @@ static int klp_apply_object_relocs(struct klp_patch *patch, return klp_write_object_relocs(patch, obj, true); } +#ifdef CONFIG_LIVEPATCH_FTRACE static void klp_clear_object_relocs(struct klp_patch *patch, struct klp_object *obj) { klp_write_object_relocs(patch, obj, false); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ /* parts of the initialization that is done only when the object is loaded */ static int klp_init_object_loaded(struct klp_patch *patch, @@ -853,10 +915,10 @@ static int klp_init_object_loaded(struct klp_patch *patch, func->old_name); return -ENOENT; } - +#ifdef CONFIG_LIVEPATCH_FTRACE if (func->nop) func->new_func = func->old_func; - +#endif ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, &func->new_size, NULL); if (!ret) { @@ -924,7 +986,9 @@ static void klp_init_patch_early(struct klp_patch *patch) INIT_LIST_HEAD(&patch->obj_list); kobject_init(&patch->kobj, &klp_ktype_patch); patch->enabled = false; +#ifdef CONFIG_LIVEPATCH_FTRACE patch->forced = false; +#endif INIT_WORK(&patch->free_work, klp_free_patch_work_fn); init_completion(&patch->finish); @@ -946,11 +1010,13 @@ static int klp_init_patch(struct klp_patch *patch) if (ret) return ret; +#ifdef CONFIG_LIVEPATCH_FTRACE if (patch->replace) { ret = klp_add_nops(patch); if (ret) return ret; } +#endif klp_for_each_object(patch, obj) { ret = klp_init_object(patch, obj); @@ -963,6 +1029,7 @@ static int klp_init_patch(struct klp_patch *patch) return 0; } +#ifdef CONFIG_LIVEPATCH_FTRACE static int __klp_disable_patch(struct klp_patch *patch) { struct klp_object *obj; @@ -1049,6 +1116,7 @@ static int __klp_enable_patch(struct klp_patch *patch) klp_cancel_transition(); return ret; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * klp_enable_patch() - enable the livepatch @@ -1093,12 +1161,14 @@ int klp_enable_patch(struct klp_patch *patch) mutex_lock(&klp_mutex); +#ifdef CONFIG_LIVEPATCH_FTRACE if (!klp_is_patch_compatible(patch)) { pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", patch->mod->name); mutex_unlock(&klp_mutex); return -EINVAL; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ if (!try_module_get(patch->mod)) { mutex_unlock(&klp_mutex); @@ -1130,6 +1200,7 @@ int klp_enable_patch(struct klp_patch *patch) } EXPORT_SYMBOL_GPL(klp_enable_patch); +#ifdef CONFIG_LIVEPATCH_FTRACE /* * This function unpatches objects from the replaced livepatches. * @@ -1313,6 +1384,7 @@ void klp_module_going(struct module *mod) mutex_unlock(&klp_mutex); } +#endif /* CONFIG_LIVEPATCH_FTRACE */ static int __init klp_init(void) { diff --git a/kernel/livepatch/core.h b/kernel/livepatch/core.h index 38209c7361b6..20f916239125 100644 --- a/kernel/livepatch/core.h +++ b/kernel/livepatch/core.h @@ -13,16 +13,19 @@ extern struct list_head klp_patches; #define klp_for_each_patch(patch) \ list_for_each_entry(patch, &klp_patches, list) +#ifdef CONFIG_LIVEPATCH_FTRACE void klp_free_patch_async(struct klp_patch *patch); void klp_free_replaced_patches_async(struct klp_patch *new_patch); void klp_unpatch_replaced_patches(struct klp_patch *new_patch); void klp_discard_nops(struct klp_patch *new_patch); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static inline bool klp_is_object_loaded(struct klp_object *obj) { return !obj->name || obj->mod; } +#ifdef CONFIG_LIVEPATCH_FTRACE static inline int klp_pre_patch_callback(struct klp_object *obj) { int ret = 0; @@ -55,5 +58,6 @@ static inline void klp_post_unpatch_callback(struct klp_object *obj) obj->callbacks.post_unpatch_enabled = false; } +#endif /* CONFIG_LIVEPATCH_FTRACE */ #endif /* _LIVEPATCH_CORE_H */ -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- support livepatch without ftrace for x86_64 supported now: livepatch relocation when init_patch after load_module; instruction patched when enable; activeness function check; enforcing the patch stacking principle; x86_64 use variable length instruction, so there's no need to consider extra implementation for long jumps. Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Li Bin <huawei.libin@huawei.com> Signed-off-by: Wang ShaoBo <bobo.shaobowang@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/x86/configs/openeuler_defconfig | 4 +- arch/x86/include/asm/livepatch.h | 33 + arch/x86/kernel/Makefile | 1 + arch/x86/kernel/livepatch.c | 289 ++++++++ include/linux/livepatch.h | 89 ++- kernel/livepatch/Kconfig | 21 +- kernel/livepatch/core.c | 1013 +++++++++++++++++++++++++- 7 files changed, 1405 insertions(+), 45 deletions(-) create mode 100644 arch/x86/include/asm/livepatch.h create mode 100644 arch/x86/kernel/livepatch.c diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 465fea71fc97..84a402352c06 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -502,9 +502,9 @@ CONFIG_LEGACY_VSYSCALL_XONLY=y # CONFIG_CMDLINE_BOOL is not set CONFIG_MODIFY_LDT_SYSCALL=y # CONFIG_STRICT_SIGALTSTACK_SIZE is not set -CONFIG_HAVE_LIVEPATCH_FTRACE=y +CONFIG_HAVE_LIVEPATCH_WO_FTRACE=y CONFIG_LIVEPATCH=y -CONFIG_LIVEPATCH_FTRACE=y +CONFIG_LIVEPATCH_WO_FTRACE=y # end of Processor type and features CONFIG_FUNCTION_PADDING_CFI=11 diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h new file mode 100644 index 000000000000..359c06ad1cc2 --- /dev/null +++ b/arch/x86/include/asm/livepatch.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * livepatch.h - x86-specific Kernel Live Patching Core + * + * Copyright (C) 2023 Huawei. + */ + +#ifndef _ASM_X86_LIVEPATCH_H +#define _ASM_X86_LIVEPATCH_H + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + +#define JMP_E9_INSN_SIZE 5 +struct arch_klp_data { + unsigned char old_insns[JMP_E9_INSN_SIZE]; +}; + +#define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) + +struct klp_func; + +#define klp_smp_isb() +int arch_klp_patch_func(struct klp_func *func); +void arch_klp_unpatch_func(struct klp_func *func); +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func); +bool arch_check_jump_insn(unsigned long func_addr); +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); +void arch_klp_code_modify_prepare(void); +void arch_klp_code_modify_post_process(void); + +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + +#endif /* _ASM_X86_LIVEPATCH_H */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 3269a0e23d3a..6fafa4560c6d 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -90,6 +90,7 @@ obj-$(CONFIG_SMP) += setup_percpu.o obj-$(CONFIG_X86_MPPARSE) += mpparse.o obj-y += apic/ obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o +obj-$(CONFIG_LIVEPATCH_WO_FTRACE) += livepatch.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace_$(BITS).o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o diff --git a/arch/x86/kernel/livepatch.c b/arch/x86/kernel/livepatch.c new file mode 100644 index 000000000000..a877ffdf96ac --- /dev/null +++ b/arch/x86/kernel/livepatch.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * livepatch.c - x86-specific Kernel Live Patching Core + * + * Copyright (C) 2023 Huawei Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/kallsyms.h> +#include <linux/livepatch.h> +#include <linux/stacktrace.h> +#include <linux/memory.h> +#include <asm/text-patching.h> +#include <asm/stacktrace.h> +#include <asm/set_memory.h> +#include <asm/insn.h> + +#include <linux/slab.h> +#include <asm/nops.h> +#include <asm/sections.h> +#include <linux/kprobes.h> + +/* + * The instruction set on x86 is CISC. + * The instructions of call in same segment are 11101000(direct), + * 11111111(register indirect) and 11111111(memory indirect). + * The instructions of call in other segment are 10011010(direct), + * 11111111(indirect). + */ +static bool is_jump_insn(u8 *insn) +{ + if ((insn[0] == 0xE8) || (insn[0] == 0x9a)) + return true; + else if ((insn[0] == 0xFF) && ((insn[1] & 0x30) == 0x10)) + return true; + return false; +} + +bool arch_check_jump_insn(unsigned long func_addr) +{ + int len = JMP_E9_INSN_SIZE; + struct insn insn; + u8 *addr = (u8 *)func_addr; + + do { + if (is_jump_insn(addr)) + return true; + insn_init(&insn, addr, MAX_INSN_SIZE, 1); + insn_get_length(&insn); + if (!insn.length || !insn_complete(&insn)) + return true; + len -= insn.length; + addr += insn.length; + } while (len > 0); + + return false; +} + +static void klp_print_stack_trace(void *trace_ptr, int trace_len) +{ + int i; +#ifdef CONFIG_ARCH_STACKWALK + unsigned long *trace = trace_ptr; +#else + struct stack_trace *trace = trace_ptr; +#endif + + pr_err("Call Trace:\n"); +#ifdef CONFIG_ARCH_STACKWALK + for (i = 0; i < trace_len; i++) { + pr_err("[<%pK>] %pS\n", + (void *)trace[i], + (void *)trace[i]); + } +#else + for (i = 0; i < trace->nr_entries; i++) { + pr_err("[<%pK>] %pS\n", + (void *)trace->entries[i], + (void *)trace->entries[i]); + } +#endif + +} + +#ifdef MAX_STACK_ENTRIES +#undef MAX_STACK_ENTRIES +#endif +#define MAX_STACK_ENTRIES 100 + +static int klp_check_stack(void *trace_ptr, int trace_len, + bool (*fn)(void *, int *, unsigned long), void *data) +{ +#ifdef CONFIG_ARCH_STACKWALK + unsigned long *trace = trace_ptr; +#else + struct stack_trace *trace = trace_ptr; +#endif + unsigned long address; + int i, ret; + +#ifdef CONFIG_ARCH_STACKWALK + for (i = 0; i < trace_len; i++) { + address = trace[i]; +#else + for (i = 0; i < trace->nr_entries; i++) { + address = trace->entries[i]; +#endif + if (!fn(data, &ret, address)) { +#ifdef CONFIG_ARCH_STACKWALK + klp_print_stack_trace(trace_ptr, trace_len); +#else + klp_print_stack_trace(trace_ptr, 0); +#endif + return ret; + } + } + + return 0; +} + +static int check_task_calltrace(struct task_struct *t, + bool (*fn)(void *, int *, unsigned long), + void *data) +{ + int ret = 0; + static unsigned long trace_entries[MAX_STACK_ENTRIES]; +#ifdef CONFIG_ARCH_STACKWALK + int trace_len; +#else + struct stack_trace trace; +#endif + +#ifdef CONFIG_ARCH_STACKWALK + ret = stack_trace_save_tsk_reliable(t, trace_entries, MAX_STACK_ENTRIES); + if (ret < 0) { + pr_err("%s:%d has an unreliable stack, ret=%d\n", + t->comm, t->pid, ret); + return ret; + } + trace_len = ret; + ret = klp_check_stack(trace_entries, trace_len, fn, data); +#else + trace.skip = 0; + trace.nr_entries = 0; + trace.max_entries = MAX_STACK_ENTRIES; + trace.entries = trace_entries; + ret = save_stack_trace_tsk_reliable(t, &trace); + if (ret) { + pr_err("%s: %s:%d has an unreliable stack, ret=%d\n", + __func__, t->comm, t->pid, ret); + return ret; + } + ret = klp_check_stack(&trace, 0, fn, data); +#endif + if (ret) { + pr_err("%s:%d check stack failed, ret=%d\n", + t->comm, t->pid, ret); + return ret; + } + return 0; +} + +static int do_check_calltrace(bool (*fn)(void *, int *, unsigned long), void *data) +{ + int ret = 0; + struct task_struct *g, *t; + unsigned int cpu; + + for_each_process_thread(g, t) { + if (klp_is_migration_thread(t->comm)) + continue; + + ret = check_task_calltrace(t, fn, data); + if (ret) + return ret; + } + for_each_online_cpu(cpu) { + ret = check_task_calltrace(idle_task(cpu), fn, data); + if (ret) + return ret; + } + return 0; +} + +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data) +{ + return do_check_calltrace(check_func, data); +} + +void arch_klp_code_modify_prepare(void) + __acquires(&text_mutex) +{ + mutex_lock(&text_mutex); +} + +void arch_klp_code_modify_post_process(void) + __releases(&text_mutex) +{ + text_poke_sync(); + mutex_unlock(&text_mutex); +} + +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + long ret; + + /* Prevent text modification */ + mutex_lock(&text_mutex); + ret = copy_from_kernel_nofault(arch_data->old_insns, + old_func, JMP_E9_INSN_SIZE); + mutex_unlock(&text_mutex); + + return ret; +} + +static void klp_patch_text(void *dst, const void *src, int len) +{ + if (len <= 1) + return; + /* skip breakpoint at first */ + text_poke(dst + 1, src + 1, len - 1); + /* + * Avoid compile optimization, make sure that instructions + * except first breakpoint has been patched. + */ + barrier(); + /* update jmp opcode */ + text_poke(dst, src, 1); +} + +static void *klp_jmp_code(unsigned long ip, unsigned long addr) +{ + return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr); +} + +int arch_klp_patch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + unsigned long ip, new_addr; + unsigned char *new; + + func_node = func->func_node; + ip = (unsigned long)func->old_func; + list_add_rcu(&func->stack_node, &func_node->func_stack); + new_addr = (unsigned long)func->new_func; + /* replace the text with the new text */ + new = (unsigned char *)klp_jmp_code(ip, new_addr); + klp_patch_text((void *)ip, (const void *)new, JMP_E9_INSN_SIZE); + return 0; +} + +void arch_klp_unpatch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + struct klp_func *next_func; + unsigned long ip, new_addr; + void *new; + + func_node = func->func_node; + ip = (unsigned long)func_node->old_func; + list_del_rcu(&func->stack_node); + if (list_empty(&func_node->func_stack)) { + new = func_node->arch_data.old_insns; + } else { + next_func = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + + new_addr = (unsigned long)next_func->new_func; + new = klp_jmp_code(ip, new_addr); + } + + /* replace the text with the new text */ + klp_patch_text((void *)ip, (const void *)new, JMP_E9_INSN_SIZE); +} diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index f635bd6b2c6f..b5ef9c2d8718 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -17,11 +17,17 @@ #if IS_ENABLED(CONFIG_LIVEPATCH) +#include <asm/livepatch.h> + /* task patch states */ #define KLP_UNDEFINED -1 #define KLP_UNPATCHED 0 #define KLP_PATCHED 1 +#define KLP_NORMAL_FORCE 0 +#define KLP_ENFORCEMENT 1 +#define KLP_STACK_OPTIMIZE 2 + /** * struct klp_func - function structure for live patching * @old_name: name of the function to be patched @@ -65,6 +71,7 @@ struct klp_func { * in kallsyms for the given object is used. */ unsigned long old_sympos; + int force; /* Only used in the solution without ftrace */ /* internal */ void *old_func; @@ -72,16 +79,14 @@ struct klp_func { struct list_head node; struct list_head stack_node; unsigned long old_size, new_size; -#ifdef CONFIG_LIVEPATCH_FTRACE - bool nop; -#endif + bool nop; /* Not used in the solution without ftrace */ bool patched; #ifdef CONFIG_LIVEPATCH_FTRACE bool transition; #endif + void *func_node; /* Only used in the solution without ftrace */ }; -#ifdef CONFIG_LIVEPATCH_FTRACE struct klp_object; /** @@ -105,7 +110,6 @@ struct klp_callbacks { void (*post_unpatch)(struct klp_object *obj); bool post_unpatch_enabled; }; -#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_object - kernel object structure for live patching @@ -124,22 +128,17 @@ struct klp_object { /* external */ const char *name; struct klp_func *funcs; -#ifdef CONFIG_LIVEPATCH_FTRACE - struct klp_callbacks callbacks; -#endif + struct klp_callbacks callbacks; /* Not used in the solution without ftrace */ /* internal */ struct kobject kobj; struct list_head func_list; struct list_head node; struct module *mod; -#ifdef CONFIG_LIVEPATCH_FTRACE - bool dynamic; -#endif + bool dynamic; /* Not used in the solution without ftrace */ bool patched; }; -#ifdef CONFIG_LIVEPATCH_FTRACE /** * struct klp_state - state of the system modified by the livepatch * @id: system state identifier (non-zero) @@ -151,7 +150,6 @@ struct klp_state { unsigned int version; void *data; }; -#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * struct klp_patch - patch structure for live patching @@ -171,20 +169,16 @@ struct klp_patch { /* external */ struct module *mod; struct klp_object *objs; -#ifdef CONFIG_LIVEPATCH_FTRACE - struct klp_state *states; - bool replace; -#endif + struct klp_state *states; /* Not used in the solution without ftrace */ + bool replace; /* Not used in the solution without ftrace */ /* internal */ struct list_head list; struct kobject kobj; struct list_head obj_list; bool enabled; -#ifdef CONFIG_LIVEPATCH_FTRACE - bool forced; -#endif - struct work_struct free_work; + bool forced; /* Not used in the solution without ftrace */ + struct work_struct free_work; /* Not used in the solution without ftrace */ struct completion finish; }; @@ -208,9 +202,9 @@ struct klp_patch { #define klp_for_each_func(obj, func) \ list_for_each_entry(func, &obj->func_list, node) +#ifdef CONFIG_LIVEPATCH_FTRACE int klp_enable_patch(struct klp_patch *); -#ifdef CONFIG_LIVEPATCH_FTRACE /* Called from the module loader during module coming/going states */ int klp_module_coming(struct module *mod); void klp_module_going(struct module *mod); @@ -249,6 +243,57 @@ struct klp_state *klp_get_prev_state(unsigned long id); #else /* !CONFIG_LIVEPATCH_FTRACE */ +struct klp_func_node { + struct list_head node; + struct list_head func_stack; + void *old_func; + struct arch_klp_data arch_data; +}; + +static inline +int klp_compare_address(unsigned long pc, unsigned long func_addr, + const char *func_name, unsigned long check_size) +{ + if (pc >= func_addr && pc < func_addr + check_size) { + pr_warn("func %s is in use!\n", func_name); + /* Return -EAGAIN for next retry */ + return -EAGAIN; + } + return 0; +} + +typedef int (*klp_add_func_t)(struct list_head *func_list, + unsigned long func_addr, unsigned long func_size, + const char *func_name, int force); + +struct walk_stackframe_args { + void *data; + int ret; + bool (*check_func)(void *data, int *ret, unsigned long pc); +}; + +#ifndef klp_smp_isb +#define klp_smp_isb() +#endif + +#define KLP_MIGRATION_NAME_PREFIX "migration/" +static inline bool klp_is_migration_thread(const char *task_name) +{ + /* + * current on other CPU + * we call this in stop_machine, so the current + * of each CPUs is migration, just compare the + * task_comm here, because we can't get the + * cpu_curr(task_cpu(t))). This assumes that no + * other thread will pretend to be a stopper via + * task_comm. + */ + return !strncmp(task_name, KLP_MIGRATION_NAME_PREFIX, + sizeof(KLP_MIGRATION_NAME_PREFIX) - 1); +} + +int klp_register_patch(struct klp_patch *patch); +int klp_unregister_patch(struct klp_patch *patch); static inline int klp_module_coming(struct module *mod) { return 0; } static inline void klp_module_going(struct module *mod) {} static inline bool klp_patch_pending(struct task_struct *task) { return false; } diff --git a/kernel/livepatch/Kconfig b/kernel/livepatch/Kconfig index 8ffe3b9ca1f9..d67a3e59a372 100644 --- a/kernel/livepatch/Kconfig +++ b/kernel/livepatch/Kconfig @@ -13,11 +13,12 @@ menu "Enable Livepatch" config LIVEPATCH bool "Kernel Live Patching" - depends on (HAVE_LIVEPATCH_FTRACE && (DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS)) || HAVE_LIVEPATCH_WO_FTRACE + depends on (HAVE_LIVEPATCH_FTRACE && (DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS)) || (HAVE_LIVEPATCH_WO_FTRACE && DEBUG_INFO) depends on MODULES depends on SYSFS depends on KALLSYMS_ALL depends on !TRIM_UNUSED_KSYMS + default n help Say Y here if you want to support kernel live patching. This option has no runtime impact until a kernel "patch" @@ -37,6 +38,7 @@ choice config LIVEPATCH_FTRACE bool "based on ftrace" depends on HAVE_LIVEPATCH_FTRACE + depends on DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS help Supports kernel live patching based on ftrace. This is the original implementation of kernel live @@ -46,6 +48,8 @@ config LIVEPATCH_FTRACE config LIVEPATCH_WO_FTRACE bool "without ftrace" depends on HAVE_LIVEPATCH_WO_FTRACE + depends on DEBUG_INFO + select LIVEPATCH_STOP_MACHINE_CONSISTENCY help Supports kernel live patching without ftrace. This solution will patch the first few instructions @@ -57,4 +61,19 @@ config LIVEPATCH_WO_FTRACE endchoice +config LIVEPATCH_STOP_MACHINE_CONSISTENCY + bool "Stop machine consistency" + depends on LIVEPATCH_WO_FTRACE + help + Use stop machine consistency model + stop-machine consistency and kpatch's stack + trace checking. + +config LIVEPATCH_STACK + bool "Enforcing the patch stacking principle" + depends on LIVEPATCH_WO_FTRACE + default y + help + Say N here if you want to remove the patch stacking principle. + endmenu diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 28705113b75c..72db3e79c303 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -4,6 +4,7 @@ * * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> * Copyright (C) 2014 SUSE + * Copyright (C) 2023 Huawei Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -26,6 +27,10 @@ #include "patch.h" #include "state.h" #include "transition.h" +#else /* !CONFIG_LIVEPATCH_FTRACE */ +#include <linux/proc_fs.h> +#include <linux/seq_file.h> +#include <linux/stop_machine.h> #endif /* CONFIG_LIVEPATCH_FTRACE */ /* @@ -53,6 +58,7 @@ static bool klp_is_module(struct klp_object *obj) return obj->name; } +#ifdef CONFIG_LIVEPATCH_FTRACE /* sets obj->mod if object is not vmlinux and module is found */ static void klp_find_object_module(struct klp_object *obj) { @@ -80,6 +86,9 @@ static void klp_find_object_module(struct klp_object *obj) rcu_read_unlock_sched(); } +#else /* !CONFIG_LIVEPATCH_FTRACE */ +static int klp_find_object_module(struct klp_object *obj); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static bool klp_initialized(void) { @@ -407,16 +416,8 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, #else /* !CONFIG_LIVEPATCH_FTRACE */ -static int __klp_enable_patch(struct klp_patch *patch) -{ - return -EINVAL; -} - static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t count) -{ - return -EINVAL; -} + const char *buf, size_t count); #endif /* CONFIG_LIVEPATCH_FTRACE */ @@ -711,6 +712,12 @@ static void __klp_free_objects(struct klp_patch *patch, bool nops_only) struct klp_object *obj, *tmp_obj; klp_for_each_object_safe(patch, obj, tmp_obj) { +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + if (klp_is_module(obj) && obj->mod) { + module_put(obj->mod); + obj->mod = NULL; + } +#endif __klp_free_funcs(obj, nops_only); #ifdef CONFIG_LIVEPATCH_FTRACE if (nops_only && !obj->dynamic) @@ -748,6 +755,7 @@ static void klp_free_patch_start(struct klp_patch *patch) klp_free_objects(patch); } +#ifdef CONFIG_LIVEPATCH_FTRACE /* * This function implements the free part that must be called outside * klp_mutex. @@ -767,13 +775,9 @@ static void klp_free_patch_finish(struct klp_patch *patch) kobject_put(&patch->kobj); wait_for_completion(&patch->finish); -#ifdef CONFIG_LIVEPATCH_FTRACE /* Put the module after the last access to struct klp_patch. */ if (!patch->forced) module_put(patch->mod); -#else - module_put(patch->mod); -#endif /* CONFIG_LIVEPATCH_FTRACE */ } /* @@ -789,7 +793,6 @@ static void klp_free_patch_work_fn(struct work_struct *work) klp_free_patch_finish(patch); } -#ifdef CONFIG_LIVEPATCH_FTRACE void klp_free_patch_async(struct klp_patch *patch) { klp_free_patch_start(patch); @@ -910,11 +913,25 @@ static int klp_init_object_loaded(struct klp_patch *patch, ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, &func->old_size, NULL); +#ifdef CONFIG_LIVEPATCH_FTRACE if (!ret) { pr_err("kallsyms size lookup failed for '%s'\n", func->old_name); return -ENOENT; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ + if (!ret || ((long)func->old_size < 0)) { + pr_err("kallsyms size lookup failed for '%s'\n", + func->old_name); + return -ENOENT; + } + if (func->old_size < KLP_MAX_REPLACE_SIZE) { + pr_err("%s size less than limit (%lu < %zu)\n", func->old_name, + func->old_size, KLP_MAX_REPLACE_SIZE); + return -EINVAL; + } +#endif /* CONFIG_LIVEPATCH_FTRACE */ + #ifdef CONFIG_LIVEPATCH_FTRACE if (func->nop) func->new_func = func->old_func; @@ -931,6 +948,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, return 0; } +#ifdef CONFIG_LIVEPATCH_FTRACE static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; @@ -961,12 +979,18 @@ static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) return ret; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ +static int klp_init_object(struct klp_patch *patch, struct klp_object *obj); +#endif /* CONFIG_LIVEPATCH_FTRACE */ static void klp_init_func_early(struct klp_object *obj, struct klp_func *func) { kobject_init(&func->kobj, &klp_ktype_func); list_add_tail(&func->node, &obj->func_list); +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + func->func_node = NULL; +#endif } static void klp_init_object_early(struct klp_patch *patch, @@ -975,6 +999,9 @@ static void klp_init_object_early(struct klp_patch *patch, INIT_LIST_HEAD(&obj->func_list); kobject_init(&obj->kobj, &klp_ktype_object); list_add_tail(&obj->node, &patch->obj_list); +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + obj->mod = NULL; +#endif } static void klp_init_patch_early(struct klp_patch *patch) @@ -988,8 +1015,8 @@ static void klp_init_patch_early(struct klp_patch *patch) patch->enabled = false; #ifdef CONFIG_LIVEPATCH_FTRACE patch->forced = false; -#endif INIT_WORK(&patch->free_work, klp_free_patch_work_fn); +#endif init_completion(&patch->finish); klp_for_each_object_static(patch, obj) { @@ -1116,7 +1143,6 @@ static int __klp_enable_patch(struct klp_patch *patch) klp_cancel_transition(); return ret; } -#endif /* CONFIG_LIVEPATCH_FTRACE */ /** * klp_enable_patch() - enable the livepatch @@ -1161,14 +1187,12 @@ int klp_enable_patch(struct klp_patch *patch) mutex_lock(&klp_mutex); -#ifdef CONFIG_LIVEPATCH_FTRACE if (!klp_is_patch_compatible(patch)) { pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", patch->mod->name); mutex_unlock(&klp_mutex); return -EINVAL; } -#endif /* CONFIG_LIVEPATCH_FTRACE */ if (!try_module_get(patch->mod)) { mutex_unlock(&klp_mutex); @@ -1200,7 +1224,6 @@ int klp_enable_patch(struct klp_patch *patch) } EXPORT_SYMBOL_GPL(klp_enable_patch); -#ifdef CONFIG_LIVEPATCH_FTRACE /* * This function unpatches objects from the replaced livepatches. * @@ -1384,7 +1407,6 @@ void klp_module_going(struct module *mod) mutex_unlock(&klp_mutex); } -#endif /* CONFIG_LIVEPATCH_FTRACE */ static int __init klp_init(void) { @@ -1395,4 +1417,955 @@ static int __init klp_init(void) return 0; } +#else /* !CONFIG_LIVEPATCH_FTRACE */ + +struct patch_data { + struct klp_patch *patch; + atomic_t cpu_count; +}; + +static bool klp_is_patch_registered(struct klp_patch *patch) +{ + struct klp_patch *mypatch; + + list_for_each_entry(mypatch, &klp_patches, list) + if (mypatch == patch) + return true; + + return false; +} + +static int state_show(struct seq_file *m, void *v) +{ + struct klp_patch *patch; + char *state; + int index = 0; + + seq_printf(m, "%-5s\t%-26s\t%-8s\n", "Index", "Patch", "State"); + seq_puts(m, "-----------------------------------------------\n"); + mutex_lock(&klp_mutex); + list_for_each_entry(patch, &klp_patches, list) { + if (patch->enabled) + state = "enabled"; + else + state = "disabled"; + + seq_printf(m, "%-5d\t%-26s\t%-8s\n", ++index, + patch->mod->name, state); + } + mutex_unlock(&klp_mutex); + seq_puts(m, "-----------------------------------------------\n"); + + return 0; +} + +static int klp_state_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, state_show, NULL); +} + +static const struct proc_ops proc_klpstate_operations = { + .proc_open = klp_state_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, +}; + +static int klp_find_object_module(struct klp_object *obj) +{ + struct module *mod; + + if (!klp_is_module(obj)) + return 0; + + rcu_read_lock_sched(); + /* + * We do not want to block removal of patched modules and therefore + * we do not take a reference here. The patches are removed by + * klp_module_going() instead. + */ + mod = find_module(obj->name); + if (!mod) { + pr_err("module '%s' not loaded\n", obj->name); + rcu_read_unlock_sched(); + return -ENOPKG; /* the deponds module is not loaded */ + } + + if (mod->state == MODULE_STATE_COMING || !try_module_get(mod)) { + rcu_read_unlock_sched(); + return -EINVAL; + } + + obj->mod = mod; + + rcu_read_unlock_sched(); + return 0; +} + +static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) +{ + struct klp_func *func; + int ret; + const char *name; + + if (klp_is_module(obj) && strnlen(obj->name, MODULE_NAME_LEN) >= MODULE_NAME_LEN) { + pr_err("obj name is too long\n"); + return -EINVAL; + } + klp_for_each_func(obj, func) { + if (!func->old_name) { + pr_err("old name is invalid\n"); + return -EINVAL; + } + /* + * NOPs get the address later. The patched module must be loaded, + * see klp_init_object_loaded(). + */ + if (!func->new_func && !func->nop) { + pr_err("new_func is invalid\n"); + return -EINVAL; + } + if (strlen(func->old_name) >= KSYM_NAME_LEN) { + pr_err("function old name is too long\n"); + return -EINVAL; + } + } + + obj->patched = false; + obj->mod = NULL; + + ret = klp_find_object_module(obj); + if (ret) + return ret; + + name = klp_is_module(obj) ? obj->name : "vmlinux"; + ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name); + if (ret) + goto out; + + /* + * For livepatch without ftrace, we need to modify the first N + * instructions of the to-be-patched func. So should check if the + * func length enough to allow this modification. + * + * We add check hook in klp_init_func and will using the old_size + * internally, so the klp_init_object_loaded should called first + * to fill the klp_func struct. + */ + if (klp_is_object_loaded(obj)) { + ret = klp_init_object_loaded(patch, obj); + if (ret) + goto out; + } + + klp_for_each_func(obj, func) { + ret = klp_init_func(obj, func); + if (ret) + goto out; + } + + return 0; + +out: + if (klp_is_module(obj)) { + module_put(obj->mod); + obj->mod = NULL; + } + return ret; +} + +int __weak arch_klp_check_calltrace(bool (*fn)(void *, int *, unsigned long), void *data) +{ + return -EINVAL; +} + +bool __weak arch_check_jump_insn(unsigned long func_addr) +{ + return true; +} + +int __weak arch_klp_check_activeness_func(struct klp_func *func, int enable, + klp_add_func_t add_func, + struct list_head *func_list) +{ + int ret; + unsigned long func_addr = 0; + unsigned long func_size; + struct klp_func_node *func_node = NULL; + unsigned long old_func = (unsigned long)func->old_func; + + func_node = func->func_node; + /* Check func address in stack */ + if (enable) { + if (func->patched || func->force == KLP_ENFORCEMENT) + return 0; + /* + * When enable, checking the currently active functions. + */ + if (list_empty(&func_node->func_stack)) { + /* + * Not patched on this function [the origin one] + */ + func_addr = old_func; + func_size = func->old_size; + } else { + /* + * Previously patched function [the active one] + */ + struct klp_func *prev; + + prev = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + func_addr = (unsigned long)prev->new_func; + func_size = prev->new_size; + } + /* + * When preemption is disabled and the replacement area + * does not contain a jump instruction, the migration + * thread is scheduled to run stop machine only after the + * execution of instructions to be replaced is complete. + */ + if (IS_ENABLED(CONFIG_PREEMPTION) || + (func->force == KLP_NORMAL_FORCE) || + arch_check_jump_insn(func_addr)) { + ret = add_func(func_list, func_addr, func_size, + func->old_name, func->force); + if (ret) + return ret; + if (func_addr != old_func) { + ret = add_func(func_list, old_func, KLP_MAX_REPLACE_SIZE, + func->old_name, func->force); + if (ret) + return ret; + } + } + } else { +#ifdef CONFIG_PREEMPTION + /* + * No scheduling point in the replacement instructions. Therefore, + * when preemption is not enabled, atomic execution is performed + * and these instructions will not appear on the stack. + */ + if (list_is_singular(&func_node->func_stack)) { + func_addr = old_func; + func_size = func->old_size; + } else { + struct klp_func *prev; + + prev = list_first_or_null_rcu( + &func_node->func_stack, + struct klp_func, stack_node); + func_addr = (unsigned long)prev->new_func; + func_size = prev->new_size; + } + ret = add_func(func_list, func_addr, + func_size, func->old_name, 0); + if (ret) + return ret; + if (func_addr != old_func) { + ret = add_func(func_list, old_func, KLP_MAX_REPLACE_SIZE, + func->old_name, 0); + if (ret) + return ret; + } +#endif + + func_addr = (unsigned long)func->new_func; + func_size = func->new_size; + ret = add_func(func_list, func_addr, + func_size, func->old_name, 0); + if (ret) + return ret; + } + return 0; +} + +static inline unsigned long klp_size_to_check(unsigned long func_size, + int force) +{ + unsigned long size = func_size; + + if (force == KLP_STACK_OPTIMIZE && size > KLP_MAX_REPLACE_SIZE) + size = KLP_MAX_REPLACE_SIZE; + return size; +} + +struct actv_func { + struct list_head list; + unsigned long func_addr; + unsigned long func_size; + const char *func_name; + int force; +}; + +static bool check_func_list(void *data, int *ret, unsigned long pc) +{ + struct list_head *func_list = (struct list_head *)data; + struct actv_func *func = NULL; + + list_for_each_entry(func, func_list, list) { + *ret = klp_compare_address(pc, func->func_addr, func->func_name, + klp_size_to_check(func->func_size, func->force)); + if (*ret) + return false; + } + return true; +} + +static int add_func_to_list(struct list_head *func_list, unsigned long func_addr, + unsigned long func_size, const char *func_name, + int force) +{ + struct actv_func *func = kzalloc(sizeof(struct actv_func), GFP_ATOMIC); + + if (!func) + return -ENOMEM; + func->func_addr = func_addr; + func->func_size = func_size; + func->func_name = func_name; + func->force = force; + list_add_tail(&func->list, func_list); + return 0; +} + +static void free_func_list(struct list_head *func_list) +{ + struct actv_func *func = NULL; + struct actv_func *tmp = NULL; + + list_for_each_entry_safe(func, tmp, func_list, list) { + list_del(&func->list); + kfree(func); + } +} + +static int klp_check_activeness_func(struct klp_patch *patch, int enable, + struct list_head *func_list) +{ + int ret; + struct klp_object *obj = NULL; + struct klp_func *func = NULL; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + ret = arch_klp_check_activeness_func(func, enable, + add_func_to_list, + func_list); + if (ret) + return ret; + } + } + return 0; +} + +static int klp_check_calltrace(struct klp_patch *patch, int enable) +{ + int ret = 0; + LIST_HEAD(func_list); + + ret = klp_check_activeness_func(patch, enable, &func_list); + if (ret) { + pr_err("collect active functions failed, ret=%d\n", ret); + goto out; + } + + if (list_empty(&func_list)) + goto out; + + ret = arch_klp_check_calltrace(check_func_list, (void *)&func_list); + +out: + free_func_list(&func_list); + return ret; +} + +static LIST_HEAD(klp_func_list); + +/* + * The caller must ensure that the klp_mutex lock is held or is in the rcu read + * critical area. + */ +static struct klp_func_node *klp_find_func_node(const void *old_func) +{ + struct klp_func_node *func_node; + + list_for_each_entry_rcu(func_node, &klp_func_list, node, + lockdep_is_held(&klp_mutex)) { + if (func_node->old_func == old_func) + return func_node; + } + + return NULL; +} + +static void klp_add_func_node(struct klp_func_node *func_node) +{ + list_add_rcu(&func_node->node, &klp_func_list); +} + +static void klp_del_func_node(struct klp_func_node *func_node) +{ + list_del_rcu(&func_node->node); +} + +void __weak *arch_klp_mem_alloc(size_t size) +{ + return kzalloc(size, GFP_ATOMIC); +} + +void __weak arch_klp_mem_free(void *mem) +{ + kfree(mem); +} + +void __weak arch_klp_code_modify_prepare(void) +{ +} + +void __weak arch_klp_code_modify_post_process(void) +{ +} + +long __weak arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + return -EINVAL; +} + +static struct klp_func_node *func_node_alloc(struct klp_func *func) +{ + long ret; + struct klp_func_node *func_node = NULL; + + func_node = klp_find_func_node(func->old_func); + if (func_node) /* The old_func has ever been patched */ + return func_node; + func_node = arch_klp_mem_alloc(sizeof(struct klp_func_node)); + if (func_node) { + INIT_LIST_HEAD(&func_node->func_stack); + func_node->old_func = func->old_func; + /* + * Module which contains 'old_func' would not be removed because + * it's reference count has been held during registration. + * But it's not in stop_machine context here, 'old_func' should + * not be modified as saving old code. + */ + ret = arch_klp_save_old_code(&func_node->arch_data, func->old_func); + if (ret) { + arch_klp_mem_free(func_node); + pr_err("save old code failed, ret=%ld\n", ret); + return NULL; + } + klp_add_func_node(func_node); + } + return func_node; +} + +static void func_node_free(struct klp_func *func) +{ + struct klp_func_node *func_node; + + func_node = func->func_node; + if (func_node) { + func->func_node = NULL; + if (list_empty(&func_node->func_stack)) { + klp_del_func_node(func_node); + synchronize_rcu(); + arch_klp_mem_free(func_node); + } + } +} + +static void klp_mem_recycle(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + func_node_free(func); + } + } +} + +static int klp_mem_prepare(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + func->func_node = func_node_alloc(func); + if (func->func_node == NULL) { + klp_mem_recycle(patch); + pr_err("alloc func_node failed\n"); + return -ENOMEM; + } + } + } + return 0; +} + +void __weak arch_klp_unpatch_func(struct klp_func *func) +{ +} + +int __weak arch_klp_patch_func(struct klp_func *func) +{ + return -EINVAL; +} + +static void klp_unpatch_func(struct klp_func *func) +{ + if (WARN_ON(!func->patched)) + return; + if (WARN_ON(!func->old_func)) + return; + if (WARN_ON(!func->func_node)) + return; + + arch_klp_unpatch_func(func); + + func->patched = false; +} + +static inline int klp_patch_func(struct klp_func *func) +{ + int ret = 0; + + if (func->patched) + return 0; + if (WARN_ON(!func->old_func)) + return -EINVAL; + if (WARN_ON(!func->func_node)) + return -EINVAL; + + ret = arch_klp_patch_func(func); + if (!ret) + func->patched = true; + + return ret; +} + +static void klp_unpatch_object(struct klp_object *obj) +{ + struct klp_func *func; + + klp_for_each_func(obj, func) { + if (func->patched) + klp_unpatch_func(func); + } + obj->patched = false; +} + +static int klp_patch_object(struct klp_object *obj) +{ + struct klp_func *func; + int ret; + + if (obj->patched) + return 0; + + klp_for_each_func(obj, func) { + ret = klp_patch_func(func); + if (ret) { + klp_unpatch_object(obj); + return ret; + } + } + obj->patched = true; + + return 0; +} + +static void klp_unpatch_objects(struct klp_patch *patch) +{ + struct klp_object *obj; + + klp_for_each_object(patch, obj) + if (obj->patched) + klp_unpatch_object(obj); +} + +static int klp_stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus) +{ + int ret; + + /* + * Cpu hotplug locking is a "percpu" rw semaphore, however write + * lock and read lock on it are globally mutual exclusive, that is + * cpus_write_lock() on one cpu can block all cpus_read_lock() + * on other cpus, vice versa. + * + * Since cpu hotplug take the cpus_write_lock() before text_mutex, + * here take cpus_read_lock() before text_mutex to avoid deadlock. + */ + cpus_read_lock(); + arch_klp_code_modify_prepare(); + ret = stop_machine_cpuslocked(fn, data, cpus); + arch_klp_code_modify_post_process(); + cpus_read_unlock(); + return ret; +} + +static int disable_patch(struct klp_patch *patch) +{ + pr_notice("disabling patch '%s'\n", patch->mod->name); + + klp_unpatch_objects(patch); + patch->enabled = false; + module_put(patch->mod); + return 0; +} + +static int klp_try_disable_patch(void *data) +{ + int ret = 0; + struct patch_data *pd = (struct patch_data *)data; + + if (atomic_inc_return(&pd->cpu_count) == 1) { + struct klp_patch *patch = pd->patch; + + ret = klp_check_calltrace(patch, 0); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + ret = disable_patch(patch); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + atomic_inc(&pd->cpu_count); + } else { + while (atomic_read(&pd->cpu_count) <= num_online_cpus()) + cpu_relax(); + + klp_smp_isb(); + } + + return ret; +} + +static int __klp_disable_patch(struct klp_patch *patch) +{ + int ret; + struct patch_data patch_data = { + .patch = patch, + .cpu_count = ATOMIC_INIT(0), + }; + + if (WARN_ON(!patch->enabled)) + return -EINVAL; + +#ifdef CONFIG_LIVEPATCH_STACK + /* enforce stacking: only the last enabled patch can be disabled */ + if (!list_is_last(&patch->list, &klp_patches) && + list_next_entry(patch, list)->enabled) { + pr_err("only the last enabled patch can be disabled\n"); + return -EBUSY; + } +#endif + + ret = klp_stop_machine(klp_try_disable_patch, &patch_data, cpu_online_mask); + if (ret) + return ret; + + klp_mem_recycle(patch); + return 0; +} + +/* + * This function is called from stop_machine() context. + */ +static int enable_patch(struct klp_patch *patch) +{ + struct klp_object *obj; + int ret; + + pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n"); + add_taint(TAINT_LIVEPATCH, LOCKDEP_STILL_OK); + + if (!patch->enabled) { + if (!try_module_get(patch->mod)) + return -ENODEV; + + patch->enabled = true; + + pr_notice("enabling patch '%s'\n", patch->mod->name); + } + + klp_for_each_object(patch, obj) { + if (!klp_is_object_loaded(obj)) + continue; + + ret = klp_patch_object(obj); + if (ret) { + pr_warn("failed to patch object '%s'\n", + klp_is_module(obj) ? obj->name : "vmlinux"); + goto disable; + } + } + + return 0; + +disable: + disable_patch(patch); + return ret; +} + +static int klp_try_enable_patch(void *data) +{ + int ret = 0; + struct patch_data *pd = (struct patch_data *)data; + + if (atomic_inc_return(&pd->cpu_count) == 1) { + struct klp_patch *patch = pd->patch; + + ret = klp_check_calltrace(patch, 1); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + ret = enable_patch(patch); + if (ret) { + atomic_inc(&pd->cpu_count); + return ret; + } + atomic_inc(&pd->cpu_count); + } else { + while (atomic_read(&pd->cpu_count) <= num_online_cpus()) + cpu_relax(); + + klp_smp_isb(); + } + + return ret; +} + +static int __klp_enable_patch(struct klp_patch *patch) +{ + int ret; + struct patch_data patch_data = { + .patch = patch, + .cpu_count = ATOMIC_INIT(0), + }; + + if (WARN_ON(patch->enabled)) + return -EINVAL; + +#ifdef CONFIG_LIVEPATCH_STACK + /* enforce stacking: only the first disabled patch can be enabled */ + if (patch->list.prev != &klp_patches && + !list_prev_entry(patch, list)->enabled) { + pr_err("only the first disabled patch can be enabled\n"); + return -EBUSY; + } +#endif + + ret = klp_mem_prepare(patch); + if (ret) + return ret; + + ret = klp_stop_machine(klp_try_enable_patch, &patch_data, cpu_online_mask); + if (ret) + goto err_out; + +#ifndef CONFIG_LIVEPATCH_STACK + /* move the enabled patch to the list tail */ + list_del(&patch->list); + list_add_tail(&patch->list, &klp_patches); +#endif + + return 0; + +err_out: + klp_mem_recycle(patch); + return ret; +} + + +static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct klp_patch *patch; + int ret; + bool enabled; + + ret = kstrtobool(buf, &enabled); + if (ret) + return ret; + + patch = container_of(kobj, struct klp_patch, kobj); + + mutex_lock(&klp_mutex); + + if (!klp_is_patch_registered(patch)) { + /* + * Module with the patch could either disappear meanwhile or is + * not properly initialized yet. + */ + ret = -EINVAL; + goto out; + } + + if (patch->enabled == enabled) { + /* already in requested state */ + ret = -EINVAL; + goto out; + } + + if (enabled) + ret = __klp_enable_patch(patch); + else + ret = __klp_disable_patch(patch); + +out: + mutex_unlock(&klp_mutex); + + if (ret) + return ret; + return count; +} + +/** + * klp_register_patch() - registers a patch + * @patch: Patch to be registered + * + * Initializes the data structure associated with the patch and + * creates the sysfs interface. + * + * Return: 0 on success, otherwise error + */ +int klp_register_patch(struct klp_patch *patch) +{ + int ret; + struct klp_object *obj; + + if (!patch) { + pr_err("patch invalid\n"); + return -EINVAL; + } + if (!patch->mod) { + pr_err("patch->mod invalid\n"); + return -EINVAL; + } + if (!patch->objs) { + pr_err("patch->objs invalid\n"); + return -EINVAL; + } + + klp_for_each_object_static(patch, obj) { + if (!obj->funcs) { + pr_err("obj->funcs invalid\n"); + return -EINVAL; + } + } + + if (!is_livepatch_module(patch->mod)) { + pr_err("module %s is not marked as a livepatch module\n", + patch->mod->name); + return -EINVAL; + } + + if (!klp_initialized()) { + pr_err("kernel live patch not available\n"); + return -ENODEV; + } + + mutex_lock(&klp_mutex); + + if (klp_is_patch_registered(patch)) { + mutex_unlock(&klp_mutex); + return -EINVAL; + } + + klp_init_patch_early(patch); + + ret = klp_init_patch(patch); + if (ret) + goto err; + + mutex_unlock(&klp_mutex); + + return 0; + +err: + klp_free_patch_start(patch); + + mutex_unlock(&klp_mutex); + + kobject_put(&patch->kobj); + wait_for_completion(&patch->finish); + + return ret; +} +EXPORT_SYMBOL_GPL(klp_register_patch); + +/** + * klp_unregister_patch() - unregisters a patch + * @patch: Disabled patch to be unregistered + * + * Frees the data structures and removes the sysfs interface. + * + * Return: 0 on success, otherwise error + */ +int klp_unregister_patch(struct klp_patch *patch) +{ + int ret = 0; + + mutex_lock(&klp_mutex); + + if (!klp_is_patch_registered(patch)) { + ret = -EINVAL; + goto out; + } + + if (patch->enabled) { + ret = -EBUSY; + goto out; + } + + klp_free_patch_start(patch); + + mutex_unlock(&klp_mutex); + + kobject_put(&patch->kobj); + wait_for_completion(&patch->finish); + + return 0; +out: + mutex_unlock(&klp_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(klp_unregister_patch); + +static int __init klp_init(void) +{ + struct proc_dir_entry *root_klp_dir, *res; + + root_klp_dir = proc_mkdir("livepatch", NULL); + if (!root_klp_dir) + goto error_out; + + res = proc_create("livepatch/state", 0, NULL, + &proc_klpstate_operations); + if (!res) + goto error_remove; + + klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj); + if (!klp_root_kobj) + goto error_remove_state; + + return 0; + +error_remove_state: + remove_proc_entry("livepatch/state", NULL); +error_remove: + remove_proc_entry("livepatch", NULL); +error_out: + return -ENOMEM; +} + +#endif /* CONFIG_LIVEPATCH_FTRACE */ + module_init(klp_init); -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- livepatch wo_ftrace and kprobe are in conflict, because kprobe may modify the instructions anywhere in the function. So it's dangerous to patched/unpatched an function when there are some kprobes registered on it. Restrict these situation. we should hold kprobe_mutex in klp_check_patch_kprobed, but it's static and can't export, so protect klp_check_patch_probe in stop_machine to avoid registing kprobes when patching. we do nothing for (un)register kprobes on the (old) function which has been patched. because there are sone engineers need this. certainly, it will not lead to hangs, but not recommended. Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Wang ShaoBo <bobo.shaobowang@huawei.com> Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- kernel/livepatch/Kconfig | 11 ++++++++++ kernel/livepatch/core.c | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/kernel/livepatch/Kconfig b/kernel/livepatch/Kconfig index d67a3e59a372..ad15685dfd53 100644 --- a/kernel/livepatch/Kconfig +++ b/kernel/livepatch/Kconfig @@ -76,4 +76,15 @@ config LIVEPATCH_STACK help Say N here if you want to remove the patch stacking principle. +config LIVEPATCH_RESTRICT_KPROBE + bool "Enforing check livepatch and kprobe restrict" + depends on LIVEPATCH_WO_FTRACE + depends on KPROBES + default y + help + Livepatch without ftrace and kprobe are conflicting. + We should not patch for the functions where registered with kprobe, + and vice versa. + Say Y here if you want to check those. + endmenu diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index fef672d7b4b6..0e745f489cee 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -31,6 +31,9 @@ #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/stop_machine.h> +#ifdef CONFIG_LIVEPATCH_RESTRICT_KPROBE +#include <linux/kprobes.h> +#endif /* CONFIG_LIVEPATCH_RESTRICT_KPROBE */ #endif /* CONFIG_LIVEPATCH_FTRACE */ /* @@ -1908,6 +1911,40 @@ static int klp_mem_prepare(struct klp_patch *patch) return 0; } +#ifdef CONFIG_LIVEPATCH_RESTRICT_KPROBE +/* + * Check whether a function has been registered with kprobes before patched. + * We can't patched this function util we unregistered the kprobes. + */ +static struct kprobe *klp_check_patch_kprobed(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + struct kprobe *kp; + int i; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + for (i = 0; i < func->old_size; i++) { + kp = get_kprobe(func->old_func + i); + if (kp) { + pr_err("func %s has been probed, (un)patch failed\n", + func->old_name); + return kp; + } + } + } + } + + return NULL; +} +#else +static inline struct kprobe *klp_check_patch_kprobed(struct klp_patch *patch) +{ + return NULL; +} +#endif /* CONFIG_LIVEPATCH_RESTRICT_KPROBE */ + void __weak arch_klp_unpatch_func(struct klp_func *func) { } @@ -2028,6 +2065,11 @@ static int klp_try_disable_patch(void *data) if (atomic_inc_return(&pd->cpu_count) == 1) { struct klp_patch *patch = pd->patch; + if (klp_check_patch_kprobed(patch)) { + atomic_inc(&pd->cpu_count); + return -EINVAL; + } + ret = klp_check_calltrace(patch, 0); if (ret) { atomic_inc(&pd->cpu_count); @@ -2124,6 +2166,11 @@ static int klp_try_enable_patch(void *data) if (atomic_inc_return(&pd->cpu_count) == 1) { struct klp_patch *patch = pd->patch; + if (klp_check_patch_kprobed(patch)) { + atomic_inc(&pd->cpu_count); + return -EINVAL; + } + ret = klp_check_calltrace(patch, 1); if (ret) { atomic_inc(&pd->cpu_count); -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- samples/Kconfig | 2 +- samples/livepatch/Makefile | 2 ++ samples/livepatch/livepatch-sample.c | 36 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/samples/Kconfig b/samples/Kconfig index db611889bd11..b0ddf5f36738 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -130,7 +130,7 @@ config SAMPLE_RPMSG_CLIENT config SAMPLE_LIVEPATCH tristate "Build live patching samples -- loadable modules only" - depends on LIVEPATCH_FTRACE && m + depends on LIVEPATCH && m help Build sample live patch demonstrations. diff --git a/samples/livepatch/Makefile b/samples/livepatch/Makefile index 9f853eeb6140..1e384d50c73f 100644 --- a/samples/livepatch/Makefile +++ b/samples/livepatch/Makefile @@ -1,8 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-sample.o +ifeq ($(CONFIG_LIVEPATCH_FTRACE), y) obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-shadow-mod.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-shadow-fix1.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-shadow-fix2.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-callbacks-demo.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-callbacks-mod.o obj-$(CONFIG_SAMPLE_LIVEPATCH) += livepatch-callbacks-busymod.o +endif diff --git a/samples/livepatch/livepatch-sample.c b/samples/livepatch/livepatch-sample.c index cd76d7ebe598..4a99769086b2 100644 --- a/samples/livepatch/livepatch-sample.c +++ b/samples/livepatch/livepatch-sample.c @@ -30,6 +30,31 @@ */ #include <linux/seq_file.h> + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +void load_hook(void) +{ + pr_info("loading\n"); +} + +void unload_hook(void) +{ + pr_info("unloading\n"); +} + +static struct klp_hook hooks_load[] = { + { + .hook = load_hook + }, { } +}; + +static struct klp_hook hooks_unload[] = { + { + .hook = unload_hook + }, { } +}; +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + static int livepatch_cmdline_proc_show(struct seq_file *m, void *v) { seq_printf(m, "%s\n", "this has been live patched"); @@ -47,6 +72,10 @@ static struct klp_object objs[] = { { /* name being NULL means vmlinux */ .funcs = funcs, +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + .hooks_load = hooks_load, + .hooks_unload = hooks_unload, +#endif }, { } }; @@ -57,11 +86,18 @@ static struct klp_patch patch = { static int livepatch_init(void) { +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + return klp_register_patch(&patch); +#else return klp_enable_patch(&patch); +#endif } static void livepatch_exit(void) { +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + WARN_ON(klp_unregister_patch(&patch)); +#endif } module_init(livepatch_init); -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- Livepatch allows users to call hook functions to perform some customized operations when insmod the .ko. During the test, it is found that in the ARM64 architecture, if accesses the global variables defined by other ko in hook functions, a crash occurs. Since relocation is performed during the livepatch insertion, instructions in the icache should be invalid. If the instructions in the icache are directly obtained, incorrect addresses may be obtained, caseing crash. Therefore, flush the icache before calling the hook functions. Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/moduleloader.h | 2 ++ kernel/livepatch/core.c | 1 + kernel/module/main.c | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index 068362e08747..176fef9870b9 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -121,6 +121,8 @@ void module_arch_cleanup(struct module *mod); /* Any cleanup before freeing mod->module_init */ void module_arch_freeing_init(struct module *mod); +void flush_module_icache(const struct module *mod); + #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ !defined(CONFIG_KASAN_VMALLOC) #include <linux/kasan.h> diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 8d323994284e..900270647a62 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -1060,6 +1060,7 @@ static int klp_init_patch(struct klp_patch *patch) } #ifdef CONFIG_LIVEPATCH_WO_FTRACE + flush_module_icache(patch->mod); set_mod_klp_rel_state(patch->mod, MODULE_KLP_REL_DONE); ret = jump_label_register(patch->mod); if (ret) { diff --git a/kernel/module/main.c b/kernel/module/main.c index d5765c6cd188..4f1592035229 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2317,7 +2317,7 @@ static int check_export_symbol_versions(struct module *mod) return 0; } -static void flush_module_icache(const struct module *mod) +void flush_module_icache(const struct module *mod) { /* * Flush the instruction cache, since we've played with text. -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- static call and static key allow user to modify instructions on call site, relate configs are: CONFIG_HAVE_STATIC_CALL_INLINE for static call, CONFIG_JUMP_LABEL for static key. When they exist in first several instruction of an old function, and livepatch could also modify there, then confliction happened. To avoid the confliction, we don't allow a livepatch module of this case to be inserted. Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- kernel/livepatch/core.c | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 0df2fc0e6c76..ab0ac54c7ef4 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -424,6 +424,7 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count); static inline int klp_load_hook(struct klp_object *obj); static inline int klp_unload_hook(struct klp_object *obj); +static int check_address_conflict(struct klp_patch *patch); #endif /* CONFIG_LIVEPATCH_FTRACE */ @@ -1080,6 +1081,11 @@ static int klp_init_patch(struct klp_patch *patch) pr_err("register static call failed, ret=%d\n", ret); return ret; } + + ret = check_address_conflict(patch); + if (ret) + return ret; + klp_for_each_object(patch, obj) klp_load_hook(obj); #endif @@ -1468,6 +1474,43 @@ static bool klp_is_patch_registered(struct klp_patch *patch) return false; } +static int check_address_conflict(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + int ret; + void *start; + void *end; + + /* + * Locks seem required as comment of jump_label_text_reserved() said: + * Caller must hold jump_label_mutex. + * But looking into implementation of jump_label_text_reserved() and + * static_call_text_reserved(), call sites of every jump_label or static_call + * are checked, and they won't be changed after corresponding module inserted, + * so no need to take jump_label_lock and static_call_lock here. + */ + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + start = func->old_func; + end = start + KLP_MAX_REPLACE_SIZE - 1; + ret = jump_label_text_reserved(start, end); + if (ret) { + pr_err("'%s' has static key in first %zu bytes, ret=%d\n", + func->old_name, KLP_MAX_REPLACE_SIZE, ret); + return -EINVAL; + } + ret = static_call_text_reserved(start, end); + if (ret) { + pr_err("'%s' has static call in first %zu bytes, ret=%d\n", + func->old_name, KLP_MAX_REPLACE_SIZE, ret); + return -EINVAL; + } + } + } + return 0; +} + static int state_show(struct seq_file *m, void *v) { struct klp_patch *patch; -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA -------------------------------- After commit d556e1be3332 ("livepatch: Remove module_disable_ro() usage") and commit 0d9fbf78fefb ("module: Remove module_disable_ro()") and commit e6eff4376e28 ("module: Make module_enable_ro() static again"), the module_disable_ro is removed and module_enable_ro is make static. It's ok for x86/ppc platform because the livepatch module relocation is done by text poke func which internally modify the text addr by remap to high virtaddr which has write permission. However for arm/arm64 platform, it's apply_relocate[_add] still directly modify the text code so we should change the module text permission before relocation. Otherwise it will lead to following problem: Unable to handle kernel write to read-only memory at virtual address ffff800008a95288 Mem abort info: ESR = 0x9600004f EC = 0x25: DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x0000004f CM = 0, WnR = 1 swapper pgtable: 4k pages, 48-bit VAs, pgdp=000000004133c000 [ffff800008a95288] pgd=00000000bdfff003, p4d=00000000bdfff003, pud=00000000bdffe003, pmd=0000000080ce7003, pte=0040000080d5d783 Internal error: Oops: 9600004f [#1] PREEMPT SMP Modules linked in: livepatch_testmod_drv(OK+) testmod_drv(O) CPU: 0 PID: 139 Comm: insmod Tainted: G O K 5.10.0-01131-gf6b4602e09b2-dirty #35 Hardware name: linux,dummy-virt (DT) pstate: 80000005 (Nzcv daif -PAN -UAO -TCO BTYPE=--) pc : reloc_insn_imm+0x54/0x78 lr : reloc_insn_imm+0x50/0x78 sp : ffff800011cf3910 ... Call trace: reloc_insn_imm+0x54/0x78 apply_relocate_add+0x464/0x680 klp_apply_section_relocs+0x11c/0x148 klp_enable_patch+0x338/0x998 patch_init+0x338/0x1000 [livepatch_testmod_drv] do_one_initcall+0x60/0x1d8 do_init_module+0x58/0x1e0 load_module+0x1fb4/0x2688 __do_sys_finit_module+0xc0/0x128 __arm64_sys_finit_module+0x20/0x30 do_el0_svc+0x84/0x1b0 el0_svc+0x14/0x20 el0_sync_handler+0x90/0xc8 el0_sync+0x158/0x180 Code: 2a0503e0 9ad42a73 97d6a499 91000673 (b90002a0) ---[ end trace 67dd2ef1203ed335 ]--- Though the permission change is not necessary to x86/ppc platform, consider that the jump_label_register api may modify the text code either, we just put the change handle here instead of putting it in arch-specific relocate. Besides, the jump_label_module_nb callback called in jump_label_register also maybe need motify the module code either, it sort and swap the jump entries if necessary. So just disable ro before jump_label handling and restore it back. Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> [Introduce klp_module_enable_ro() and klp_module_disable_ro() to solve conflicts and compile issues, and only implement them on arm and arm64] Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/livepatch.h | 2 ++ kernel/livepatch/core.c | 27 ++++++++++++++++++++++++++- kernel/module/strict_rwx.c | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 864e75727200..fc5fac56cc58 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -310,6 +310,8 @@ static inline bool klp_patch_pending(struct task_struct *task) { return false; } static inline void klp_update_patch_state(struct task_struct *task) {} static inline void klp_copy_process(struct task_struct *child) {} static inline bool klp_have_reliable_stack(void) { return true; } +extern void module_enable_ro(const struct module *mod, bool after_init); +extern void module_disable_ro(const struct module *mod); #endif /* CONFIG_LIVEPATCH_FTRACE */ diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index ab0ac54c7ef4..053e66ad6e7c 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -418,6 +418,9 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } +static inline void klp_module_enable_ro(const struct module *mod, bool after_init) {} +static inline void klp_module_disable_ro(const struct module *mod) {} + #else /* !CONFIG_LIVEPATCH_FTRACE */ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, @@ -426,6 +429,20 @@ static inline int klp_load_hook(struct klp_object *obj); static inline int klp_unload_hook(struct klp_object *obj); static int check_address_conflict(struct klp_patch *patch); +static void klp_module_enable_ro(const struct module *mod, bool after_init) +{ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) + module_enable_ro(mod, after_init); +#endif +} + +static void klp_module_disable_ro(const struct module *mod) +{ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) + module_disable_ro(mod); +#endif +} + #endif /* CONFIG_LIVEPATCH_FTRACE */ static ssize_t enabled_show(struct kobject *kobj, @@ -899,6 +916,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, struct klp_func *func; int ret; + klp_module_disable_ro(patch->mod); if (klp_is_module(obj)) { /* * Only write module-specific relocations here @@ -907,9 +925,12 @@ static int klp_init_object_loaded(struct klp_patch *patch, * itself. */ ret = klp_apply_object_relocs(patch, obj); - if (ret) + if (ret) { + klp_module_enable_ro(patch->mod, true); return ret; + } } + klp_module_enable_ro(patch->mod, true); klp_for_each_func(obj, func) { ret = klp_find_object_symbol(obj->name, func->old_name, @@ -1064,8 +1085,10 @@ static int klp_init_patch(struct klp_patch *patch) #ifdef CONFIG_LIVEPATCH_WO_FTRACE flush_module_icache(patch->mod); set_mod_klp_rel_state(patch->mod, MODULE_KLP_REL_DONE); + klp_module_disable_ro(patch->mod); ret = jump_label_register(patch->mod); if (ret) { + klp_module_enable_ro(patch->mod, true); pr_err("register jump label failed, ret=%d\n", ret); return ret; } @@ -1078,9 +1101,11 @@ static int klp_init_patch(struct klp_patch *patch) * do_init_module * fail_free_freeinit: <-- notify GOING here */ + klp_module_enable_ro(patch->mod, true); pr_err("register static call failed, ret=%d\n", ret); return ret; } + klp_module_enable_ro(patch->mod, true); ret = check_address_conflict(patch); if (ret) diff --git a/kernel/module/strict_rwx.c b/kernel/module/strict_rwx.c index a2b656b4e3d2..397e18f50517 100644 --- a/kernel/module/strict_rwx.c +++ b/kernel/module/strict_rwx.c @@ -32,6 +32,23 @@ void module_enable_x(const struct module *mod) module_set_memory(mod, type, set_memory_x); } +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +void module_disable_ro(const struct module *mod) +{ + if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) + return; +#ifdef CONFIG_STRICT_MODULE_RWX + if (!rodata_enabled) + return; +#endif + + module_set_memory(mod, MOD_TEXT, set_memory_rw); + module_set_memory(mod, MOD_INIT_TEXT, set_memory_rw); + module_set_memory(mod, MOD_RODATA, set_memory_rw); + module_set_memory(mod, MOD_INIT_RODATA, set_memory_rw); +} +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + void module_enable_ro(const struct module *mod, bool after_init) { if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA --------------------------- We are planning to add livepatch without ftrace support for arm in the next commit. However after commit 425595a7fc20 ("livepatch: reuse module loader code to write relocations") merged, the klp relocations is done by apply_relocate function. The mod->arch.{core,init}.plt pointers were problematic for livepatch because they pointed within temporary section headers (provided by the module loader via info->sechdrs) that would be freed after module load. Here we take same modification based on commit c8ebf64eab74 ("arm64/module: use plt section indices for relocations") to solve. Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/arm/include/asm/module.h | 4 +++- arch/arm/kernel/ftrace.c | 4 ++-- arch/arm/kernel/module-plts.c | 22 ++++++++++++++-------- arch/arm/kernel/module.c | 4 ++-- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/arch/arm/include/asm/module.h b/arch/arm/include/asm/module.h index 07c51a34f77d..761305646831 100644 --- a/arch/arm/include/asm/module.h +++ b/arch/arm/include/asm/module.h @@ -21,6 +21,7 @@ struct plt_entries { struct mod_plt_sec { struct elf32_shdr *plt; struct plt_entries *plt_ent; + int plt_shndx; int plt_count; }; @@ -36,7 +37,8 @@ struct mod_arch_specific { }; struct module; -u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val); +u32 get_module_plt(struct module *mod, Elf32_Shdr *sechdrs, + unsigned long loc, Elf32_Addr val); #ifdef CONFIG_ARM_MODULE_PLTS bool in_module_plt(unsigned long loc); #else diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index a0b6d1e3812f..d99874cc9b2e 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -158,7 +158,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) new = ftrace_call_replace(ip, aaddr, !mod); #ifdef CONFIG_ARM_MODULE_PLTS if (!new && mod) { - aaddr = get_module_plt(mod, ip, aaddr); + aaddr = get_module_plt(mod, NULL, ip, aaddr); new = ftrace_call_replace(ip, aaddr, true); } #endif @@ -204,7 +204,7 @@ int ftrace_make_nop(struct module *mod, !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod); #ifdef CONFIG_ARM_MODULE_PLTS if (!old && mod) { - aaddr = get_module_plt(mod, ip, aaddr); + aaddr = get_module_plt(mod, NULL, ip, aaddr); old = ftrace_call_replace(ip, aaddr, true); } #endif diff --git a/arch/arm/kernel/module-plts.c b/arch/arm/kernel/module-plts.c index da2ee8d6ef1a..630388dbbf25 100644 --- a/arch/arm/kernel/module-plts.c +++ b/arch/arm/kernel/module-plts.c @@ -43,16 +43,19 @@ static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) memcpy(plt->lit, fixed_plts, sizeof(fixed_plts)); } -u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) +u32 get_module_plt(struct module *mod, Elf32_Shdr *sechdrs, + unsigned long loc, Elf32_Addr val) { struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ? &mod->arch.core : &mod->arch.init; + Elf32_Shdr *plt_shdr = sechdrs ? &sechdrs[pltsec->plt_shndx] : + pltsec->plt; struct plt_entries *plt; int idx; /* cache the address, ELF header is available only during module load */ if (!pltsec->plt_ent) - pltsec->plt_ent = (struct plt_entries *)pltsec->plt->sh_addr; + pltsec->plt_ent = (struct plt_entries *)plt_shdr->sh_addr; plt = pltsec->plt_ent; prealloc_fixed(pltsec, plt); @@ -80,7 +83,7 @@ u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) } pltsec->plt_count++; - BUG_ON(pltsec->plt_count * PLT_ENT_SIZE > pltsec->plt->sh_size); + BUG_ON(pltsec->plt_count * PLT_ENT_SIZE > plt_shdr->sh_size); if (!idx) /* Populate a new set of entries */ @@ -213,21 +216,24 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, unsigned long init_plts = ARRAY_SIZE(fixed_plts); Elf32_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum; Elf32_Sym *syms = NULL; + int i = 0; /* * To store the PLTs, we expand the .text section for core module code * and for initialization code. */ - for (s = sechdrs; s < sechdrs_end; ++s) { - if (strcmp(".plt", secstrings + s->sh_name) == 0) + for (s = sechdrs; s < sechdrs_end; ++s, ++i) { + if (strcmp(".plt", secstrings + s->sh_name) == 0) { mod->arch.core.plt = s; - else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) + mod->arch.core.plt_shndx = i; + } else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) { mod->arch.init.plt = s; - else if (s->sh_type == SHT_SYMTAB) + mod->arch.init.plt_shndx = i; + } else if (s->sh_type == SHT_SYMTAB) syms = (Elf32_Sym *)s->sh_addr; } - if (!mod->arch.core.plt || !mod->arch.init.plt) { + if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) { pr_err("%s: module PLT section(s) missing\n", mod->name); return -ENOEXEC; } diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index e74d84f58b77..6c310d49f71c 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -182,7 +182,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && (offset <= (s32)0xfe000000 || offset >= (s32)0x02000000)) - offset = get_module_plt(module, loc, + offset = get_module_plt(module, sechdrs, loc, offset + loc + 8) - loc - 8; @@ -353,7 +353,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && (offset <= (s32)0xff000000 || offset >= (s32)0x01000000)) - offset = get_module_plt(module, loc, + offset = get_module_plt(module, sechdrs, loc, offset + loc + 4) - loc - 4; -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 CVE: NA ---------------------------------------- support livepatch without ftrace for ARM supported now: livepatch relocation when init_patch after load_module; instruction patched when enable; activeness function check; enforcing the patch stacking principle; Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Wang ShaoBo <bobo.shaobowang@huawei.com> Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/arm/Kconfig | 3 + arch/arm/include/asm/livepatch.h | 51 ++++++ arch/arm/kernel/Makefile | 1 + arch/arm/kernel/livepatch.c | 256 +++++++++++++++++++++++++++++++ 4 files changed, 311 insertions(+) create mode 100644 arch/arm/include/asm/livepatch.h create mode 100644 arch/arm/kernel/livepatch.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 9557808e8937..e093c9ba408a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -139,6 +139,7 @@ config ARM select RTC_LIB select SPARSE_IRQ if !(ARCH_FOOTBRIDGE || ARCH_RPC) select SYS_SUPPORTS_APM_EMULATION + select HAVE_LIVEPATCH_WO_FTRACE select THREAD_INFO_IN_TASK select TIMER_OF if OF select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS @@ -1811,3 +1812,5 @@ config ARCH_HIBERNATION_POSSIBLE endmenu source "arch/arm/Kconfig.assembler" + +source "kernel/livepatch/Kconfig" diff --git a/arch/arm/include/asm/livepatch.h b/arch/arm/include/asm/livepatch.h new file mode 100644 index 000000000000..892231ca55ce --- /dev/null +++ b/arch/arm/include/asm/livepatch.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * livepatch.h - arm-specific Kernel Live Patching Core + * + * Copyright (C) 2018 Huawei Technologies Co., Ltd. + * Copyright (C) 2023 Huawei Technologies Co., Ltd. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ASM_ARM_LIVEPATCH_H +#define _ASM_ARM_LIVEPATCH_H + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + +#ifdef CONFIG_ARM_MODULE_PLTS +#define LJMP_INSN_SIZE 3 +#else +#define LJMP_INSN_SIZE 1 +#endif /* CONFIG_ARM_MODULE_PLTS */ + +struct arch_klp_data { + u32 old_insns[LJMP_INSN_SIZE]; +}; + +#define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) + +struct klp_func; + +/* kernel livepatch instruction barrier */ +#define klp_smp_isb() isb() +int arch_klp_patch_func(struct klp_func *func); +void arch_klp_unpatch_func(struct klp_func *func); +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func); +bool arch_check_jump_insn(unsigned long func_addr); +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); + +#endif /* CONFIG_LIVEPATCH_WO_FTRACE */ + +#endif /* _ASM_ARM_LIVEPATCH_H */ diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index d53f56d6f840..3d1f8ff10a86 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -58,6 +58,7 @@ obj-$(CONFIG_ARM_ARCH_TIMER) += arch_timer.o obj-$(CONFIG_FUNCTION_TRACER) += entry-ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o patch.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o patch.o +obj-$(CONFIG_LIVEPATCH_WO_FTRACE) += livepatch.o insn.o patch.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o # Main staffs in KPROBES are in arch/arm/probes/ . diff --git a/arch/arm/kernel/livepatch.c b/arch/arm/kernel/livepatch.c new file mode 100644 index 000000000000..193cfcd0aca5 --- /dev/null +++ b/arch/arm/kernel/livepatch.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * livepatch.c - arm-specific Kernel Live Patching Core + * + * Copyright (C) 2018 Huawei Technologies Co., Ltd. + * Copyright (C) 2023 Huawei Technologies Co., Ltd. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/uaccess.h> +#include <linux/livepatch.h> +#include <linux/sched/debug.h> +#include <asm/livepatch.h> +#include <asm/stacktrace.h> +#include <asm/cacheflush.h> +#include <linux/slab.h> +#include <linux/ptrace.h> +#include <asm/traps.h> +#include <asm/insn.h> +#include <asm/patch.h> + +#ifdef ARM_INSN_SIZE +#error "ARM_INSN_SIZE have been redefined, please check" +#else +#define ARM_INSN_SIZE 4 +#endif + +#define CHECK_JUMP_RANGE LJMP_INSN_SIZE + +/* + * The instruction set on arm is A32. + * The instruction of BL is xxxx1011xxxxxxxxxxxxxxxxxxxxxxxx, and first four + * bits could not be 1111. + * The instruction of BLX(immediate) is 1111101xxxxxxxxxxxxxxxxxxxxxxxxx. + * The instruction of BLX(register) is xxxx00010010xxxxxxxxxxxx0011xxxx, and + * first four bits could not be 1111. + */ +static bool is_jump_insn(u32 insn) +{ + if (((insn & 0x0f000000) == 0x0b000000) && + ((insn & 0xf0000000) != 0xf0000000)) + return true; + if ((insn & 0xfe000000) == 0xfa000000) + return true; + if (((insn & 0x0ff000f0) == 0x01200030) && + ((insn & 0xf0000000) != 0xf0000000)) + return true; + return false; +} + +bool arch_check_jump_insn(unsigned long func_addr) +{ + unsigned long i; + u32 *insn = (u32 *)func_addr; + + for (i = 0; i < CHECK_JUMP_RANGE; i++) { + if (is_jump_insn(*insn)) + return true; + insn++; + } + return false; +} + +static bool klp_check_jump_func(void *ws_args, unsigned long pc) +{ + struct walk_stackframe_args *args = ws_args; + + return args->check_func(args->data, &args->ret, pc); +} + +static int check_task_calltrace(struct task_struct *t, + struct walk_stackframe_args *args, + bool (*fn)(void *, unsigned long)) +{ + struct stackframe frame; + + if (t == current) { + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_stack_pointer; + frame.lr = (unsigned long)__builtin_return_address(0); + frame.pc = (unsigned long)check_task_calltrace; + } else { + frame.fp = thread_saved_fp(t); + frame.sp = thread_saved_sp(t); + frame.lr = 0; /* recovered from the stack */ + frame.pc = thread_saved_pc(t); + } + walk_stackframe(&frame, fn, args); + if (args->ret) { + pr_info("PID: %d Comm: %.20s\n", t->pid, t->comm); + show_stack(t, NULL, KERN_INFO); + return args->ret; + } + return 0; +} + +static int do_check_calltrace(struct walk_stackframe_args *args, + bool (*fn)(void *, unsigned long)) +{ + int ret; + struct task_struct *g, *t; + unsigned int cpu; + + for_each_process_thread(g, t) { + if (klp_is_migration_thread(t->comm)) + continue; + ret = check_task_calltrace(t, args, fn); + if (ret) + return ret; + } + for_each_online_cpu(cpu) { + ret = check_task_calltrace(idle_task(cpu), args, fn); + if (ret) + return ret; + } + return 0; +} + +int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data) +{ + struct walk_stackframe_args args = { + .data = data, + .ret = 0, + .check_func = check_func, + }; + + return do_check_calltrace(&args, klp_check_jump_func); +} + +static inline bool offset_in_range(unsigned long pc, unsigned long addr, + long range) +{ + long offset = addr - pc; + + return (offset >= -range && offset < range); +} + +long arm_insn_read(void *addr, u32 *insnp) +{ + long ret; + u32 val; + + ret = copy_from_kernel_nofault(&val, addr, ARM_INSN_SIZE); + if (!ret) + *insnp = le32_to_cpu(val); + + return ret; +} + +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + long ret; + int i; + + for (i = 0; i < LJMP_INSN_SIZE; i++) { + ret = arm_insn_read((u32 *)old_func + i, &arch_data->old_insns[i]); + if (ret) + break; + } + return ret; +} + +static void klp_patch_text(u32 *dst, const u32 *src, int len) +{ + int i; + + if (len <= 0) + return; + /* skip breakpoint at first */ + for (i = 1; i < len; i++) + __patch_text(dst + i, src[i]); + /* + * Avoid compile optimization, make sure that instructions + * except first breakpoint has been patched. + */ + barrier(); + __patch_text(dst, src[0]); +} + +static int do_patch(unsigned long pc, unsigned long new_addr) +{ + u32 insns[LJMP_INSN_SIZE]; + + if (!offset_in_range(pc, new_addr, SZ_32M)) { +#ifdef CONFIG_ARM_MODULE_PLTS + /* + * [0] LDR PC, [PC+8] + * [4] nop + * [8] new_addr_to_jump + */ + insns[0] = __opcode_to_mem_arm(0xe59ff000); + insns[1] = __opcode_to_mem_arm(0xe320f000); + insns[2] = new_addr; + + klp_patch_text((u32 *)pc, insns, LJMP_INSN_SIZE); +#else + /* + * When offset from 'new_addr' to 'pc' is out of SZ_32M range but + * CONFIG_ARM_MODULE_PLTS not enabled, we should stop patching. + */ + pr_err("new address out of range\n"); + return -EFAULT; +#endif + } else { + insns[0] = arm_gen_branch(pc, new_addr); + klp_patch_text((u32 *)pc, insns, 1); + } + return 0; +} + +int arch_klp_patch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + int ret; + + func_node = func->func_node; + list_add_rcu(&func->stack_node, &func_node->func_stack); + ret = do_patch((unsigned long)func->old_func, (unsigned long)func->new_func); + if (ret) + list_del_rcu(&func->stack_node); + return ret; +} + +void arch_klp_unpatch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + struct klp_func *next_func; + unsigned long pc; + + func_node = func->func_node; + pc = (unsigned long)func_node->old_func; + list_del_rcu(&func->stack_node); + if (list_empty(&func_node->func_stack)) { + klp_patch_text((u32 *)pc, func_node->arch_data.old_insns, LJMP_INSN_SIZE); + } else { + next_func = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + + do_patch(pc, (unsigned long)next_func->new_func); + } +} -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 -------------------------------- The commit 86e35fae15bb ("livepatch: checks only if the replaced instruction is on the stack") optimizes stack checking. However, for extremely hot functions, the replaced instruction may still be on the stack, and there is room for further optimization. By inserting a breakpoint exception instruction at the entry of the patched old function, we can divert calls from the old function to the new function. In this way, during stack check, only tasks that have entered the old function before the breakpoint is inserted need to be considered. This increases the probability of passing the stack check. If the stack check fails, we sleep for a period of time and try again, giving the task entering the old function a chance to run out of the instruction replacement area. We first enable the patch using the normal process, that is, do not insert breakpoints. If the first enable fails and the force flag KLP_STACK_OPTIMIZE is set for all functions of the patch, then we use breakpoint exception optimization. Signed-off-by: Li Huafei <lihuafei1@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/livepatch.h | 10 ++ kernel/livepatch/core.c | 265 +++++++++++++++++++++++++++++++++++++- kernel/livepatch/core.h | 12 ++ 3 files changed, 280 insertions(+), 7 deletions(-) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index fc5fac56cc58..bc2f1ba9b766 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -258,8 +258,18 @@ struct klp_func_node { struct list_head func_stack; void *old_func; struct arch_klp_data arch_data; + /* + * Used in breakpoint exception handling functions. + * If 'brk_func' is NULL, no breakpoint is inserted into the entry of + * the old function. + * If it is not NULL, the value is the new function that will jump to + * when the breakpoint exception is triggered. + */ + void *brk_func; }; +void *klp_get_brk_func(void *addr); + static inline int klp_compare_address(unsigned long pc, unsigned long func_addr, const char *func_name, unsigned long check_size) diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 652fff3e8716..0633fe1c20d9 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -30,6 +30,7 @@ #else /* !CONFIG_LIVEPATCH_FTRACE */ #include <linux/proc_fs.h> #include <linux/seq_file.h> +#include <linux/delay.h> #include <linux/stop_machine.h> #ifdef CONFIG_LIVEPATCH_RESTRICT_KPROBE #include <linux/kprobes.h> @@ -1498,6 +1499,7 @@ static int __init klp_init(void) struct patch_data { struct klp_patch *patch; atomic_t cpu_count; + bool rollback; }; static bool klp_is_patch_registered(struct klp_patch *patch) @@ -1947,6 +1949,37 @@ static void klp_del_func_node(struct klp_func_node *func_node) list_del_rcu(&func_node->node); } +/* + * Called from the breakpoint exception handler function. + */ +void *klp_get_brk_func(void *addr) +{ + struct klp_func_node *func_node; + void *brk_func = NULL; + + if (!addr) + return NULL; + + rcu_read_lock(); + + func_node = klp_find_func_node(addr); + if (!func_node) + goto unlock; + + /* + * Corresponds to smp_wmb() in {add, remove}_breakpoint(). If the + * current breakpoint exception belongs to us, we have observed the + * breakpoint instruction, so brk_func must be observed. + */ + smp_rmb(); + + brk_func = func_node->brk_func; + +unlock: + rcu_read_unlock(); + return brk_func; +} + void __weak *arch_klp_mem_alloc(size_t size) { return kzalloc(size, GFP_ATOMIC); @@ -1970,6 +2003,25 @@ long __weak arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_fu return -EINVAL; } +int __weak arch_klp_check_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ + return 0; +} + +int __weak arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ + return -EOPNOTSUPP; +} + +void __weak arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ +} + +void __weak arch_klp_set_brk_func(struct klp_func_node *func_node, void *new_func) +{ + func_node->brk_func = new_func; +} + static struct klp_func_node *func_node_alloc(struct klp_func *func) { long ret; @@ -2044,6 +2096,109 @@ static int klp_mem_prepare(struct klp_patch *patch) return 0; } +static void remove_breakpoint(struct klp_func *func, bool restore) +{ + struct klp_func_node *func_node = klp_find_func_node(func->old_func); + struct arch_klp_data *arch_data = &func_node->arch_data; + + if (!func_node->brk_func) + return; + + if (restore) + arch_klp_remove_breakpoint(arch_data, func->old_func); + + /* Wait for all breakpoint exception handler functions to exit. */ + synchronize_rcu(); + + /* 'brk_func' cannot be set to NULL before the breakpoint is removed. */ + smp_wmb(); + + arch_klp_set_brk_func(func_node, NULL); +} + +static void __klp_breakpoint_post_process(struct klp_patch *patch, bool restore) +{ + struct klp_object *obj; + struct klp_func *func; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + remove_breakpoint(func, restore); + } + } +} + +static int add_breakpoint(struct klp_func *func) +{ + struct klp_func_node *func_node = klp_find_func_node(func->old_func); + struct arch_klp_data *arch_data = &func_node->arch_data; + int ret; + + if (WARN_ON_ONCE(func_node->brk_func)) + return -EINVAL; + + ret = arch_klp_check_breakpoint(arch_data, func->old_func); + if (ret) + return ret; + + arch_klp_set_brk_func(func_node, func->new_func); + + /* + * When entering an exception, we must see 'brk_func' or the kernel + * will not be able to handle the breakpoint exception we are about + * to insert. + */ + smp_wmb(); + + ret = arch_klp_add_breakpoint(arch_data, func->old_func); + if (ret) + arch_klp_set_brk_func(func_node, NULL); + + return ret; +} + +static int klp_add_breakpoint(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + int ret; + + /* + * Ensure that the module is not uninstalled before the breakpoint is + * removed. After the breakpoint is removed, it can be ensured that the + * new function will not be jumped through the handler function of the + * breakpoint. + */ + if (!try_module_get(patch->mod)) + return -ENODEV; + + arch_klp_code_modify_prepare(); + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + ret = add_breakpoint(func); + if (ret) { + __klp_breakpoint_post_process(patch, true); + arch_klp_code_modify_post_process(); + module_put(patch->mod); + return ret; + } + } + } + + arch_klp_code_modify_post_process(); + + return 0; +} + +static void klp_breakpoint_post_process(struct klp_patch *patch, bool restore) +{ + arch_klp_code_modify_prepare(); + __klp_breakpoint_post_process(patch, restore); + arch_klp_code_modify_post_process(); + module_put(patch->mod); +} + #ifdef CONFIG_LIVEPATCH_RESTRICT_KPROBE /* * Check whether a function has been registered with kprobes before patched. @@ -2130,7 +2285,7 @@ static void klp_unpatch_object(struct klp_object *obj) obj->patched = false; } -static int klp_patch_object(struct klp_object *obj) +static int klp_patch_object(struct klp_object *obj, bool rollback) { struct klp_func *func; int ret; @@ -2140,7 +2295,7 @@ static int klp_patch_object(struct klp_object *obj) klp_for_each_func(obj, func) { ret = klp_patch_func(func); - if (ret) { + if (ret && klp_need_rollback(ret, rollback)) { klp_unpatch_object(obj); return ret; } @@ -2255,7 +2410,7 @@ static int __klp_disable_patch(struct klp_patch *patch) /* * This function is called from stop_machine() context. */ -static int enable_patch(struct klp_patch *patch) +static int enable_patch(struct klp_patch *patch, bool rollback) { struct klp_object *obj; int ret; @@ -2276,8 +2431,8 @@ static int enable_patch(struct klp_patch *patch) if (!klp_is_object_loaded(obj)) continue; - ret = klp_patch_object(obj); - if (ret) { + ret = klp_patch_object(obj, rollback); + if (ret && klp_need_rollback(ret, rollback)) { pr_warn("failed to patch object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto disable; @@ -2309,7 +2464,7 @@ static int klp_try_enable_patch(void *data) atomic_inc(&pd->cpu_count); return ret; } - ret = enable_patch(patch); + ret = enable_patch(patch, pd->rollback); if (ret) { atomic_inc(&pd->cpu_count); return ret; @@ -2325,12 +2480,97 @@ static int klp_try_enable_patch(void *data) return ret; } +/* + * When the stop_machine is used to enable the patch, if the patch fails to be + * enabled because the stack check fails, a certain number of retries are + * allowed. The maximum number of retries is KLP_RETRY_COUNT. + * + * Sleeps for KLP_RETRY_INTERVAL milliseconds before each retry to give tasks + * that fail the stack check a chance to run out of the instruction replacement + * area. + */ +#define KLP_RETRY_COUNT 5 +#define KLP_RETRY_INTERVAL 100 + +static bool klp_use_breakpoint(struct klp_patch *patch) +{ + struct klp_object *obj; + struct klp_func *func; + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + if (func->force != KLP_STACK_OPTIMIZE) + return false; + } + } + + return true; +} + +static int klp_breakpoint_enable_patch(struct klp_patch *patch, int *cnt) +{ + int ret = -EINVAL; + int i; + int retry_cnt = 0; + + for (i = 0; i < KLP_RETRY_COUNT; i++) { + struct patch_data patch_data = { + .patch = patch, + .cpu_count = ATOMIC_INIT(0), + .rollback = false, + }; + + if (i == KLP_RETRY_COUNT - 1) + patch_data.rollback = true; + + retry_cnt++; + + ret = klp_stop_machine(klp_try_enable_patch, &patch_data, + cpu_online_mask); + if (!ret || ret != -EAGAIN) + break; + + pr_notice("try again in %d ms\n", KLP_RETRY_INTERVAL); + + msleep(KLP_RETRY_INTERVAL); + } + *cnt = retry_cnt; + return ret; +} + +static int klp_breakpoint_optimize(struct klp_patch *patch) +{ + int ret; + int cnt = 0; + + ret = klp_add_breakpoint(patch); + if (ret) { + pr_err("failed to add breakpoints, ret=%d\n", ret); + return ret; + } + + ret = klp_breakpoint_enable_patch(patch, &cnt); + + pr_notice("patching %s, tried %d times, ret=%d.\n", + ret ? "failed" : "success", cnt, ret); + /* + * If the patch is enabled successfully, the breakpoint instruction + * has been replaced with the jump instruction. However, if the patch + * fails to be enabled, we need to delete the previously inserted + * breakpoint to restore the instruction at the old function entry. + */ + klp_breakpoint_post_process(patch, !!ret); + + return ret; +} + static int __klp_enable_patch(struct klp_patch *patch) { int ret; struct patch_data patch_data = { .patch = patch, .cpu_count = ATOMIC_INIT(0), + .rollback = true, }; if (WARN_ON(patch->enabled)) @@ -2350,9 +2590,21 @@ static int __klp_enable_patch(struct klp_patch *patch) return ret; ret = klp_stop_machine(klp_try_enable_patch, &patch_data, cpu_online_mask); + if (!ret) + goto move_patch_to_tail; + if (ret != -EAGAIN) + goto err_out; + + if (!klp_use_breakpoint(patch)) { + pr_debug("breakpoint exception optimization is not used.\n"); + goto err_out; + } + + ret = klp_breakpoint_optimize(patch); if (ret) goto err_out; +move_patch_to_tail: #ifndef CONFIG_LIVEPATCH_STACK /* move the enabled patch to the list tail */ list_del(&patch->list); @@ -2366,7 +2618,6 @@ static int __klp_enable_patch(struct klp_patch *patch) return ret; } - static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { diff --git a/kernel/livepatch/core.h b/kernel/livepatch/core.h index 20f916239125..fbf5ff50a5d7 100644 --- a/kernel/livepatch/core.h +++ b/kernel/livepatch/core.h @@ -18,6 +18,18 @@ void klp_free_patch_async(struct klp_patch *patch); void klp_free_replaced_patches_async(struct klp_patch *new_patch); void klp_unpatch_replaced_patches(struct klp_patch *new_patch); void klp_discard_nops(struct klp_patch *new_patch); +#else +/* + * In the enable_patch() process, we do not need to roll back the patch + * immediately if the patch fails to enabled. In this way, the function that has + * been successfully patched does not need to be enabled repeatedly during + * retry. However, if it is the last retry (rollback == true) or not because of + * stack check failure (patch_err != -EAGAIN), rollback is required immediately. + */ +static inline bool klp_need_rollback(int patch_err, bool rollback) +{ + return patch_err != -EAGAIN || rollback; +} #endif /* CONFIG_LIVEPATCH_FTRACE */ static inline bool klp_is_object_loaded(struct klp_object *obj) -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 -------------------------------- For the ARM architecture, need to register the callback function for processing BRK exception in advance. Therefore, the architecture-related init interface needs to be provided. Signed-off-by: Yang Jihong <yangjihong1@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- include/linux/livepatch.h | 2 ++ kernel/livepatch/core.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index bc2f1ba9b766..e6d9a794ad77 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -282,6 +282,8 @@ int klp_compare_address(unsigned long pc, unsigned long func_addr, return 0; } +void arch_klp_init(void); + typedef int (*klp_add_func_t)(struct list_head *func_list, unsigned long func_addr, unsigned long func_size, const char *func_name, int force); diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 0633fe1c20d9..35a016c83ae2 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -2003,6 +2003,10 @@ long __weak arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_fu return -EINVAL; } +void __weak arch_klp_init(void) +{ +} + int __weak arch_klp_check_breakpoint(struct arch_klp_data *arch_data, void *old_func) { return 0; @@ -2794,6 +2798,8 @@ static int __init klp_init(void) if (!klp_root_kobj) goto error_remove_state; + arch_klp_init(); + return 0; error_remove_state: -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 -------------------------------- Add breakpoint exception optimization support to improve livepatch success rate for arm. Signed-off-by: Yang Jihong <yangjihong1@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/arm/include/asm/livepatch.h | 9 +++++++ arch/arm/kernel/livepatch.c | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/arch/arm/include/asm/livepatch.h b/arch/arm/include/asm/livepatch.h index 892231ca55ce..62a65bdc85a2 100644 --- a/arch/arm/include/asm/livepatch.h +++ b/arch/arm/include/asm/livepatch.h @@ -30,8 +30,15 @@ #define LJMP_INSN_SIZE 1 #endif /* CONFIG_ARM_MODULE_PLTS */ +#define KLP_ARM_BREAKPOINT_INSTRUCTION 0xe7f001f9 + struct arch_klp_data { u32 old_insns[LJMP_INSN_SIZE]; + /* + * Saved opcode at the entry of the old func (which maybe replaced + * with breakpoint). + */ + u32 saved_opcode; }; #define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) @@ -45,6 +52,8 @@ void arch_klp_unpatch_func(struct klp_func *func); long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func); bool arch_check_jump_insn(unsigned long func_addr); int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); +int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func); +void arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func); #endif /* CONFIG_LIVEPATCH_WO_FTRACE */ diff --git a/arch/arm/kernel/livepatch.c b/arch/arm/kernel/livepatch.c index 193cfcd0aca5..6a041aa003be 100644 --- a/arch/arm/kernel/livepatch.c +++ b/arch/arm/kernel/livepatch.c @@ -142,6 +142,48 @@ int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), v return do_check_calltrace(&args, klp_check_jump_func); } +int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ + u32 *addr = (u32 *)old_func; + + arch_data->saved_opcode = le32_to_cpu(*addr); + patch_text(old_func, KLP_ARM_BREAKPOINT_INSTRUCTION); + return 0; +} + +void arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ + patch_text(old_func, arch_data->saved_opcode); +} + +static int klp_trap_handler(struct pt_regs *regs, unsigned int instr) +{ + void *brk_func = NULL; + unsigned long addr = regs->ARM_pc; + + brk_func = klp_get_brk_func((void *)addr); + if (!brk_func) { + pr_warn("Unrecoverable livepatch detected.\n"); + BUG(); + } + + regs->ARM_pc = (unsigned long)brk_func; + return 0; +} + +static struct undef_hook klp_arm_break_hook = { + .instr_mask = 0x0fffffff, + .instr_val = (KLP_ARM_BREAKPOINT_INSTRUCTION & 0x0fffffff), + .cpsr_mask = MODE_MASK, + .cpsr_val = SVC_MODE, + .fn = klp_trap_handler, +}; + +void arch_klp_init(void) +{ + register_undef_hook(&klp_arm_break_hook); +} + static inline bool offset_in_range(unsigned long pc, unsigned long addr, long range) { -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 -------------------------------- Add arch_klp_module_check_calltrace to check whether stacks of all tasks are within the code segment of module. Signed-off-by: Yang Jihong <yangjihong1@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/x86/include/asm/livepatch.h | 1 + arch/x86/kernel/livepatch.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h index cf1d1523462e..8b030adcb53f 100644 --- a/arch/x86/include/asm/livepatch.h +++ b/arch/x86/include/asm/livepatch.h @@ -36,6 +36,7 @@ int arch_klp_check_breakpoint(struct arch_klp_data *arch_data, void *old_func); int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func); void arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func); int klp_int3_handler(struct pt_regs *regs); +int arch_klp_module_check_calltrace(void *data); #endif /* CONFIG_LIVEPATCH_WO_FTRACE */ diff --git a/arch/x86/kernel/livepatch.c b/arch/x86/kernel/livepatch.c index 57f1034a88c8..660c25f2cf89 100644 --- a/arch/x86/kernel/livepatch.c +++ b/arch/x86/kernel/livepatch.c @@ -202,6 +202,23 @@ int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), v return do_check_calltrace(check_func, data); } +static bool check_module_calltrace(void *data, int *ret, unsigned long pc) +{ + struct module *mod = (struct module *)data; + + if (within_module_core(pc, mod)) { + pr_err("module %s is in use!\n", mod->name); + *ret = -EBUSY; + return false; + } + return true; +} + +int arch_klp_module_check_calltrace(void *data) +{ + return do_check_calltrace(check_module_calltrace, data); +} + int arch_klp_check_breakpoint(struct arch_klp_data *arch_data, void *old_func) { int ret; -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 -------------------------------- Add arch_klp_module_check_calltrace to check whether stacks of all tasks are within the code segment of module. Signed-off-by: Yang Jihong <yangjihong1@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/arm/include/asm/livepatch.h | 1 + arch/arm/kernel/livepatch.c | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/arch/arm/include/asm/livepatch.h b/arch/arm/include/asm/livepatch.h index 62a65bdc85a2..e228921e1034 100644 --- a/arch/arm/include/asm/livepatch.h +++ b/arch/arm/include/asm/livepatch.h @@ -54,6 +54,7 @@ bool arch_check_jump_insn(unsigned long func_addr); int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func); void arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func); +int arch_klp_module_check_calltrace(void *data); #endif /* CONFIG_LIVEPATCH_WO_FTRACE */ diff --git a/arch/arm/kernel/livepatch.c b/arch/arm/kernel/livepatch.c index 6a041aa003be..cf3cf73cea5c 100644 --- a/arch/arm/kernel/livepatch.c +++ b/arch/arm/kernel/livepatch.c @@ -142,6 +142,28 @@ int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), v return do_check_calltrace(&args, klp_check_jump_func); } +static bool check_module_calltrace(void *ws_args, unsigned long pc) +{ + struct walk_stackframe_args *args = ws_args; + struct module *mod = args->data; + + if (within_module_core(pc, mod)) { + pr_err("module %s is in use!\n", mod->name); + return (args->ret = -EBUSY); + } + return 0; +} + +int arch_klp_module_check_calltrace(void *data) +{ + struct walk_stackframe_args args = { + .data = data, + .ret = 0 + }; + + return do_check_calltrace(&args, check_module_calltrace); +} + int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func) { u32 *addr = (u32 *)old_func; -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 --------------------------- Initially completed the livepatch for ppc64be, the call from the old function to the new function using stub space. We call from old func to new func, when return form new func, we need to restore R2. The previous module relocation was by adding an extra nop space after the call (bxxx) instruction to restore R2, but it is impossible to use extra space here, because we will not return after calling new func, so we need to use a trampoline space. We will call new func in trampoline and then restore R2 when we return. Please note that we can also use old func as trampoline as a solution, but we are afraid that old func often does not have that much space to store trampoline instruction fragments. The trampoline can be implemented as global. However we need to implement a trampoline for each function and improve its stack check. Our call chain to the new function looks like this: CALLER old_func | old_func | -=> trampoline | -=> new_func So we can't simply check that new_func, old_func and trampoline are both possible on the stack. Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> Signed-off-by: Dong Kai <dongkai11@huawei.com> Signed-off-by: Ye Weihua <yeweihua4@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/powerpc/Kconfig | 2 +- arch/powerpc/include/asm/livepatch.h | 50 ++++- arch/powerpc/include/asm/module.h | 3 + arch/powerpc/kernel/Makefile | 1 + arch/powerpc/kernel/livepatch_64.c | 264 ++++++++++++++++++++++++++ arch/powerpc/kernel/livepatch_tramp.S | 92 +++++++++ arch/powerpc/kernel/module_64.c | 109 +++++++++++ include/linux/livepatch.h | 5 + kernel/livepatch/core.c | 28 +++ 9 files changed, 551 insertions(+), 3 deletions(-) create mode 100644 arch/powerpc/kernel/livepatch_64.c create mode 100644 arch/powerpc/kernel/livepatch_tramp.S diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 59eaab656ab4..64b312a9384d 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -257,7 +257,7 @@ config PPC select HAVE_KRETPROBES select HAVE_LD_DEAD_CODE_DATA_ELIMINATION if HAVE_OBJTOOL_MCOUNT && (!ARCH_USING_PATCHABLE_FUNCTION_ENTRY || (!CC_IS_GCC || GCC_VERSION >= 110100)) select HAVE_LIVEPATCH_FTRACE if HAVE_DYNAMIC_FTRACE_WITH_REGS - select HAVE_LIVEPATCH_WO_FTRACE if PPC32 + select HAVE_LIVEPATCH_WO_FTRACE if (PPC64 && CPU_BIG_ENDIAN && PPC64_ELF_ABI_V1) || PPC32 select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI if PERF_EVENTS || (PPC64 && PPC_BOOK3S) select HAVE_OPTPROBES diff --git a/arch/powerpc/include/asm/livepatch.h b/arch/powerpc/include/asm/livepatch.h index aa36c6e09110..3b2d089eb180 100644 --- a/arch/powerpc/include/asm/livepatch.h +++ b/arch/powerpc/include/asm/livepatch.h @@ -23,16 +23,62 @@ static inline void klp_init_thread_info(struct task_struct *p) { } #ifdef CONFIG_LIVEPATCH_WO_FTRACE +struct klp_func; + +#ifdef CONFIG_PPC64 +/* + * use the livepatch stub to jump to the trampoline. + * It is similar to stub, but does not need to save + * and load R2. + * struct ppc64_klp_bstub_entry + */ +struct ppc64_klp_bstub_entry { + u32 jump[5]; + u32 magic; + /* address for livepatch trampoline */ + u64 trampoline; +}; + +struct ppc64_klp_btramp_entry { + u32 jump[18]; + u32 magic; + union { + func_desc_t funcdata; + unsigned long saved_entry[3]; + }; +}; + +#define PPC64_INSN_SIZE 4 +#define LJMP_INSN_SIZE (sizeof(struct ppc64_klp_bstub_entry) / PPC64_INSN_SIZE) + +/* STUB_MAGIC 0x73747562 "stub" */ +#define BRANCH_STUB_MAGIC 0x73747563 /* stub + 1 */ +#define BRANCH_TRAMPOLINE_MAGIC 0x73747564 /* stub + 2 */ +void livepatch_branch_stub(void); +void livepatch_branch_stub_end(void); +void livepatch_branch_trampoline(void); +void livepatch_branch_trampoline_end(void); + +int livepatch_create_branch(unsigned long pc, unsigned long trampoline, + unsigned long addr, struct module *me); +struct klp_object; +int arch_klp_init_func(struct klp_object *obj, struct klp_func *func); +void *arch_klp_mem_alloc(size_t size); +void arch_klp_mem_free(void *mem); +#else /* !CONFIG_PPC64 */ #define PPC32_INSN_SIZE 4 #define LJMP_INSN_SIZE 4 +#endif /* CONFIG_PPC64 */ + struct arch_klp_data { u32 old_insns[LJMP_INSN_SIZE]; +#ifdef CONFIG_PPC64 + struct ppc64_klp_btramp_entry trampoline; +#endif }; #define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) -struct klp_func; - /* kernel livepatch instruction barrier */ #define klp_smp_isb() __smp_lwsync() int arch_klp_patch_func(struct klp_func *func); diff --git a/arch/powerpc/include/asm/module.h b/arch/powerpc/include/asm/module.h index a8e2e8339fb7..963352a9b2f4 100644 --- a/arch/powerpc/include/asm/module.h +++ b/arch/powerpc/include/asm/module.h @@ -38,6 +38,9 @@ struct mod_arch_specific { /* For module function descriptor dereference */ unsigned long start_opd; unsigned long end_opd; +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + unsigned long toc; +#endif #else /* powerpc64 */ /* Indices of PLT sections within module. */ unsigned int core_plt_section; diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index ad7f7183cae0..2a66f1dd810a 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -211,6 +211,7 @@ obj-$(CONFIG_ALTIVEC) += vector.o obj-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o obj64-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_entry_64.o extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init_check +obj64-$(CONFIG_LIVEPATCH_WO_FTRACE) += livepatch_tramp.o obj-$(CONFIG_PPC64) += $(obj64-y) obj-$(CONFIG_PPC32) += $(obj32-y) diff --git a/arch/powerpc/kernel/livepatch_64.c b/arch/powerpc/kernel/livepatch_64.c new file mode 100644 index 000000000000..c286da982558 --- /dev/null +++ b/arch/powerpc/kernel/livepatch_64.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * livepatch.c - powerpc-specific Kernel Live Patching Core + * + * Copyright (C) 2018 Huawei Technologies Co., Ltd. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/moduleloader.h> +#include <linux/uaccess.h> +#include <linux/livepatch.h> +#include <linux/slab.h> +#include <linux/sizes.h> +#include <linux/kallsyms.h> +#include <asm/livepatch.h> +#include <asm/cacheflush.h> +#include <asm/code-patching.h> +#include <asm/elf.h> + +int arch_klp_check_activeness_func(struct klp_func *func, int enable, + klp_add_func_t add_func, struct list_head *func_list) +{ + int ret; + unsigned long func_addr, func_size; + struct klp_func_node *func_node = NULL; + + func_node = func->func_node; + /* Check func address in stack */ + if (enable) { + if (func->patched || func->force == KLP_ENFORCEMENT) + return 0; + /* + * When enable, checking the currently + * active functions. + */ + if (list_empty(&func_node->func_stack)) { + /* + * No patched on this function + * [ the origin one ] + */ + func_addr = (unsigned long)func->old_func; + func_size = func->old_size; + } else { + /* + * Previously patched function + * [ the active one ] + */ + struct klp_func *prev; + + prev = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + func_addr = ppc_function_entry((void *)prev->new_func); + func_size = prev->new_size; + } + /* + * When preemption is disabled and the + * replacement area does not contain a jump + * instruction, the migration thread is + * scheduled to run stop machine only after the + * excution of instructions to be repalced is + * complete. + */ + if (IS_ENABLED(CONFIG_PREEMPTION) || + IS_ENABLED(CONFIG_LIVEPATCH_BREAKPOINT_NO_STOP_MACHINE) || + (func->force == KLP_NORMAL_FORCE) || + arch_check_jump_insn(func_addr)) { + ret = add_func(func_list, func_addr, func_size, + func->old_name, func->force); + if (ret) + return ret; + } + } else { + /* + * When disable, check for the function itself + * which to be unpatched. + */ + func_addr = ppc_function_entry((void *)func->new_func); + func_size = func->new_size; + ret = add_func(func_list, func_addr, + func_size, func->old_name, 0); + if (ret) + return ret; + } + + /* + * Check trampoline in stack + * new_func callchain: + * old_func + * -=> trampoline + * -=> new_func + * so, we should check all the func in the callchain + */ + if (func_addr != (unsigned long)func->old_func) { +#ifdef CONFIG_PREEMPTION + /* + * No scheduling point in the replacement + * instructions. Therefore, when preemption is + * not enabled, atomic execution is performed + * and these instructions will not appear on + * the stack. + */ + func_addr = (unsigned long)func->old_func; + func_size = func->old_size; + ret = add_func(func_list, func_addr, + func_size, "OLD_FUNC", 0); + if (ret) + return ret; +#endif /* CONFIG_PREEMPTION */ + + if (func_node->arch_data.trampoline.magic != BRANCH_TRAMPOLINE_MAGIC) + return 0; + + func_addr = (unsigned long)&func_node->arch_data.trampoline; + func_size = sizeof(struct ppc64_klp_btramp_entry); + ret = add_func(func_list, func_addr, + func_size, "trampoline", 0); + if (ret) + return ret; + } + return 0; +} + +long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func) +{ + long ret; + int i; + + for (i = 0; i < LJMP_INSN_SIZE; i++) { + ret = copy_from_kernel_nofault(&arch_data->old_insns[i], + ((u32 *)old_func) + i, PPC64_INSN_SIZE); + if (ret) + break; + } + return ret; +} + +static int do_patch(unsigned long pc, unsigned long new_addr, + struct arch_klp_data *arch_data, struct module *old_mod) +{ + int ret; + + ret = livepatch_create_branch(pc, (unsigned long)&arch_data->trampoline, + new_addr, old_mod); + if (ret) { + pr_err("create branch failed, ret=%d\n", ret); + return -EPERM; + } + flush_icache_range(pc, pc + LJMP_INSN_SIZE * PPC64_INSN_SIZE); + pr_debug("[%s %d] old = 0x%lx/0x%lx/%pS, new = 0x%lx/0x%lx/%pS\n", + __func__, __LINE__, + pc, ppc_function_entry((void *)pc), (void *)pc, + new_addr, ppc_function_entry((void *)new_addr), + (void *)ppc_function_entry((void *)new_addr)); + return 0; +} + +int arch_klp_patch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + int ret; + + func_node = func->func_node; + list_add_rcu(&func->stack_node, &func_node->func_stack); + ret = do_patch((unsigned long)func->old_func, + (unsigned long)func->new_func, + &func_node->arch_data, func->old_mod); + if (ret) + list_del_rcu(&func->stack_node); + return ret; +} + +void arch_klp_unpatch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + struct klp_func *next_func; + unsigned long pc; + int ret; + + func_node = func->func_node; + pc = (unsigned long)func_node->old_func; + list_del_rcu(&func->stack_node); + if (list_empty(&func_node->func_stack)) { + ret = klp_patch_text((u32 *)pc, func_node->arch_data.old_insns, + LJMP_INSN_SIZE); + if (ret) { + pr_err("restore instruction failed, ret=%d\n", ret); + return; + } + + pr_debug("[%s %d] restore insns at 0x%lx\n", __func__, __LINE__, pc); + flush_icache_range(pc, pc + LJMP_INSN_SIZE * PPC64_INSN_SIZE); + } else { + next_func = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + do_patch(pc, (unsigned long)next_func->new_func, + &func_node->arch_data, func->old_mod); + } +} + +int arch_klp_init_func(struct klp_object *obj, struct klp_func *func) +{ + unsigned long new_addr = (unsigned long)func->new_func; + + /* + * ABI v1 address is address of the OPD entry, + * which contains address of fn. ABI v2 An address + * is simply the address of the function. + * + * The function descriptor is in the data section. So + * If new_addr is in the code segment, we think it is + * a function address, if addr isn't in the code segment, + * we consider it to be a function descriptor. + */ + if (!is_module_text_address(new_addr)) { + new_addr = (unsigned long)ppc_function_entry((void *)new_addr); + if (!kallsyms_lookup_size_offset((unsigned long)new_addr, + &func->new_size, NULL)) + return -ENOENT; + } + + func->this_mod = __module_text_address(new_addr); + if (!func->this_mod) + return -EINVAL; + + func->new_func_descr.addr = new_addr; + func->new_func_descr.toc = func->this_mod->arch.toc; + func->new_func_descr.env = 0; + func->new_func = (void *)&func->new_func_descr; + + return 0; +} + +/* + * Trampoline would be stored in the allocated memory and it need + * executable permission, so ppc64 use 'module_alloc' but not 'kmalloc'. + */ +void *arch_klp_mem_alloc(size_t size) +{ + void *mem = module_alloc(size); + + if (mem) + memset(mem, 0, size); /* initially clear the memory */ + return mem; +} + +void arch_klp_mem_free(void *mem) +{ + module_memfree(mem); +} diff --git a/arch/powerpc/kernel/livepatch_tramp.S b/arch/powerpc/kernel/livepatch_tramp.S new file mode 100644 index 000000000000..d8baf8a98a97 --- /dev/null +++ b/arch/powerpc/kernel/livepatch_tramp.S @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * livepatch.c - powerpc-specific Kernel Live Patching Core + * + * Copyright (C) 2023 Huawei Technologies Co., Ltd. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include <asm/asm-offsets.h> +#include <asm/ppc_asm.h> +#include <asm/ptrace.h> + +.section ".text","ax",@progbits + + /* + * Livepatch function branch stub. + * see struct ppc64_klp_bstub_entry + * use it jump to livepatch trampoline + */ +_GLOBAL(livepatch_branch_stub) + addis r11,r2, 0 /* <high> */ + addi r11,r11, 0 /* <low> */ + ld r12,24(r11) + mtctr r12 + bctr +_GLOBAL(livepatch_branch_stub_end) + nop /* for magic */ + + /* + * This function runs in the livepatch context, between two functions. + * As such it can only clobber registers which are volatile and used in + * function linkage. + * + * We get here when a function A, calls another function B, but B has + * been live patched with a new function C. + * + * On entry: + * - we have no stack frame and can not allocate one + * - LR points back to the original caller (in A) + * - CTR used to hold the new NIP for call + * - r0, r11 & r12 are free + * -- r11 point back to the bstub data which store (func descr) + * ---- 0(saved_entry) : new function address + * ---- 8(r0) : new R2(toc) for new function + * -- tag livepatch stack with r11 + * -- save temporary variables with r12 + */ +_GLOBAL(livepatch_branch_trampoline) + mflr r0 + std r0, 16(r1) + std r2, 24(r1) + stdu r1, -STACK_FRAME_MIN_SIZE(r1) + + /* Load func descr address to R11 */ + lis r11, 0 /* saved_entry@highest */ + ori r11,r11,0 /* saved_entry@higher */ + rldicr r11,r11,32,31 + oris r11,r11,0 /* saved_entry@high */ + ori r11,r11,0 /* saved_entry@low */ + + /* Call NEW_FUNC */ + ld r12, 0(r11) /* load new func address to R12 */ + ld r2, 8(r11) /* set up new R2 */ + mtctr r12 /* load R12(new func address) to CTR */ + bctrl /* call new func */ + + /* + * Now we are returning from the patched function to the original + * caller A. We are free to use r11, r12 and we can use r2 until we + * restore it. + */ + addi r1, r1, STACK_FRAME_MIN_SIZE + ld r2, 24(r1) + ld r0, 16(r1) + mtlr r0 + + /* Return to original caller of live patched function */ + blr +_GLOBAL(livepatch_branch_trampoline_end) + nop diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 7112adc597a8..a1715d70d348 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -1046,6 +1046,9 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, } } +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + me->arch.toc = my_r2(sechdrs, me); +#endif return 0; } @@ -1107,3 +1110,109 @@ int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs) return 0; } #endif + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +#include <asm/livepatch.h> +#include <asm/cacheflush.h> + +#define PPC_LIVEPATCH_BITMASK(v, n) (((v) >> (n)) & 0xffff) +#define PPC_LIVEPATCH_HIGHEST(v) PPC_LIVEPATCH_BITMASK(v, 48) +#define PPC_LIVEPATCH_HIGHER(v) PPC_LIVEPATCH_BITMASK(v, 32) +#define PPC_LIVEPATCH_HIGH(v) PPC_LIVEPATCH_BITMASK(v, 16) +#define PPC_LIVEPATCH_LOW(v) PPC_LIVEPATCH_BITMASK(v, 0) + +/* + * Patch jump stub to reference trampoline + * without saved the old R2 and load the new R2. + */ +static int livepatch_create_bstub(void *pc, unsigned long addr, struct module *me) +{ + long reladdr; + unsigned long my_r2; + unsigned long stub_start, stub_end, stub_size; + struct ppc64_klp_bstub_entry entry; + + /* Stub uses address relative to r2. */ + my_r2 = me ? me->arch.toc : kernel_toc_addr(); + reladdr = (unsigned long)pc - my_r2; + if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { + pr_err("%s: Address %p of jump stub out of range of %p.\n", + me ? me->name : "kernel", + (void *)reladdr, (void *)my_r2); + return 0; + } + + + stub_start = ppc_function_entry((void *)livepatch_branch_stub); + stub_end = ppc_function_entry((void *)livepatch_branch_stub_end); + stub_size = stub_end - stub_start; + memcpy(entry.jump, (u32 *)stub_start, stub_size); + + entry.jump[0] |= PPC_HA(reladdr); + entry.jump[1] |= PPC_LO(reladdr); + entry.magic = BRANCH_STUB_MAGIC; + entry.trampoline = addr; + + + /* skip breakpoint at first */ + memcpy(pc + PPC64_INSN_SIZE, (void *)&entry + PPC64_INSN_SIZE, + sizeof(entry) - PPC64_INSN_SIZE); + /* + * Avoid compile optimization, make sure that instructions + * except first breakpoint has been patched. + */ + barrier(); + memcpy(pc, (void *)&entry, PPC64_INSN_SIZE); + pr_debug("Create livepatch branch stub 0x%p with reladdr 0x%lx r2 0x%lx to trampoline 0x%lx\n", + pc, reladdr, my_r2, addr); + + return 1; +} + +static void livepatch_create_btramp(struct ppc64_klp_btramp_entry *entry, + unsigned long addr) +{ + unsigned long reladdr, tramp_start, tramp_end, tramp_size; + + tramp_start = ppc_function_entry((void *)livepatch_branch_trampoline); + tramp_end = ppc_function_entry((void *)livepatch_branch_trampoline_end); + tramp_size = tramp_end - tramp_start; + + if (entry->magic != BRANCH_TRAMPOLINE_MAGIC) { + reladdr = (unsigned long)entry->saved_entry; + + memcpy(entry->jump, (u32 *)tramp_start, tramp_size); + + entry->jump[4] |= PPC_LIVEPATCH_HIGHEST(reladdr); + entry->jump[5] |= PPC_LIVEPATCH_HIGHER(reladdr); + entry->jump[7] |= PPC_LIVEPATCH_HIGH(reladdr); + entry->jump[8] |= PPC_LIVEPATCH_LOW(reladdr); + + entry->magic = BRANCH_TRAMPOLINE_MAGIC; + } + entry->funcdata = func_desc(addr); + + flush_icache_range((unsigned long)entry, (unsigned long)entry + tramp_size); + + pr_debug("Create livepatch trampoline 0x%p+%lu/0x%lx to 0x%lx/0x%lx/%pS\n", + (void *)entry, tramp_size, (unsigned long)entry->saved_entry, + addr, ppc_function_entry((void *)addr), + (void *)ppc_function_entry((void *)addr)); +} + +int livepatch_create_branch(unsigned long pc, + unsigned long trampoline, + unsigned long addr, + struct module *me) +{ + + /* Create trampoline to addr(new func) */ + livepatch_create_btramp((struct ppc64_klp_btramp_entry *)trampoline, addr); + + /* Create stub to trampoline */ + if (!livepatch_create_bstub((void *)pc, trampoline, me)) + return -EINVAL; + + return 0; +} +#endif diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 9e0915201268..f04f96b99d50 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -83,6 +83,11 @@ struct klp_func { bool patched; #ifdef CONFIG_LIVEPATCH_FTRACE bool transition; +#endif +#if defined(CONFIG_LIVEPATCH_WO_FTRACE) && defined(CONFIG_PPC64) + struct module *old_mod; + struct module *this_mod; + struct func_desc new_func_descr; #endif void *func_node; /* Only used in the solution without ftrace */ }; diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 2abf8db039d0..5fcf5712de3b 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -93,6 +93,11 @@ static void klp_find_object_module(struct klp_object *obj) } #else /* !CONFIG_LIVEPATCH_FTRACE */ static int klp_find_object_module(struct klp_object *obj); + +int __weak arch_klp_init_func(struct klp_object *obj, struct klp_func *func) +{ + return 0; +} #endif /* CONFIG_LIVEPATCH_FTRACE */ static bool klp_initialized(void) @@ -850,6 +855,9 @@ void klp_free_replaced_patches_async(struct klp_patch *new_patch) static int klp_init_func(struct klp_object *obj, struct klp_func *func) { +#ifndef CONFIG_LIVEPATCH_FTRACE + int ret; +#endif if (!func->old_name) return -EINVAL; @@ -872,6 +880,16 @@ static int klp_init_func(struct klp_object *obj, struct klp_func *func) func->patched = false; #ifdef CONFIG_LIVEPATCH_FTRACE func->transition = false; +#else +#ifdef CONFIG_PPC64 + if (klp_is_module(obj)) + func->old_mod = obj->mod; + else + func->old_mod = NULL; +#endif + ret = arch_klp_init_func(obj, func); + if (ret) + return ret; #endif /* The format for the sysfs directory is <function,sympos> where sympos @@ -952,6 +970,16 @@ static int klp_init_object_loaded(struct klp_patch *patch, if (ret) return ret; +#ifdef CONFIG_PPC64 + /* + * PPC64 big endian binary format is 'elfv1' defaultly, actual + * symbol name of old function need a prefix '.' (related + * feature 'function descriptor'), otherwise size found by + * 'kallsyms_lookup_size_offset' may be abnormal. + */ + if (func->old_name[0] != '.') + pr_warn("old_name '%s' may miss the prefix '.'\n", func->old_name); +#endif ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, &func->old_size, NULL); #ifdef CONFIG_LIVEPATCH_FTRACE -- 2.25.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8MGE6 -------------------------------- Add breakpoint exception optimization support to improve livepatch success rate for ppc64/ppc32. Signed-off-by: Yang Jihong <yangjihong1@huawei.com> Signed-off-by: Li Huafei <lihuafei1@huawei.com> Signed-off-by: Zheng Yejian <zhengyejian1@huawei.com> --- arch/powerpc/include/asm/livepatch.h | 9 ++++++ arch/powerpc/kernel/livepatch.c | 40 +++++++++++++++++++++++++++ arch/powerpc/kernel/livepatch_tramp.S | 34 +++++++++++++++++++++++ arch/powerpc/kernel/traps.c | 8 ++++++ 4 files changed, 91 insertions(+) diff --git a/arch/powerpc/include/asm/livepatch.h b/arch/powerpc/include/asm/livepatch.h index 3b2d089eb180..e83f0cc7b267 100644 --- a/arch/powerpc/include/asm/livepatch.h +++ b/arch/powerpc/include/asm/livepatch.h @@ -58,6 +58,7 @@ void livepatch_branch_stub(void); void livepatch_branch_stub_end(void); void livepatch_branch_trampoline(void); void livepatch_branch_trampoline_end(void); +void livepatch_brk_trampoline(void); int livepatch_create_branch(unsigned long pc, unsigned long trampoline, unsigned long addr, struct module *me); @@ -75,6 +76,11 @@ struct arch_klp_data { #ifdef CONFIG_PPC64 struct ppc64_klp_btramp_entry trampoline; #endif + /* + * Saved opcode at the entry of the old func (which maybe replaced + * with breakpoint). + */ + u32 saved_opcode; }; #define KLP_MAX_REPLACE_SIZE sizeof_field(struct arch_klp_data, old_insns) @@ -87,6 +93,9 @@ long arch_klp_save_old_code(struct arch_klp_data *arch_data, void *old_func); int arch_klp_check_calltrace(bool (*check_func)(void *, int *, unsigned long), void *data); bool arch_check_jump_insn(unsigned long func_addr); int klp_patch_text(u32 *dst, const u32 *src, int len); +int klp_brk_handler(struct pt_regs *regs); +int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func); +void arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func); #endif /* CONFIG_LIVEPATCH_WO_FTRACE */ diff --git a/arch/powerpc/kernel/livepatch.c b/arch/powerpc/kernel/livepatch.c index 09b405740cb5..c1f6c200d711 100644 --- a/arch/powerpc/kernel/livepatch.c +++ b/arch/powerpc/kernel/livepatch.c @@ -282,3 +282,43 @@ int klp_patch_text(u32 *dst, const u32 *src, int len) barrier(); return patch_instruction(dst, ppc_inst(src[0])); } + +int arch_klp_add_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ + ppc_inst_t insn = ppc_inst_read((const u32 *)old_func); + + arch_data->saved_opcode = ppc_inst_val(insn); + patch_instruction((u32 *)old_func, ppc_inst(BREAKPOINT_INSTRUCTION)); + return 0; +} + +void arch_klp_remove_breakpoint(struct arch_klp_data *arch_data, void *old_func) +{ + patch_instruction((u32 *)old_func, ppc_inst(arch_data->saved_opcode)); +} + +int klp_brk_handler(struct pt_regs *regs) +{ + void *brk_func = NULL; + unsigned long addr = regs->nip; + + if (user_mode(regs)) + return 0; + + brk_func = klp_get_brk_func((void *)addr); + if (!brk_func) + return 0; + +#ifdef CONFIG_PPC64 + /* + * Only static trampoline can be used here to prevent + * resource release caused by rollback. + */ + regs->gpr[PT_R11] = (unsigned long)brk_func; + regs->nip = ppc_function_entry((void *)livepatch_brk_trampoline); +#else + regs->nip = (unsigned long)brk_func; +#endif + + return 1; +} diff --git a/arch/powerpc/kernel/livepatch_tramp.S b/arch/powerpc/kernel/livepatch_tramp.S index d8baf8a98a97..643a3f6d1aae 100644 --- a/arch/powerpc/kernel/livepatch_tramp.S +++ b/arch/powerpc/kernel/livepatch_tramp.S @@ -90,3 +90,37 @@ _GLOBAL(livepatch_branch_trampoline) blr _GLOBAL(livepatch_branch_trampoline_end) nop + +/* + * This function is the trampoline of livepatch brk handler. + * + * brk -> traps + * - klp_brk_handler + * - set R11 to new_func address + * - set NIP to livepatch_brk_trampoline address + * see arch/powerpc/kernel/livepatch.c + */ +_GLOBAL(livepatch_brk_trampoline) + mflr r0 + std r0, 16(r1) + std r2, 24(r1) + stdu r1, -STACK_FRAME_MIN_SIZE(r1) + + /* Call NEW_FUNC */ + ld r12, 0(r11) /* load new func address to R12 */ + ld r2, 8(r11) /* set up new R2 */ + mtctr r12 /* load R12(new func address) to CTR */ + bctrl /* call new func */ + + /* + * Now we are returning from the patched function to the original + * caller A. We are free to use r11, r12 and we can use r2 until we + * restore it. + */ + addi r1, r1, STACK_FRAME_MIN_SIZE + ld r2, 24(r1) + ld r0, 16(r1) + mtlr r0 + + /* Return to original caller of live patched function */ + blr diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index fe3f720c9cd6..057f6167199b 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -70,6 +70,10 @@ #include <asm/disassemble.h> #include <asm/udbg.h> +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +#include <asm/livepatch.h> +#endif + #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE) int (*__debugger)(struct pt_regs *regs) __read_mostly; int (*__debugger_ipi)(struct pt_regs *regs) __read_mostly; @@ -1496,6 +1500,10 @@ static void do_program_check(struct pt_regs *regs) if (kprobe_handler(regs)) return; +#ifdef CONFIG_LIVEPATCH_WO_FTRACE + if (klp_brk_handler(regs)) + return; +#endif /* trap exception */ if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) -- 2.25.1