From: Cheng Jian cj.chengjian@huawei.com
hulk inclusion category: feature bugzilla: 5507 CVE: NA
----------------------------------------
support livepatch without ftrace for x86_64
supported now: livepatch relocation when init_patch after load_module; instruction patched when enable; activeness function check; enforcing the patch stacking principle;
x86_64 use variable length instruction, so there's no need to consider extra implementation for long jumps.
Signed-off-by: Cheng Jian cj.chengjian@huawei.com Signed-off-by: Li Bin huawei.libin@huawei.com Tested-by: Yang ZuoTing yangzuoting@huawei.com Tested-by: Cheng Jian cj.chengjian@huawei.com Reviewed-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: zhangyi (F) yi.zhang@huawei.com Reviewed-By: Xie XiuQi xiexiuqi@huawei.com Reviewed-By: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- arch/x86/Kconfig | 1 + arch/x86/include/asm/livepatch.h | 33 ++++- arch/x86/kernel/livepatch.c | 280 +++++++++++++++++++++++++++++++++++++++ kernel/livepatch/core.c | 13 ++ 4 files changed, 326 insertions(+), 1 deletion(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e297c79..454d411 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -166,6 +166,7 @@ config X86 select HAVE_KRETPROBES select HAVE_KVM select HAVE_LIVEPATCH_FTRACE if X86_64 + select HAVE_LIVEPATCH_WO_FTRACE if X86_64 select HAVE_MEMBLOCK select HAVE_MEMBLOCK_NODE_MAP select HAVE_MIXED_BREAKPOINTS_REGS diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h index ed80003..8f8533d 100644 --- a/arch/x86/include/asm/livepatch.h +++ b/arch/x86/include/asm/livepatch.h @@ -24,17 +24,48 @@ #include <asm/setup.h> #include <linux/ftrace.h>
+#ifdef CONFIG_LIVEPATCH static inline int klp_check_compiler_support(void) { -#ifndef CC_USING_FENTRY +#if defined(CONFIG_LIVEPATCH_FTRACE) && !defined(CC_USING_FENTRY) return 1; #endif return 0; }
+#ifdef CONFIG_LIVEPATCH_FTRACE static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) { regs->ip = ip; }
+static inline unsigned long klp_arch_stub_ip(unsigned long addr) +{ + return addr; +} +#else /* CONFIG_LIVEPATCH_WO_FTRACE */ +#define klp_smp_isb() + +static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) +{ + BUG(); +} + +static inline unsigned long klp_arch_stub_ip(unsigned long addr) +{ + BUG(); + return 0; +} + +struct klp_patch; +struct klp_func; +int arch_klp_patch_func(struct klp_func *func); +void arch_klp_unpatch_func(struct klp_func *func); +int klp_check_calltrace(struct klp_patch *patch, int enable); +#endif + +#else +#error Live patching support is disabled; check CONFIG_LIVEPATCH +#endif + #endif /* _ASM_X86_LIVEPATCH_H */ diff --git a/arch/x86/kernel/livepatch.c b/arch/x86/kernel/livepatch.c index e9d252d..1fbfa62 100644 --- a/arch/x86/kernel/livepatch.c +++ b/arch/x86/kernel/livepatch.c @@ -15,10 +15,14 @@ * along with this program; if not, see http://www.gnu.org/licenses/. */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/module.h> #include <linux/kallsyms.h> #include <linux/livepatch.h> #include <asm/text-patching.h> +#include <asm/stacktrace.h> +#include <asm/set_memory.h>
/* Apply per-object alternatives. Based on x86 module_finalize() */ void arch_klp_init_object_loaded(struct klp_patch *patch, @@ -63,3 +67,279 @@ void arch_klp_init_object_loaded(struct klp_patch *patch, apply_paravirt(pseg, pseg + para->sh_size); } } + + +#ifdef CONFIG_LIVEPATCH_WO_FTRACE +static inline int klp_compare_address(unsigned long stack_addr, + unsigned long func_addr, unsigned long func_size, + const char *func_name) +{ + if (stack_addr >= func_addr && stack_addr < func_addr + func_size) { + pr_err("func %s is in use!\n", func_name); + return -EBUSY; + } + return 0; +} + +static int klp_check_stack_func(struct klp_func *func, + struct stack_trace *trace, int enable) +{ + unsigned long func_addr, func_size, address; + const char *func_name; + int i; + + for (i = 0; i < trace->nr_entries; i++) { + address = trace->entries[i]; + + if (enable) { + if (func->force) + continue; + func_addr = func->old_addr; + func_size = func->old_size; + } else { + func_addr = (unsigned long)func->new_func; + func_size = func->new_size; + } + func_name = func->old_name; + + if (klp_compare_address(address, func_addr, + func_size, func_name)) + return -EAGAIN; + } + + return 0; +} + +static void klp_print_stack_trace(struct stack_trace *trace) +{ + int i; + + pr_err("Call Trace:\n"); + for (i = 0; i < trace->nr_entries; i++) { + pr_err("[<%pK>] %pS\n", + (void *)trace->entries[i], + (void *)trace->entries[i]); + } + +} + +#ifdef MAX_STACK_ENTRIES +#undef MAX_STACK_ENTRIES +#endif +#define MAX_STACK_ENTRIES 100 + +/* + * Determine whether it's safe to transition the task to the target patch state + * by looking for any to-be-patched or to-be-unpatched functions on its stack. + */ +static int klp_check_stack(struct task_struct *task, + struct klp_patch *patch, int enable) + +{ + static unsigned long entries[MAX_STACK_ENTRIES]; + struct stack_trace trace; + struct klp_object *obj; + struct klp_func *func; + int ret; + + trace.skip = 0; + trace.nr_entries = 0; + trace.max_entries = MAX_STACK_ENTRIES; + trace.entries = entries; + ret = save_stack_trace_tsk_reliable(task, &trace); + WARN_ON_ONCE(ret == -ENOSYS); + if (ret) { + pr_info("%s: %s:%d has an unreliable stack\n", + __func__, task->comm, task->pid); + return ret; + } + + klp_for_each_object(patch, obj) { + klp_for_each_func(obj, func) { + ret = klp_check_stack_func(func, &trace, enable); + if (ret) { + pr_info("%s: %s:%d is sleeping on function %s\n", + __func__, task->comm, task->pid, + func->old_name); + + klp_print_stack_trace(&trace); + + return ret; + + } + } + } + + return 0; +} + +int klp_check_calltrace(struct klp_patch *patch, int enable) +{ + struct task_struct *g, *t; + int ret = 0; + + for_each_process_thread(g, t) { + ret = klp_check_stack(t, patch, enable); + if (ret) + goto out; + } + +out: + return ret; +} + +#include <linux/slab.h> +#include <asm/nops.h> +#include <asm/sections.h> + +#define JMP_E9_INSN_SIZE 5 +union klp_code_union { + char code[JMP_E9_INSN_SIZE]; + struct { + unsigned char e9; + int offset; + } __packed; +}; + +struct klp_func_node { + struct list_head node; + struct list_head func_stack; + unsigned long old_addr; + unsigned char old_code[JMP_E9_INSN_SIZE]; +}; + +static LIST_HEAD(klp_func_list); + +static struct klp_func_node *klp_find_func_node(unsigned long old_addr) +{ + struct klp_func_node *func_node; + + list_for_each_entry(func_node, &klp_func_list, node) { + if (func_node->old_addr == old_addr) + return func_node; + } + + return NULL; +} + +int arch_klp_init_func(struct klp_object *obj, struct klp_func *func) +{ + return 0; +} + +void arch_klp_free_func(struct klp_object *obj, struct klp_func *limit) +{ + +} + +static int klp_calc_offset(long pc, long addr) +{ + return (int)(addr - pc); +} + +static unsigned char *klp_jmp_code(unsigned long ip, unsigned long addr) +{ + static union klp_code_union calc; + + calc.e9 = 0xe9; + calc.offset = klp_calc_offset(ip + JMP_E9_INSN_SIZE, addr); + + return calc.code; +} + +static unsigned char *klp_old_code(unsigned char *code) +{ + static union klp_code_union old_code; + + strncpy(old_code.code, code, JMP_E9_INSN_SIZE); + return old_code.code; +} + +void arch_klp_code_modify_prepare(void) +{ + set_kernel_text_rw(); + set_all_modules_text_rw(); +} + +void arch_klp_code_modify_post_process(void) +{ + set_all_modules_text_ro(); + set_kernel_text_ro(); +} + +static inline int within(unsigned long addr, unsigned long start, + unsigned long end) +{ + return addr >= start && addr < end; +} + +static unsigned long text_ip_addr(unsigned long ip) +{ + if (within(ip, (unsigned long)_text, (unsigned long)_etext)) + ip = (unsigned long)__va(__pa_symbol(ip)); + + return ip; +} + +int arch_klp_patch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + unsigned long ip, new_addr; + const unsigned char *new; + + func_node = klp_find_func_node(func->old_addr); + ip = func->old_addr; + if (!func_node) { + func_node = kzalloc(sizeof(*func_node), GFP_ATOMIC); + if (!func_node) + return -ENOMEM; + + INIT_LIST_HEAD(&func_node->func_stack); + func_node->old_addr = func->old_addr; + probe_kernel_read(func_node->old_code, + (void *)ip, JMP_E9_INSN_SIZE); + list_add_rcu(&func_node->node, &klp_func_list); + } + + list_add_rcu(&func->stack_node, &func_node->func_stack); + + new_addr = (unsigned long)func->new_func; + new = klp_jmp_code(ip, new_addr); + + ip = text_ip_addr(ip); + if (probe_kernel_write((void *)ip, new, JMP_E9_INSN_SIZE)) + return -EPERM; + + sync_core(); + + return 0; +} + +void arch_klp_unpatch_func(struct klp_func *func) +{ + struct klp_func_node *func_node; + struct klp_func *next_func; + unsigned long ip, new_addr; + const unsigned char *new; + + func_node = klp_find_func_node(func->old_addr); + ip = func_node->old_addr; + if (list_is_singular(&func_node->func_stack)) { + list_del_rcu(&func->stack_node); + list_del_rcu(&func_node->node); + new = klp_old_code(func_node->old_code); + kfree(func_node); + } else { + list_del_rcu(&func->stack_node); + next_func = list_first_or_null_rcu(&func_node->func_stack, + struct klp_func, stack_node); + + new_addr = (unsigned long)next_func->new_func; + new = klp_jmp_code(ip, new_addr); + } + + ip = text_ip_addr(ip); + probe_kernel_write((void *)ip, new, JMP_E9_INSN_SIZE); + sync_core(); +} +#endif diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index ca74203..1349063 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -438,6 +438,15 @@ int klp_try_disable_patch(void *data) return ret; }
+void __weak arch_klp_code_modify_prepare(void) +{ +} + +void __weak arch_klp_code_modify_post_process(void) +{ +} + + static int __klp_disable_patch(struct klp_patch *patch) { int ret; @@ -458,7 +467,9 @@ static int __klp_disable_patch(struct klp_patch *patch) } #endif
+ arch_klp_code_modify_prepare(); ret = stop_machine(klp_try_disable_patch, &patch_data, cpu_online_mask); + arch_klp_code_modify_post_process();
return ret; } @@ -662,7 +673,9 @@ static int __klp_enable_patch(struct klp_patch *patch) } #endif
+ arch_klp_code_modify_prepare(); ret = stop_machine(klp_try_enable_patch, &patch_data, cpu_online_mask); + arch_klp_code_modify_prepare(); if (ret) return ret;