From: Xu Kuohai xukuohai@huawei.com
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9FGRE
--------------------------------
For kernel modules copmiled without DIRECT_CALL, there is no literal space reserved on the function entry, this patch add support for these functions.
Signed-off-by: Xu Kuohai xukuohai@huawei.com Signed-off-by: Pu Lehui pulehui@huawei.com --- arch/arm64/kernel/ftrace.c | 74 +++++++++++++++++++++++++++++++------- include/linux/ftrace.h | 2 ++ kernel/trace/ftrace.c | 8 +++++ 3 files changed, 72 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index bccb938d52f7..c7d4764a553f 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -117,6 +117,9 @@ enum ftrace_callsite_action {
static unsigned long ftrace_literal_call_addr(struct dyn_ftrace *rec) { + if (rec->arch.func == 0UL) + return 0UL; + return rec->arch.func - 2 * AARCH64_INSN_SIZE; }
@@ -125,6 +128,9 @@ static unsigned long ftrace_literal_addr(struct dyn_ftrace *rec) unsigned long addr = 0;
addr = ftrace_literal_call_addr(rec); + if (addr == 0UL) + return 0UL; + if (addr % sizeof(long)) addr -= 3 * AARCH64_INSN_SIZE; else @@ -154,6 +160,9 @@ static int ftrace_init_literal(struct module *mod, struct dyn_ftrace *rec) old = aarch64_insn_gen_nop();
addr = ftrace_literal_addr(rec); + if (addr == 0UL) + return 0UL; + ftrace_update_literal(addr, 0, FC_INIT);
pc = ftrace_literal_call_addr(rec); @@ -222,9 +231,11 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, unsigned long literal_addr;
literal_addr = ftrace_literal_addr(rec); - ftrace_update_literal(literal_addr, *addr, action); - *addr = ftrace_literal_call_addr(rec); - return true; + if (literal_addr != 0UL) { + ftrace_update_literal(literal_addr, *addr, action); + *addr = ftrace_literal_call_addr(rec); + return true; + } }
/* @@ -343,27 +354,51 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) } #endif
+static int ftrace_nop_count(unsigned long addr) +{ + u32 insn; + u32 nop = aarch64_insn_gen_nop(); + int count = 0; + + for (;;) { + if (aarch64_insn_read((void *)addr, &insn)) + return -1; + + if (insn != nop) + break; + + count++; + addr += AARCH64_INSN_SIZE; + } + + return count; +} + unsigned long ftrace_call_adjust(unsigned long addr) { if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)) { u32 insn; u32 nop = aarch64_insn_gen_nop(); + int count = ftrace_nop_count(addr);
- /* Skip the first 5 NOPS */ - addr += 5 * AARCH64_INSN_SIZE; - - if (aarch64_insn_read((void *)addr, &insn)) + if (count != 5 && count != 7 && count != 2) return 0;
- if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) { - if (insn != nop) { - addr += AARCH64_INSN_SIZE; + if (count == 5 || count == 7) { + /* Skip the first 5 NOPS */ + addr += 5 * AARCH64_INSN_SIZE; + + /* Skip bti c */ + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) { if (aarch64_insn_read((void *)addr, &insn)) return 0; + + if (insn != nop) + addr += AARCH64_INSN_SIZE; } }
- if (WARN_ON_ONCE(insn != nop)) + if (ftrace_nop_count(addr) != 2) return 0;
return addr + AARCH64_INSN_SIZE; @@ -421,10 +456,25 @@ void arch_ftrace_update_code(int command) ftrace_modify_all_code(command); }
+bool ftrace_directable(struct dyn_ftrace *rec) +{ +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + return rec->arch.func != 0UL; +#else + return false; +#endif +} + void ftrace_rec_arch_init(struct dyn_ftrace *rec, unsigned long func) { #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS - rec->arch.func = func + 5 * AARCH64_INSN_SIZE; + int count; + + count = ftrace_nop_count(func); + if (count == 5 || count == 7) + rec->arch.func = func + 5 * AARCH64_INSN_SIZE; + else + rec->arch.func = 0UL; #endif }
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index d979242f44fe..9585a29ace3e 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -280,6 +280,8 @@ struct ftrace_func_entry {
struct dyn_ftrace;
+bool ftrace_directable(struct dyn_ftrace *rec); + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS extern int ftrace_direct_func_count; int register_ftrace_direct(unsigned long ip, unsigned long addr); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index ba2cce21bb83..b2888890add0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5092,6 +5092,11 @@ static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr) return direct; }
+bool __weak ftrace_directable(struct dyn_ftrace *rec) +{ + return true; +} + /** * register_ftrace_direct - Call a custom trampoline directly * @ip: The address of the nop at the beginning of a function @@ -5133,6 +5138,9 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr) if (!rec) goto out_unlock;
+ if (!ftrace_directable(rec)) + goto out_unlock; + /* * Check if the rec says it has a direct call but we didn't * find one earlier?