From: Jisheng Zhang Jisheng.Zhang@synaptics.com
maillist inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4XWBS
Reference: https://lore.kernel.org/all/20210524172433.015b3b6b@xhacker.debian/
--------------------------------
Jisheng Zhang has another way of saving memory, we combine his two patches into one and made some adaptations with dynamic kfence objects.
Description of the original patch:
Some architectures may want to allocate the __kfence_pool differently for example, allocate the __kfence_pool earlier before paging_init(). We also delay the memset() to kfence_init_pool().
KFENCE requires linear map to be mapped at page granularity, so that it is possible to protect/unprotect single pages in the KFENCE pool. Currently if KFENCE is enabled, arm64 maps all pages at page granularity, it seems overkilled. In fact, we only need to map the pages in KFENCE pool itself at page granularity. We acchieve this goal by allocating KFENCE pool before paging_init() so we know the KFENCE pool address, then we take care to map the pool at page granularity during map_mem().
Signed-off-by: Jisheng Zhang Jisheng.Zhang@synaptics.com Conflicts: mm/kfence/core.c Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/include/asm/kfence.h | 8 +++++++- arch/arm64/kernel/setup.c | 3 +++ arch/arm64/mm/mmu.c | 33 +++++++++++++++++++++++++-------- include/linux/kfence.h | 2 ++ mm/kfence/core.c | 16 ++++++++++++++-- 5 files changed, 51 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/include/asm/kfence.h b/arch/arm64/include/asm/kfence.h index d061176d57ea..64d7cbfe067b 100644 --- a/arch/arm64/include/asm/kfence.h +++ b/arch/arm64/include/asm/kfence.h @@ -8,9 +8,15 @@ #ifndef __ASM_KFENCE_H #define __ASM_KFENCE_H
+#include <linux/kfence.h> #include <asm/cacheflush.h>
-static inline bool arch_kfence_init_pool(void) { return true; } +static inline bool arch_kfence_init_pool(void) +{ + memset(__kfence_pool, 0, KFENCE_POOL_SIZE); + + return true; +}
static inline bool kfence_protect_page(unsigned long addr, bool protect) { diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index f6e56847a4e9..ddca8d27fca6 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -18,6 +18,7 @@ #include <linux/screen_info.h> #include <linux/init.h> #include <linux/kexec.h> +#include <linux/kfence.h> #include <linux/root_dev.h> #include <linux/cpu.h> #include <linux/interrupt.h> @@ -374,6 +375,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
arm64_memblock_init();
+ kfence_early_alloc_pool(); + efi_fake_memmap(); efi_find_mirror();
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 7314a7a3613f..56bfd692ebd9 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -13,6 +13,7 @@ #include <linux/init.h> #include <linux/ioport.h> #include <linux/kexec.h> +#include <linux/kfence.h> #include <linux/libfdt.h> #include <linux/mman.h> #include <linux/nodemask.h> @@ -477,10 +478,19 @@ static void __init map_mem(pgd_t *pgdp) int flags = 0, eflags = 0; u64 i;
- if (rodata_full || debug_pagealloc_enabled() || - IS_ENABLED(CONFIG_KFENCE)) + if (rodata_full || debug_pagealloc_enabled()) flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+#ifdef CONFIG_KFENCE + /* + * KFENCE requires linear map to be mapped at page granularity, so + * temporarily skip mapping for __kfence_pool in the following + * for-loop + */ + if (__kfence_pool) + memblock_mark_nomap(__pa(__kfence_pool), KFENCE_POOL_SIZE); +#endif + /* * Take care not to create a writable alias for the * read-only text and rodata sections of the kernel image. @@ -553,6 +563,18 @@ static void __init map_mem(pgd_t *pgdp) resource_size(&crashk_res)); } #endif +#ifdef CONFIG_KFENCE + /* + * Map the __kfence_pool at page granularity now. + */ + if (__kfence_pool) { + __map_memblock(pgdp, __pa(__kfence_pool), + __pa(__kfence_pool + KFENCE_POOL_SIZE), + pgprot_tagged(PAGE_KERNEL), + NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); + memblock_clear_nomap(__pa(__kfence_pool), KFENCE_POOL_SIZE); + } +#endif }
void mark_rodata_ro(void) @@ -1483,12 +1505,7 @@ int arch_add_memory(int nid, u64 start, u64 size, }
- /* - * KFENCE requires linear map to be mapped at page granularity, so that - * it is possible to protect/unprotect single pages in the KFENCE pool. - */ - if (rodata_full || debug_pagealloc_enabled() || - IS_ENABLED(CONFIG_KFENCE)) + if (rodata_full || debug_pagealloc_enabled()) flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
__create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 4b5e3679a72c..00dee7459cd9 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -56,6 +56,7 @@ static __always_inline bool is_kfence_address(const void *addr) return unlikely((unsigned long)((char *)addr - __kfence_pool) < KFENCE_POOL_SIZE && __kfence_pool); }
+void __init kfence_early_alloc_pool(void); /** * kfence_alloc_pool() - allocate the KFENCE pool via memblock */ @@ -205,6 +206,7 @@ bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write, st #else /* CONFIG_KFENCE */
static inline bool is_kfence_address(const void *addr) { return false; } +static inline void kfence_early_alloc_pool(void) { } static inline void kfence_alloc_pool(void) { } static inline void kfence_init(void) { } static inline void kfence_shutdown_cache(struct kmem_cache *s) { } diff --git a/mm/kfence/core.c b/mm/kfence/core.c index a19154a8d196..370721509958 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -752,14 +752,26 @@ static void toggle_allocation_gate(struct work_struct *work) static DECLARE_DELAYED_WORK(kfence_timer, toggle_allocation_gate);
/* === Public interface ===================================================== */ +void __init kfence_early_alloc_pool(void) +{ + if (!kfence_sample_interval) + return; + + __kfence_pool = memblock_alloc_raw(KFENCE_POOL_SIZE, PAGE_SIZE); + + if (!__kfence_pool) { + kfence_sample_interval = 0; + pr_err("failed to early allocate pool, disable KFENCE\n"); + } +}
void __init kfence_alloc_pool(void) { if (!kfence_sample_interval) return;
- __kfence_pool = memblock_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); - + if (!__kfence_pool) + __kfence_pool = memblock_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); if (!__kfence_pool) pr_err("failed to allocate pool\n"); }