shrink and reclaim memory when allocate hugepage failed in mg_sp_alloc_nodemask(). Signed-off-by: Yuan Can <yuancan@huawei.com> Signed-off-by: Wang Wensheng <wangwensheng4@huawei.com> Signed-off-by: Yin Tirui <yintirui@huawei.com> --- fs/drop_caches.c | 7 +++ include/linux/share_pool.h | 2 + mm/share_pool.c | 117 ++++++++++++++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/fs/drop_caches.c b/fs/drop_caches.c index b9575957a7c2..3363a26f5093 100644 --- a/fs/drop_caches.c +++ b/fs/drop_caches.c @@ -77,3 +77,10 @@ int drop_caches_sysctl_handler(struct ctl_table *table, int write, } return 0; } + +#ifdef CONFIG_SHARE_POOL +void sp_drop_caches_handler(void) +{ + iterate_supers(drop_pagecache_sb, NULL); +} +#endif \ No newline at end of file diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 4102d0dcf25b..540cd307e3e5 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -214,6 +214,8 @@ static inline bool sp_check_mmap_addr(unsigned long addr, unsigned long flags) return sp_is_enabled() && mg_is_sharepool_addr(addr) && !(flags & MAP_SHARE_POOL); } +extern void sp_drop_caches_handler(void); + int sp_register_hugepage_allocator(sp_alloc_hugepage_fn alloc_func); #else /* CONFIG_SHARE_POOL */ diff --git a/mm/share_pool.c b/mm/share_pool.c index 8041bd727948..b6e543f1cefc 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -51,6 +51,7 @@ #include <linux/time64.h> #include <linux/pagewalk.h> #include <linux/workqueue.h> +#include <linux/compaction.h> /* Use spa va address as mmap offset. This can work because spa_file * is setup with 64-bit address space. So va shall be well covered. @@ -70,6 +71,14 @@ #define SEC2US(sec) ((sec) * 1000000) #define NS2US(ns) ((ns) / 1000) +#define SP_RECLAIM_RATIO (100) +#define SP_RECLAIM_RETRY_DEFAULT (10) + +static int sysctl_hugetlb_reclaim_ext_enable = 0; +static unsigned int sysctl_sp_reclaim_max_retry = SP_RECLAIM_RETRY_DEFAULT; +static unsigned int sysctl_sp_reclaim_ratio = SP_RECLAIM_RATIO; +static unsigned int sysctl_sp_reclaim_defer_threshold = UINT_MAX; +static unsigned int sysctl_sp_reclaim_drop_all = 0; static int sysctl_sp_dump_enable = 1; static int sysctl_sp_skip_clear_huge_page = 1; @@ -887,6 +896,9 @@ struct sp_alloc_context { int preferred_node_id; bool have_mbind; enum spa_type type; + int retry_count; + int nr_reclaimed; + int drop_all; }; static int sp_map_spa_to_mm(struct mm_struct *mm, struct sp_area *spa, @@ -2352,12 +2364,70 @@ static int sp_alloc_prepare(unsigned long size, unsigned long sp_flags, ac->have_mbind = false; ac->size_aligned = (sp_flags & SP_HUGEPAGE) ? ALIGN(size, PMD_SIZE) : ALIGN(size, PAGE_SIZE); + ac->retry_count = (sp_flags & SP_HUGEPAGE) ? sysctl_sp_reclaim_max_retry : 0; return 0; } +#ifdef CONFIG_PAGE_CACHE_LIMIT +extern unsigned long shrink_memory(unsigned long nr_to_reclaim, bool may_swap); +#else +static unsigned long shrink_memory(unsigned long nr_to_reclaim, bool may_swap) +{ + return 0; +} +#endif + +static void sp_reset_compact_defer(unsigned int i) +{ + struct zone *zone; + if (i < sysctl_sp_reclaim_defer_threshold) + return; + + for_each_populated_zone(zone) + compaction_defer_reset(zone, PMD_SHIFT - PAGE_SHIFT, false); +} + +static bool sp_reclaim_hugepages(struct sp_alloc_context *ac) +{ + unsigned long nr_to_reclaim = DIV_ROUND_UP(ac->size_aligned / PAGE_SIZE * + sysctl_sp_reclaim_ratio, SP_RECLAIM_RATIO); + + if (!(ac->sp_flags & SP_HUGEPAGE)) + return false; + + if (!ac->retry_count) { + /* print end info if we have exhaust retry_count */ + if (sysctl_sp_reclaim_max_retry) + pr_info_ratelimited("process %s(%d) reclaim hugepages failed, len: %ld, reclaimed: %d, drop_all: %s", + current->comm, current->tgid, ac->size_aligned, ac->nr_reclaimed, + ac->drop_all ? "y" : "n"); + + return false; + } + + if (ac->retry_count == sysctl_sp_reclaim_max_retry) + pr_info_ratelimited("process %s(%d) reclaim hugepages start.\n", + current->comm, current->tgid); + + wakeup_flusher_threads(WB_REASON_SYNC); + ac->nr_reclaimed += shrink_memory(nr_to_reclaim, 0); + if (sysctl_sp_reclaim_drop_all && ac->retry_count == (sysctl_sp_reclaim_max_retry - 1)) { + ac->drop_all = 1; + sp_drop_caches_handler(); + } + + sp_reset_compact_defer(sysctl_sp_reclaim_max_retry - ac->retry_count); + + cond_resched(); + ac->retry_count--; + return true; +} + static bool sp_alloc_fallback(struct sp_area *spa, struct sp_alloc_context *ac) { + if (sysctl_hugetlb_reclaim_ext_enable && sp_reclaim_hugepages(ac)) + return true; /* * If hugepage allocation fails, this will transfer to normal page * and try again. (only if SP_HUGEPAGE_ONLY is not flagged @@ -2686,7 +2756,7 @@ static void *__mg_sp_alloc_nodemask(unsigned long size, unsigned long sp_flags, struct sp_area *spa; struct sp_group *spg; nodemask_t __nodemask; - struct sp_alloc_context ac; + struct sp_alloc_context ac = {0}; struct sp_group_node *spg_node; /** @@ -4301,6 +4371,51 @@ static struct ctl_table sp_ctl_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sharepool_hugetlb_reclaim_extend", + .data = &sysctl_hugetlb_reclaim_ext_enable, + .maxlen = sizeof(sysctl_hugetlb_reclaim_ext_enable), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sharepool_hugetlb_reclaim_ratio", + .data = &sysctl_sp_reclaim_ratio, + .maxlen = sizeof(sysctl_sp_reclaim_ratio), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sharepool_hugetlb_reclaim_max_retry", + .data = &sysctl_sp_reclaim_max_retry, + .maxlen = sizeof(sysctl_sp_reclaim_max_retry), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sharepool_hugetlb_reclaim_defer_threshold", + .data = &sysctl_sp_reclaim_defer_threshold, + .maxlen = sizeof(sysctl_sp_reclaim_defer_threshold), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sharepool_hugetlb_reclaim_drop_all", + .data = &sysctl_sp_reclaim_drop_all, + .maxlen = sizeof(sysctl_sp_reclaim_drop_all), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, }; static int __init share_pool_init(void) -- 2.43.0