hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8S9BY CVE: NA
--------------------------------
Support to demote pages in 2M pages pool to 4K size. Support to promote pages in 4K pages pool to 2M size.
Signed-off-by: Liu Shixin liushixin2@huawei.com --- mm/dynamic_pool.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++ mm/internal.h | 5 +++ mm/page_alloc.c | 6 ++-- 3 files changed, 100 insertions(+), 3 deletions(-)
diff --git a/mm/dynamic_pool.c b/mm/dynamic_pool.c index 7924c2d28ecb..da442a9afdc8 100644 --- a/mm/dynamic_pool.c +++ b/mm/dynamic_pool.c @@ -134,6 +134,34 @@ static int dpool_demote_gigantic_page(struct pages_pool *src_pool, return 0; }
+static int dpool_demote_huge_page(struct pages_pool *src_pool, + struct pages_pool *dst_pool, + struct page *page) +{ + struct folio *folio = page_folio(page); + int nr_pages = 1 << PMD_ORDER; + struct page *subpage; + int i; + + if (PageHWPoison(page)) + return -EHWPOISON; + + list_del(&page->lru); + __ClearPageDpool(page); + src_pool->free_pages--; + + clear_compound_page(page_folio(page), PMD_ORDER); + for (i = 0; i < nr_pages; i++) { + subpage = folio_page(folio, i); + free_pages_prepare(subpage, 0, 0); + __SetPageDpool(subpage); + list_add_tail(&subpage->lru, &dst_pool->freelist); + dst_pool->free_pages++; + } + + return 0; +} + static int dpool_demote_pool_locked(struct dynamic_pool *dpool, int type) { struct pages_pool *src_pool, *dst_pool; @@ -161,6 +189,9 @@ static int dpool_demote_pool_locked(struct dynamic_pool *dpool, int type) case PAGES_POOL_1G: ret = dpool_demote_gigantic_page(src_pool, dst_pool, page); break; + case PAGES_POOL_2M: + ret = dpool_demote_huge_page(src_pool, dst_pool, page); + break; default: BUG(); } @@ -218,6 +249,43 @@ static int dpool_promote_gigantic_page(struct pages_pool *src_pool, return 0; }
+static int dpool_promote_huge_page(struct pages_pool *src_pool, + struct pages_pool *dst_pool, + struct split_page *spage) +{ + struct hstate *h = size_to_hstate(PMD_SIZE); + int nr_pages = 1 << PMD_ORDER; + struct page *page, *subpage; + int i; + + for (i = 0; i < nr_pages; i++) { + subpage = pfn_to_page(spage->start_pfn + i); + if (!PageDpool(subpage)) + return -EBUSY; + + if (PageHWPoison(subpage)) + return -EHWPOISON; + } + + for (i = 0; i < nr_pages; i++) { + subpage = pfn_to_page(spage->start_pfn + i); + __ClearPageDpool(subpage); + list_del(&subpage->lru); + src_pool->free_pages--; + } + + page = pfn_to_page(spage->start_pfn); + prep_new_page(page, PMD_ORDER, __GFP_COMP, 0); + set_page_count(page, 0); + folio_change_private(page_folio(page), NULL); + __SetPageDpool(page); + __prep_new_hugetlb_folio(h, page_folio(page)); + list_add_tail(&page->lru, &dst_pool->freelist); + dst_pool->free_pages++; + + return 0; +} + static int dpool_promote_pool(struct dynamic_pool *dpool, int type) { struct pages_pool *src_pool, *dst_pool; @@ -241,6 +309,18 @@ static int dpool_promote_pool(struct dynamic_pool *dpool, int type) case PAGES_POOL_1G: ret = dpool_promote_gigantic_page(src_pool, dst_pool, spage); break; + case PAGES_POOL_2M: { + /* + * Since the dpool_mutex is already locked, + * there is no way to free spage_next, so + * it is safe to unlock here. + */ + spin_unlock(&dpool->lock); + cond_resched(); + spin_lock(&dpool->lock); + ret = dpool_promote_huge_page(src_pool, dst_pool, spage); + break; + } default: BUG(); } @@ -744,6 +824,18 @@ int dynamic_pool_reserve_hugepage(struct mem_cgroup *memcg, while (delta > pool->free_pages && !dpool_demote_pool_locked(dpool, type - 1)) cond_resched_lock(&dpool->lock); + /* Only try merge pages for 2M pages */ + if (type == PAGES_POOL_2M) { + while (delta > pool->free_pages) { + spin_unlock(&dpool->lock); + cond_resched(); + if (dpool_promote_pool(dpool, type)) { + spin_lock(&dpool->lock); + break; + } + spin_lock(&dpool->lock); + } + } delta = min(delta, pool->free_pages); pool->nr_huge_pages += delta; pool->free_huge_pages += delta; diff --git a/mm/internal.h b/mm/internal.h index 1ebba69437d6..a0f252baa67f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -434,10 +434,15 @@ static inline void prep_compound_tail(struct page *head, int tail_idx) set_page_private(p, 0); }
+typedef int __bitwise fpi_t; extern void prep_compound_page(struct page *page, unsigned int order);
extern void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); +extern bool free_pages_prepare(struct page *page, unsigned int order, + fpi_t fpi_flags); +extern void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, + unsigned int alloc_flags); extern int user_min_free_kbytes;
extern void free_unref_page(struct page *page, unsigned int order); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 42e434224e67..04f4e5fc7d69 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1073,7 +1073,7 @@ static void kernel_init_pages(struct page *page, int numpages) kasan_enable_current(); }
-static __always_inline bool free_pages_prepare(struct page *page, +__always_inline bool free_pages_prepare(struct page *page, unsigned int order, fpi_t fpi_flags) { int bad = 0; @@ -1538,8 +1538,8 @@ inline void post_alloc_hook(struct page *page, unsigned int order, page_table_check_alloc(page, order); }
-static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, - unsigned int alloc_flags) +void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, + unsigned int alloc_flags) { post_alloc_hook(page, order, gfp_flags);