hulk inclusion category: feature bugzilla: NA -------------------------------- Introduce sp_register_hugepage_allocator() to let drivers replace the default sharepool hugepage allocation functions. Signed-off-by: Yin Tirui <yintirui@huawei.com> --- include/linux/share_pool.h | 9 +++++++ mm/share_pool.c | 51 +++++++++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index a3caedb99c5c..147e99620b9d 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -88,6 +88,8 @@ struct sp_walk_data { #define MMAP_SHARE_POOL_DYNAMIC_DVPP_END (MMAP_SHARE_POOL_DYNAMIC_DVPP_BASE + \ MMAP_SHARE_POOL_16G_SIZE * 64) +typedef unsigned long (*sp_alloc_hugepage_fn)(unsigned long size, int nid, nodemask_t *nodemask); + #ifdef CONFIG_SHARE_POOL /* @@ -161,6 +163,8 @@ static inline bool sp_check_mmap_addr(unsigned long addr, unsigned long flags) return sp_is_enabled() && mg_is_sharepool_addr(addr) && !(flags & MAP_SHARE_POOL); } +int sp_register_hugepage_allocator(sp_alloc_hugepage_fn alloc_func); + #else /* CONFIG_SHARE_POOL */ static inline int mg_sp_group_add_task(int tgid, unsigned long prot, int spg_id) @@ -260,6 +264,11 @@ static inline bool sp_check_mmap_addr(unsigned long addr, unsigned long flags) { return false; } + +static inline int sp_register_hugepage_allocator(sp_alloc_hugepage_fn alloc_func) +{ + return -EPERM; +} #endif /* !CONFIG_SHARE_POOL */ #endif /* LINUX_SHARE_POOL_H */ diff --git a/mm/share_pool.c b/mm/share_pool.c index d380fa74bb95..cec673f18160 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -88,6 +88,8 @@ static DEFINE_MUTEX(master_list_lock); static LIST_HEAD(spm_dvpp_list); /* mutex to protect insert/delete ops from master_list */ static DEFINE_MUTEX(spm_list_lock); +/* mutex to protect hugepage allocator registration */ +static DEFINE_MUTEX(sp_hugepage_reg_mutex); #define SEQ_printf(m, x...) \ do { \ @@ -1513,6 +1515,36 @@ static bool sp_group_delete_area(struct sp_group *spg, struct sp_area *spa) return atomic_dec_and_test(&spa->spg->spa_num); } +unsigned long sp_alloc_hugepage(unsigned long size, int nid, nodemask_t *nodemask) +{ + struct folio *folio = alloc_hugetlb_folio_nodemask_size(size, nid, nodemask); + + if (unlikely(!folio)) + return -ENOMEM; + + return folio_pfn(folio); +} + +static sp_alloc_hugepage_fn sp_alloc_hugepage_func = sp_alloc_hugepage; +static bool allow_hugepage_allocator_register = true; + +int sp_register_hugepage_allocator(sp_alloc_hugepage_fn alloc_func) +{ + if (unlikely(!alloc_func)) + return -EINVAL; + + mutex_lock(&sp_hugepage_reg_mutex); + if (!allow_hugepage_allocator_register) { + mutex_unlock(&sp_hugepage_reg_mutex); + return -EBUSY; + } + sp_alloc_hugepage_func = alloc_func; + allow_hugepage_allocator_register = false; + mutex_unlock(&sp_hugepage_reg_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(sp_register_hugepage_allocator); + static bool spa_remap_huge __ro_after_init = true; static bool spa_remap_normal __ro_after_init; @@ -1544,13 +1576,16 @@ static bool sp_area_use_remap(struct sp_area *spa) static struct page *sp_area_alloc_hugepages(int nid, nodemask_t *nodemask) { struct page *page; + unsigned long pfn; - page = (struct page *)alloc_hugetlb_folio_nodemask_size(PMD_SIZE, nid, nodemask); - if (!page) + pfn = sp_alloc_hugepage_func(PMD_SIZE, nid, nodemask); + if (IS_ERR_VALUE(pfn)) { page = (struct page *)alloc_temporary_hugetlb_folio_nodemask(nid, nodemask, GFP_HIGHUSER_MOVABLE | __GFP_ACCOUNT | __GFP_COMP); + return page; + } - return page; + return pfn_to_page(pfn); } static bool sp_area_prepare_pages(struct sp_area *spa, int nid, nodemask_t *nodemask) @@ -2447,6 +2482,16 @@ static void *__mg_sp_alloc_nodemask(unsigned long size, unsigned long sp_flags, struct sp_alloc_context ac; struct sp_group_node *spg_node; + /** + * To avoid atomic risks, once share pool alloc memory, + * registering hugepage allocation hook is no longer supported. + */ + if (unlikely(allow_hugepage_allocator_register)) { + mutex_lock(&sp_hugepage_reg_mutex); + allow_hugepage_allocator_register = false; + mutex_unlock(&sp_hugepage_reg_mutex); + } + ret = sp_alloc_prepare(size, sp_flags, spg_id, &ac); if (ret) return ERR_PTR(ret); -- 2.43.0