hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IDAIXT ------------------------------------------------- Commit 2896144b31df ("mempolicy: alloc_pages_mpol() for NUMA policy without vma") inadvertently extended the "smart grid scheduling policy" to all memory allocation paths that call alloc_pages_mpol(). This policy was originally designed to optimize page placement for user-space memory regions with associated VMAs. Applying it generically to kernel-internal paths, file caches, and others can lead to suboptimal NUMA node selection or unexpected results. To address this, we introduce the __alloc_pages_mpol() function and add a boolean parameter use_smart_grid. This flag is set to true only when the allocation request originates from vma_alloc_folio(). For the original alloc_pages_mpol, it is set to false to preserve the original allocation logic. Fixes: 2896144b31df ("mempolicy: alloc_pages_mpol() for NUMA policy without vma") Signed-off-by: Ze Zuo <zuoze1@huawei.com> --- include/linux/gfp.h | 13 +++++++++---- mm/mempolicy.c | 32 +++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 558c3f6bee08..3e6e2160a8a0 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -296,8 +296,8 @@ static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask, #ifdef CONFIG_NUMA struct page *alloc_pages(gfp_t gfp, unsigned int order); -struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, - struct mempolicy *mpol, pgoff_t ilx, int nid); +struct page *__alloc_pages_mpol(gfp_t gfp, unsigned int order, + struct mempolicy *mpol, pgoff_t ilx, int nid, bool use_smart_grid); struct folio *folio_alloc(gfp_t gfp, unsigned int order); struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma, unsigned long addr, bool hugepage); @@ -306,8 +306,8 @@ static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order) { return alloc_pages_node(numa_node_id(), gfp_mask, order); } -static inline struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, - struct mempolicy *mpol, pgoff_t ilx, int nid) +static inline struct page *__alloc_pages_mpol(gfp_t gfp, unsigned int order, + struct mempolicy *mpol, pgoff_t ilx, int nid, bool use_smart_grid) { return alloc_pages(gfp, order); } @@ -318,6 +318,11 @@ static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order) #define vma_alloc_folio(gfp, order, vma, addr, hugepage) \ folio_alloc(gfp, order) #endif +static inline struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, + struct mempolicy *mpol, pgoff_t ilx, int nid) +{ + return __alloc_pages_mpol(gfp, order, mpol, ilx, nid, false); +} #define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0) static inline struct page *alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 324e354ac077..2ae02bf860bf 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2267,13 +2267,9 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol, WARN_ON_ONCE(gfp & __GFP_THISNODE); break; case MPOL_INTERLEAVE: - *nid = NUMA_NO_NODE; - if (smart_grid_used()) - *nid = sched_grid_preferred_interleave_nid(pol); /* Override input node id */ - if (*nid == NUMA_NO_NODE) - *nid = (ilx == NO_INTERLEAVE_INDEX) ? - interleave_nodes(pol) : interleave_nid(pol, ilx); + *nid = (ilx == NO_INTERLEAVE_INDEX) ? + interleave_nodes(pol) : interleave_nid(pol, ilx); break; case MPOL_WEIGHTED_INTERLEAVE: *nid = (ilx == NO_INTERLEAVE_INDEX) ? @@ -2408,17 +2404,18 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, } /** - * alloc_pages_mpol - Allocate pages according to NUMA mempolicy. + * __alloc_pages_mpol - Allocate pages according to NUMA mempolicy. * @gfp: GFP flags. * @order: Order of the page allocation. * @pol: Pointer to the NUMA mempolicy. * @ilx: Index for interleave mempolicy (also distinguishes alloc_pages()). * @nid: Preferred node (usually numa_node_id() but @mpol may override it). + * @use_smart_grid: Control whether to apply the smart grid scheduling policy. * * Return: The page on success or NULL if allocation fails. */ -struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, - struct mempolicy *pol, pgoff_t ilx, int nid) +struct page *__alloc_pages_mpol(gfp_t gfp, unsigned int order, + struct mempolicy *pol, pgoff_t ilx, int nid, bool use_smart_grid) { nodemask_t *nodemask; struct page *page; @@ -2461,6 +2458,19 @@ struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } + if (use_smart_grid && smart_grid_used()) { + if ((pol->mode == MPOL_INTERLEAVE) || + (pol->mode == MPOL_WEIGHTED_INTERLEAVE)) { + int preferred_nid; + + preferred_nid = sched_grid_preferred_interleave_nid(pol); + if (preferred_nid != NUMA_NO_NODE) + nid = preferred_nid; + } else { + nid = sched_grid_preferred_nid(nid, nodemask); + } + } + page = __alloc_pages(gfp, order, nid, nodemask); if (unlikely(pol->mode == MPOL_INTERLEAVE || @@ -2501,8 +2511,8 @@ struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma, struct page *page; pol = get_vma_policy(vma, addr, order, &ilx); - page = alloc_pages_mpol(gfp | __GFP_COMP, order, - pol, ilx, numa_node_id()); + page = __alloc_pages_mpol(gfp | __GFP_COMP, order, + pol, ilx, numa_node_id(), true); mpol_cond_put(pol); return page_rmappable_folio(page); } -- 2.25.1