From: Ze Zuo <zuoze1@huawei.com> Introduce the function alloc_misplaced_dst_page_thp to support THP NUMA page migration in migrate_misplaced_page(). Signed-off-by: Ze Zuo <zuoze1@huawei.com> --- mm/migrate.c | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index c8491a744e8c..d55d0d1a6b89 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2082,6 +2082,23 @@ static struct page *alloc_misplaced_dst_page(struct page *page, return newpage; } +static struct page *alloc_misplaced_dst_page_thp(struct page *page, + unsigned long data) +{ + int nid = (int) data; + struct page *newpage; + + newpage = alloc_pages_node(nid, (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE), + HPAGE_PMD_ORDER); + if (!newpage) + goto out; + + prep_transhuge_page(newpage); + +out: + return newpage; +} + static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) { int page_lru; @@ -2141,6 +2158,20 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, int isolated; int nr_remaining; LIST_HEAD(migratepages); + new_page_t *new; + bool compound; + + /* + * PTE mapped THP or HugeTLB page can't reach here so the page could + * be either base page or THP. And it must be head page if it is + * THP. + */ + compound = PageTransHuge(page); + + if (compound) + new = alloc_misplaced_dst_page_thp; + else + new = alloc_misplaced_dst_page; /* * Don't migrate file pages that are mapped in multiple processes @@ -2162,9 +2193,8 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, goto out; list_add(&page->lru, &migratepages); - nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page, - NULL, node, MIGRATE_ASYNC, - MR_NUMA_MISPLACED); + nr_remaining = migrate_pages(&migratepages, *new, NULL, node, + MIGRATE_ASYNC, MR_NUMA_MISPLACED); if (nr_remaining) { if (!list_empty(&migratepages)) { list_del(&page->lru); -- 2.25.1

From: Ze Zuo <zuoze1@huawei.com> The NUMA fault behavior didn't split THP if migration is failed due to lack of memory on the target node. But the THP migration does split THP, so keep the old behavior for misplaced NUMA page migration. Signed-off-by: Ze Zuo <zuoze1@huawei.com> --- mm/migrate.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index a3f9cbbfbfe5..4c2a1d99db53 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1487,6 +1487,7 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, struct page *page2; int swapwrite = current->flags & PF_SWAPWRITE; int rc, nr_subpages; + bool nosplit = (reason == MR_NUMA_MISPLACED); if (!swapwrite) current->flags |= PF_SWAPWRITE; @@ -1527,8 +1528,9 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, * pages are added to the tail of the list so * we encounter them after the rest of the list * is processed. + * THP NUMA acces migrate doesn't split THP to retry. */ - if (is_thp) { + if (is_thp && !nosplit) { lock_page(page); rc = split_huge_page_to_list(page, from); unlock_page(page); -- 2.25.1

From: Ze Zuo <zuoze1@huawei.com> In the NUMA balancing scenario, support for PMD-level page migration, similar to the "do_huge_pmd_numa_page" function in the auto NUMA scheme. Signed-off-by: Ze Zuo <zuoze1@huawei.com> --- mm/mem_sampling.c | 66 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/mm/mem_sampling.c b/mm/mem_sampling.c index 0eaea2680d83..23f3eb28c5e6 100644 --- a/mm/mem_sampling.c +++ b/mm/mem_sampling.c @@ -144,6 +144,41 @@ static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, return mpol_misplaced(page, vma, addr); } +/* NUMA hinting page access point for trans huge pmds */ +void do_huge_pmd_numa_access(struct vm_area_struct *vma, u64 vaddr, struct page *page) +{ + int page_nid = NUMA_NO_NODE; + int target_nid, last_cpupid = (-1 & LAST_CPUPID_MASK); + bool migrated = false; + int flags = 0; + u64 haddr = vaddr & HPAGE_PMD_MASK; + + page = compound_head(page); + page_nid = page_to_nid(page); + last_cpupid = page_cpupid_last(page); + target_nid = numa_migrate_prep(page, vma, haddr, page_nid, + &flags); + + if (target_nid == NUMA_NO_NODE) { + put_page(page); + goto out; + } + + migrated = migrate_misplaced_page(page, vma, target_nid); + if (migrated) { + flags |= TNF_MIGRATED; + page_nid = target_nid; + } else { + flags |= TNF_MIGRATE_FAIL; + } + +out: + trace_mm_numa_migrating(haddr, page_nid, target_nid, flags&TNF_MIGRATED); + if (page_nid != NUMA_NO_NODE) + task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, + flags); +} + /* * Called from task_work context to act upon the page access. * @@ -161,6 +196,11 @@ static void do_numa_access(struct task_struct *p, u64 vaddr, u64 paddr) int last_cpupid; int target_nid; int flags = 0; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; + pmd_t pmde; if (!mm) return; @@ -190,9 +230,31 @@ static void do_numa_access(struct task_struct *p, u64 vaddr, u64 paddr) if (unlikely(!PageLRU(page))) goto out_unlock; - /* TODO: handle PTE-mapped THP or PMD-mapped THP*/ - if (PageCompound(page)) + if (PageCompound(page)) { + pgd = pgd_offset(mm, vaddr); + if (!pgd_present(*pgd)) + goto out_unlock; + + p4d = p4d_offset(pgd, vaddr); + if (!p4d_present(*p4d)) + goto out_unlock; + + pud = pud_offset(p4d, vaddr); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, vaddr); + if (!pmd) + goto out_unlock; + pmde = *pmd; + + barrier(); + if (pmd_trans_huge(pmde) || pmd_devmap(pmde)) + /* handle PMD-mapped THP */ + do_huge_pmd_numa_access(vma, vaddr, page); + /* TODO: handle PTE-mapped THP */ goto out_unlock; + } /* * Flag if the page is shared between multiple address spaces. This -- 2.25.1