From: David Hildenbrand david@redhat.com
mainline inclusion from mainline-6.8-rc1 commit e135826b2da0cf25305086dc9ac1e91718a148e1 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9CESE CVE: NA
-------------------------------------------------
hugetlb rmap handling differs quite a lot from "ordinary" rmap code. For example, hugetlb currently only supports entire mappings, and treats any mapping as mapped using a single "logical PTE". Let's move it out of the way so we can overhaul our "ordinary" rmap. implementation/interface.
Let's introduce and use hugetlb_remove_rmap() and remove the hugetlb code from page_remove_rmap(). This effectively removes one check on the small-folio path as well.
Add sanity checks that we end up with the right folios in the right functions.
Note: all possible candidates that need care are page_remove_rmap() that pass compound=true.
Link: https://lkml.kernel.org/r/20231220224504.646757-3-david@redhat.com Signed-off-by: David Hildenbrand david@redhat.com Reviewed-by: Yin Fengwei fengwei.yin@intel.com Reviewed-by: Ryan Roberts ryan.roberts@arm.com Reviewed-by: Matthew Wilcox (Oracle) willy@infradead.org Reviewed-by: Muchun Song songmuchun@bytedance.com Cc: Hugh Dickins hughd@google.com Cc: Muchun Song muchun.song@linux.dev Cc: Peter Xu peterx@redhat.com Signed-off-by: Andrew Morton akpm@linux-foundation.org (cherry picked from commit e135826b2da0cf25305086dc9ac1e91718a148e1) Signed-off-by: Kefeng Wang wangkefeng.wang@huawei.com --- include/linux/rmap.h | 7 +++++++ mm/hugetlb.c | 4 ++-- mm/rmap.c | 18 +++++++++--------- 3 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 456b8b82249e..556366dafa8f 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -208,6 +208,13 @@ void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *, void hugetlb_add_new_anon_rmap(struct folio *, struct vm_area_struct *, unsigned long address);
+static inline void hugetlb_remove_rmap(struct folio *folio) +{ + VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); + + atomic_dec(&folio->_entire_mapcount); +} + static inline void __page_dup_rmap(struct page *page, bool compound) { if (compound) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4a7b1260721c..c6e9b3f7d542 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5628,7 +5628,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, make_pte_marker(PTE_MARKER_UFFD_WP), sz); hugetlb_count_sub(pages_per_huge_page(h), mm); - page_remove_rmap(page, vma, true); + hugetlb_remove_rmap(page_folio(page));
spin_unlock(ptl); tlb_remove_page_size(tlb, page, huge_page_size(h)); @@ -5939,7 +5939,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
/* Break COW or unshare */ huge_ptep_clear_flush(vma, haddr, ptep); - page_remove_rmap(&old_folio->page, vma, true); + hugetlb_remove_rmap(old_folio); hugetlb_add_new_anon_rmap(new_folio, vma, haddr); if (huge_pte_uffd_wp(pte)) newpte = huge_pte_mkuffd_wp(newpte); diff --git a/mm/rmap.c b/mm/rmap.c index 2796c07dcde1..460c589b7720 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1450,15 +1450,9 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma, bool last; enum node_stat_item idx;
+ VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); VM_BUG_ON_PAGE(compound && !PageHead(page), page);
- /* Hugetlb pages are not counted in NR_*MAPPED */ - if (unlikely(folio_test_hugetlb(folio))) { - /* hugetlb pages are always mapped with pmds */ - atomic_dec(&folio->_entire_mapcount); - return; - } - /* Is page being unmapped by PTE? Is this its last map to be removed? */ if (likely(!compound)) { last = atomic_add_negative(-1, &page->_mapcount); @@ -1821,7 +1815,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_reliable_folio_counter(folio, mm, -1); } discard: - page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); + if (unlikely(folio_test_hugetlb(folio))) + hugetlb_remove_rmap(folio); + else + page_remove_rmap(subpage, vma, false); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put(folio); @@ -2176,7 +2173,10 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ }
- page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); + if (unlikely(folio_test_hugetlb(folio))) + hugetlb_remove_rmap(folio); + else + page_remove_rmap(subpage, vma, false); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put(folio);