From: Weilong Chen chenweilong@huawei.com
ascend inclusion category: feature bugzilla: NA CVE: NA
-------------------------------------------------
Sharepool applies for a dedicated interface for large pages, which optimizes the efficiency of memory application
Reviewed-by: Ding Tianhong dingtianhong@huawei.com Signed-off-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- include/linux/hugetlb.h | 4 ++ include/linux/share_pool.h | 5 +++ mm/hugetlb.c | 22 +++++++--- mm/share_pool.c | 90 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 6 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index bd658f44e133..be649e5ba322 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -645,4 +645,8 @@ static inline spinlock_t *huge_pte_lock(struct hstate *h, return ptl; }
+#ifdef CONFIG_ASCEND_SHARE_POOL +pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, int writable); +#endif + #endif /* _LINUX_HUGETLB_H */ diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 2557ef138122..47d8579c23ec 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -220,6 +220,11 @@ static inline void sp_dump_stack(void) dump_stack(); }
+vm_fault_t sharepool_no_page(struct mm_struct *mm, + struct vm_area_struct *vma, + struct address_space *mapping, pgoff_t idx, + unsigned long address, pte_t *ptep, unsigned int flags); + #else
static inline int sp_group_add_task(int pid, int spg_id) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 9d2035632aed..a68e7e1afd82 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3352,8 +3352,13 @@ const struct vm_operations_struct hugetlb_vm_ops = { .pagesize = hugetlb_vm_op_pagesize, };
+#ifdef CONFIG_ASCEND_SHARE_POOL +pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, + int writable) +#else static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, int writable) +#endif { pte_t entry;
@@ -3370,6 +3375,9 @@ static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
return entry; } +#ifdef CONFIG_ASCEND_SHARE_POOL +EXPORT_SYMBOL(make_huge_pte); +#endif
static void set_huge_ptep_writable(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) @@ -3962,12 +3970,6 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, }
page = alloc_huge_page(vma, haddr, 0); - if (IS_ERR(page) && sp_check_vm_share_pool(vma->vm_flags)) { - page = alloc_huge_page_node(hstate_file(vma->vm_file), - numa_mem_id()); - if (!page) - page = ERR_PTR(-ENOMEM); - } if (IS_ERR(page)) { /* * Returning error will result in faulting task being @@ -4155,7 +4157,15 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
entry = huge_ptep_get(ptep); if (huge_pte_none(entry)) { +#ifdef CONFIG_ASCEND_SHARE_POOL + if (sp_check_vm_share_pool(vma->vm_flags)) { + ret = sharepool_no_page(mm, vma, mapping, idx, address, ptep, flags); + } else { + ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); + } +#else ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); +#endif goto out_mutex; }
diff --git a/mm/share_pool.c b/mm/share_pool.c index 23195a76c2c6..286e74f99360 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -41,6 +41,8 @@ #include <linux/idr.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> +#include <linux/rmap.h> +#include <linux/hugetlb.h>
/* access control mode macros */ #define AC_NONE 0 @@ -2399,6 +2401,94 @@ static int spa_stat_show(struct seq_file *seq, void *offset) return 0; }
+vm_fault_t sharepool_no_page(struct mm_struct *mm, + struct vm_area_struct *vma, + struct address_space *mapping, pgoff_t idx, + unsigned long address, pte_t *ptep, unsigned int flags) +{ + struct hstate *h = hstate_vma(vma); + vm_fault_t ret = VM_FAULT_SIGBUS; + unsigned long size; + struct page *page; + pte_t new_pte; + spinlock_t *ptl; + unsigned long haddr = address & huge_page_mask(h); + bool new_page = false; + int err; + +retry: + page = find_lock_page(mapping, idx); + if (!page) { + size = i_size_read(mapping->host) >> huge_page_shift(h); + if (idx >= size) + goto out; + + page = alloc_huge_page(vma, haddr, 0); + if (IS_ERR(page)) { + page = alloc_huge_page_node(hstate_file(vma->vm_file), + numa_mem_id()); + if (!page) + page = ERR_PTR(-ENOMEM); + } + if (IS_ERR(page)) { + ptl = huge_pte_lock(h, mm, ptep); + if (!huge_pte_none(huge_ptep_get(ptep))) { + ret = 0; + spin_unlock(ptl); + goto out; + } + spin_unlock(ptl); + ret = vmf_error(PTR_ERR(page)); + goto out; + } + __SetPageUptodate(page); + new_page = true; + + /* sharepool pages are all shared */ + err = huge_add_to_page_cache(page, mapping, idx); + if (err) { + put_page(page); + if (err == -EEXIST) + goto retry; + goto out; + } + } + + + ptl = huge_pte_lock(h, mm, ptep); + size = i_size_read(mapping->host) >> huge_page_shift(h); + if (idx >= size) + goto backout; + + ret = 0; + if (!huge_pte_none(huge_ptep_get(ptep))) + goto backout; + + page_dup_rmap(page, true); + new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) + && (vma->vm_flags & VM_SHARED))); + set_huge_pte_at(mm, haddr, ptep, new_pte); + + hugetlb_count_add(pages_per_huge_page(h), mm); + + spin_unlock(ptl); + + if (new_page) { + SetPagePrivate(&page[1]); + } + + unlock_page(page); +out: + return ret; + +backout: + spin_unlock(ptl); + unlock_page(page); + put_page(page); + goto out; +} +EXPORT_SYMBOL(sharepool_no_page); + /* * Called by proc_root_init() to initialize the /proc/sharepool subtree */