From: Wang Wensheng wangwensheng4@huawei.com
ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA
-------------------
Introduce function hugetlb_alloc_hugepage that alloc hugepages from static hugepages first. When the static hugepage is used up, it attempts to apply for hugepages from buddy system. Two additional modes are supported: static hugepages only and buddy hugepages only.
When the driver gets huge pages by alloc_huge_page_node, it attempts to apply for migrate hugepages after the reserved memory hugepages are used up. We expect that the migrated hugepages that are applied for can be charged in memcg to limit the memory usage. So we enable charge migrate hugepages, and default enable it.
Add hugetlb_insert_hugepage_pte[_by_pa] to insert hugepages into page table. The by_pa version performs like remap_pfn_range() that make the pte special and can be used for reserved physical memory.
Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Signed-off-by: Zhang Jian zhangjian210@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/Kconfig | 12 ++++ include/linux/hugetlb.h | 62 ++++++++++++++++++ mm/hugetlb.c | 139 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 212 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3afabc81551c..85610dd3ac0c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2022,6 +2022,18 @@ config ASCEND_OOM 0: disable oom killer 1: enable oom killer (default,compatible with mainline)
+config ASCEND_CHARGE_MIGRATE_HUGEPAGES + bool "Enable support for migrate hugepages" + depends on HUGETLBFS + default y + help + When reserved hugepages are used up, we attempts to apply for migrate + hugepages. We expect that the migrated hugepages that are applied for + can be charged in memcg to limit the memory usage. + + This option enable the feature to charge migrate hugepages to memory + cgroup. + endif
endmenu diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index fd9635a6a92f..397e6bfa8268 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -607,6 +607,45 @@ struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, int huge_add_to_page_cache(struct page *page, struct address_space *mapping, pgoff_t idx);
+#ifdef CONFIG_ASCEND_FEATURES +#define HUGETLB_ALLOC_NONE 0x00 +#define HUGETLB_ALLOC_NORMAL 0x01 /* normal hugepage */ +#define HUGETLB_ALLOC_BUDDY 0x02 /* buddy hugepage */ +#define HUGETLB_ALLOC_MASK (HUGETLB_ALLOC_NONE | \ + HUGETLB_ALLOC_NORMAL | \ + HUGETLB_ALLOC_BUDDY) + +const struct hstate *hugetlb_get_hstate(void); +struct page *hugetlb_alloc_hugepage(int nid, int flag); +int hugetlb_insert_hugepage_pte(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, struct page *hpage); +int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr); +#else +static inline const struct hstate *hugetlb_get_hstate(void) +{ + return NULL; +} + +static inline struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + return NULL; +} + +static inline int hugetlb_insert_hugepage_pte(struct mm_struct *mm, + unsigned long addr, pgprot_t prot, struct page *hpage) +{ + return -EPERM; +} +static inline int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr) +{ + return -EPERM; +} +#endif + /* arch callback */ int __init __alloc_bootmem_huge_page(struct hstate *h); int __init alloc_bootmem_huge_page(struct hstate *h); @@ -1028,6 +1067,29 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr pte_t *ptep, pte_t pte, unsigned long sz) { } + +static inline const struct hstate *hugetlb_get_hstate(void) +{ + return NULL; +} + +static inline struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + return NULL; +} + +static inline int hugetlb_insert_hugepage_pte(struct mm_struct *mm, + unsigned long addr, pgprot_t prot, struct page *hpage) +{ + return -EPERM; +} + +static inline int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr) +{ + return -EPERM; +} #endif /* CONFIG_HUGETLB_PAGE */
#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d0672e482879..eaddb18d58e1 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -67,7 +67,6 @@ static struct hstate * __initdata parsed_hstate; static unsigned long __initdata default_hstate_max_huge_pages; static bool __initdata parsed_valid_hugepagesz = true; static bool __initdata parsed_default_hugepagesz; - /* * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, * free_huge_pages, and surplus_huge_pages. @@ -5907,3 +5906,141 @@ void __init hugetlb_cma_check(void) }
#endif /* CONFIG_CMA */ + +#ifdef CONFIG_ASCEND_FEATURES +static int enable_charge_mighp __read_mostly; + +const struct hstate *hugetlb_get_hstate(void) +{ + return &default_hstate; +} +EXPORT_SYMBOL_GPL(hugetlb_get_hstate); + +static struct page *hugetlb_alloc_hugepage_normal(struct hstate *h, + gfp_t gfp_mask, int nid) +{ + struct page *page = NULL; + + spin_lock(&hugetlb_lock); + if (h->free_huge_pages - h->resv_huge_pages > 0) + page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL, NULL); + spin_unlock(&hugetlb_lock); + + return page; +} + +/* + * Allocate hugepage without reserve + */ +struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + struct hstate *h = &default_hstate; + gfp_t gfp_mask = htlb_alloc_mask(h); + struct page *page = NULL; + + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + if (nid < 0 || nid >= MAX_NUMNODES) + return NULL; + + if (flag & ~HUGETLB_ALLOC_MASK) + return NULL; + + gfp_mask |= __GFP_THISNODE; + if (enable_charge_mighp) + gfp_mask |= __GFP_ACCOUNT; + + if (flag & HUGETLB_ALLOC_NORMAL) + page = hugetlb_alloc_hugepage_normal(h, gfp_mask, nid); + else if (flag & HUGETLB_ALLOC_BUDDY) + page = alloc_migrate_huge_page(h, gfp_mask, nid, NULL); + else + page = alloc_huge_page_nodemask(h, nid, NULL, gfp_mask); + + return page; +} +EXPORT_SYMBOL_GPL(hugetlb_alloc_hugepage); + +static int __hugetlb_insert_hugepage(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, unsigned long pfn, bool special) +{ + int ret = 0; + pte_t *ptep, entry; + struct hstate *h; + struct vm_area_struct *vma; + struct address_space *mapping; + spinlock_t *ptl; + + h = size_to_hstate(PMD_SIZE); + if (!h) + return -EINVAL; + + if (!IS_ALIGNED(addr, PMD_SIZE)) + return -EINVAL; + + vma = find_vma(mm, addr); + if (!vma || !range_in_vma(vma, addr, addr + PMD_SIZE)) + return -EINVAL; + + mapping = vma->vm_file->f_mapping; + i_mmap_lock_read(mapping); + ptep = huge_pte_alloc(mm, addr, huge_page_size(h)); + if (!ptep) { + ret = -ENXIO; + goto out_unlock; + } + + if (WARN_ON(ptep && !pte_none(*ptep) && !pmd_huge(*(pmd_t *)ptep))) { + ret = -ENXIO; + goto out_unlock; + } + + entry = pfn_pte(pfn, prot); + entry = huge_pte_mkdirty(entry); + if (!(pgprot_val(prot) & PTE_RDONLY)) + entry = huge_pte_mkwrite(entry); + entry = pte_mkyoung(entry); + entry = pte_mkhuge(entry); + if (special) + entry = pte_mkspecial(entry); + + ptl = huge_pte_lockptr(h, mm, ptep); + spin_lock(ptl); + set_huge_pte_at(mm, addr, ptep, entry); + spin_unlock(ptl); + +out_unlock: + i_mmap_unlock_read(mapping); + + return ret; +} + +int hugetlb_insert_hugepage_pte(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, struct page *hpage) +{ + return __hugetlb_insert_hugepage(mm, addr, prot, page_to_pfn(hpage), false); +} +EXPORT_SYMBOL_GPL(hugetlb_insert_hugepage_pte); + +int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, unsigned long phy_addr) +{ + return __hugetlb_insert_hugepage(mm, addr, prot, phy_addr >> PAGE_SHIFT, true); +} +EXPORT_SYMBOL_GPL(hugetlb_insert_hugepage_pte_by_pa); + +#ifdef CONFIG_ASCEND_CHARGE_MIGRATE_HUGEPAGES + +static int __init ascend_enable_charge_migrate_hugepages(char *s) +{ + enable_charge_mighp = 1; + + pr_info("Ascend enable charge migrate hugepage\n"); + + return 1; +} +__setup("enable_charge_mighp", ascend_enable_charge_migrate_hugepages); + +#endif +#endif