From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Dynamic hugetlb is a self-developed feature based on the hugetlb and memcontrol. It supports to split huge page dynamically in a memory cgroup. There is a new structure dhugetlb_pool in every mem_cgroup to manage the pages configured to the mem_cgroup. For the mem_cgroup configured with dhugetlb_pool, processes in the mem_cgroup will preferentially use the pages in dhugetlb_pool.

Dynamic hugetlb supports three types of pages, including 1G/2M huge pages and 4K pages. For the mem_cgroup configured with dhugetlb_pool, processes will be limited to alloc 1G/2M huge pages only from dhugetlb_pool. But there is no such constraint for 4K pages. If there are insufficient 4K pages in the dhugetlb_pool, pages can also be allocated from buddy system. So before using dynamic hugetlb, user must know how many huge pages they need.

Usage: 1. Add 'dynamic_hugetlb=on' in cmdline to enable dynamic hugetlb feature. 2. Prealloc some 1G hugepages through hugetlb. 3. Create a mem_cgroup and configure dhugetlb_pool to mem_cgroup. 4. Configure the count of 1G/2M hugepages, and the remaining pages in dhugetlb_pool will be used as basic pages. 5. Bound a process to mem_cgroup. then the memory for it will be allocated from dhugetlb_pool.

This patch add the corresponding structure dhugetlb_pool for dynamic hugetlb feature, the interface 'dhugetlb.nr_pages' in mem_cgroup to configure dhugetlb_pool and the cmdline 'dynamic_hugetlb=on' to enable dynamic hugetlb feature.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- fs/Kconfig | 10 + include/linux/dynamic_hugetlb.h | 106 +++++++++ include/linux/memcontrol.h | 5 +- kernel/cgroup/cgroup.c | 11 + mm/Makefile | 1 + mm/dynamic_hugetlb.c | 375 ++++++++++++++++++++++++++++++++ mm/hugetlb.c | 3 + mm/memcontrol.c | 10 + 8 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 include/linux/dynamic_hugetlb.h create mode 100644 mm/dynamic_hugetlb.c

diff --git a/fs/Kconfig b/fs/Kconfig index 3cc647e00f3c..20bd86b65dcc 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -251,6 +251,16 @@ config HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON to enable freeing vmemmap pages of HugeTLB by default. It can then be disabled on the command line via hugetlb_free_vmemmap=off.

+config DYNAMIC_HUGETLB + bool "Dynamic HugeTLB" + depends on X86_64 + depends on HUGETLBFS + depends on MEMCG && CGROUP_HUGETLB + help + Dynamic hugepage are used in memcg and can be splited into small + pages automatically. The tasks in the memcg prefer to alloc dynamic + hugepage. + config MEMFD_CREATE def_bool TMPFS || HUGETLBFS

diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h new file mode 100644 index 000000000000..30ccbd9f1853 --- /dev/null +++ b/include/linux/dynamic_hugetlb.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __LINUX_DYNAMIC_HUGETLB_H +#define __LINUX_DYNAMIC_HUGETLB_H + +#include <linux/hugetlb.h> +#include <linux/memcontrol.h> + +#ifdef CONFIG_DYNAMIC_HUGETLB + +extern struct static_key_false dhugetlb_enabled_key; +#define dhugetlb_enabled (static_branch_unlikely(&dhugetlb_enabled_key)) + +#define NR_PERCPU_POOL num_possible_cpus() +#define PERCPU_POOL_PAGE_MAX 1024 +#define PERCPU_POOL_PAGE_BATCH (PERCPU_POOL_PAGE_MAX >> 2) + +struct split_hugepage { + struct list_head head_pages; + unsigned long start_pfn; +}; + +struct percpu_pages_pool { + spinlock_t lock; + unsigned long free_pages; + long used_pages; + struct list_head head_page; +}; + +struct huge_pages_pool { + /* + * This four counts is used for huge page allocation. + */ + unsigned long nr_huge_pages; + unsigned long free_huge_pages; + unsigned long resv_huge_pages; + unsigned long used_huge_pages; + /* + * free_normal_pages means how many huge pages can be split to + * smaller pages or reserved for huge page allocation. + */ + unsigned long free_normal_pages; + /* + * split_normal_pages means how many huge pages have already been + * split. + */ + unsigned long split_normal_pages; + struct list_head hugepage_freelists; + /* Used to record which hugepages have been split */ + struct list_head hugepage_splitlists; +}; + +enum huge_pages_pool_type { + HUGE_PAGES_POOL_1G, + HUGE_PAGES_POOL_2M, + HUGE_PAGES_POOL_4K, + HUGE_PAGES_POOL_MAX, +}; +/* + * Dynamic hugetlb pool data structure. Each Dynamic hugetlb pool is + * associated with one memory cgroup and controls the allocation of memory + * resources for both processes and files which belongs to the memory cgroup. + */ +struct dhugetlb_pool { + int nid; + spinlock_t lock; + spinlock_t reserved_lock; + atomic_t refcnt; + unsigned long normal_pages_disabled; + + struct mem_cgroup *attach_memcg; + + unsigned long total_huge_pages; + struct huge_pages_pool hpages_pool[HUGE_PAGES_POOL_MAX]; + struct percpu_pages_pool percpu_pool[0]; +}; + +bool dhugetlb_hide_files(struct cftype *cft); +ssize_t write_hugepage_to_hpool(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); +int hugetlb_pool_info_show(struct seq_file *m, void *v); +void hugetlb_pool_inherit(struct mem_cgroup *memcg, struct mem_cgroup *parent); +int hugetlb_pool_destroy(struct cgroup *cgrp); +void __init dynamic_hugetlb_init(void); + +#else + +#define dhugetlb_enabled 0 + +struct dhugetlb_pool {}; + +static inline bool dhugetlb_hide_files(struct cftype *cft) +{ + return false; +} +static inline void hugetlb_pool_inherit(struct mem_cgroup *memcg, struct mem_cgroup *parent) +{ +} +static inline int hugetlb_pool_destroy(struct cgroup *cgrp) +{ + return 0; +} +static inline void __init dynamic_hugetlb_init(void) +{ +} +#endif /* CONFIG_DYNAMIC_HUGETLB */ +#endif /* __LINUX_DYNAMIC_HUGETLB_H */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 2e0a480a8665..7cc7cfe55d9a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -22,6 +22,7 @@ #include <linux/writeback.h> #include <linux/page-flags.h> #include <linux/kabi.h> +#include <linux/dynamic_hugetlb.h>

struct mem_cgroup; struct obj_cgroup; @@ -370,6 +371,9 @@ struct mem_cgroup { struct deferred_split deferred_split_queue; #endif

+#ifdef CONFIG_DYNAMIC_HUGETLB + struct dhugetlb_pool *hpool; +#endif KABI_RESERVE(1) KABI_RESERVE(2) KABI_RESERVE(3) @@ -1238,7 +1242,6 @@ void split_page_memcg(struct page *head, unsigned int nr); unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, gfp_t gfp_mask, unsigned long *total_scanned); - /* * Test whether @memcg has children, dead or alive. Note that this * function doesn't care whether @memcg has use_hierarchy enabled and diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 8ef51ddfb301..59cc82ef52a6 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -57,6 +57,7 @@ #include <linux/fs_parser.h> #include <linux/sched/cputime.h> #include <linux/psi.h> +#include <linux/dynamic_hugetlb.h> #include <net/sock.h>

#define CREATE_TRACE_POINTS @@ -4009,6 +4010,9 @@ static int cgroup_addrm_files(struct cgroup_subsys_state *css, continue; if ((cft->flags & CFTYPE_DEBUG) && !cgroup_debug) continue; + /* if dynamic hugetlb is not enabled, hide the interfaces */ + if (dhugetlb_hide_files(cft)) + continue; if (is_add) { ret = cgroup_add_file(css, cgrp, cft); if (ret) { @@ -5609,6 +5613,13 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) if (css_has_online_children(&cgrp->self)) return -EBUSY;

+ /* + * If dynamic hugetlb is enabled, make sure dhugtlb_pool is free + * before removing the corresponding memory cgroup. + */ + if (hugetlb_pool_destroy(cgrp)) + return -EBUSY; + /* * Mark @cgrp and the associated csets dead. The former prevents * further task migration and child creation by disabling diff --git a/mm/Makefile b/mm/Makefile index ec3d0ab14a6a..f3dce99ee62f 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -72,6 +72,7 @@ obj-$(CONFIG_ZSWAP) += zswap.o obj-$(CONFIG_HAS_DMA) += dmapool.o obj-$(CONFIG_HUGETLBFS) += hugetlb.o obj-$(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP) += hugetlb_vmemmap.o +obj-$(CONFIG_DYNAMIC_HUGETLB) += dynamic_hugetlb.o obj-$(CONFIG_NUMA) += mempolicy.o obj-$(CONFIG_SPARSEMEM) += sparse.o obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c new file mode 100644 index 000000000000..8881e9e1a032 --- /dev/null +++ b/mm/dynamic_hugetlb.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dynamic hugetlb core file + */ + +#include <linux/dynamic_hugetlb.h> + +static bool enable_dhugetlb = false; +DEFINE_STATIC_KEY_FALSE(dhugetlb_enabled_key); + +#define hugepage_index(pfn) ((pfn) >> (PUD_SHIFT - PAGE_SHIFT)) + +static bool get_hpool_unless_zero(struct dhugetlb_pool *hpool) +{ + if (!dhugetlb_enabled || !hpool) + return false; + return atomic_inc_not_zero(&hpool->refcnt); +} + +static void put_hpool(struct dhugetlb_pool *hpool) +{ + if (!dhugetlb_enabled || !hpool) + return; + if (atomic_dec_and_test(&hpool->refcnt)) { + css_put(&hpool->attach_memcg->css); + kfree(hpool); + } +} + +struct dhugetlb_pagelist { + unsigned long count; + struct dhugetlb_pool *hpool[0]; +}; + +static struct dhugetlb_pagelist *dhugetlb_pagelist_t; +static DEFINE_RWLOCK(dhugetlb_pagelist_rwlock); + +static int set_hpool_in_dhugetlb_pagelist(unsigned long idx, struct dhugetlb_pool *hpool) +{ + /* + * There is not conflit when write to dhugetlb_pagelist_t->hpool, so just + * need read_lock here. + */ + read_lock(&dhugetlb_pagelist_rwlock); + + /* + * If page's pfn is greater than dhugetlb_pagelist_t->count (which may + * occurs due to memory hotplug) then dhugetlb_pagelist_t need to be + * reallocated, so need write_lock here. + */ + if (idx >= dhugetlb_pagelist_t->count) { + unsigned long size; + struct dhugetlb_pagelist *tmp; + + read_unlock(&dhugetlb_pagelist_rwlock); + write_lock(&dhugetlb_pagelist_rwlock); + + size = sizeof(struct dhugetlb_pagelist) + + (idx + 1) * sizeof(struct dhugetlb_pool *); + tmp = krealloc(dhugetlb_pagelist_t, size, GFP_ATOMIC); + if (!tmp) { + write_unlock(&dhugetlb_pagelist_rwlock); + return -ENOMEM; + } + tmp->count = idx + 1; + dhugetlb_pagelist_t = tmp; + + write_unlock(&dhugetlb_pagelist_rwlock); + read_lock(&dhugetlb_pagelist_rwlock); + } + dhugetlb_pagelist_t->hpool[idx] = hpool; + read_unlock(&dhugetlb_pagelist_rwlock); + + return 0; +} + +static int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, + unsigned long nid, unsigned long nr_pages) +{ + struct hstate *h = size_to_hstate(PUD_SIZE); + struct huge_pages_pool *hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_1G]; + struct page *page, *next; + unsigned long count = 0, idx; + int ret = 0; + + if (!h) + return -ENOMEM; + + spin_lock(&hpool->lock); + spin_lock(&hugetlb_lock); + if (h->free_huge_pages_node[nid] - h->resv_huge_pages_node[nid] < nr_pages) { + ret = -ENOMEM; + goto out_unlock; + } + + list_for_each_entry_safe(page, next, &h->hugepage_freelists[nid], lru) { + idx = hugepage_index(page_to_pfn(page)); + ret = set_hpool_in_dhugetlb_pagelist(idx, hpool); + if (ret) + continue; + + list_move_tail(&page->lru, &hpages_pool->hugepage_freelists); + h->free_huge_pages--; + h->free_huge_pages_node[nid]--; + hpool->total_huge_pages++; + hpages_pool->free_normal_pages++; + + if (++count == nr_pages) + break; + } + +out_unlock: + spin_unlock(&hugetlb_lock); + spin_unlock(&hpool->lock); + return ret; +} + +static int free_hugepage_to_hugetlb(struct dhugetlb_pool *hpool) +{ + struct hstate *h = size_to_hstate(PUD_SIZE); + struct huge_pages_pool *hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_1G]; + struct page *page, *next, *p; + unsigned long pfn, idx; + unsigned int nr_pages; + int nid, ret = 0; + + spin_lock(&hpool->lock); + spin_lock(&hugetlb_lock); + list_for_each_entry_safe(page, next, &hpages_pool->hugepage_freelists, lru) { + nr_pages = 1 << huge_page_order(h); + pfn = page_to_pfn(page); + for (; nr_pages--; pfn++) { + p = pfn_to_page(pfn); + p->mapping = NULL; + } + set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); + + nid = page_to_nid(page); + list_move(&page->lru, &h->hugepage_freelists[nid]); + hpool->total_huge_pages--; + hpages_pool->free_normal_pages--; + h->free_huge_pages++; + h->free_huge_pages_node[nid]++; + + idx = hugepage_index(page_to_pfn(page)); + ret = set_hpool_in_dhugetlb_pagelist(idx, NULL); + if (ret) + break; + } + spin_unlock(&hugetlb_lock); + spin_unlock(&hpool->lock); + return ret; +} + +void hugetlb_pool_inherit(struct mem_cgroup *memcg, struct mem_cgroup *parent) +{ + if (!dhugetlb_enabled || !memcg || !parent) + return; + memcg->hpool = parent->hpool; +} + +static int hugetlb_pool_create(struct mem_cgroup *memcg, unsigned long nid) +{ + struct dhugetlb_pool *hpool; + int i; + + if (memcg_has_children(memcg)) + return -EINVAL; + + hpool = kzalloc(sizeof(struct dhugetlb_pool) + + NR_PERCPU_POOL * sizeof(struct percpu_pages_pool), GFP_KERNEL); + if (!hpool) + return -ENOMEM; + + spin_lock_init(&hpool->lock); + spin_lock_init(&hpool->reserved_lock); + hpool->nid = nid; + atomic_set(&hpool->refcnt, 1); + + for (i = 0; i < HUGE_PAGES_POOL_MAX; i++) { + INIT_LIST_HEAD(&hpool->hpages_pool[i].hugepage_freelists); + INIT_LIST_HEAD(&hpool->hpages_pool[i].hugepage_splitlists); + } + for (i = 0; i < NR_PERCPU_POOL; i++) { + spin_lock_init(&hpool->percpu_pool[i].lock); + INIT_LIST_HEAD(&hpool->percpu_pool[i].head_page); + } + + hpool->attach_memcg = memcg; + css_get(&memcg->css); + memcg->hpool = hpool; + + return 0; +} + +int hugetlb_pool_destroy(struct cgroup *cgrp) +{ + struct cgroup_subsys_state *css = cgrp->subsys[memory_cgrp_id]; + struct mem_cgroup *memcg = mem_cgroup_from_css(css); + struct dhugetlb_pool *hpool = memcg ? memcg->hpool : NULL; + int ret = 0; + + if (!dhugetlb_enabled) + return 0; + + if (!hpool || hpool->attach_memcg != memcg) + return 0; + + ret = free_hugepage_to_hugetlb(hpool); + memcg->hpool = NULL; + + put_hpool(hpool); + return ret; +} + +static int hugetlb_pool_update(struct mem_cgroup *memcg, + unsigned long nid, unsigned long size) +{ + struct dhugetlb_pool *hpool; + bool new_create = false; + int ret = -EINVAL; + +again: + hpool = memcg->hpool; + if (!hpool) { + ret = hugetlb_pool_create(memcg, nid); + if (ret) + return ret; + new_create = true; + goto again; + } + if (!get_hpool_unless_zero(hpool)) + return -EINVAL; + + if (hpool->attach_memcg != memcg || hpool->nid != nid) + goto out; + ret = alloc_hugepage_from_hugetlb(hpool, nid, size); + /* + * if create a new hpool here but alloc hugepages failed, + * destroy it directly here. + */ + if (ret && new_create) { + memcg->hpool = NULL; + put_hpool(hpool); + } +out: + put_hpool(hpool); + return ret; +} + +bool dhugetlb_hide_files(struct cftype *cft) +{ + if (!dhugetlb_enabled && strstr(cft->name, "dhugetlb")) + return true; + return false; +} + +ssize_t write_hugepage_to_hpool(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + unsigned long nid, size; + char *endp; + int ret; + + if (!dhugetlb_enabled || !memcg) + return -EINVAL; + + buf = strstrip(buf); + nid = memparse(buf, &endp); + if (*endp != ' ' || nid < 0 || nid >= MAX_NUMNODES) + return -EINVAL; + + buf = endp + 1; + size = memparse(buf, &endp); + if (*endp != '\0' || size == 0) + return -EINVAL; + + ret = hugetlb_pool_update(memcg, nid, size); + + return ret ? : nbytes; +} + +int hugetlb_pool_info_show(struct seq_file *m, void *v) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); + struct dhugetlb_pool *hpool = memcg ? memcg->hpool : NULL; + unsigned long free_pages; + long used_pages = 0; + int i; + + if (!dhugetlb_enabled) + return 0; + + if (!hpool) { + seq_printf(m, "Curent hierarchial have not memory pool.\n"); + return 0; + } + + if (!get_hpool_unless_zero(hpool)) + return 0; + + for (i = 0; i < NR_PERCPU_POOL; i++) + spin_lock(&hpool->percpu_pool[i].lock); + spin_lock(&hpool->lock); + + free_pages = hpool->hpages_pool[HUGE_PAGES_POOL_4K].free_normal_pages; + for (i = 0; i < NR_PERCPU_POOL; i++) { + free_pages += hpool->percpu_pool[i].free_pages; + used_pages += hpool->percpu_pool[i].used_pages; + } + + seq_printf(m, + "dhugetlb_total_pages %ld\n" + "1G_total_reserved_pages %ld\n" + "1G_free_reserved_pages %ld\n" + "1G_mmap_reserved_pages %ld\n" + "1G_used_pages %ld\n" + "2M_total_reserved_pages %ld\n" + "2M_free_reserved_pages %ld\n" + "2M_mmap_reserved_pages %ld\n" + "2M_used_pages %ld\n" + "1G_free_unreserved_pages %ld\n" + "2M_free_unreserved_pages %ld\n" + "4K_free_pages %ld\n" + "4K_used_pages %ld\n", + hpool->total_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_1G].nr_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_1G].free_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_1G].resv_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_1G].used_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_2M].nr_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_2M].free_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_2M].resv_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_2M].used_huge_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_1G].free_normal_pages, + hpool->hpages_pool[HUGE_PAGES_POOL_2M].free_normal_pages, + free_pages, + used_pages); + + spin_unlock(&hpool->lock); + for (i = NR_PERCPU_POOL - 1; i >= 0; i--) + spin_unlock(&hpool->percpu_pool[i].lock); + put_hpool(hpool); + return 0; +} + +#define DEFAULT_PAGELIST_COUNT 4096 +void __init dynamic_hugetlb_init(void) +{ + unsigned long count, size; + + if (!enable_dhugetlb) + return; + + count = max(hugepage_index(max_pfn), (unsigned long)DEFAULT_PAGELIST_COUNT); + size = sizeof(struct dhugetlb_pagelist) + count * sizeof(struct dhugetlb_pool *); + dhugetlb_pagelist_t = kzalloc(size, GFP_KERNEL); + if (!dhugetlb_pagelist_t) { + pr_info("Dynamic hugetlb init failed, need %lu memory\n", size); + return; + } + + dhugetlb_pagelist_t->count = count; + static_branch_enable(&dhugetlb_enabled_key); + pr_info("Dynamic hugetlb is enabled\n"); +} + +static int __init dynamic_hugetlb_setup(char *s) +{ + if (!strcmp(s, "on")) + enable_dhugetlb = true; + return 1; +} +__setup("dynamic_hugetlb=", dynamic_hugetlb_setup); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 1528a12ab3a9..6049fd4a9050 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -43,6 +43,7 @@ #include <linux/node.h> #include <linux/userfaultfd_k.h> #include <linux/page_owner.h> +#include <linux/dynamic_hugetlb.h> #include "internal.h" #include "hugetlb_vmemmap.h"

@@ -3460,6 +3461,8 @@ static int __init hugetlb_init(void) hugetlb_register_all_nodes(); hugetlb_cgroup_file_init();

+ dynamic_hugetlb_init(); + #ifdef CONFIG_SMP num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); #else diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 011aff396af2..1a292d54e7ad 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5195,6 +5195,14 @@ static struct cftype mem_cgroup_legacy_files[] = { .write_s64 = memcg_qos_write, }, #endif +#ifdef CONFIG_DYNAMIC_HUGETLB + { + .name = "dhugetlb.nr_pages", + .write = write_hugepage_to_hpool, + .seq_show = hugetlb_pool_info_show, + .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE | CFTYPE_NOT_ON_ROOT, + }, +#endif #ifdef CONFIG_NUMA { .name = "numa_stat", @@ -5523,6 +5531,8 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) return &memcg->css; }

+ hugetlb_pool_inherit(memcg, parent); + error = memcg_online_kmem(memcg); if (error) goto fail;

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Add two interfaces in mem_cgroup to configure the count of 1G/2M hugepages in dhugetlb_pool.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/dynamic_hugetlb.h | 4 +++ mm/dynamic_hugetlb.c | 56 +++++++++++++++++++++++++++++++++ mm/memcontrol.c | 10 ++++++ 3 files changed, 70 insertions(+)

diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h index 30ccbd9f1853..d0f6c1dd2361 100644 --- a/include/linux/dynamic_hugetlb.h +++ b/include/linux/dynamic_hugetlb.h @@ -75,6 +75,10 @@ struct dhugetlb_pool { };

bool dhugetlb_hide_files(struct cftype *cft); +ssize_t write_2M_reserved_pages(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); +ssize_t write_1G_reserved_pages(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); ssize_t write_hugepage_to_hpool(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); int hugetlb_pool_info_show(struct seq_file *m, void *v); diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 8881e9e1a032..d12e07ca90c2 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -255,6 +255,62 @@ bool dhugetlb_hide_files(struct cftype *cft) return false; }

+static ssize_t update_reserved_pages(struct mem_cgroup *memcg, char *buf, int hpages_pool_idx) +{ + struct dhugetlb_pool *hpool = memcg->hpool; + struct huge_pages_pool *hpages_pool; + unsigned long nr_pages; + unsigned long delta; + char *endp; + + if (!dhugetlb_enabled) + return -EINVAL; + + buf = strstrip(buf); + nr_pages = memparse(buf, &endp); + if (*endp != '\0') + return -EINVAL; + + if (!get_hpool_unless_zero(hpool)) + return -EINVAL; + + spin_lock(&hpool->reserved_lock); + spin_lock(&hpool->lock); + hpages_pool = &hpool->hpages_pool[hpages_pool_idx]; + if (nr_pages > hpages_pool->nr_huge_pages) { + delta = min(nr_pages - hpages_pool->nr_huge_pages, hpages_pool->free_normal_pages); + hpages_pool->nr_huge_pages += delta; + hpages_pool->free_huge_pages += delta; + hpages_pool->free_normal_pages -= delta; + } else { + delta = min(hpages_pool->nr_huge_pages - nr_pages, + hpages_pool->free_huge_pages - hpages_pool->resv_huge_pages); + hpages_pool->nr_huge_pages -= delta; + hpages_pool->free_huge_pages -= delta; + hpages_pool->free_normal_pages += delta; + } + spin_unlock(&hpool->lock); + spin_unlock(&hpool->reserved_lock); + put_hpool(hpool); + return 0; +} + +ssize_t write_2M_reserved_pages(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + + return update_reserved_pages(memcg, buf, HUGE_PAGES_POOL_2M) ?: nbytes; +} + +ssize_t write_1G_reserved_pages(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + + return update_reserved_pages(memcg, buf, HUGE_PAGES_POOL_1G) ?: nbytes; +} + ssize_t write_hugepage_to_hpool(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1a292d54e7ad..6381de898f31 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5202,6 +5202,16 @@ static struct cftype mem_cgroup_legacy_files[] = { .seq_show = hugetlb_pool_info_show, .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE | CFTYPE_NOT_ON_ROOT, }, + { + .name = "dhugetlb.1G.reserved_pages", + .write = write_1G_reserved_pages, + .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE | CFTYPE_NOT_ON_ROOT, + }, + { + .name = "dhugetlb.2M.reserved_pages", + .write = write_2M_reserved_pages, + .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE | CFTYPE_NOT_ON_ROOT, + }, #endif #ifdef CONFIG_NUMA {

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

When destroying hpool or alloc huge pages, the pages has been split may need to be merged to huge pages. Add functions to merge pages in dhugetlb_pool. The information about split huge pages has been recorded in hugepage_splitlists and can traverse it to merge huge pages.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/dynamic_hugetlb.c | 127 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+)

diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 2050f6093544..c03ea40ad7b7 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -119,6 +119,126 @@ static int hpool_split_page(struct dhugetlb_pool *hpool, int hpages_pool_idx) return 0; }

+static void reclaim_pages_from_percpu_pool(struct dhugetlb_pool *hpool, + struct percpu_pages_pool *percpu_pool, + unsigned long nr_pages) +{ + struct huge_pages_pool *hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_4K]; + struct page *page, *next; + int i = 0; + + list_for_each_entry_safe(page, next, &percpu_pool->head_page, lru) { + list_del(&page->lru); + percpu_pool->free_pages--; + list_add(&page->lru, &hpages_pool->hugepage_freelists); + hpages_pool->free_normal_pages++; + if (++i == nr_pages) + break; + } +} + +static void clear_percpu_pools(struct dhugetlb_pool *hpool) +{ + struct percpu_pages_pool *percpu_pool; + int i; + + lockdep_assert_held(&hpool->lock); + + spin_unlock(&hpool->lock); + for (i = 0; i < NR_PERCPU_POOL; i++) + spin_lock(&hpool->percpu_pool[i].lock); + spin_lock(&hpool->lock); + for (i = 0; i < NR_PERCPU_POOL; i++) { + percpu_pool = &hpool->percpu_pool[i]; + reclaim_pages_from_percpu_pool(hpool, percpu_pool, percpu_pool->free_pages); + } + for (i = 0; i < NR_PERCPU_POOL; i++) + spin_unlock(&hpool->percpu_pool[i].lock); +} + +static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx) +{ + struct huge_pages_pool *hpages_pool, *src_hpages_pool; + struct split_hugepage *split_page, *split_next; + unsigned long nr_pages, block_size; + struct page *page; + int i; + + lockdep_assert_held(&hpool->lock); + + if (hpages_pool_idx < 0 || hpages_pool_idx >= HUGE_PAGES_POOL_MAX - 1) + return -EINVAL; + + switch (hpages_pool_idx) { + case HUGE_PAGES_POOL_1G: + nr_pages = 1 << (PUD_SHIFT - PMD_SHIFT); + block_size = 1 << (PMD_SHIFT - PAGE_SHIFT); + break; + case HUGE_PAGES_POOL_2M: + nr_pages = 1 << (PMD_SHIFT - PAGE_SHIFT); + block_size = 1; + break; + } + + hpages_pool = &hpool->hpages_pool[hpages_pool_idx]; + src_hpages_pool = &hpool->hpages_pool[hpages_pool_idx + 1]; + if (!hpages_pool->split_normal_pages) + return -ENOMEM; + + list_for_each_entry_safe(split_page, split_next, &hpages_pool->hugepage_splitlists, head_pages) { + clear_percpu_pools(hpool); + page = pfn_to_page(split_page->start_pfn); + for (i = 0; i < nr_pages; i+= block_size) { + if (PagePool(&page[i])) + goto next; + } + list_del(&split_page->head_pages); + hpages_pool->split_normal_pages--; + kfree(split_page); + for (i = 0; i < nr_pages; i+= block_size) { + list_del(&page[i].lru); + src_hpages_pool->free_normal_pages--; + } + add_new_page_to_pool(hpool, page, hpages_pool_idx); + return 0; +next: + continue; + } + return -ENOMEM; +} + +static int hugetlb_pool_merge_all_pages(struct dhugetlb_pool *hpool) +{ + int ret = 0; + + spin_lock(&hpool->lock); + while (hpool->hpages_pool[HUGE_PAGES_POOL_2M].split_normal_pages) { + ret = hpool_merge_page(hpool, HUGE_PAGES_POOL_2M); + if (ret) { + pr_err("dynamic_hugetlb: some 4K pages are still in use, delete memcg: %s failed!\n", + hpool->attach_memcg->css.cgroup->kn->name); + goto out; + } + } + while (hpool->hpages_pool[HUGE_PAGES_POOL_1G].split_normal_pages) { + ret = hpool_merge_page(hpool, HUGE_PAGES_POOL_1G); + if (ret) { + pr_err("dynamic_hugetlb: some 2M pages are still in use, delete memcg: %s failed!\n", + hpool->attach_memcg->css.cgroup->kn->name); + goto out; + } + } + if (hpool->hpages_pool[HUGE_PAGES_POOL_1G].used_huge_pages) { + ret = -ENOMEM; + pr_err("dynamic_hugetlb: some 1G pages are still in use, delete memcg: %s failed!\n", + hpool->attach_memcg->css.cgroup->kn->name); + goto out; + } +out: + spin_unlock(&hpool->lock); + return ret; +} + static bool get_hpool_unless_zero(struct dhugetlb_pool *hpool) { if (!dhugetlb_enabled || !hpool) @@ -315,6 +435,9 @@ int hugetlb_pool_destroy(struct cgroup *cgrp) if (!hpool || hpool->attach_memcg != memcg) return 0;

+ ret = hugetlb_pool_merge_all_pages(hpool); + if (ret) + return -ENOMEM; ret = free_hugepage_to_hugetlb(hpool); memcg->hpool = NULL;

@@ -392,6 +515,10 @@ static ssize_t update_reserved_pages(struct mem_cgroup *memcg, char *buf, int hp if (hpool_split_page(hpool, hpages_pool_idx - 1)) break; } + while (delta > hpages_pool->free_normal_pages) { + if (hpool_merge_page(hpool, hpages_pool_idx)) + break; + } delta = min(nr_pages - hpages_pool->nr_huge_pages, hpages_pool->free_normal_pages); hpages_pool->nr_huge_pages += delta; hpages_pool->free_huge_pages += delta;

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Add function to alloc page from dhugetlb_pool. When process is bound to a mem_cgroup configured with dhugtlb_pool, alloc page from dhugetlb_pool firstly. If there is no page in dhugetlb_pool, fallback to alloc page from buddy system.

As the process will alloc pages from dhugetlb_pool in the mem_cgroup, process is not allowed to migrate to other mem_cgroup.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/dynamic_hugetlb.h | 15 ++++ mm/dynamic_hugetlb.c | 135 ++++++++++++++++++++++++++++++++ mm/memcontrol.c | 4 + mm/page_alloc.c | 6 ++ 4 files changed, 160 insertions(+)

diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h index 2004f174f7d3..9af8ed6ec96b 100644 --- a/include/linux/dynamic_hugetlb.h +++ b/include/linux/dynamic_hugetlb.h @@ -88,6 +88,10 @@ void hugetlb_pool_inherit(struct mem_cgroup *memcg, struct mem_cgroup *parent); int hugetlb_pool_destroy(struct cgroup *cgrp); void __init dynamic_hugetlb_init(void);

+struct page *alloc_page_from_dhugetlb_pool(gfp_t gfp, unsigned int order, + unsigned int flags); +int task_has_mem_in_hpool(struct task_struct *tsk); + #else

#define dhugetlb_enabled 0 @@ -108,5 +112,16 @@ static inline int hugetlb_pool_destroy(struct cgroup *cgrp) static inline void __init dynamic_hugetlb_init(void) { } + +static inline struct page *alloc_page_from_dhugetlb_pool(gfp_t gfp, unsigned int order, + unsigned int flags) +{ + return NULL; +} +static inline int task_has_mem_in_hpool(struct task_struct *tsk) +{ + return 0; +} + #endif /* CONFIG_DYNAMIC_HUGETLB */ #endif /* __LINUX_DYNAMIC_HUGETLB_H */ diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 2dd58cbee610..c54c99627994 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -122,6 +122,34 @@ static int hpool_split_page(struct dhugetlb_pool *hpool, int hpages_pool_idx) return 0; }

+static int add_pages_to_percpu_pool(struct dhugetlb_pool *hpool, + struct percpu_pages_pool *percpu_pool, + unsigned long nr_pages) +{ + struct huge_pages_pool *hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_4K]; + struct page *page, *next; + int ret, i = 0; + + while (hpages_pool->free_normal_pages < nr_pages) { + ret = hpool_split_page(hpool, HUGE_PAGES_POOL_2M); + if (ret) + break; + } + + list_for_each_entry_safe(page, next, &hpages_pool->hugepage_freelists, lru) { + list_del(&page->lru); + hpages_pool->free_normal_pages--; + list_add_tail(&page->lru, &percpu_pool->head_page); + percpu_pool->free_pages++; + if (++i == nr_pages) + break; + } + + if (percpu_pool->free_pages == 0) + return -ENOMEM; + return 0; +} + static void reclaim_pages_from_percpu_pool(struct dhugetlb_pool *hpool, struct percpu_pages_pool *percpu_pool, unsigned long nr_pages) @@ -350,6 +378,113 @@ static int set_hpool_in_dhugetlb_pagelist(unsigned long idx, struct dhugetlb_poo return 0; }

+static struct dhugetlb_pool *find_hpool_by_task(struct task_struct *tsk) +{ + struct mem_cgroup *memcg; + + if (!dhugetlb_enabled) + return NULL; + + rcu_read_lock(); + memcg = mem_cgroup_from_task(tsk); + rcu_read_unlock(); + + if (!memcg) + return NULL; + + return memcg->hpool; +} + +int task_has_mem_in_hpool(struct task_struct *tsk) +{ + struct dhugetlb_pool *hpool; + + if (!dhugetlb_enabled) + return 0; + + hpool = find_hpool_by_task(tsk); + + return hpool ? -EPERM : 0; +} + +static bool should_allocate_from_dhugetlb_pool(gfp_t gfp_mask) +{ + gfp_t gfp = gfp_mask & GFP_HIGHUSER_MOVABLE; + + if (current->flags & PF_KTHREAD) + return false; + + /* + * The cgroup only charges anonymous and file pages from usespage. + * some filesystem maybe has masked out the __GFP_IO | __GFP_FS + * to avoid recursive memory request. eg: loop device, xfs. + */ + if ((gfp | __GFP_IO | __GFP_FS) != GFP_HIGHUSER_MOVABLE) + return false; + + return true; +} + +static struct page *__alloc_page_from_dhugetlb_pool(void) +{ + struct percpu_pages_pool *percpu_pool; + struct dhugetlb_pool *hpool; + struct page *page = NULL; + unsigned long flags; + + hpool = find_hpool_by_task(current); + + if (!get_hpool_unless_zero(hpool)) + return NULL; + + percpu_pool = &hpool->percpu_pool[smp_processor_id()]; + /* + * Before we lock percpu_pool, must be sure hpool is unlocked. + */ + spin_lock_irqsave(&percpu_pool->lock, flags); + + if (percpu_pool->free_pages == 0) { + int ret; + + spin_lock(&hpool->lock); + ret = add_pages_to_percpu_pool(hpool, percpu_pool, + PERCPU_POOL_PAGE_BATCH); + spin_unlock(&hpool->lock); + if (ret) + goto unlock; + } + + page = list_entry(percpu_pool->head_page.next, struct page, lru); + list_del(&page->lru); + percpu_pool->free_pages--; + percpu_pool->used_pages++; + SetPagePool(page); + +unlock: + spin_unlock_irqrestore(&percpu_pool->lock, flags); + put_hpool(hpool); + return page; +} + +struct page *alloc_page_from_dhugetlb_pool(gfp_t gfp, unsigned int order, + unsigned int flags) +{ + struct page *page = NULL; + + if (!dhugetlb_enabled) + return NULL; + + if (order != 0) + return NULL; + + if (should_allocate_from_dhugetlb_pool(gfp)) + page = __alloc_page_from_dhugetlb_pool(); + + if (page) + prep_new_page(page, order, gfp, flags); + return page; +} + static int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, unsigned long nid, unsigned long nr_pages) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6381de898f31..f06c7349d9a4 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6254,6 +6254,10 @@ static int mem_cgroup_can_attach(struct cgroup_taskset *tset) if (!p) return 0;

+ ret = task_has_mem_in_hpool(p); + if (ret) + return ret; + /* * We are now commited to this value whatever it is. Changes in this * tunable will only affect upcoming migrations, not the current one. diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 0ff4f4e3a538..403898a3ab1e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -72,6 +72,7 @@ #include <linux/khugepaged.h> #include <linux/buffer_head.h> #include <linux/vmalloc.h> +#include <linux/dynamic_hugetlb.h>

#include <asm/sections.h> #include <asm/tlbflush.h> @@ -5160,6 +5161,11 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, */ alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp);

+ /* Before alloc from buddy system, alloc from hpool firstly */ + page = alloc_page_from_dhugetlb_pool(alloc_gfp, order, alloc_flags); + if (page) + goto out; + /* First allocation attempt */ page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); if (likely(page))

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Add new interface "dhugetlb.disable_normal_pages" to disable the allocation of normal pages from a hpool. This makes dynamic hugetlb more flexible.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/dynamic_hugetlb.h | 2 ++ mm/dynamic_hugetlb.c | 30 ++++++++++++++++++++++++++++++ mm/memcontrol.c | 6 ++++++ 3 files changed, 38 insertions(+)

diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h index ebb5a36a4a26..2b2c90562bcf 100644 --- a/include/linux/dynamic_hugetlb.h +++ b/include/linux/dynamic_hugetlb.h @@ -81,6 +81,8 @@ ssize_t write_2M_reserved_pages(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); ssize_t write_1G_reserved_pages(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); +int normal_pages_disabled_write(struct cgroup_subsys_state *css, struct cftype *cft, u64 val); +u64 normal_pages_disabled_read(struct cgroup_subsys_state *css, struct cftype *cft); ssize_t write_hugepage_to_hpool(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); int hugetlb_pool_info_show(struct seq_file *m, void *v); diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 40f79f3e6aeb..423d9624d4f0 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -449,6 +449,8 @@ static struct page *__alloc_page_from_dhugetlb_pool(void) if (!get_hpool_unless_zero(hpool)) return NULL;

+ if (hpool->normal_pages_disabled) + goto out; percpu_pool = &hpool->percpu_pool[smp_processor_id()]; /* * Before we lock percpu_pool, must be sure hpool is unlocked. @@ -474,6 +476,7 @@ static struct page *__alloc_page_from_dhugetlb_pool(void)

unlock: spin_unlock_irqrestore(&percpu_pool->lock, flags); +out: put_hpool(hpool); return page; } @@ -810,6 +813,33 @@ ssize_t write_1G_reserved_pages(struct kernfs_open_file *of, return update_reserved_pages(memcg, buf, HUGE_PAGES_POOL_1G) ?: nbytes; }

+int normal_pages_disabled_write(struct cgroup_subsys_state *css, + struct cftype *cft, u64 val) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(css); + struct dhugetlb_pool *hpool = memcg->hpool; + + if (!dhugetlb_enabled || !hpool) + return -EINVAL; + if (!((val == 0) || (val == 1))) + return -EINVAL; + + hpool->normal_pages_disabled = val; + return 0; +} + +u64 normal_pages_disabled_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(css); + struct dhugetlb_pool *hpool = memcg->hpool; + + if (!dhugetlb_enabled || !hpool) + return 0; + + return hpool->normal_pages_disabled; +} + ssize_t write_hugepage_to_hpool(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index f06c7349d9a4..2804fe9d3dae 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5212,6 +5212,12 @@ static struct cftype mem_cgroup_legacy_files[] = { .write = write_2M_reserved_pages, .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE | CFTYPE_NOT_ON_ROOT, }, + { + .name = "dhugetlb.disable_normal_pages", + .read_u64 = normal_pages_disabled_read, + .write_u64 = normal_pages_disabled_write, + .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE | CFTYPE_NOT_ON_ROOT, + }, #endif #ifdef CONFIG_NUMA {

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Add function to alloc huge page from dhugetlb_pool. When process is bound to a mem_cgroup configured with dhugetlb_pool, only allowed to alloc huge page from dhugetlb_pool. If there is no huge pages in dhugetlb_pool, the mmap() will failed due to the reserve count introduced in previous patch.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/dynamic_hugetlb.h | 8 +++++++ mm/dynamic_hugetlb.c | 39 ++++++++++++++++++++++++++++++++- mm/hugetlb.c | 14 ++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/include/linux/dynamic_hugetlb.h b/include/linux/dynamic_hugetlb.h index 8512f509899b..65d4b5dbf3f6 100644 --- a/include/linux/dynamic_hugetlb.h +++ b/include/linux/dynamic_hugetlb.h @@ -100,6 +100,8 @@ void link_hpool(struct hugetlbfs_inode_info *p); void unlink_hpool(struct hugetlbfs_inode_info *p); bool file_has_mem_in_hpool(struct hugetlbfs_inode_info *p); int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_info *p); +struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetlb_pool *hpool, + bool need_unreserved);

#else

@@ -154,6 +156,12 @@ static inline int dhugetlb_acct_memory(struct hstate *h, long delta, struct huge { return 0; } +static inline +struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetlb_pool *hpool, + bool need_unreserved) +{ + return NULL; +} #endif

#endif /* CONFIG_DYNAMIC_HUGETLB */ diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index f8ae9ba90bcb..126b3d9d3754 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -103,7 +103,7 @@ static int hpool_split_page(struct dhugetlb_pool *hpool, int hpages_pool_idx) if (!split_page) return -ENOMEM;

- page = list_entry(hpages_pool->hugepage_freelists.next, struct page, lru); + page = list_entry(hpages_pool->hugepage_freelists.prev, struct page, lru); list_del(&page->lru); hpages_pool->free_normal_pages--;

@@ -612,6 +612,43 @@ int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_in return ret; }

+struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetlb_pool *hpool, + bool need_unreserved) +{ + struct huge_pages_pool *hpages_pool; + struct page *page = NULL; + unsigned long flags; + + if (!dhugetlb_enabled) + return NULL; + + spin_lock_irqsave(&hpool->lock, flags); + if (hstate_is_gigantic(h)) + hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_1G]; + else + hpages_pool = &hpool->hpages_pool[HUGE_PAGES_POOL_2M]; + + if (hpages_pool->free_huge_pages) { + page = list_entry(hpages_pool->hugepage_freelists.next, struct page, lru); + list_del(&page->lru); + hpages_pool->free_huge_pages--; + hpages_pool->used_huge_pages++; + if (need_unreserved) { + SetHPageRestoreReserve(page); + hpages_pool->resv_huge_pages--; + } + } + if (page) { + INIT_LIST_HEAD(&page->lru); + set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); + set_page_refcounted(page); + SetPagePool(page); + } + spin_unlock_irqrestore(&hpool->lock, flags); + + return page; +} + static int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, unsigned long nid, unsigned long nr_pages) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d26f0a7ca780..031ad320f10c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2534,6 +2534,19 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, if (ret) goto out_uncharge_cgroup_reservation;

+ if (file_has_mem_in_hpool(info)) { + bool need_unreserved = false; + + if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) + need_unreserved = true; + page = alloc_huge_page_from_dhugetlb_pool(h, info->hpool, need_unreserved); + if (!page) + goto out_uncharge_cgroup; + spin_lock_irq(&hugetlb_lock); + list_add(&page->lru, &h->hugepage_activelist); + goto out; + } + spin_lock_irq(&hugetlb_lock); /* * glb_chg is passed to indicate whether or not a page must be taken @@ -2554,6 +2567,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, list_add(&page->lru, &h->hugepage_activelist); /* Fall through */ } +out: hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); /* If allocation is not consuming a reservation, also store the * hugetlb_cgroup pointer on the page.

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Add tracepoints for dynamic_hugetlb to track the process of page split, page merge, page migration, page allocation and page free.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/trace/events/dynamic_hugetlb.h | 121 +++++++++++++++++++++++++ mm/dynamic_hugetlb.c | 22 ++++- 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 include/trace/events/dynamic_hugetlb.h

diff --git a/include/trace/events/dynamic_hugetlb.h b/include/trace/events/dynamic_hugetlb.h new file mode 100644 index 000000000000..1de0df5df793 --- /dev/null +++ b/include/trace/events/dynamic_hugetlb.h @@ -0,0 +1,121 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dynamic_hugetlb + +#if !defined(_TRACE_DHUGETLB_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DHUGETLB_H + +#include <linux/tracepoint.h> +#include <trace/events/mmflags.h> + +#define DHUGETLB_SPLIT 0x01u +#define DHUGETLB_MERGE 0x02u +#define DHUGETLB_MIGRATE 0x04u +#define DHUGETLB_RESV 0x08u +#define DHUGETLB_UNRESV 0x10u +#define DHUGETLB_ALLOC 0x20u +#define DHUGETLB_FREE 0x40u + +#define __def_action_names \ + {(unsigned long)DHUGETLB_SPLIT, "split page"}, \ + {(unsigned long)DHUGETLB_MERGE, "merge page"}, \ + {(unsigned long)DHUGETLB_MIGRATE, "migrate page"}, \ + {(unsigned long)DHUGETLB_RESV, "resv page"}, \ + {(unsigned long)DHUGETLB_UNRESV, "unresv page"}, \ + {(unsigned long)DHUGETLB_ALLOC, "alloc page"}, \ + {(unsigned long)DHUGETLB_FREE, "free page"} + +#define show_action(action) \ + (action) ? __print_flags(action, "", \ + __def_action_names \ + ) : "none" + +TRACE_EVENT(dynamic_hugetlb_split_merge, + + TP_PROTO(const void *hpool, struct page *page, unsigned long action, unsigned long size), + + TP_ARGS(hpool, page, action, size), + + TP_STRUCT__entry( + __field( const void *, hpool ) + __field( unsigned long, pfn ) + __field( unsigned long, action ) + __field( unsigned long, size ) + ), + + TP_fast_assign( + __entry->hpool = hpool; + __entry->pfn = page ? page_to_pfn(page) : -1UL; + __entry->action = action; + __entry->size = size; + ), + + TP_printk("hpool=%p page=%p pfn=%lu action=%s size=%lu", + __entry->hpool, + __entry->pfn != -1UL ? pfn_to_page(__entry->pfn) : NULL, + __entry->pfn, + show_action(__entry->action), + __entry->size) +); + +TRACE_EVENT(dynamic_hugetlb_acct_memory, + + TP_PROTO(const void *hpool, unsigned long count, unsigned long action, unsigned long size), + + TP_ARGS(hpool, count, action, size), + + TP_STRUCT__entry( + __field( const void *, hpool ) + __field( unsigned long, count ) + __field( unsigned long, action ) + __field( unsigned long, size ) + ), + + TP_fast_assign( + __entry->hpool = hpool; + __entry->size = size; + __entry->count = count; + __entry->action = action; + ), + + TP_printk("hpool=%p action=%s size = %lu mmap_count=%lu", + __entry->hpool, + show_action(__entry->action), + __entry->size, + __entry->count) +); + +TRACE_EVENT(dynamic_hugetlb_alloc_free, + + TP_PROTO(const void *hpool, struct page *page, unsigned long count, unsigned long action, unsigned long size), + + TP_ARGS(hpool, page, count, action, size), + + TP_STRUCT__entry( + __field( const void *, hpool ) + __field( unsigned long, pfn ) + __field( unsigned long, count ) + __field( unsigned long, action ) + __field( unsigned long, size ) + ), + + TP_fast_assign( + __entry->hpool = hpool; + __entry->pfn = page ? page_to_pfn(page) : -1UL; + __entry->count = count; + __entry->action = action; + __entry->size = size; + ), + + TP_printk("hpool=%p page=%p pfn=%lu action=%s size = %lu free_count=%lu", + __entry->hpool, + __entry->pfn != -1UL ? pfn_to_page(__entry->pfn) : NULL, + __entry->pfn, + show_action(__entry->action), + __entry->size, + __entry->count) +); + +#endif /* _TRACE_DHUGETLB_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 73795d533f7e..1faac3b4572b 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -10,6 +10,11 @@

#include "internal.h"

+#if (defined CONFIG_DYNAMIC_HUGETLB) && (!defined __GENKSYMS__) +#define CREATE_TRACE_POINTS +#include <trace/events/dynamic_hugetlb.h> +#endif + static bool enable_dhugetlb = false; DEFINE_STATIC_KEY_FALSE(dhugetlb_enabled_key);

@@ -110,6 +115,7 @@ static int hpool_split_page(struct dhugetlb_pool *hpool, int hpages_pool_idx) split_page->start_pfn = page_to_pfn(page); list_add(&split_page->head_pages, &hpages_pool->hugepage_splitlists); hpages_pool->split_normal_pages++; + trace_dynamic_hugetlb_split_merge(hpool, page, DHUGETLB_SPLIT, page_size(page));

switch (hpages_pool_idx) { case HUGE_PAGES_POOL_1G: @@ -245,6 +251,7 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo src_hpages_pool->free_normal_pages--; } add_new_page_to_pool(hpool, page, hpages_pool_idx); + trace_dynamic_hugetlb_split_merge(hpool, page, DHUGETLB_MERGE, page_size(page)); return 0; next: continue; @@ -602,10 +609,14 @@ int dhugetlb_acct_memory(struct hstate *h, long delta, struct hugetlbfs_inode_in if (delta > 0 && delta <= hpages_pool->free_huge_pages - hpages_pool->resv_huge_pages) { hpages_pool->resv_huge_pages += delta; ret = 0; + trace_dynamic_hugetlb_acct_memory(hpool, hpages_pool->resv_huge_pages, + DHUGETLB_RESV, huge_page_size(h)); } else if (delta < 0) { hpages_pool->resv_huge_pages -= (unsigned long)(-delta); WARN_ON(hpages_pool->resv_huge_pages < 0); ret = 0; + trace_dynamic_hugetlb_acct_memory(hpool, hpages_pool->resv_huge_pages, + DHUGETLB_UNRESV, huge_page_size(h)); } spin_unlock(&hpool->lock);

@@ -636,7 +647,11 @@ struct page *alloc_huge_page_from_dhugetlb_pool(struct hstate *h, struct dhugetl if (need_unreserved) { SetHPageRestoreReserve(page); hpages_pool->resv_huge_pages--; + trace_dynamic_hugetlb_acct_memory(hpool, hpages_pool->resv_huge_pages, + DHUGETLB_UNRESV, huge_page_size(h)); } + trace_dynamic_hugetlb_alloc_free(hpool, page, hpages_pool->free_huge_pages, + DHUGETLB_ALLOC, huge_page_size(h)); } if (page) { INIT_LIST_HEAD(&page->lru); @@ -673,8 +688,13 @@ void free_huge_page_to_dhugetlb_pool(struct page *page, bool restore_reserve) list_add(&page->lru, &hpages_pool->hugepage_freelists); hpages_pool->free_huge_pages++; hpages_pool->used_huge_pages--; - if (restore_reserve) + if (restore_reserve) { hpages_pool->resv_huge_pages++; + trace_dynamic_hugetlb_acct_memory(hpool, hpages_pool->resv_huge_pages, + DHUGETLB_RESV, huge_page_size(h)); + } + trace_dynamic_hugetlb_alloc_free(hpool, page, hpages_pool->free_huge_pages, + DHUGETLB_FREE, huge_page_size(h)); spin_unlock(&hpool->lock); put_hpool(hpool); }

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

The dynamic_hugetlb feature need to split and merge pages frequently. hugetlb_vmemmap will affects the perforemance of page split and merge. If want to use dynamic hugetlb, please disable hugetlb_vmemmap.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/dynamic_hugetlb.c | 11 +++++++++++ 1 file changed, 11 insertions(+)

diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index e9f5348bcccd..f20e654cc856 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -1081,6 +1081,17 @@ void __init dynamic_hugetlb_init(void) if (!enable_dhugetlb) return;

+ /* + * The dynamic_hugetlb feature need to split and merge pages frequently. + * hugetlb_vmemmap will affects the perforemance of page split and merge. + * If want to use dynamic hugetlb, please disable hugetlb_vmemmap. + */ + if (hugetlb_free_vmemmap_enabled) { + enable_dhugetlb = false; + pr_info("Please set hugetlb_free_vmemmap=off if want to enable dynamic hugetlb\n"); + return; + } + count = max(hugepage_index(max_pfn), (unsigned long)DEFAULT_PAGELIST_COUNT); size = sizeof(struct dhugetlb_pagelist) + count * sizeof(struct dhugetlb_pool *); dhugetlb_pagelist_t = kzalloc(size, GFP_KERNEL);

From: Liu Shixin liushixin2@huawei.com

hulk inclusion category: feature bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I4QSHG CVE: NA

--------------------------------

Enable CONFIG_DYNAMIC_HUGETLB for x86 by default.

Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/x86/configs/openeuler_defconfig | 1 + 1 file changed, 1 insertion(+)

diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 0200c289ccec..fc7be06e8054 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -7547,6 +7547,7 @@ CONFIG_TMPFS_XATTR=y # CONFIG_TMPFS_INODE64 is not set CONFIG_HUGETLBFS=y CONFIG_HUGETLB_PAGE=y +CONFIG_DYNAMIC_HUGETLB=y CONFIG_MEMFD_CREATE=y CONFIG_ARCH_HAS_GIGANTIC_PAGE=y CONFIG_CONFIGFS_FS=y

From: Di Zhu zhudi21@huawei.com

mainline-inclusion from mainline-v5.16-rc7 commit 3c9ef511b9fa128a4c62e3aa0aac4c6b190f0d55 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4RADY CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

-------------------------------------------------

The following steps will definitely cause the kernel to crash: ip link add vrf1 type vrf table 1 modprobe bonding.ko max_bonds=1 echo "+vrf1" >/sys/class/net/bond0/bonding/slaves rmmod bonding

The root cause is that: When the VRF is added to the slave device, it will fail, and some cleaning work will be done. because VRF device has IFF_MASTER flag, cleanup process will not clear the IFF_BONDING flag. Then, when we unload the bonding module, unregister_netdevice_notifier() will treat the VRF device as a bond master device and treat netdev_priv() as struct bonding{} which actually is struct net_vrf{}.

By analyzing the processing logic of bond_enslave(), it seems that it is not allowed to add the slave device with the IFF_MASTER flag, so we need to add a code check for this situation.

Signed-off-by: Di Zhu zhudi21@huawei.com Signed-off-by: David S. Miller davem@davemloft.net Signed-off-by: Aichun Li liaichun@huawei.com Signed-off-by: Aichun Li liaichun@huawei.com Reviewed-by: Yue Haibing yuehaibing@huawei.com Reviewed-by: wuchangye wuchangye@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/net/bonding/bond_main.c | 6 ++++++ 1 file changed, 6 insertions(+)

diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 4f0894748347..97b9187bc8af 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1700,6 +1700,12 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, int link_reporting; int res = 0, i;

+ if (slave_dev->flags & IFF_MASTER) { + netdev_err(bond_dev, + "Error: Device with IFF_MASTER cannot be enslaved\n"); + return -EPERM; + } + if (!bond->params.use_carrier && slave_dev->ethtool_ops->get_link == NULL && slave_ops->ndo_do_ioctl == NULL) {

From: nifujia nifujia1@hisilicon.com

mainline inclusion from mainline-v5.16-rc1 commit 00aeaf329a3a category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4QKP3 CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/dr...

-----------------------------------------------------------------------------------

Export sas_phy_enable() so LLDDs can directly use it to control remote phys.

We already do this for companion function sas_phy_reset().

Link: https://lore.kernel.org/r/1634041588-74824-4-git-send-email-john.garry@huawe... Signed-off-by: Luo Jiaxing luojiaxing@huawei.com Signed-off-by: John Garry john.garry@huawei.com Signed-off-by: Martin K. Petersen martin.petersen@oracle.com Reviewed-by: Ouyangdelong ouyangdelong@huawei.com Signed-off-by: Nifujia nifujia1@hisilicon.com Reviewed-by: Jason Yan yanaijie@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/scsi/libsas/sas_init.c | 3 ++- include/scsi/libsas.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c index 2b0f98ca6ec3..23f8d34ccb0d 100644 --- a/drivers/scsi/libsas/sas_init.c +++ b/drivers/scsi/libsas/sas_init.c @@ -252,7 +252,7 @@ static int transport_sas_phy_reset(struct sas_phy *phy, int hard_reset) } }

-static int sas_phy_enable(struct sas_phy *phy, int enable) +int sas_phy_enable(struct sas_phy *phy, int enable) { int ret; enum phy_func cmd; @@ -284,6 +284,7 @@ static int sas_phy_enable(struct sas_phy *phy, int enable) } return ret; } +EXPORT_SYMBOL_GPL(sas_phy_enable);

int sas_phy_reset(struct sas_phy *phy, int hard_reset) { diff --git a/include/scsi/libsas.h b/include/scsi/libsas.h index 6fe125a71b60..79e4903bd414 100644 --- a/include/scsi/libsas.h +++ b/include/scsi/libsas.h @@ -664,6 +664,7 @@ extern void sas_suspend_ha(struct sas_ha_struct *sas_ha);

int sas_set_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates); int sas_phy_reset(struct sas_phy *phy, int hard_reset); +int sas_phy_enable(struct sas_phy *phy, int enable); extern int sas_queuecommand(struct Scsi_Host *, struct scsi_cmnd *); extern int sas_target_alloc(struct scsi_target *); extern int sas_slave_configure(struct scsi_device *);