hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9H66Y CVE: NA
--------------------------------
After bound the zram with a memcg which has a hpool as memory provider. The memory usage of zram should prefer to allocate from hpool. zspool is allocated by alloc_page() before, replace it with zs_alloc_page() to achieve the effect. zram->table is allocated by vzalloc(), and vzalloc() is calling alloc_pages_node() to allocate memory. Add vzalloc_with_memcg() and __vmalloc_alloc_pages() to replace vzalloc() and alloc_pages_node() respectively to achieve the effect.
Signed-off-by: Liu Shixin liushixin2@huawei.com --- drivers/block/zram/zram_drv.c | 14 +++++- include/linux/vmalloc.h | 3 ++ mm/dynamic_hugetlb.c | 3 +- mm/vmalloc.c | 90 +++++++++++++++++++++++++++++++++-- mm/zsmalloc.c | 22 ++++++++- 5 files changed, 125 insertions(+), 7 deletions(-)
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index a56a549ee92e..90e2c989a553 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1368,6 +1368,12 @@ static inline void zram_uncharge_memory(struct zram *zram, unsigned long size)
memcg_uncharge_zram(zram->memcg, nr_pages); } + +static inline struct zram_table_entry *zram_table_alloc(struct zram *zram, + unsigned long size) +{ + return vzalloc_with_memcg(size, zram->memcg); +} #else static inline void reset_memcg(struct zram *zram) { @@ -1385,6 +1391,12 @@ static inline void zram_charge_memory(struct zram *zram, unsigned long size) static inline void zram_uncharge_memory(struct zram *zram, unsigned long size) { } + +static inline struct zram_table_entry *zram_table_alloc(struct zram *zram, + unsigned long size) +{ + return vzalloc(size); +} #endif
static void zram_meta_free(struct zram *zram, u64 disksize) @@ -1407,7 +1419,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) size_t num_pages = disksize >> PAGE_SHIFT; unsigned long size = array_size(num_pages, sizeof(*zram->table));
- zram->table = vzalloc(size); + zram->table = zram_table_alloc(zram, size); if (!zram->table) return false;
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 1ebe364ed29a..c0c9a3476700 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -131,6 +131,9 @@ static inline unsigned long vmalloc_nr_pages(void) { return 0; }
extern void *vmalloc(unsigned long size); extern void *vzalloc(unsigned long size); +#ifdef CONFIG_MEMCG_ZRAM +extern void *vzalloc_with_memcg(unsigned long size, void *memcg); +#endif extern void *vmalloc_user(unsigned long size); extern void *vmalloc_node(unsigned long size, int node); extern void *vzalloc_node(unsigned long size, int node); diff --git a/mm/dynamic_hugetlb.c b/mm/dynamic_hugetlb.c index 95a2a82eb2b0..149b5fa54d80 100644 --- a/mm/dynamic_hugetlb.c +++ b/mm/dynamic_hugetlb.c @@ -582,8 +582,6 @@ static struct page *__alloc_page_from_dpool(struct dhugetlb_pool *hpool) struct page *page = NULL; unsigned long flags;
- hpool = find_hpool_by_task(current); - if (!get_hpool_unless_zero(hpool)) return NULL;
@@ -663,6 +661,7 @@ struct page *alloc_page_from_dhugetlb_pool(struct mem_cgroup *memcg,
return page; } +EXPORT_SYMBOL_GPL(alloc_page_from_dhugetlb_pool);
static void __free_page_to_dhugetlb_pool(struct page *page) { diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 6d802924d9e8..f0aaae496ec4 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2889,9 +2889,36 @@ void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot) EXPORT_SYMBOL_GPL(vmap_pfn); #endif /* CONFIG_VMAP_PFN */
+#include <linux/dynamic_hugetlb.h> + +#ifdef CONFIG_MEMCG_ZRAM +static inline struct page *__vmalloc_alloc_pages(int node, gfp_t gfp_mask, + unsigned int order, + struct mem_cgroup *memcg) +{ + struct page *page; + + if (!memcg) + return alloc_pages_node(node, gfp_mask, order); + + page = alloc_page_from_dhugetlb_pool(memcg, gfp_mask, order, 0); + if (!page) + page = alloc_pages_node(node, gfp_mask, order); + + return page; +} +#else +static inline struct page *__vmalloc_alloc_pages(int node, gfp_t gfp_mask, + unsigned int order, + struct mem_cgroup *memcg) +{ + return alloc_pages_node(node, gfp_mask, order); +} +#endif + static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot, unsigned int page_shift, - int node) + int node, struct mem_cgroup *memcg) { const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; unsigned long addr = (unsigned long)area->addr; @@ -2940,7 +2967,8 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, int p;
/* Compound pages required for remap_vmalloc_page */ - page = alloc_pages_node(node, gfp_mask | __GFP_COMP, page_order); + page = __vmalloc_alloc_pages(node, gfp_mask | __GFP_COMP, + page_order, memcg); if (unlikely(!page)) { /* Successfully allocated i pages, free them in __vfree() */ area->nr_pages = i; @@ -3050,7 +3078,7 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, goto fail; }
- addr = __vmalloc_area_node(area, gfp_mask, prot, shift, node); + addr = __vmalloc_area_node(area, gfp_mask, prot, shift, node, NULL); if (!addr) goto fail;
@@ -3180,6 +3208,62 @@ void *vzalloc(unsigned long size) } EXPORT_SYMBOL(vzalloc);
+#ifdef CONFIG_MEMCG_ZRAM +static void *__vmalloc_with_memcg(unsigned long size, gfp_t gfp_mask, + struct mem_cgroup *memcg, const void *caller) +{ + struct vm_struct *area; + void *addr; + + if (WARN_ON_ONCE(!size)) + return NULL; + + if ((size >> PAGE_SHIFT) > totalram_pages()) { + warn_alloc(gfp_mask, NULL, + "vmalloc size %lu: exceeds total pages", size); + return NULL; + } + + area = __get_vm_area_node(size, 1, PAGE_SHIFT, VM_ALLOC | + VM_UNINITIALIZED, VMALLOC_START, + VMALLOC_END, NUMA_NO_NODE, + gfp_mask, caller); + if (!area) { + warn_alloc(gfp_mask, NULL, + "vmalloc size %lu: vm_struct allocation failed", size); + return NULL; + } + + addr = __vmalloc_area_node(area, gfp_mask, PAGE_KERNEL, PAGE_SHIFT, + NUMA_NO_NODE, memcg); + if (!addr) + return NULL; + + /* + * In this function, newly allocated vm_struct has VM_UNINITIALIZED + * flag. It means that vm_struct is not fully initialized. + * Now, it is fully initialized, so remove this flag here. + */ + clear_vm_uninitialized_flag(area); + + size = PAGE_ALIGN(size); + kmemleak_vmalloc(area, size, gfp_mask); + + return addr; +} + +void *vzalloc_with_memcg(unsigned long size, void *memcg) +{ + if (!memcg) + return vzalloc(size); + + return __vmalloc_with_memcg(size, GFP_KERNEL | __GFP_ZERO, + (struct mem_cgroup *)memcg, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(vzalloc_with_memcg); +#endif + /** * vmalloc_user - allocate zeroed virtually contiguous memory for userspace * @size: allocation size diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 934101f9f09e..55a1d9ea16ce 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -952,6 +952,21 @@ static inline void zs_uncharge_memory(struct zs_pool *pool, /* See zs_charge_memory() for detail */ memcg_uncharge_zram(pool->memcg, nr_pages); } + +static inline struct page *zs_alloc_page(struct zs_pool *pool, gfp_t gfp) +{ + struct mem_cgroup *memcg = pool->memcg; + struct page *page; + + if (!memcg) + return alloc_page(gfp); + + page = alloc_page_from_dhugetlb_pool(memcg, gfp, 0, 0); + if (!page) + page = alloc_page(gfp); + + return page; +} #else static inline void zs_charge_memory(struct zs_pool *pool, unsigned long nr_pages) @@ -962,6 +977,11 @@ static inline void zs_uncharge_memory(struct zs_pool *pool, unsigned long nr_pages) { } + +static inline struct page *zs_alloc_page(struct zs_pool *pool, gfp_t gfp) +{ + return alloc_page(gfp); +} #endif
static void __free_zspage(struct zs_pool *pool, struct size_class *class, @@ -1111,7 +1131,7 @@ static struct zspage *alloc_zspage(struct zs_pool *pool, for (i = 0; i < class->pages_per_zspage; i++) { struct page *page;
- page = alloc_page(gfp); + page = zs_alloc_page(pool, gfp); if (!page) { while (--i >= 0) { dec_zone_page_state(pages[i], NR_ZSPAGES);