hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAFWQH ---------------------------------------- Add a sysfs file for each zspool to bind memcg. The file path is /sys/block/zramX/zs_pool/zspool-xxx/mem_cgroup. For zspool 0, this file isn't exported, because zspool 0 is used if write IO isn't bounded to memcg or the memcg isn't bounded to any zpool. Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- .../multi_pool_zram/multi_pool_zram_drv.c | 213 +++++++++++++++++- .../multi_pool_zram/multi_pool_zram_drv.h | 13 ++ include/linux/swap_slots.h | 4 + include/linux/zsmalloc.h | 1 + mm/swap_slots.c | 9 + mm/zsmalloc.c | 3 +- 6 files changed, 241 insertions(+), 2 deletions(-) diff --git a/drivers/block/multi_pool_zram/multi_pool_zram_drv.c b/drivers/block/multi_pool_zram/multi_pool_zram_drv.c index a5f8a7ba799e..9890afa6e4ee 100644 --- a/drivers/block/multi_pool_zram/multi_pool_zram_drv.c +++ b/drivers/block/multi_pool_zram/multi_pool_zram_drv.c @@ -24,6 +24,7 @@ #include <linux/debugfs.h> #include <linux/cpuhotplug.h> #include <linux/part_stat.h> +#include <linux/swap_slots.h> #include "multi_pool_zram_drv.h" @@ -133,6 +134,63 @@ static void zram_set_obj_size(struct zram *zram, zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; } +static int zspool_bind_memcg(struct zram *zram, int pool_index, + struct mem_cgroup *memcg) +{ + bool drained = false; + int ret = -EBUSY; + + down_write(&zram->init_lock); + if (!init_done(zram)) { + if (zram->pool_memcg[pool_index]) + css_put(&zram->pool_memcg[pool_index]->css); + zram->pool_memcg[pool_index] = memcg; + ret = 0; + goto out_uninited; + } + +restart: + down_write(&zram->rebind_memcg_lock); + spin_lock(&zram->rebind_memcg_spinlock); + if (zs_get_total_pages(zram->mem_pool[pool_index]) != 0) { + if (!drained) { + spin_unlock(&zram->rebind_memcg_spinlock); + up_write(&zram->rebind_memcg_lock); + drain_swap_slots_cache(); + drained = true; + goto restart; + } + goto out; + } + + if (zram->pool_memcg[pool_index]) + css_put(&zram->pool_memcg[pool_index]->css); + zram->pool_memcg[pool_index] = memcg; + zs_set_memcg(zram->mem_pool[pool_index], memcg); + ret = 0; +out: + spin_unlock(&zram->rebind_memcg_spinlock); + up_write(&zram->rebind_memcg_lock); +out_uninited: + up_write(&zram->init_lock); + return ret; +} + +static void reset_memcg(struct zram *zram) +{ + int i; + + for (i = 0; i < NR_ZPOOL; i++) { + if (i == DEFAULT_ZPOOL_INDEX) + continue; + + if (zram->pool_memcg[i]) { + css_put(&zram->pool_memcg[i]->css); + zram->pool_memcg[i] = NULL; + } + } +} + #if PAGE_SIZE != 4096 static inline bool is_partial_io(struct bio_vec *bvec) { @@ -298,7 +356,10 @@ static int zram_pool_create(struct zram *zram) zram_pool_destroy(zram); return -ENOMEM; } + zs_set_memcg(zram->mem_pool[i], zram->pool_memcg[i]); } + init_rwsem(&zram->rebind_memcg_lock); + spin_lock_init(&zram->rebind_memcg_spinlock); return 0; } @@ -723,6 +784,7 @@ static void zram_submit_bio(struct bio *bio) { struct zram *zram = bio->bi_bdev->bd_disk->private_data; + down_read(&zram->rebind_memcg_lock); switch (bio_op(bio)) { case REQ_OP_READ: zram_bio_read(zram, bio); @@ -738,6 +800,7 @@ static void zram_submit_bio(struct bio *bio) WARN_ON_ONCE(1); bio_endio(bio); } + up_read(&zram->rebind_memcg_lock); } static void zram_slot_free_notify(struct block_device *bdev, @@ -746,11 +809,12 @@ static void zram_slot_free_notify(struct block_device *bdev, struct zram *zram; zram = bdev->bd_disk->private_data; - atomic64_inc(&zram->stats.notify_free); + spin_lock(&zram->rebind_memcg_spinlock); zram_slot_lock(zram, index); zram_free_page(zram, index); zram_slot_unlock(zram, index); + spin_unlock(&zram->rebind_memcg_spinlock); } static void zram_reset_device(struct zram *zram) @@ -769,6 +833,7 @@ static void zram_reset_device(struct zram *zram) } memset(&zram->stats, 0, sizeof(zram->stats)); strscpy(zram->compressor, default_compressor, sizeof(zram->compressor)); + reset_memcg(zram); up_write(&zram->init_lock); } @@ -895,6 +960,145 @@ static struct attribute *zram_disk_attrs[] = { ATTRIBUTE_GROUPS(zram_disk); +static int kobj_to_pool_index(struct zram *zram, struct kobject *kobj) +{ + int i; + + for (i = 0; i < NR_ZPOOL; i++) + if (zram->pool_kobj[i] == kobj) + return i; + + VM_BUG_ON(1); + return 0; +} + +static ssize_t mem_cgroup_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct kobject *kobj = &dev->kobj; + struct mem_cgroup *memcg; + struct zram *zram; + int pool_index; + + zram = dev_to_zram(kobj_to_dev(kobj->parent->parent)); + pool_index = kobj_to_pool_index(zram, kobj); + memcg = zram->pool_memcg[pool_index]; + + if (!memcg) + return scnprintf(buf, PAGE_SIZE, "none\n"); + + if (!cgroup_path(memcg->css.cgroup, buf, PATH_MAX)) + return scnprintf(buf, PAGE_SIZE, "none\n"); + + return scnprintf(buf, PAGE_SIZE, "%s\n", buf); +} + +static ssize_t mem_cgroup_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t len) +{ + struct kobject *kobj = &dev->kobj; + struct mem_cgroup *memcg; + struct zram *zram; + int pool_index; + char *kbuf; + size_t sz; + int ret = 0; + + kbuf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + + strlcpy(kbuf, buf, PATH_MAX); + sz = strlen(kbuf); + if (sz > 0 && kbuf[sz - 1] == '\n') + kbuf[sz - 1] = 0x00; + + if (!strcmp(kbuf, "none")) { + memcg = NULL; + } else { + memcg = memcg_get_from_path(kbuf, PATH_MAX); + if (!memcg) { + ret = -EINVAL; + goto out; + } + } + + zram = dev_to_zram(kobj_to_dev(kobj->parent->parent)); + pool_index = kobj_to_pool_index(zram, kobj); + ret = zspool_bind_memcg(zram, pool_index, memcg); + if (ret && memcg) + css_put(&memcg->css); +out: + kfree(kbuf); + return ret ? ret : len; +} +static DEVICE_ATTR_RW(mem_cgroup); + +static struct attribute *mem_pool_attrs[] = { + &dev_attr_mem_cgroup.attr, + NULL, +}; + +static const struct attribute_group mem_pool_attr_group = { + .attrs = mem_pool_attrs, +}; + +static void zram_pool_kobject_destroy(struct zram *zram) +{ + int i; + + for (i = 0; i < NR_ZPOOL; i++) { + if (i == DEFAULT_ZPOOL_INDEX) + continue; + if (!zram->pool_kobj[i]) + continue; + + sysfs_remove_group(zram->pool_kobj[i], &mem_pool_attr_group); + kobject_put(zram->pool_kobj[i]); + zram->pool_kobj[i] = NULL; + } + + if (zram->pool_parent_kobj) { + kobject_put(zram->pool_parent_kobj); + zram->pool_parent_kobj = NULL; + } +} + +static int zram_pool_kobject_create(struct zram *zram) +{ + char name[64]; + int ret, i; + + zram->pool_parent_kobj = kobject_create_and_add("zs_pool", + &disk_to_dev(zram->disk)->kobj); + if (!zram->pool_parent_kobj) + return -ENOMEM; + + for (i = 0; i < NR_ZPOOL; i++) { + if (i == DEFAULT_ZPOOL_INDEX) + continue; + + snprintf(name, 64, "zspool-%03d", i); + zram->pool_kobj[i] = kobject_create_and_add(name, zram->pool_parent_kobj); + if (!zram->pool_kobj[i]) { + ret = -ENOMEM; + goto err; + } + + ret = sysfs_create_group(zram->pool_kobj[i], &mem_pool_attr_group); + if (ret) { + kobject_put(zram->pool_kobj[i]); + zram->pool_kobj[i] = NULL; + goto err; + } + } + + return 0; +err: + zram_pool_kobject_destroy(zram); + return ret; +} + /* * Allocate and initialize new zram device. the function returns * '>= 0' device_id upon success, and negative value otherwise. @@ -967,9 +1171,15 @@ static int zram_add(void) if (ret) goto out_cleanup_disk; + ret = zram_pool_kobject_create(zram); + if (ret < 0) + goto out_del_disk; + pr_info("Added device: %s\n", zram->disk->disk_name); return device_id; +out_del_disk: + del_gendisk(zram->disk); out_cleanup_disk: put_disk(zram->disk); out_free_idr: @@ -1019,6 +1229,7 @@ static int zram_remove(struct zram *zram) * anything allocated with disksize_store() */ zram_reset_device(zram); + zram_pool_kobject_destroy(zram); put_disk(zram->disk); kfree(zram); diff --git a/drivers/block/multi_pool_zram/multi_pool_zram_drv.h b/drivers/block/multi_pool_zram/multi_pool_zram_drv.h index 0304149b1697..e8298e9cfd7b 100644 --- a/drivers/block/multi_pool_zram/multi_pool_zram_drv.h +++ b/drivers/block/multi_pool_zram/multi_pool_zram_drv.h @@ -80,6 +80,16 @@ struct zram_stats { struct zram { struct zram_table_entry *table; struct zs_pool *mem_pool[NR_ZPOOL]; + /* + * Use the below two locks to serialize zspool update + * and use. + * + * zram_slot_free_notify() may be called with holding + * spinlock, so can't use rebind_memcg_lock, use + * rebind_memcg_spinlock instead. + */ + struct rw_semaphore rebind_memcg_lock; + spinlock_t rebind_memcg_spinlock; struct zcomp *comp; struct gendisk *disk; /* Prevent concurrent execution of device init */ @@ -96,5 +106,8 @@ struct zram { * zram is claimed so open request will be failed */ bool claim; /* Protected by disk->open_mutex */ + struct kobject *pool_parent_kobj; + struct kobject *pool_kobj[NR_ZPOOL]; + struct mem_cgroup *pool_memcg[NR_ZPOOL]; }; #endif diff --git a/include/linux/swap_slots.h b/include/linux/swap_slots.h index 77521ac11dca..ccd6753af7b5 100644 --- a/include/linux/swap_slots.h +++ b/include/linux/swap_slots.h @@ -26,6 +26,10 @@ void reenable_swap_slots_cache_unlock(void); void enable_swap_slots_cache(int type); void free_swap_slot(swp_entry_t entry); +#ifdef CONFIG_MEMCG_ZRAM +void drain_swap_slots_cache(void); +#endif + extern bool swap_slot_cache_enabled; #endif /* _LINUX_SWAP_SLOTS_H */ diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h index 2c09676f9178..9a5d6c24d62d 100644 --- a/include/linux/zsmalloc.h +++ b/include/linux/zsmalloc.h @@ -43,6 +43,7 @@ struct zs_pool; struct zs_pool *zs_create_pool(const char *name); #ifdef CONFIG_MEMCG_ZRAM struct zs_pool *zs_create_pool_with_memcg(const char *name, void *memcg); +void zs_set_memcg(struct zs_pool *pool, void *memcg); #endif void zs_destroy_pool(struct zs_pool *pool); diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 7af3b93d4c8c..04c22efee496 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -93,6 +93,15 @@ void reenable_swap_slots_cache_unlock(void) mutex_unlock(&swap_slots_cache_enable_mutex); } +#ifdef CONFIG_MEMCG_ZRAM +void drain_swap_slots_cache(void) +{ + disable_swap_slots_cache_lock(); + reenable_swap_slots_cache_unlock(); +} +EXPORT_SYMBOL_GPL(drain_swap_slots_cache); +#endif + static bool check_cache_active(void) { long pages; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 1fb34d1dd563..b106b4f9981b 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -2280,11 +2280,12 @@ struct zs_pool *zs_create_pool(const char *name) EXPORT_SYMBOL_GPL(zs_create_pool); #ifdef CONFIG_MEMCG_ZRAM -static inline void zs_set_memcg(struct zs_pool *pool, void *memcg) +void zs_set_memcg(struct zs_pool *pool, void *memcg) { if (pool) pool->memcg = memcg; } +EXPORT_SYMBOL_GPL(zs_set_memcg); struct zs_pool *zs_create_pool_with_memcg(const char *name, void *memcg) { -- 2.43.0