hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAFWQH ---------------------------------------- Initialize multiple pool zram driver by copying from zram and delete unused code. The following code is deleted: * CONFIG_ZRAM_WRITEBACK related code * idle related code * CONFIG_ZRAM_MEMORY_TRACKING and ZRAM_TRACK_ENTRY_ACTIME related code * mem_limit and max_used_pages related code * CONFIG_MEMCG_ZRAM related code * max_comp_streams interface * CONFIG_ZRAM_MULTI_COMP related code * compact interface The following interfaces need to be adapted to multiple pools, so delete them in this patch and add them back in the later patch. * mm_stat interface Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- drivers/block/Kconfig | 2 + drivers/block/Makefile | 1 + drivers/block/multi_pool_zram/Kconfig | 12 + drivers/block/multi_pool_zram/Makefile | 4 + .../multi_pool_zram/multi_pool_zram_drv.c | 1128 +++++++++++++++++ .../multi_pool_zram/multi_pool_zram_drv.h | 89 ++ drivers/block/multi_pool_zram/zcomp.c | 233 ++++ drivers/block/multi_pool_zram/zcomp.h | 43 + 8 files changed, 1512 insertions(+) create mode 100644 drivers/block/multi_pool_zram/Kconfig create mode 100644 drivers/block/multi_pool_zram/Makefile create mode 100644 drivers/block/multi_pool_zram/multi_pool_zram_drv.c create mode 100644 drivers/block/multi_pool_zram/multi_pool_zram_drv.h create mode 100644 drivers/block/multi_pool_zram/zcomp.c create mode 100644 drivers/block/multi_pool_zram/zcomp.h diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 5b9d4aaebb81..861c959d16f6 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -107,6 +107,8 @@ source "drivers/block/mtip32xx/Kconfig" source "drivers/block/zram/Kconfig" +source "drivers/block/multi_pool_zram/Kconfig" + config BLK_DEV_UBD bool "Virtual block device" depends on UML diff --git a/drivers/block/Makefile b/drivers/block/Makefile index 101612cba303..bb5c00991e75 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -33,6 +33,7 @@ obj-$(CONFIG_BLK_DEV_RBD) += rbd.o obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/ obj-$(CONFIG_ZRAM) += zram/ +obj-$(CONFIG_ZRAM) += multi_pool_zram/ obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/ obj-$(CONFIG_BLK_DEV_NULL_BLK) += null_blk/ diff --git a/drivers/block/multi_pool_zram/Kconfig b/drivers/block/multi_pool_zram/Kconfig new file mode 100644 index 000000000000..a7fb0986b3de --- /dev/null +++ b/drivers/block/multi_pool_zram/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +config MULTI_POOL_ZRAM + tristate "Compressed RAM block device with multiple zspools" + depends on BLOCK && SYSFS && MMU + depends on CRYPTO_LZO || CRYPTO_ZSTD || CRYPTO_LZ4 || CRYPTO_LZ4HC || CRYPTO_842 + depends on MEMCG_ZRAM + select ZSMALLOC + help + Support zram driver with multiple zspools. Each zspool can + bind to a memcg. The memory allocated by the zspool will + be from the dynamoc pool of the bounded memcg, and IO from + the memcg will use the bounded zspool. diff --git a/drivers/block/multi_pool_zram/Makefile b/drivers/block/multi_pool_zram/Makefile new file mode 100644 index 000000000000..30b74db73730 --- /dev/null +++ b/drivers/block/multi_pool_zram/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +multi_pool_zram-y := zcomp.o multi_pool_zram_drv.o + +obj-$(CONFIG_MULTI_POOL_ZRAM) += multi_pool_zram.o diff --git a/drivers/block/multi_pool_zram/multi_pool_zram_drv.c b/drivers/block/multi_pool_zram/multi_pool_zram_drv.c new file mode 100644 index 000000000000..69a899ae98b5 --- /dev/null +++ b/drivers/block/multi_pool_zram/multi_pool_zram_drv.c @@ -0,0 +1,1128 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Compressed RAM block device with multiple zspool support + */ + +#define KMSG_COMPONENT "multi-pool zram" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/bio.h> +#include <linux/bitops.h> +#include <linux/blkdev.h> +#include <linux/buffer_head.h> +#include <linux/device.h> +#include <linux/highmem.h> +#include <linux/slab.h> +#include <linux/backing-dev.h> +#include <linux/string.h> +#include <linux/vmalloc.h> +#include <linux/err.h> +#include <linux/idr.h> +#include <linux/sysfs.h> +#include <linux/debugfs.h> +#include <linux/cpuhotplug.h> +#include <linux/part_stat.h> + +#include "multi_pool_zram_drv.h" + +static DEFINE_IDR(zram_index_idr); +/* idr index must be protected */ +static DEFINE_MUTEX(zram_index_mutex); + +static int zram_major; +static const char *default_compressor = "lzo-rle"; + +/* Module params (documentation at end) */ +static unsigned int num_devices = 1; +/* + * Pages that compress to sizes equals or greater than this are stored + * uncompressed in memory. + */ +static size_t huge_class_size; + +static const struct block_device_operations zram_devops; + +static void zram_free_page(struct zram *zram, size_t index); +static int zram_read_page(struct zram *zram, struct page *page, u32 index, + struct bio *parent); + +static int zram_slot_trylock(struct zram *zram, u32 index) +{ + return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags); +} + +static void zram_slot_lock(struct zram *zram, u32 index) +{ + bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags); +} + +static void zram_slot_unlock(struct zram *zram, u32 index) +{ + bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); +} + +static inline bool init_done(struct zram *zram) +{ + return zram->disksize; +} + +static inline struct zram *dev_to_zram(struct device *dev) +{ + return (struct zram *)dev_to_disk(dev)->private_data; +} + +static unsigned long zram_get_handle(struct zram *zram, u32 index) +{ + return zram->table[index].handle; +} + +static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle) +{ + zram->table[index].handle = handle; +} + +/* flag operations require table entry bit_spin_lock() being held */ +static bool zram_test_flag(struct zram *zram, u32 index, + enum zram_pageflags flag) +{ + return zram->table[index].flags & BIT(flag); +} + +static void zram_set_flag(struct zram *zram, u32 index, + enum zram_pageflags flag) +{ + zram->table[index].flags |= BIT(flag); +} + +static void zram_clear_flag(struct zram *zram, u32 index, + enum zram_pageflags flag) +{ + zram->table[index].flags &= ~BIT(flag); +} + +static inline void zram_set_element(struct zram *zram, u32 index, + unsigned long element) +{ + zram->table[index].element = element; +} + +static unsigned long zram_get_element(struct zram *zram, u32 index) +{ + return zram->table[index].element; +} + +static size_t zram_get_obj_size(struct zram *zram, u32 index) +{ + return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1); +} + +static void zram_set_obj_size(struct zram *zram, + u32 index, size_t size) +{ + unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; + + zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; +} + +#if PAGE_SIZE != 4096 +static inline bool is_partial_io(struct bio_vec *bvec) +{ + return bvec->bv_len != PAGE_SIZE; +} +#define ZRAM_PARTIAL_IO 1 +#else +static inline bool is_partial_io(struct bio_vec *bvec) +{ + return false; +} +#endif + +static inline void zram_fill_page(void *ptr, unsigned long len, + unsigned long value) +{ + WARN_ON_ONCE(!IS_ALIGNED(len, sizeof(unsigned long))); + memset_l(ptr, value, len / sizeof(unsigned long)); +} + +static bool page_same_filled(void *ptr, unsigned long *element) +{ + unsigned long *page; + unsigned long val; + unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1; + + page = (unsigned long *)ptr; + val = page[0]; + + if (val != page[last_pos]) + return false; + + for (pos = 1; pos < last_pos; pos++) { + if (val != page[pos]) + return false; + } + + *element = val; + + return true; +} + +static ssize_t initstate_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u32 val; + struct zram *zram = dev_to_zram(dev); + + down_read(&zram->init_lock); + val = init_done(zram); + up_read(&zram->init_lock); + + return scnprintf(buf, PAGE_SIZE, "%u\n", val); +} + +static ssize_t disksize_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct zram *zram = dev_to_zram(dev); + + return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize); +} + +static ssize_t comp_algorithm_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t len) +{ + struct zram *zram = dev_to_zram(dev); + char compressor[ARRAY_SIZE(zram->compressor)]; + size_t sz; + + sz = strlen(buf); + if (sz >= CRYPTO_MAX_ALG_NAME) + return -E2BIG; + + strscpy(compressor, buf, sizeof(compressor)); + /* ignore trailing newline */ + if (sz > 0 && compressor[sz - 1] == '\n') + compressor[sz - 1] = 0x00; + + if (!zcomp_available_algorithm(compressor)) + return -EINVAL; + + down_write(&zram->init_lock); + if (init_done(zram)) { + up_write(&zram->init_lock); + pr_info("Can't change algorithm for initialized device\n"); + return -EBUSY; + } + + strcpy(zram->compressor, compressor); + up_write(&zram->init_lock); + return 0; +} + +static ssize_t comp_algorithm_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct zram *zram = dev_to_zram(dev); + ssize_t sz; + + down_read(&zram->init_lock); + sz = zcomp_available_show(zram->compressor, buf); + up_read(&zram->init_lock); + + return sz; +} + +static ssize_t io_stat_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct zram *zram = dev_to_zram(dev); + ssize_t ret; + + down_read(&zram->init_lock); + ret = scnprintf(buf, PAGE_SIZE, + "%8llu %8llu 0 %8llu\n", + (u64)atomic64_read(&zram->stats.failed_reads), + (u64)atomic64_read(&zram->stats.failed_writes), + (u64)atomic64_read(&zram->stats.notify_free)); + up_read(&zram->init_lock); + + return ret; +} + +static ssize_t debug_stat_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int version = 1; + struct zram *zram = dev_to_zram(dev); + ssize_t ret; + + down_read(&zram->init_lock); + ret = scnprintf(buf, PAGE_SIZE, + "version: %d\n%8llu %8llu\n", + version, + (u64)atomic64_read(&zram->stats.writestall), + (u64)atomic64_read(&zram->stats.miss_free)); + up_read(&zram->init_lock); + + return ret; +} + +static DEVICE_ATTR_RO(io_stat); +static DEVICE_ATTR_RO(debug_stat); + +static void zram_meta_free(struct zram *zram, u64 disksize) +{ + size_t num_pages = disksize >> PAGE_SHIFT; + size_t index; + + if (!zram->table) + return; + + /* Free all pages that are still in this zram device */ + for (index = 0; index < num_pages; index++) + zram_free_page(zram, index); + + zs_destroy_pool(zram->mem_pool); + vfree(zram->table); + zram->table = NULL; +} + +static bool zram_meta_alloc(struct zram *zram, u64 disksize) +{ + size_t num_pages = disksize >> PAGE_SHIFT; + unsigned long size = array_size(num_pages, sizeof(*zram->table)); + + zram->table = vzalloc(size); + if (!zram->table) + return false; + + zram->mem_pool = zs_create_pool(zram->disk->disk_name); + if (!zram->mem_pool) { + vfree(zram->table); + zram->table = NULL; + return false; + } + + if (!huge_class_size) + huge_class_size = zs_huge_class_size(zram->mem_pool); + return true; +} + +/* + * To protect concurrent access to the same index entry, + * caller should hold this table index entry's bit_spinlock to + * indicate this index entry is accessing. + */ +static void zram_free_page(struct zram *zram, size_t index) +{ + unsigned long handle; + + if (zram_test_flag(zram, index, ZRAM_HUGE)) { + zram_clear_flag(zram, index, ZRAM_HUGE); + atomic64_dec(&zram->stats.huge_pages); + } + + /* + * No memory is allocated for same element filled pages. + * Simply clear same page flag. + */ + if (zram_test_flag(zram, index, ZRAM_SAME)) { + zram_clear_flag(zram, index, ZRAM_SAME); + atomic64_dec(&zram->stats.same_pages); + goto out; + } + + handle = zram_get_handle(zram, index); + if (!handle) + return; + + zs_free(zram->mem_pool, handle); + + atomic64_sub(zram_get_obj_size(zram, index), + &zram->stats.compr_data_size); +out: + atomic64_dec(&zram->stats.pages_stored); + zram_set_handle(zram, index, 0); + zram_set_obj_size(zram, index, 0); + WARN_ON_ONCE(zram->table[index].flags & + ~(1UL << ZRAM_LOCK)); +} + +/* + * Reads (decompresses if needed) a page from zspool (zsmalloc). + * Corresponding ZRAM slot should be locked. + */ +static int zram_read_from_zspool(struct zram *zram, struct page *page, + u32 index) +{ + struct zcomp_strm *zstrm; + unsigned long handle; + unsigned int size; + void *src, *dst; + int ret; + + handle = zram_get_handle(zram, index); + if (!handle || zram_test_flag(zram, index, ZRAM_SAME)) { + unsigned long value; + void *mem; + + value = handle ? zram_get_element(zram, index) : 0; + mem = kmap_atomic(page); + zram_fill_page(mem, PAGE_SIZE, value); + kunmap_atomic(mem); + return 0; + } + + size = zram_get_obj_size(zram, index); + + if (size != PAGE_SIZE) + zstrm = zcomp_stream_get(zram->comp); + + src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO); + if (size == PAGE_SIZE) { + dst = kmap_atomic(page); + memcpy(dst, src, PAGE_SIZE); + kunmap_atomic(dst); + ret = 0; + } else { + dst = kmap_atomic(page); + ret = zcomp_decompress(zstrm, src, size, dst); + kunmap_atomic(dst); + zcomp_stream_put(zram->comp); + } + zs_unmap_object(zram->mem_pool, handle); + return ret; +} + +static int zram_read_page(struct zram *zram, struct page *page, u32 index, + struct bio *parent) +{ + int ret; + + zram_slot_lock(zram, index); + /* Slot should be locked through out the function call */ + ret = zram_read_from_zspool(zram, page, index); + zram_slot_unlock(zram, index); + + /* Should NEVER happen. Return bio error if it does. */ + if (WARN_ON(ret < 0)) + pr_err("Decompression failed! err=%d, page=%u\n", ret, index); + + return ret; +} + +/* + * Use a temporary buffer to decompress the page, as the decompressor + * always expects a full page for the output. + */ +static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, + u32 index, int offset) +{ + struct page *page = alloc_page(GFP_NOIO); + int ret; + + if (!page) + return -ENOMEM; + ret = zram_read_page(zram, page, index, NULL); + if (likely(!ret)) + memcpy_to_bvec(bvec, page_address(page) + offset); + __free_page(page); + return ret; +} + +static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, + u32 index, int offset, struct bio *bio) +{ + if (is_partial_io(bvec)) + return zram_bvec_read_partial(zram, bvec, index, offset); + return zram_read_page(zram, bvec->bv_page, index, bio); +} + +static int zram_write_page(struct zram *zram, struct page *page, u32 index) +{ + int ret = 0; + unsigned long handle = -ENOMEM; + unsigned int comp_len = 0; + void *src, *dst, *mem; + struct zcomp_strm *zstrm; + unsigned long element = 0; + enum zram_pageflags flags = 0; + + mem = kmap_atomic(page); + if (page_same_filled(mem, &element)) { + kunmap_atomic(mem); + /* Free memory associated with this sector now. */ + flags = ZRAM_SAME; + atomic64_inc(&zram->stats.same_pages); + goto out; + } + kunmap_atomic(mem); + +compress_again: + zstrm = zcomp_stream_get(zram->comp); + src = kmap_atomic(page); + ret = zcomp_compress(zstrm, src, &comp_len); + kunmap_atomic(src); + + if (unlikely(ret)) { + zcomp_stream_put(zram->comp); + pr_err("Compression failed! err=%d\n", ret); + zs_free(zram->mem_pool, handle); + return ret; + } + + if (comp_len >= huge_class_size) + comp_len = PAGE_SIZE; + /* + * handle allocation has 2 paths: + * a) fast path is executed with preemption disabled (for + * per-cpu streams) and has __GFP_DIRECT_RECLAIM bit clear, + * since we can't sleep; + * b) slow path enables preemption and attempts to allocate + * the page with __GFP_DIRECT_RECLAIM bit set. we have to + * put per-cpu compression stream and, thus, to re-do + * the compression once handle is allocated. + * + * if we have a 'non-null' handle here then we are coming + * from the slow path and handle has already been allocated. + */ + if (IS_ERR_VALUE(handle)) + handle = zs_malloc(zram->mem_pool, comp_len, + __GFP_KSWAPD_RECLAIM | + __GFP_NOWARN | + __GFP_HIGHMEM | + __GFP_MOVABLE); + if (IS_ERR_VALUE(handle)) { + zcomp_stream_put(zram->comp); + atomic64_inc(&zram->stats.writestall); + handle = zs_malloc(zram->mem_pool, comp_len, + GFP_NOIO | __GFP_HIGHMEM | + __GFP_MOVABLE); + if (IS_ERR_VALUE(handle)) + return PTR_ERR((void *)handle); + + if (comp_len != PAGE_SIZE) + goto compress_again; + /* + * If the page is not compressible, you need to acquire the + * lock and execute the code below. The zcomp_stream_get() + * call is needed to disable the cpu hotplug and grab the + * zstrm buffer back. It is necessary that the dereferencing + * of the zstrm variable below occurs correctly. + */ + zstrm = zcomp_stream_get(zram->comp); + } + + dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO); + src = zstrm->buffer; + if (comp_len == PAGE_SIZE) + src = kmap_atomic(page); + memcpy(dst, src, comp_len); + if (comp_len == PAGE_SIZE) + kunmap_atomic(src); + + zcomp_stream_put(zram->comp); + zs_unmap_object(zram->mem_pool, handle); + atomic64_add(comp_len, &zram->stats.compr_data_size); +out: + /* + * Free memory associated with this sector + * before overwriting unused sectors. + */ + zram_slot_lock(zram, index); + zram_free_page(zram, index); + + if (comp_len == PAGE_SIZE) { + zram_set_flag(zram, index, ZRAM_HUGE); + atomic64_inc(&zram->stats.huge_pages); + atomic64_inc(&zram->stats.huge_pages_since); + } + + if (flags) { + zram_set_flag(zram, index, flags); + zram_set_element(zram, index, element); + } else { + zram_set_handle(zram, index, handle); + zram_set_obj_size(zram, index, comp_len); + } + zram_slot_unlock(zram, index); + + /* Update stats */ + atomic64_inc(&zram->stats.pages_stored); + return ret; +} + +/* + * This is a partial IO. Read the full page before writing the changes. + */ +static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, + u32 index, int offset, struct bio *bio) +{ + struct page *page = alloc_page(GFP_NOIO); + int ret; + + if (!page) + return -ENOMEM; + + ret = zram_read_page(zram, page, index, bio); + if (!ret) { + memcpy_from_bvec(page_address(page) + offset, bvec); + ret = zram_write_page(zram, page, index); + } + __free_page(page); + return ret; +} + +static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, + u32 index, int offset, struct bio *bio) +{ + if (is_partial_io(bvec)) + return zram_bvec_write_partial(zram, bvec, index, offset, bio); + return zram_write_page(zram, bvec->bv_page, index); +} + +static void zram_bio_discard(struct zram *zram, struct bio *bio) +{ + size_t n = bio->bi_iter.bi_size; + u32 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + u32 offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) << + SECTOR_SHIFT; + + /* + * zram manages data in physical block size units. Because logical block + * size isn't identical with physical block size on some arch, we + * could get a discard request pointing to a specific offset within a + * certain physical block. Although we can handle this request by + * reading that physiclal block and decompressing and partially zeroing + * and re-compressing and then re-storing it, this isn't reasonable + * because our intent with a discard request is to save memory. So + * skipping this logical block is appropriate here. + */ + if (offset) { + if (n <= (PAGE_SIZE - offset)) + return; + + n -= (PAGE_SIZE - offset); + index++; + } + + while (n >= PAGE_SIZE) { + zram_slot_lock(zram, index); + zram_free_page(zram, index); + zram_slot_unlock(zram, index); + atomic64_inc(&zram->stats.notify_free); + index++; + n -= PAGE_SIZE; + } + + bio_endio(bio); +} + +static void zram_bio_read(struct zram *zram, struct bio *bio) +{ + unsigned long start_time = bio_start_io_acct(bio); + struct bvec_iter iter = bio->bi_iter; + + do { + u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << + SECTOR_SHIFT; + struct bio_vec bv = bio_iter_iovec(bio, iter); + + bv.bv_len = min_t(u32, bv.bv_len, PAGE_SIZE - offset); + + if (zram_bvec_read(zram, &bv, index, offset, bio) < 0) { + atomic64_inc(&zram->stats.failed_reads); + bio->bi_status = BLK_STS_IOERR; + break; + } + flush_dcache_page(bv.bv_page); + bio_advance_iter_single(bio, &iter, bv.bv_len); + } while (iter.bi_size); + + bio_end_io_acct(bio, start_time); + bio_endio(bio); +} + +static void zram_bio_write(struct zram *zram, struct bio *bio) +{ + unsigned long start_time = bio_start_io_acct(bio); + struct bvec_iter iter = bio->bi_iter; + + do { + u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << + SECTOR_SHIFT; + struct bio_vec bv = bio_iter_iovec(bio, iter); + + bv.bv_len = min_t(u32, bv.bv_len, PAGE_SIZE - offset); + + if (zram_bvec_write(zram, &bv, index, offset, bio) < 0) { + atomic64_inc(&zram->stats.failed_writes); + bio->bi_status = BLK_STS_IOERR; + break; + } + bio_advance_iter_single(bio, &iter, bv.bv_len); + } while (iter.bi_size); + + bio_end_io_acct(bio, start_time); + bio_endio(bio); +} + +/* + * Handler function for all zram I/O requests. + */ +static void zram_submit_bio(struct bio *bio) +{ + struct zram *zram = bio->bi_bdev->bd_disk->private_data; + + switch (bio_op(bio)) { + case REQ_OP_READ: + zram_bio_read(zram, bio); + break; + case REQ_OP_WRITE: + zram_bio_write(zram, bio); + break; + case REQ_OP_DISCARD: + case REQ_OP_WRITE_ZEROES: + zram_bio_discard(zram, bio); + break; + default: + WARN_ON_ONCE(1); + bio_endio(bio); + } +} + +static void zram_slot_free_notify(struct block_device *bdev, + unsigned long index) +{ + struct zram *zram; + + zram = bdev->bd_disk->private_data; + + atomic64_inc(&zram->stats.notify_free); + if (!zram_slot_trylock(zram, index)) { + atomic64_inc(&zram->stats.miss_free); + return; + } + + zram_free_page(zram, index); + zram_slot_unlock(zram, index); +} + +static void zram_reset_device(struct zram *zram) +{ + down_write(&zram->init_lock); + + set_capacity_and_notify(zram->disk, 0); + part_stat_set_all(zram->disk->part0, 0); + + /* I/O operation under all of CPU are done so let's free */ + zram_meta_free(zram, zram->disksize); + zram->disksize = 0; + if (zram->comp) { + zcomp_destroy(zram->comp); + zram->comp = NULL; + } + memset(&zram->stats, 0, sizeof(zram->stats)); + strscpy(zram->compressor, default_compressor, sizeof(zram->compressor)); + up_write(&zram->init_lock); +} + +static ssize_t disksize_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t len) +{ + u64 disksize; + struct zcomp *comp; + struct zram *zram = dev_to_zram(dev); + int err; + + disksize = memparse(buf, NULL); + if (!disksize) + return -EINVAL; + + down_write(&zram->init_lock); + if (init_done(zram)) { + pr_info("Cannot change disksize for initialized device\n"); + err = -EBUSY; + goto out_unlock; + } + + disksize = PAGE_ALIGN(disksize); + if (!zram_meta_alloc(zram, disksize)) { + err = -ENOMEM; + goto out_unlock; + } + + comp = zcomp_create(zram->compressor); + if (IS_ERR(comp)) { + pr_err("Cannot initialise %s compressing backend\n", + zram->compressor); + err = PTR_ERR(comp); + goto out_free_meta; + } + + zram->comp = comp; + zram->disksize = disksize; + set_capacity_and_notify(zram->disk, zram->disksize >> SECTOR_SHIFT); + up_write(&zram->init_lock); + + return len; + +out_free_meta: + zram_meta_free(zram, disksize); +out_unlock: + up_write(&zram->init_lock); + return err; +} + +static ssize_t reset_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t len) +{ + int ret; + unsigned short do_reset; + struct zram *zram; + struct gendisk *disk; + + ret = kstrtou16(buf, 10, &do_reset); + if (ret) + return ret; + + if (!do_reset) + return -EINVAL; + + zram = dev_to_zram(dev); + disk = zram->disk; + + mutex_lock(&disk->open_mutex); + /* Do not reset an active device or claimed device */ + if (disk_openers(disk) || zram->claim) { + mutex_unlock(&disk->open_mutex); + return -EBUSY; + } + + /* From now on, anyone can't open /dev/zram[0-9] */ + zram->claim = true; + mutex_unlock(&disk->open_mutex); + + /* Make sure all the pending I/O are finished */ + sync_blockdev(disk->part0); + zram_reset_device(zram); + + mutex_lock(&disk->open_mutex); + zram->claim = false; + mutex_unlock(&disk->open_mutex); + + return len; +} + +static int zram_open(struct gendisk *disk, blk_mode_t mode) +{ + struct zram *zram = disk->private_data; + + WARN_ON(!mutex_is_locked(&disk->open_mutex)); + + /* zram was claimed to reset so open request fails */ + if (zram->claim) + return -EBUSY; + return 0; +} + +static const struct block_device_operations zram_devops = { + .open = zram_open, + .submit_bio = zram_submit_bio, + .swap_slot_free_notify = zram_slot_free_notify, + .owner = THIS_MODULE +}; + +static DEVICE_ATTR_RW(disksize); +static DEVICE_ATTR_RO(initstate); +static DEVICE_ATTR_WO(reset); +static DEVICE_ATTR_RW(comp_algorithm); + +static struct attribute *zram_disk_attrs[] = { + &dev_attr_disksize.attr, + &dev_attr_initstate.attr, + &dev_attr_reset.attr, + &dev_attr_comp_algorithm.attr, + &dev_attr_io_stat.attr, + &dev_attr_debug_stat.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(zram_disk); + +/* + * Allocate and initialize new zram device. the function returns + * '>= 0' device_id upon success, and negative value otherwise. + */ +static int zram_add(void) +{ + struct zram *zram; + int ret, device_id; + + zram = kzalloc(sizeof(struct zram), GFP_KERNEL); + if (!zram) + return -ENOMEM; + + ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL); + if (ret < 0) + goto out_free_dev; + device_id = ret; + + init_rwsem(&zram->init_lock); + + /* gendisk structure */ + zram->disk = blk_alloc_disk(NUMA_NO_NODE); + if (!zram->disk) { + pr_err("Error allocating disk structure for device %d\n", + device_id); + ret = -ENOMEM; + goto out_free_idr; + } + + zram->disk->major = zram_major; + zram->disk->first_minor = device_id; + zram->disk->minors = 1; + zram->disk->flags |= GENHD_FL_NO_PART; + zram->disk->fops = &zram_devops; + zram->disk->private_data = zram; + snprintf(zram->disk->disk_name, 16, "zram%d", device_id); + strscpy(zram->compressor, default_compressor, sizeof(zram->compressor)); + + /* Actual capacity set using sysfs (/sys/block/zram<id>/disksize */ + set_capacity(zram->disk, 0); + /* zram devices sort of resembles non-rotational disks */ + blk_queue_flag_set(QUEUE_FLAG_NONROT, zram->disk->queue); + blk_queue_flag_set(QUEUE_FLAG_SYNCHRONOUS, zram->disk->queue); + + /* + * To ensure that we always get PAGE_SIZE aligned + * and n*PAGE_SIZED sized I/O requests. + */ + blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE); + blk_queue_logical_block_size(zram->disk->queue, + ZRAM_LOGICAL_BLOCK_SIZE); + blk_queue_io_min(zram->disk->queue, PAGE_SIZE); + blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); + zram->disk->queue->limits.discard_granularity = PAGE_SIZE; + blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); + + /* + * zram_bio_discard() will clear all logical blocks if logical block + * size is identical with physical block size(PAGE_SIZE). But if it is + * different, we will skip discarding some parts of logical blocks in + * the part of the request range which isn't aligned to physical block + * size. So we can't ensure that all discarded logical blocks are + * zeroed. + */ + if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE) + blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX); + + blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue); + ret = device_add_disk(NULL, zram->disk, zram_disk_groups); + if (ret) + goto out_cleanup_disk; + + pr_info("Added device: %s\n", zram->disk->disk_name); + return device_id; + +out_cleanup_disk: + put_disk(zram->disk); +out_free_idr: + idr_remove(&zram_index_idr, device_id); +out_free_dev: + kfree(zram); + return ret; +} + +static int zram_remove(struct zram *zram) +{ + bool claimed; + + mutex_lock(&zram->disk->open_mutex); + if (disk_openers(zram->disk)) { + mutex_unlock(&zram->disk->open_mutex); + return -EBUSY; + } + + claimed = zram->claim; + if (!claimed) + zram->claim = true; + mutex_unlock(&zram->disk->open_mutex); + + if (claimed) { + /* + * If we were claimed by reset_store(), del_gendisk() will + * wait until reset_store() is done, so nothing need to do. + */ + ; + } else { + /* Make sure all the pending I/O are finished */ + sync_blockdev(zram->disk->part0); + zram_reset_device(zram); + } + + pr_info("Removed device: %s\n", zram->disk->disk_name); + + del_gendisk(zram->disk); + + /* del_gendisk drains pending reset_store */ + WARN_ON_ONCE(claimed && zram->claim); + + /* + * disksize_store() may be called in between zram_reset_device() + * and del_gendisk(), so run the last reset to avoid leaking + * anything allocated with disksize_store() + */ + zram_reset_device(zram); + + put_disk(zram->disk); + kfree(zram); + return 0; +} + +/* zram-control sysfs attributes */ + +/* + * NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a + * sense that reading from this file does alter the state of your system -- it + * creates a new un-initialized zram device and returns back this device's + * device_id (or an error code if it fails to create a new device). + */ +static ssize_t hot_add_show(const struct class *class, + const struct class_attribute *attr, + char *buf) +{ + int ret; + + mutex_lock(&zram_index_mutex); + ret = zram_add(); + mutex_unlock(&zram_index_mutex); + + if (ret < 0) + return ret; + return scnprintf(buf, PAGE_SIZE, "%d\n", ret); +} +/* This attribute must be set to 0400, so CLASS_ATTR_RO() can not be used */ +static struct class_attribute class_attr_hot_add = + __ATTR(hot_add, 0400, hot_add_show, NULL); + +static ssize_t hot_remove_store(const struct class *class, + const struct class_attribute *attr, + const char *buf, + size_t count) +{ + struct zram *zram; + int ret, dev_id; + + /* dev_id is gendisk->first_minor, which is `int' */ + ret = kstrtoint(buf, 10, &dev_id); + if (ret) + return ret; + if (dev_id < 0) + return -EINVAL; + + mutex_lock(&zram_index_mutex); + + zram = idr_find(&zram_index_idr, dev_id); + if (zram) { + ret = zram_remove(zram); + if (!ret) + idr_remove(&zram_index_idr, dev_id); + } else { + ret = -ENODEV; + } + + mutex_unlock(&zram_index_mutex); + return ret ? ret : count; +} +static CLASS_ATTR_WO(hot_remove); + +static struct attribute *zram_control_class_attrs[] = { + &class_attr_hot_add.attr, + &class_attr_hot_remove.attr, + NULL, +}; +ATTRIBUTE_GROUPS(zram_control_class); + +static struct class zram_control_class = { + .name = "zram-control", + .class_groups = zram_control_class_groups, +}; + +static int zram_remove_cb(int id, void *ptr, void *data) +{ + WARN_ON_ONCE(zram_remove(ptr)); + return 0; +} + +static void destroy_devices(void) +{ + class_unregister(&zram_control_class); + idr_for_each(&zram_index_idr, &zram_remove_cb, NULL); + idr_destroy(&zram_index_idr); + unregister_blkdev(zram_major, "zram"); + cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE); +} + +static int __init zram_init(void) +{ + int ret; + + BUILD_BUG_ON(__NR_ZRAM_PAGEFLAGS > BITS_PER_LONG); + + ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare", + zcomp_cpu_up_prepare, zcomp_cpu_dead); + if (ret < 0) + return ret; + + ret = class_register(&zram_control_class); + if (ret) { + pr_err("Unable to register zram-control class\n"); + cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE); + return ret; + } + + zram_major = register_blkdev(0, "zram"); + if (zram_major <= 0) { + pr_err("Unable to get major number\n"); + class_unregister(&zram_control_class); + cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE); + return -EBUSY; + } + + while (num_devices != 0) { + mutex_lock(&zram_index_mutex); + ret = zram_add(); + mutex_unlock(&zram_index_mutex); + if (ret < 0) + goto out_error; + num_devices--; + } + + return 0; + +out_error: + destroy_devices(); + return ret; +} + +static void __exit zram_exit(void) +{ + destroy_devices(); +} + +module_init(zram_init); +module_exit(zram_exit); + +module_param(num_devices, uint, 0); +MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices"); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>"); +MODULE_DESCRIPTION("Compressed RAM Block Device with multiple zspool support"); diff --git a/drivers/block/multi_pool_zram/multi_pool_zram_drv.h b/drivers/block/multi_pool_zram/multi_pool_zram_drv.h new file mode 100644 index 000000000000..e320f395d121 --- /dev/null +++ b/drivers/block/multi_pool_zram/multi_pool_zram_drv.h @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Compressed RAM block device with multiple zspool support + */ + +#ifndef _ZRAM_DRV_H_ +#define _ZRAM_DRV_H_ + +#include <linux/rwsem.h> +#include <linux/zsmalloc.h> +#include <linux/crypto.h> +#include <linux/memcontrol.h> + +#include "zcomp.h" + +#define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) +#define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT) +#define ZRAM_LOGICAL_BLOCK_SHIFT 12 +#define ZRAM_LOGICAL_BLOCK_SIZE (1 << ZRAM_LOGICAL_BLOCK_SHIFT) +#define ZRAM_SECTOR_PER_LOGICAL_BLOCK \ + (1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT)) + + +/* + * ZRAM is mainly used for memory efficiency so we want to keep memory + * footprint small and thus squeeze size and zram pageflags into a flags + * member. The lower ZRAM_FLAG_SHIFT bits is for object size (excluding + * header), which cannot be larger than PAGE_SIZE (requiring PAGE_SHIFT + * bits), the higher bits are for zram_pageflags. + * + * We use BUILD_BUG_ON() to make sure that zram pageflags don't overflow. + */ +#define ZRAM_FLAG_SHIFT (PAGE_SHIFT + 1) + +/* Flags for zram pages (table[page_no].flags) */ +enum zram_pageflags { + /* zram slot is locked */ + ZRAM_LOCK = ZRAM_FLAG_SHIFT, + ZRAM_SAME, /* Page consists the same element */ + ZRAM_HUGE, /* Incompressible page */ + + __NR_ZRAM_PAGEFLAGS, +}; + +/*-- Data structures */ + +/* Allocated for each disk page */ +struct zram_table_entry { + union { + unsigned long handle; + unsigned long element; + }; + unsigned long flags; +}; + +struct zram_stats { + atomic64_t compr_data_size; /* compressed size of pages stored */ + atomic64_t failed_reads; /* can happen when memory is too low */ + atomic64_t failed_writes; /* can happen when memory is too low */ + atomic64_t notify_free; /* no. of swap slot free notifications */ + atomic64_t same_pages; /* no. of same element filled pages */ + atomic64_t huge_pages; /* no. of huge pages */ + atomic64_t huge_pages_since; /* no. of huge pages since zram set up */ + atomic64_t pages_stored; /* no. of pages currently stored */ + atomic64_t writestall; /* no. of write slow paths */ + atomic64_t miss_free; /* no. of missed free */ +}; + +struct zram { + struct zram_table_entry *table; + struct zs_pool *mem_pool; + struct zcomp *comp; + struct gendisk *disk; + /* Prevent concurrent execution of device init */ + struct rw_semaphore init_lock; + + struct zram_stats stats; + /* + * This is the limit on amount of *uncompressed* worth of data + * we can store in a disk. + */ + u64 disksize; /* bytes */ + char compressor[CRYPTO_MAX_ALG_NAME]; + /* + * zram is claimed so open request will be failed + */ + bool claim; /* Protected by disk->open_mutex */ +}; +#endif diff --git a/drivers/block/multi_pool_zram/zcomp.c b/drivers/block/multi_pool_zram/zcomp.c new file mode 100644 index 000000000000..55af4efd7983 --- /dev/null +++ b/drivers/block/multi_pool_zram/zcomp.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2014 Sergey Senozhatsky. + */ + +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/wait.h> +#include <linux/sched.h> +#include <linux/cpu.h> +#include <linux/crypto.h> + +#include "zcomp.h" + +static const char * const backends[] = { +#if IS_ENABLED(CONFIG_CRYPTO_LZO) + "lzo", + "lzo-rle", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4) + "lz4", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) + "lz4hc", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_842) + "842", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_ZSTD) + "zstd", +#endif +}; + +static void zcomp_strm_free(struct zcomp_strm *zstrm) +{ + if (!IS_ERR_OR_NULL(zstrm->tfm)) + crypto_free_comp(zstrm->tfm); + free_pages((unsigned long)zstrm->buffer, 1); + zstrm->tfm = NULL; + zstrm->buffer = NULL; +} + +/* + * Initialize zcomp_strm structure with ->tfm initialized by backend, and + * ->buffer. Return a negative value on error. + */ +static int zcomp_strm_init(struct zcomp_strm *zstrm, struct zcomp *comp) +{ + zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); + /* + * allocate 2 pages. 1 for compressed data, plus 1 extra for the + * case when compressed size is larger than the original one + */ + zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { + zcomp_strm_free(zstrm); + return -ENOMEM; + } + return 0; +} + +bool zcomp_available_algorithm(const char *comp) +{ + /* + * Crypto does not ignore a trailing new line symbol, + * so make sure you don't supply a string containing + * one. + * This also means that we permit zcomp initialisation + * with any compressing algorithm known to crypto api. + */ + return crypto_has_comp(comp, 0, 0) == 1; +} + +/* show available compressors */ +ssize_t zcomp_available_show(const char *comp, char *buf) +{ + bool known_algorithm = false; + ssize_t sz = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(backends); i++) { + if (!strcmp(comp, backends[i])) { + known_algorithm = true; + sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, + "[%s] ", backends[i]); + } else { + sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, + "%s ", backends[i]); + } + } + + /* + * Out-of-tree module known to crypto api or a missing + * entry in `backends'. + */ + if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) + sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, + "[%s] ", comp); + + sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); + return sz; +} + +struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) +{ + local_lock(&comp->stream->lock); + return this_cpu_ptr(comp->stream); +} + +void zcomp_stream_put(struct zcomp *comp) +{ + local_unlock(&comp->stream->lock); +} + +int zcomp_compress(struct zcomp_strm *zstrm, + const void *src, unsigned int *dst_len) +{ + /* + * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized + * because sometimes we can endup having a bigger compressed data + * due to various reasons: for example compression algorithms tend + * to add some padding to the compressed buffer. Speaking of padding, + * comp algorithm `842' pads the compressed length to multiple of 8 + * and returns -ENOSP when the dst memory is not big enough, which + * is not something that ZRAM wants to see. We can handle the + * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we + * receive -ERRNO from the compressing backend we can't help it + * anymore. To make `842' happy we need to tell the exact size of + * the dst buffer, zram_drv will take care of the fact that + * compressed buffer is too big. + */ + *dst_len = PAGE_SIZE * 2; + + return crypto_comp_compress(zstrm->tfm, + src, PAGE_SIZE, + zstrm->buffer, dst_len); +} + +int zcomp_decompress(struct zcomp_strm *zstrm, + const void *src, unsigned int src_len, void *dst) +{ + unsigned int dst_len = PAGE_SIZE; + + return crypto_comp_decompress(zstrm->tfm, + src, src_len, + dst, &dst_len); +} + +int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) +{ + struct zcomp *comp = hlist_entry(node, struct zcomp, node); + struct zcomp_strm *zstrm; + int ret; + + zstrm = per_cpu_ptr(comp->stream, cpu); + local_lock_init(&zstrm->lock); + + ret = zcomp_strm_init(zstrm, comp); + if (ret) + pr_err("Can't allocate a compression stream\n"); + return ret; +} + +int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) +{ + struct zcomp *comp = hlist_entry(node, struct zcomp, node); + struct zcomp_strm *zstrm; + + zstrm = per_cpu_ptr(comp->stream, cpu); + zcomp_strm_free(zstrm); + return 0; +} + +static int zcomp_init(struct zcomp *comp) +{ + int ret; + + comp->stream = alloc_percpu(struct zcomp_strm); + if (!comp->stream) + return -ENOMEM; + + ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node); + if (ret < 0) + goto cleanup; + return 0; + +cleanup: + free_percpu(comp->stream); + return ret; +} + +void zcomp_destroy(struct zcomp *comp) +{ + cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node); + free_percpu(comp->stream); + kfree(comp); +} + +/* + * search available compressors for requested algorithm. + * allocate new zcomp and initialize it. return compressing + * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) + * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in + * case of allocation error, or any other error potentially + * returned by zcomp_init(). + */ +struct zcomp *zcomp_create(const char *alg) +{ + struct zcomp *comp; + int error; + + /* + * Crypto API will execute /sbin/modprobe if the compression module + * is not loaded yet. We must do it here, otherwise we are about to + * call /sbin/modprobe under CPU hot-plug lock. + */ + if (!zcomp_available_algorithm(alg)) + return ERR_PTR(-EINVAL); + + comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); + if (!comp) + return ERR_PTR(-ENOMEM); + + comp->name = alg; + error = zcomp_init(comp); + if (error) { + kfree(comp); + return ERR_PTR(error); + } + return comp; +} diff --git a/drivers/block/multi_pool_zram/zcomp.h b/drivers/block/multi_pool_zram/zcomp.h new file mode 100644 index 000000000000..cdefdef93da8 --- /dev/null +++ b/drivers/block/multi_pool_zram/zcomp.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2014 Sergey Senozhatsky. + */ + +#ifndef _ZCOMP_H_ +#define _ZCOMP_H_ +#include <linux/local_lock.h> + +struct zcomp_strm { + /* The members ->buffer and ->tfm are protected by ->lock. */ + local_lock_t lock; + /* compression/decompression buffer */ + void *buffer; + struct crypto_comp *tfm; +}; + +/* dynamic per-device compression frontend */ +struct zcomp { + struct zcomp_strm __percpu *stream; + const char *name; + struct hlist_node node; +}; + +int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node); +int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node); +ssize_t zcomp_available_show(const char *comp, char *buf); +bool zcomp_available_algorithm(const char *comp); + +struct zcomp *zcomp_create(const char *alg); +void zcomp_destroy(struct zcomp *comp); + +struct zcomp_strm *zcomp_stream_get(struct zcomp *comp); +void zcomp_stream_put(struct zcomp *comp); + +int zcomp_compress(struct zcomp_strm *zstrm, + const void *src, unsigned int *dst_len); + +int zcomp_decompress(struct zcomp_strm *zstrm, + const void *src, unsigned int src_len, void *dst); + +bool zcomp_set_max_streams(struct zcomp *comp, int num_strm); +#endif /* _ZCOMP_H_ */ -- 2.43.0