From: Wang Wensheng wangwensheng4@huawei.com

We provide a char device and the user could use ioctl in userspace to access the method supported by kernel.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- include/uapi/linux/xshmem_framework.h | 33 +++ lib/Kconfig | 7 + lib/Makefile | 1 + lib/xshmem/Makefile | 1 + lib/xshmem/xshmem_framework.c | 514 ++++++++++++++++++++++++++++++++++ lib/xshmem/xshmem_framework.h | 46 +++ 6 files changed, 602 insertions(+) create mode 100644 include/uapi/linux/xshmem_framework.h create mode 100644 lib/xshmem/Makefile create mode 100644 lib/xshmem/xshmem_framework.c create mode 100644 lib/xshmem/xshmem_framework.h

diff --git a/include/uapi/linux/xshmem_framework.h b/include/uapi/linux/xshmem_framework.h new file mode 100644 index 0000000..caf782b --- /dev/null +++ b/include/uapi/linux/xshmem_framework.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2021. All rights reserved. + * Author: Huawei OS Kernel Lab + * Create: Thu Jun 17 03:19:19 2021 + */ +#ifndef __USR_XSHMEM_FRAMEWORK_H +#define __USR_XSHMEM_FRAMEWORK_H + +#define XSHM_KEY_SIZE 253U + +struct xshm_reg_arg { + int algo; + unsigned int pool_size; + unsigned int key_len; + char *key; +}; + +struct xshm_alloc_arg { + int pool_id; + int size; +}; + +struct xshm_free_arg { + int pool_id; + unsigned int offset; +}; + +#define XSHMEM_POOL_REGISTER _IOW('X', 1, struct xshm_reg_arg) +#define XSHMEM_POOL_UNREGISTER _IO ('X', 2) +#define XSHMEM_POOL_ALLOC _IOW('X', 3, struct xshm_alloc_arg) +#define XSHMEM_POOL_FREE _IOW('X', 4, struct xshm_free_arg) + +#endif diff --git a/lib/Kconfig b/lib/Kconfig index edb7d40..070b3df 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -632,3 +632,10 @@ config GENERIC_LIB_CMPDI2

config GENERIC_LIB_UCMPDI2 bool + +config XSHMEM_FRAMEWORK + tristate "A region-manage mechanism in kernel" + help + Supply a region-manage mechanism in kernel. The user can register a + region and alloc/get/put/delete BLOCKs from the region via ioctl of + the associated chardev. diff --git a/lib/Makefile b/lib/Makefile index f5ee8e8..57a7897 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -285,3 +285,4 @@ obj-$(CONFIG_GENERIC_LIB_LSHRDI3) += lshrdi3.o obj-$(CONFIG_GENERIC_LIB_MULDI3) += muldi3.o obj-$(CONFIG_GENERIC_LIB_CMPDI2) += cmpdi2.o obj-$(CONFIG_GENERIC_LIB_UCMPDI2) += ucmpdi2.o +obj-$(CONFIG_XSHMEM_FRAMEWORK) += xshmem/ diff --git a/lib/xshmem/Makefile b/lib/xshmem/Makefile new file mode 100644 index 0000000..a60db51 --- /dev/null +++ b/lib/xshmem/Makefile @@ -0,0 +1 @@ +obj-m+=xshmem_framework.o diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c new file mode 100644 index 0000000..7209cc0 --- /dev/null +++ b/lib/xshmem/xshmem_framework.c @@ -0,0 +1,514 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2021. All rights reserved. + * Author: Huawei OS Kernel Lab + * Create: Tue Jun 15 06:34:28 2021 + */ +#define pr_fmt(fmt) "XSHMEM: " fmt + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/mutex.h> +#include <linux/idr.h> +#include <linux/fs.h> +#include <linux/hashtable.h> +#include <linux/miscdevice.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/jhash.h> +#include <linux/rcupdate.h> + +#include "xshmem_framework.h" + +#define XSHM_HLIST_TABLE_BIT 13 +#define XSHMEM_MAX_ID (1 << 14) + +/* Protect the lifetime for all POOL */ +static DEFINE_MUTEX(xshmem_mutex); +static DEFINE_IDR(xshmem_idr); +static DEFINE_HASHTABLE(xshmem_key_list, XSHM_HLIST_TABLE_BIT); +static int next_xshmem_id; + +static struct miscdevice xshmem_dev; + +/* FIXME: use lock to portect the list_head */ +static LIST_HEAD(registered_algo); + +static struct xshm_pool_algo *xshmem_find_algo(int algo) +{ + struct xshm_pool_algo *tmp; + + list_for_each_entry(tmp, &registered_algo, algo_node) + if (tmp->num == algo) + return tmp; + + return NULL; +} + +static struct hlist_head *hash_bucket(char *key, int len) +{ + u32 bucket = jhash(key, len, 0); + return &xshmem_key_list[bucket & ((1U << XSHM_HLIST_TABLE_BIT) - 1U)]; +} + +struct xshm_task_pool_node { + struct list_head task_node; // list node in TASK + struct list_head pool_node; // list node in POOL + struct xshm_task *task; + struct xshm_pool *pool; +}; + +struct xshm_task { + pid_t pid; + int pool_count; + struct list_head list_head; + struct mutex task_mutex; +}; + +static int xshmem_open(struct inode *inode, struct file *file) +{ + struct xshm_task *ptask; + + ptask = kmalloc(sizeof(*ptask), GFP_KERNEL); + if (unlikely(!ptask)) + return -ENOMEM; + + ptask->pid = current->pid; + ptask->pool_count = 0; + INIT_LIST_HEAD(&ptask->list_head); + mutex_init(&ptask->task_mutex); + + file->private_data = ptask; + + return 0; +} + +static bool task_is_attached(struct xshm_task *task, struct xshm_pool *xp) +{ + struct xshm_task_pool_node *node; + + list_for_each_entry(node, &task->list_head, task_node) + if (node->pool == xp) + return true; + + return false; +} + +/* the refcnt of the xp should be increased by one on success */ +static int xshmem_pool_attach(struct xshm_task *task, struct xshm_pool *xp) +{ + struct xshm_task_pool_node *node; + + if (task_is_attached(task, xp)) { + pr_err("TASK has been attached already\n"); + return -EEXIST; + } + + node = kmalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) { + pr_err("alloc xshm_task_pool_node failed\n"); + return -ENOMEM; + } + + node->task = task; + node->pool = xp; + list_add_tail(&node->task_node, &task->list_head); + spin_lock(&xp->xp_task_spinlock); + list_add_tail(&node->pool_node, &xp->list_head); + spin_unlock(&xp->xp_task_spinlock); + + atomic_inc(&xp->refcnt); + task->pool_count++; + + return 0; +} + +static int xshmem_pool_detach(struct xshm_task *task, struct xshm_pool *xp) +{ + struct xshm_task_pool_node *node; + + list_for_each_entry(node, &task->list_head, task_node) + if (node->pool == xp) { + spin_lock(&xp->xp_task_spinlock); + list_del(&node->pool_node); + spin_unlock(&xp->xp_task_spinlock); + + list_del(&node->task_node); + /* Use xshmem_pool_put() ?? */ + atomic_dec(&xp->refcnt); + task->pool_count--; + kfree(node); + return 0; + } + + pr_err("the POOL has already been detached\n"); + return -ESRCH; +} + +/* + * get the POOL specified by key, create one if it not exist. + */ +static struct xshm_pool *xshmem_pool_create(char *key, unsigned int key_len, + struct xshm_pool_algo *algo, unsigned int pool_size) +{ + int id, ret; + struct xshm_pool *xp; + struct hlist_head *bucket; + + bucket = hash_bucket(key, key_len); + + mutex_lock(&xshmem_mutex); + hlist_for_each_entry(xp, bucket, hnode) + if (key_len == xp->key_len && !strncmp(key, xp->key, key_len)) { + if (xp->size != pool_size || xp->algo != algo) { + mutex_unlock(&xshmem_mutex); + pr_err("the pool size or algorithm invalid\n"); + return ERR_PTR(-EINVAL); + } else + goto exist; + } + + xp = kmalloc(sizeof(*xp) + key_len + 1, GFP_KERNEL); + if (unlikely(!xp)) { + mutex_unlock(&xshmem_mutex); + return ERR_PTR(-ENOMEM); + } + + xp->algo = algo; + xp->size = pool_size; + ret = algo->xshm_pool_init(xp); + if (ret < 0) { + mutex_unlock(&xshmem_mutex); + kfree(xp); + pr_err("init hook failed\n"); + return ERR_PTR(ret); + } + + id = idr_alloc(&xshmem_idr, xp, next_xshmem_id, XSHMEM_MAX_ID, GFP_KERNEL); + if (id < 0) { + if (next_xshmem_id) + id = idr_alloc(&xshmem_idr, xp, 0, next_xshmem_id, GFP_KERNEL); + if (id < 0) { + mutex_unlock(&xshmem_mutex); + algo->xshm_pool_free(xp); + kfree(xp); + pr_err("idr alloc failed\n"); + return ERR_PTR(id); + } + } + + next_xshmem_id = (id + 1) & (XSHMEM_MAX_ID - 1); + + xp->key_len = key_len; + xp->pool_id = id; + atomic_set(&xp->refcnt, 0); + INIT_LIST_HEAD(&xp->list_head); + strncpy(xp->key, key, key_len); + xp->key[key_len] = '\0'; + mutex_init(&xp->xp_block_mutex); + spin_lock_init(&xp->xp_task_spinlock); + + hlist_add_head(&xp->hnode, bucket); + +exist: + /* + * Here increase the POOL's refcnt by one, for the convenience of fallback in + * error branch in ioctl_xshmem_pool_register(). + */ + atomic_inc(&xp->refcnt); + mutex_unlock(&xshmem_mutex); + + return xp; +} + +static struct xshm_pool *xshmem_pool_get(int pool_id) +{ + struct xshm_pool *xp; + + mutex_lock(&xshmem_mutex); + xp = idr_find(&xshmem_idr, pool_id); + if (xp) + atomic_inc(&xp->refcnt); + mutex_unlock(&xshmem_mutex); + + return xp; +} + +static void xshmem_pool_put(struct xshm_pool *xp) +{ + mutex_lock(&xshmem_mutex); + if (!atomic_dec_and_test(&xp->refcnt)) { + mutex_unlock(&xshmem_mutex); + return; + } + + hlist_del(&xp->hnode); + idr_remove(&xshmem_idr, xp->pool_id); + mutex_unlock(&xshmem_mutex); + + xp->algo->xshm_pool_free(xp); + kfree(xp); +} + +static int ioctl_xshmem_pool_register(struct xshm_task *task, unsigned long arg) +{ + int ret; + struct xshm_pool *xp; + unsigned int key_len; + char key[XSHM_KEY_SIZE]; + struct xshm_reg_arg reg_arg; + struct xshm_pool_algo *algo; + + if (copy_from_user(&reg_arg, (struct xshmem_reg_arg __user *)arg, sizeof(reg_arg))) { + pr_err("copy_from_user failed\n"); + return -EFAULT; + } + + algo = xshmem_find_algo(reg_arg.algo); + if (unlikely(!algo)) { + pr_err("unsupported algorithm\n"); + return -ENODEV; + } + + key_len = reg_arg.key_len; + if (!key_len || key_len > XSHM_KEY_SIZE) + return -EINVAL; + + if (copy_from_user(key, (char __user *)reg_arg.key, key_len)) { + pr_err("copy key from user failed\n"); + return -EFAULT; + } + + xp = xshmem_pool_create(key, key_len, algo, reg_arg.pool_size); + if (IS_ERR(xp)) + return PTR_ERR(xp); + + ret = xshmem_pool_attach(task, xp); + xshmem_pool_put(xp); + + return ret < 0 ? ret : xp->pool_id; +} + +static int ioctl_xshmem_pool_unregister(struct xshm_task *task, unsigned long arg) +{ + int ret; + struct xshm_pool *xp; + + xp = xshmem_pool_get(arg); + if (!xp) { + pr_err("couldn't find the pool\n"); + return -ENODEV; + } + + ret = xshmem_pool_detach(task, xp); + + xshmem_pool_put(xp); + + return ret; +} + +static int ioctl_xshmem_pool_alloc(struct xshm_task *task, unsigned long arg) +{ + int ret; + struct xshm_pool *xp; + struct xshm_alloc_arg alloc_arg; + + if (copy_from_user(&alloc_arg, (struct xshm_alloc_arg __user*)arg, sizeof(alloc_arg))) { + pr_err("copy_from_user failed\n"); + return -EFAULT; + } + + xp = xshmem_pool_get(alloc_arg.pool_id); + if (!xp) { + pr_err("invalid pool_id\n"); + return -EINVAL; + } + + if (!task_is_attached(task, xp)) { + xshmem_pool_put(xp); + pr_err("TASK is not attached to POOL\n"); + return -EINVAL; + } + + mutex_lock(&xp->xp_block_mutex); + ret = xp->algo->xshm_block_alloc(xp, alloc_arg.size); + mutex_unlock(&xp->xp_block_mutex); + + xshmem_pool_put(xp); + + return ret; +} + +static int ioctl_xshmem_pool_free(struct xshm_task *task, unsigned long arg) +{ + int ret; + struct xshm_pool *xp; + struct xshm_free_arg free_arg; + + if (copy_from_user(&free_arg, (struct xshm_free_arg __user*)arg, sizeof(free_arg))) { + pr_err("copy_from_user failed\n"); + return -EFAULT; + } + + xp = xshmem_pool_get(free_arg.pool_id); + if (!xp) { + pr_err("invalid pool_id\n"); + return -EINVAL; + } + + if (!task_is_attached(task, xp)) { + xshmem_pool_put(xp); + pr_err("TASK is not attached to POOL\n"); + return -EINVAL; + } + + mutex_lock(&xp->xp_block_mutex); + ret = xp->algo->xshm_block_free(xp, free_arg.offset); + mutex_unlock(&xp->xp_block_mutex); + + xshmem_pool_put(xp); + + return ret; +} + +static long xshmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + int ret; + struct xshm_task *task = file->private_data; + + mutex_lock(&task->task_mutex); + switch(cmd) { + case XSHMEM_POOL_REGISTER: + ret = ioctl_xshmem_pool_register(task, arg); + break; + case XSHMEM_POOL_UNREGISTER: + ret = ioctl_xshmem_pool_unregister(task, arg); + break; + case XSHMEM_POOL_ALLOC: + ret = ioctl_xshmem_pool_alloc(task, arg); + break; + case XSHMEM_POOL_FREE: + ret = ioctl_xshmem_pool_free(task, arg); + break; + default: + mutex_unlock(&task->task_mutex); + pr_err("unsupported command\n"); + return -EINVAL; + } + mutex_unlock(&task->task_mutex); + + return ret; +} + +static int xshmem_release(struct inode *inode, struct file *file) +{ + struct xshm_task_pool_node *node, *tmp; + struct xshm_task *task = file->private_data; + + list_for_each_entry_safe(node, tmp, &task->list_head, task_node) { + list_del(&node->task_node); + spin_lock(&node->pool->xp_task_spinlock); + list_del(&node->pool_node); + spin_unlock(&node->pool->xp_task_spinlock); + xshmem_pool_put(node->pool); + kfree(node); + } + + kfree(task); + + return 0; +} + +static struct file_operations xshmem_fops = { + .owner = THIS_MODULE, + .open = xshmem_open, + .release = xshmem_release, + .unlocked_ioctl = xshmem_ioctl, +}; + +int xshmem_register_algo(struct xshm_pool_algo *algo) +{ + struct xshm_pool_algo *tmp; + + if (!algo || !algo->xshm_pool_init || !algo->xshm_pool_free || + !algo->xshm_block_alloc || !algo->xshm_block_free) + return -EINVAL; + + list_for_each_entry(tmp, &registered_algo, algo_node) + if (algo->num == tmp->num) { + pr_err("algorithm with the same id(%d,%s) has been registered\n", + tmp->num, tmp->name); + return -EEXIST; + } + + list_add_tail(&algo->algo_node, &registered_algo); + + pr_info("algo: no: %d, name: %s, registered\n", algo->num, algo->name); + + return 0; +} +EXPORT_SYMBOL_GPL(xshmem_register_algo); + +static int empty_algo_pool_init(struct xshm_pool *xp) +{ + pr_info("pool_init_hook: algo:%s, pool_size: %d\n", xp->algo->name, xp->size); + return 0; +} + +static int empty_algo_pool_free(struct xshm_pool *xp) +{ + pr_info("pool_free_hook: algo:%s, pool_size: %d, pool_id:%d\n", xp->algo->name, xp->size, xp->pool_id); + return 0; +} + +static int empty_algo_block_alloc(struct xshm_pool *xp, int size) +{ + pr_info("block_alloc_hook:pool_id:%d, alloc_size:%d\n", xp->pool_id, size); + return 0; +} + +static int empty_algo_block_free(struct xshm_pool *xp, int offset) +{ + pr_info("block_free_hook:pool_id:%d, offset:%d\n", xp->pool_id, offset); + return 0; +} + +/* Use just for test */ +static struct xshm_pool_algo empty_algo = { + .num = 0, + .name = "empty_algo", + .xshm_pool_init = empty_algo_pool_init, + .xshm_pool_free = empty_algo_pool_free, + .xshm_block_alloc = empty_algo_block_alloc, + .xshm_block_free = empty_algo_block_free, +}; + +static int __init xshmem_init(void) +{ + int ret; + + xshmem_dev.minor = MISC_DYNAMIC_MINOR; + xshmem_dev.name = "xshmem_dev"; + xshmem_dev.fops = &xshmem_fops; + + ret = misc_register(&xshmem_dev); + if (ret) { + pr_err("misc_register failed, %d\n", ret); + return ret; + } + + xshmem_register_algo(&empty_algo); + + return 0; +}; +module_init(xshmem_init); + +static void __exit xshmem_exit(void) +{ + misc_deregister(&xshmem_dev); + pr_info("module exit\n"); +} +module_exit(xshmem_exit); + +MODULE_AUTHOR("Wang Wensheng wangwensheng4@huawei.com"); +MODULE_LICENSE("GPL v2"); diff --git a/lib/xshmem/xshmem_framework.h b/lib/xshmem/xshmem_framework.h new file mode 100644 index 0000000..7308ce4 --- /dev/null +++ b/lib/xshmem/xshmem_framework.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2021. All rights reserved. + * Author: Huawei OS Kernel Lab + * Create: Tue Jun 15 06:49:07 2021 + */ +#ifndef __XSHMEM_FRAMEWORK_H +#define __XSHMEM_FRAMEWORK_H + +#include <linux/types.h> +#include <linux/xshmem_framework.h> + +struct xshm_pool { + int pool_id; + int size; + atomic_t refcnt; + + /* Used to protect the list of TASK attached */ + spinlock_t xp_task_spinlock; + struct list_head list_head; + struct hlist_node hnode; + + struct xshm_pool_algo *algo; + + /* Used to serialize the alloc and free operation on the POOL */ + struct mutex xp_block_mutex; + void *private; + + int key_len; + /* MUST be the last element */ + char key[0]; +}; + +#define ALGO_NAME_MAX 20 +struct xshm_pool_algo { + int num; + char name[ALGO_NAME_MAX]; + struct list_head algo_node; + int (*xshm_pool_init)(struct xshm_pool *xp); + int (*xshm_pool_free)(struct xshm_pool *xp); + int (*xshm_block_alloc)(struct xshm_pool *xp, int size); + int (*xshm_block_free)(struct xshm_pool *xp, int offset); +}; + +int xshmem_register_algo(struct xshm_pool_algo *algo); + +#endif

From: Wang Wensheng wangwensheng4@huawei.com

FSC algorithms orgainzes all free blocks in servial list depend on the size of the free blocks.

On allocation, we take the first block from the freelist whose size is bigger than the alloc size and split it.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- lib/xshmem/Makefile | 3 +- lib/xshmem/xshmem_framework.c | 7 +- lib/xshmem/xshmem_framework.h | 4 +- lib/xshmem/xshmem_fsc.c | 267 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 lib/xshmem/xshmem_fsc.c

diff --git a/lib/xshmem/Makefile b/lib/xshmem/Makefile index a60db51..e879f82 100644 --- a/lib/xshmem/Makefile +++ b/lib/xshmem/Makefile @@ -1 +1,2 @@ -obj-m+=xshmem_framework.o +obj-m+=xshmem.o +xshmem-objs+=xshmem_fsc.o xshmem_framework.o diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index 7209cc0..f6e0772 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -461,13 +461,13 @@ static int empty_algo_pool_free(struct xshm_pool *xp) return 0; }

-static int empty_algo_block_alloc(struct xshm_pool *xp, int size) +static int empty_algo_block_alloc(struct xshm_pool *xp, u32 size) { pr_info("block_alloc_hook:pool_id:%d, alloc_size:%d\n", xp->pool_id, size); return 0; }

-static int empty_algo_block_free(struct xshm_pool *xp, int offset) +static int empty_algo_block_free(struct xshm_pool *xp, u32 offset) { pr_info("block_free_hook:pool_id:%d, offset:%d\n", xp->pool_id, offset); return 0; @@ -483,6 +483,8 @@ static struct xshm_pool_algo empty_algo = { .xshm_block_free = empty_algo_block_free, };

+extern struct xshm_pool_algo fsc_algo; + static int __init xshmem_init(void) { int ret; @@ -497,6 +499,7 @@ static int __init xshmem_init(void) return ret; }

+ xshmem_register_algo(&fsc_algo); xshmem_register_algo(&empty_algo);

return 0; diff --git a/lib/xshmem/xshmem_framework.h b/lib/xshmem/xshmem_framework.h index 7308ce4..c781e95 100644 --- a/lib/xshmem/xshmem_framework.h +++ b/lib/xshmem/xshmem_framework.h @@ -37,8 +37,8 @@ struct xshm_pool_algo { struct list_head algo_node; int (*xshm_pool_init)(struct xshm_pool *xp); int (*xshm_pool_free)(struct xshm_pool *xp); - int (*xshm_block_alloc)(struct xshm_pool *xp, int size); - int (*xshm_block_free)(struct xshm_pool *xp, int offset); + int (*xshm_block_alloc)(struct xshm_pool *xp, u32 size); + int (*xshm_block_free)(struct xshm_pool *xp, u32 offset); };

int xshmem_register_algo(struct xshm_pool_algo *algo); diff --git a/lib/xshmem/xshmem_fsc.c b/lib/xshmem/xshmem_fsc.c new file mode 100644 index 0000000..bb01926 --- /dev/null +++ b/lib/xshmem/xshmem_fsc.c @@ -0,0 +1,267 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2021. All rights reserved. + * Author: Huawei OS Kernel Lab + * Create: Fri Jun 18 07:38:11 2021 + */ +#define pr_fmt(fmt) "XSHMEM_FSC: " fmt + +#include <linux/types.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/module.h> + +#include "xshmem_framework.h" + +#define XSHMEM_FSC_MAX_SHIFT 31U +#define XSHMEM_FSC_MIN_SHIFT 4U +#define XSHMEM_FSC_MAX_TYPE (XSHMEM_FSC_MAX_SHIFT - XSHMEM_FSC_MIN_SHIFT + 1) // exclude +#define XSHMEM_FSC_ALIGN (1U << XSHMEM_FSC_MIN_SHIFT) +#define XSHMEM_FSC_MAX_ALLOC_SIZE (1U << XSHMEM_FSC_MAX_SHIFT) // exclude + +/* The valid input size is above 0b10000. + * valid return is 0,1,...,MAX_FSC_MAX_TYPE - 1 */ +static int fsc_size2type(u32 size) +{ + return fls(size) - XSHMEM_FSC_MIN_SHIFT - 1; +} + +/* the block reagion is [start, start + size) */ +struct fsc_block { + bool is_free; + u32 start; + u32 size; + struct list_head block_node; + struct list_head free_node; +}; + +struct fsc_ctrl { + u32 total_size; + u32 free_size; + DECLARE_BITMAP(free_map, XSHMEM_FSC_MAX_TYPE); /* bit0~bit27 represent type0~type27 */ + struct list_head block_head; + struct list_head free_list[XSHMEM_FSC_MAX_TYPE]; +}; + +/* insert the block to proper free_list */ +static void fsc_insert_block(struct fsc_ctrl *ctrl, struct fsc_block *block) +{ + int type = fsc_size2type(block->size); + + block->is_free = true; + list_add_tail(&block->free_node, &ctrl->free_list[type]); + ctrl->free_size += block->size; + set_bit(type, ctrl->free_map); +} + +/* delete the block from free_list */ +static void fsc_delete_block(struct fsc_ctrl *ctrl, struct fsc_block *block) +{ + int type = fsc_size2type(block->size); + + block->is_free = false; + list_del(&block->free_node); + ctrl->free_size -= block->size; + if (list_empty(&ctrl->free_list[type])) + clear_bit(type, ctrl->free_map); +} + +static int fsc_algo_pool_init(struct xshm_pool *xp) +{ + int i; + struct fsc_ctrl *ctrl; + struct fsc_block *block; + + if (!IS_ALIGNED(xp->size, XSHMEM_FSC_ALIGN) || !xp->size) { + pr_err("input xshm_pool size invalid\n"); + return -EINVAL; + } + + ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); + block = kmalloc(sizeof(*block), GFP_KERNEL); + if (!ctrl || !block) { + kfree(ctrl); + kfree(block); + return -ENOMEM; + } + + ctrl->total_size = xp->size; + ctrl->free_size = 0; + INIT_LIST_HEAD(&ctrl->block_head); + for (i = 0; i < ARRAY_SIZE(ctrl->free_list); i++) + INIT_LIST_HEAD(&ctrl->free_list[i]); + + block->start = 0; + block->size = xp->size; + list_add_tail(&block->block_node, &ctrl->block_head); + fsc_insert_block(ctrl, block); + + xp->private = ctrl; + + return 0; +} + +static int fsc_algo_pool_free(struct xshm_pool *xp) +{ + struct fsc_block *block, *tmp; + struct fsc_ctrl *ctrl = xp->private; + + WARN(ctrl->total_size != ctrl->free_size, "The alloced block are not all free\n"); + + list_for_each_entry_safe(block, tmp, &ctrl->block_head, block_node) { + list_del(&block->block_node); + kfree(block); + } + + kfree(ctrl); + + return 0; +} + +static struct fsc_block *find_free_block(struct fsc_ctrl *ctrl, u32 size) +{ + u32 type; + struct fsc_block *block; + + type = fsc_size2type(size); + type = find_next_bit(ctrl->free_map, XSHMEM_FSC_MAX_TYPE, type + 1); + if (type == XSHMEM_FSC_MAX_TYPE) { + /* The list of which the significant bits is more than required, + * contains no more free block. We need to traverse the same + * level list */ + type = fsc_size2type(size); + list_for_each_entry(block, &ctrl->free_list[type], free_node) + if (block->size >= size) + return block; + + pr_err("cannot find a xshmem_block\n"); + return NULL; + } else + return list_first_entry(&ctrl->free_list[type], struct fsc_block, free_node); +} + +static int split_new_block(struct fsc_ctrl *ctrl, struct fsc_block *block, u32 size) +{ + fsc_delete_block(ctrl, block); + + if (block->size < size + XSHMEM_FSC_ALIGN) + /* Remaining space is not enough for new blk */ + return block->start; + else { + struct fsc_block *new; + + new = kmalloc(sizeof(*new), GFP_KERNEL); + if (!new) { + fsc_insert_block(ctrl, block); + pr_err("split: alloc new block memory failed\n"); + return -ENOMEM; + } + new->start = block->start + size; + new->size = block->size - size; + block->size = size; + + list_add(&new->block_node, &block->block_node); + fsc_insert_block(ctrl, new); + + return block->start; + } +} + +static int fsc_algo_block_alloc(struct xshm_pool *xp, u32 size) +{ + u32 aligned_size; + struct fsc_block *block; + struct fsc_ctrl *ctrl = xp->private; + + if (!size) { + pr_err("alloc size invalid\n"); + return -EINVAL; + } + + aligned_size = ALIGN(size, XSHMEM_FSC_ALIGN); + if (aligned_size < size || aligned_size >= XSHMEM_FSC_MAX_ALLOC_SIZE) { + pr_err("alloc size too big\n"); + return -EOVERFLOW; + } + + block = find_free_block(ctrl, aligned_size); + if (!block) + return -ENOSPC; + + return split_new_block(ctrl, block, aligned_size); +} + +static void check_and_combine_next_block(struct fsc_ctrl *ctrl, struct fsc_block *block) +{ + struct fsc_block *next; + + if (list_is_last(&block->block_node, &ctrl->block_head)) + return; + + next = list_next_entry(block, block_node); + if (!next->is_free) + return; + + fsc_delete_block(ctrl, next); + block->size += next->size; + + list_del(&next->block_node); + kfree(next); +} + +static void check_and_combine_prev_block(struct fsc_ctrl *ctrl, struct fsc_block *block) +{ + struct fsc_block *prev; + + if (block->block_node.prev == &ctrl->block_head) + return; + + prev = list_prev_entry(block, block_node); + if (!prev->is_free) + return; + + fsc_delete_block(ctrl, prev); + block->size += prev->size; + block->start = prev->start; + + list_del(&prev->block_node); + kfree(prev); +} + +static int fsc_algo_block_free(struct xshm_pool *xp, u32 offset) +{ + bool found = false; + struct fsc_block *block; + struct fsc_ctrl *ctrl = xp->private; + + list_for_each_entry(block, &ctrl->block_head, block_node) + if (block->start == offset) { + found = true; + break; + } + + if (!found) { + pr_err("the block is not alloced\n"); + return -EINVAL; + } + + if (block->is_free) { + pr_err("free unalloced block\n"); + return -EFAULT; + } + + check_and_combine_next_block(ctrl, block); + check_and_combine_prev_block(ctrl, block); + + fsc_insert_block(ctrl, block); + + return 0; +} + +struct xshm_pool_algo fsc_algo = { + .num = 1, + .name = "fsc_algo", + .xshm_pool_init = fsc_algo_pool_init, + .xshm_pool_free = fsc_algo_pool_free, + .xshm_block_alloc = fsc_algo_block_alloc, + .xshm_block_free = fsc_algo_block_free, +};

From: Wang Wensheng wangwensheng4@huawei.com

The user should get the BLOCK before use and put it after. The kernel would put all the BLOCKs assocated with a TASK when the TASK exit.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- include/uapi/linux/xshmem_framework.h | 4 + lib/xshmem/xshmem_framework.c | 165 ++++++++++++++++++++++++++-------- lib/xshmem/xshmem_fsc.c | 2 +- 3 files changed, 133 insertions(+), 38 deletions(-)

diff --git a/include/uapi/linux/xshmem_framework.h b/include/uapi/linux/xshmem_framework.h index 79a9815..2ac2467 100644 --- a/include/uapi/linux/xshmem_framework.h +++ b/include/uapi/linux/xshmem_framework.h @@ -29,4 +29,8 @@ struct xshm_block_arg { #define XSHMEM_BLOCK_GET _IOW('X', 4, struct xshm_block_arg) #define XSHMEM_BLOCK_PUT _IOW('X', 5, struct xshm_block_arg)

+#define XSHMEM_ALGO_EMPTY -1 +#define XSHMEM_ALGO_BLOCK 0 +#define XSHMEM_ALGO_FSC 1 + #endif diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index 9b4b390..561f235 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -51,18 +51,27 @@ static struct hlist_head *hash_bucket(char *key, int len) return &xshmem_key_list[bucket & ((1U << XSHM_HLIST_TABLE_BIT) - 1U)]; }

+struct xshm_task_block_cnt { + struct list_head list; + struct xshm_block *blk; + int refcnt; +}; + struct xshm_task_pool_node { - struct list_head task_node; // list node in TASK - struct list_head pool_node; // list node in POOL - struct xshm_task *task; - struct xshm_pool *pool; + struct list_head task_node; // list node in TASK + struct list_head pool_node; // list node in POOL + struct xshm_task *task; + struct xshm_pool *pool; + + /* list_head for all BLOCKs' refcnt in the POOL allocated by the TASK */ + struct list_head block_head; };

struct xshm_task { - pid_t pid; - int pool_count; - struct list_head list_head; - struct mutex task_mutex; + pid_t pid; + int pool_count; + struct list_head list_head; + struct mutex task_mutex; };

static int xshmem_open(struct inode *inode, struct file *file) @@ -83,15 +92,15 @@ static int xshmem_open(struct inode *inode, struct file *file) return 0; }

-static bool task_is_attached(struct xshm_task *task, struct xshm_pool *xp) +static struct xshm_task_pool_node *find_task_pool_node(struct xshm_task *task, struct xshm_pool *xp) { struct xshm_task_pool_node *node;

list_for_each_entry(node, &task->list_head, task_node) if (node->pool == xp) - return true; + return node;

- return false; + return NULL; }

/* @@ -102,7 +111,7 @@ static int xshmem_pool_attach(struct xshm_task *task, struct xshm_pool *xp) { struct xshm_task_pool_node *node;

- if (task_is_attached(task, xp)) { + if (find_task_pool_node(task, xp)) { pr_err("TASK has been attached already\n"); return -EEXIST; } @@ -116,6 +125,9 @@ static int xshmem_pool_attach(struct xshm_task *task, struct xshm_pool *xp) node->task = task; node->pool = xp; list_add_tail(&node->task_node, &task->list_head); + INIT_LIST_HEAD(&node->block_head); + + /* Do not add node to xp->list_head until all its elements initilized */ spin_lock(&xp->xp_task_spinlock); list_add_tail(&node->pool_node, &xp->list_head); spin_unlock(&xp->xp_task_spinlock); @@ -126,6 +138,31 @@ static int xshmem_pool_attach(struct xshm_task *task, struct xshm_pool *xp) return 0; }

+static void xshmem_block_destroy(struct xshm_pool *xp, struct xshm_block *blk) +{ + xp->algo->xshm_block_free(xp, blk); // should not fail + list_del(&blk->block_node); + kfree(blk); +} + +static void xshmem_task_clear_block(struct xshm_task_pool_node *node) +{ + struct xshm_pool *xp = node->pool; + struct xshm_task_block_cnt *cnt, *tmp; + + mutex_lock(&xp->xp_block_mutex); + list_for_each_entry_safe(cnt, tmp, &node->block_head, list) { + struct xshm_block *blk = cnt->blk; + + blk->refcnt -= cnt->refcnt; + if (!blk->refcnt) + xshmem_block_destroy(xp, blk); + + kfree(cnt); + } + mutex_unlock(&xp->xp_block_mutex); +} + /* The caller should increase xp->refcnt to protect the POOL from being destroyed. */ static int xshmem_pool_detach(struct xshm_task *task, struct xshm_pool *xp) { @@ -137,6 +174,8 @@ static int xshmem_pool_detach(struct xshm_task *task, struct xshm_pool *xp) list_del(&node->pool_node); spin_unlock(&xp->xp_task_spinlock);

+ xshmem_task_clear_block(node); + list_del(&node->task_node); atomic_dec(&xp->refcnt); task->pool_count--; @@ -243,8 +282,6 @@ static struct xshm_pool *xshmem_pool_get(int pool_id)

static void xshmem_pool_put(struct xshm_pool *xp) { - struct xshm_block *blk, *tmp; - mutex_lock(&xshmem_mutex); if (!atomic_dec_and_test(&xp->refcnt)) { mutex_unlock(&xshmem_mutex); @@ -255,11 +292,7 @@ static void xshmem_pool_put(struct xshm_pool *xp) idr_remove(&xshmem_idr, xp->pool_id); mutex_unlock(&xshmem_mutex);

- list_for_each_entry_safe(blk, tmp, &xp->block_head, block_node) { - list_del(&blk->block_node); - WARN(blk->refcnt, "POOL deleted with referenced BLOCK\n"); - kfree(blk); - } + WARN(!list_empty(&xp->block_head), "POOL deleted with referenced BLOCK\n");

xp->algo->xshm_pool_free(xp); kfree(xp); @@ -336,10 +369,11 @@ static struct xshm_block *xshmem_find_block(struct xshm_pool *xp, u32 offset) return NULL; }

-static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, int size) +static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_node *node, int size) { int ret; struct xshm_block *blk; + struct xshm_task_block_cnt *cnt;

blk = kmalloc(sizeof(*blk), GFP_KERNEL); if (unlikely(!blk)) { @@ -347,30 +381,71 @@ static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, int size) return -ENOMEM; }

+ cnt = kmalloc(sizeof(*cnt), GFP_KERNEL); + if (unlikely(!cnt)) { + pr_err("alloc xshm_task_block_cnt memory failed\n"); + kfree(blk); + return -ENOMEM; + } + mutex_lock(&xp->xp_block_mutex); ret = xp->algo->xshm_block_alloc(xp, blk, size); - if (ret < 0) + if (ret < 0) { kfree(blk); - else { + kfree(cnt); + } else { blk->refcnt = 1; blk->offset = ret; list_add_tail(&blk->block_node, &xp->block_head); + + cnt->refcnt = 1; + cnt->blk = blk; + list_add_tail(&cnt->list, &node->block_head); } mutex_unlock(&xp->xp_block_mutex);

return ret; }

-static int ioctl_xshmem_block_get(struct xshm_pool *xp, u32 offset) +static struct xshm_task_block_cnt * +find_task_block_cnt(struct xshm_task_pool_node *node, struct xshm_block *blk) +{ + struct xshm_task_block_cnt *cnt; + + list_for_each_entry(cnt, &node->block_head, list) + if (cnt->blk == blk) + return cnt; + + return NULL; +} + +static int ioctl_xshmem_block_get(struct xshm_pool *xp, struct xshm_task_pool_node *node, u32 offset) { int ret = 0; struct xshm_block *blk;

mutex_lock(&xp->xp_block_mutex); blk = xshmem_find_block(xp, offset); - if (blk) + if (blk) { + struct xshm_task_block_cnt *cnt; + + cnt = find_task_block_cnt(node, blk); + if (!cnt) { + cnt = kmalloc(sizeof(*cnt), GFP_KERNEL); + if (!cnt) { + mutex_unlock(&xp->xp_block_mutex); + pr_err("get:alloc xshm_task_block_cnt memory failed\n"); + return -ENOMEM; + } + + cnt->refcnt = 1; + cnt->blk = blk; + list_add_tail(&cnt->list, &node->block_head); + } else + cnt->refcnt++; + blk->refcnt++; - else { + } else { pr_err("get unalloced block\n"); ret = -ENODEV; } @@ -379,7 +454,7 @@ static int ioctl_xshmem_block_get(struct xshm_pool *xp, u32 offset) return ret; }

-static int ioctl_xshmem_block_put(struct xshm_pool *xp, u32 offset) +static int ioctl_xshmem_block_put(struct xshm_pool *xp, struct xshm_task_pool_node *node, u32 offset) { int ret = 0; struct xshm_block *blk; @@ -387,12 +462,24 @@ static int ioctl_xshmem_block_put(struct xshm_pool *xp, u32 offset) mutex_lock(&xp->xp_block_mutex); blk = xshmem_find_block(xp, offset); if (blk) { - blk->refcnt--; - if (!blk->refcnt) { - ret = xp->algo->xshm_block_free(xp, blk); // should not fail - list_del(&blk->block_node); - kfree(blk); + struct xshm_task_block_cnt *cnt; + + cnt = find_task_block_cnt(node, blk); + if (!cnt) { + mutex_unlock(&xp->xp_block_mutex); + pr_err("the TASK cannot put the block\n"); + return -EPERM; } + + cnt->refcnt--; + if (!cnt->refcnt) { + list_del(&cnt->list); + kfree(cnt); + } + + blk->refcnt--; + if (!blk->refcnt) + xshmem_block_destroy(xp, blk); } else { pr_err("free unalloced block\n"); ret = -ENODEV; @@ -407,6 +494,7 @@ static int ioctl_xshmem_block_common(struct xshm_task *task, unsigned int cmd, u int ret; struct xshm_pool *xp; struct xshm_block_arg block_arg; + struct xshm_task_pool_node *node;

if (copy_from_user(&block_arg, (struct xshm_free_arg __user*)arg, sizeof(block_arg))) { pr_err("copy_from_user failed\n"); @@ -419,7 +507,7 @@ static int ioctl_xshmem_block_common(struct xshm_task *task, unsigned int cmd, u return -EINVAL; }

- if (!task_is_attached(task, xp)) { + if (!(node = find_task_pool_node(task, xp))) { xshmem_pool_put(xp); pr_err("TASK is not attached to POOL\n"); return -EINVAL; @@ -427,13 +515,13 @@ static int ioctl_xshmem_block_common(struct xshm_task *task, unsigned int cmd, u

switch(cmd) { case XSHMEM_BLOCK_ALLOC: - ret = ioctl_xshmem_block_alloc(xp, block_arg.size); + ret = ioctl_xshmem_block_alloc(xp, node, block_arg.size); break; case XSHMEM_BLOCK_PUT: - ret = ioctl_xshmem_block_put(xp, block_arg.offset); + ret = ioctl_xshmem_block_put(xp, node, block_arg.offset); break; case XSHMEM_BLOCK_GET: - ret = ioctl_xshmem_block_get(xp, block_arg.offset); + ret = ioctl_xshmem_block_get(xp, node, block_arg.offset); break; }

@@ -476,10 +564,13 @@ static int xshmem_release(struct inode *inode, struct file *file) struct xshm_task *task = file->private_data;

list_for_each_entry_safe(node, tmp, &task->list_head, task_node) { - list_del(&node->task_node); spin_lock(&node->pool->xp_task_spinlock); list_del(&node->pool_node); spin_unlock(&node->pool->xp_task_spinlock); + + xshmem_task_clear_block(node); + + list_del(&node->task_node); xshmem_pool_put(node->pool); kfree(node); } @@ -545,7 +636,7 @@ static int empty_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk)

/* Use just for test */ static struct xshm_pool_algo empty_algo = { - .num = 0, + .num = XSHMEM_ALGO_EMPTY, .name = "empty_algo", .xshm_pool_init = empty_algo_pool_init, .xshm_pool_free = empty_algo_pool_free, diff --git a/lib/xshmem/xshmem_fsc.c b/lib/xshmem/xshmem_fsc.c index 87d7533..2b18a25 100644 --- a/lib/xshmem/xshmem_fsc.c +++ b/lib/xshmem/xshmem_fsc.c @@ -252,7 +252,7 @@ static int fsc_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) }

struct xshm_pool_algo fsc_algo = { - .num = 1, + .num = XSHMEM_ALGO_FSC, .name = "fsc_algo", .xshm_pool_init = fsc_algo_pool_init, .xshm_pool_free = fsc_algo_pool_free,

From: Wang Wensheng wangwensheng4@huawei.com

The element of the pool_reg_arg could be defferent between different algorithm, so we could not judge if a newly registered POOL confilicts with an original one in the framework. Add a hook for each algorithm to solve the problem.

Pass the pool_reg_arg not only the POOL size to the init hook of the specified algorithm for the same reason above.

Delete the size element in xshm_pool structrue.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- lib/xshmem/xshmem_framework.c | 33 ++++++++++++++++++++++----------- lib/xshmem/xshmem_framework.h | 4 ++-- lib/xshmem/xshmem_fsc.c | 16 ++++++++++++---- 3 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index ae6899f..61fe44a 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -199,21 +199,22 @@ static int xshmem_pool_detach(struct xshm_task *task, struct xshm_pool *xp) /* * get the POOL specified by key, create one if it not exist. */ -static struct xshm_pool *xshmem_pool_create(char *key, unsigned int key_len, - struct xshm_pool_algo *algo, unsigned int pool_size) +static struct xshm_pool *xshmem_pool_create(struct xshm_pool_algo *algo, struct xshm_reg_arg *arg) { int id, ret; struct xshm_pool *xp; struct hlist_head *bucket; + char *key = arg->key; + unsigned int key_len = arg->key_len;

bucket = hash_bucket(key, key_len);

mutex_lock(&xshmem_mutex); hlist_for_each_entry(xp, bucket, hnode) if (key_len == xp->key_len && !strncmp(key, xp->key, key_len)) { - if (xp->size != pool_size || xp->algo != algo) { + if (xp->algo != algo || !algo->xshm_pool_same(xp, arg)) { mutex_unlock(&xshmem_mutex); - pr_err("the pool size or algorithm invalid\n"); + pr_err("the pool reg arg check failed\n"); return ERR_PTR(-EINVAL); } else goto exist; @@ -226,8 +227,7 @@ static struct xshm_pool *xshmem_pool_create(char *key, unsigned int key_len, }

xp->algo = algo; - xp->size = pool_size; - ret = algo->xshm_pool_init(xp); + ret = algo->xshm_pool_init(xp, arg); if (ret < 0) { mutex_unlock(&xshmem_mutex); kfree(xp); @@ -338,7 +338,8 @@ static int ioctl_xshmem_pool_register(struct xshm_task *task, unsigned long arg) return -EFAULT; }

- xp = xshmem_pool_create(key, key_len, algo, reg_arg.pool_size); + reg_arg.key = key; + xp = xshmem_pool_create(algo, &reg_arg); if (IS_ERR(xp)) return PTR_ERR(xp);

@@ -620,7 +621,8 @@ int xshmem_register_algo(struct xshm_pool_algo *algo) struct xshm_pool_algo *tmp;

if (!algo || !algo->xshm_pool_init || !algo->xshm_pool_free || - !algo->xshm_block_alloc || !algo->xshm_block_free) + !algo->xshm_block_alloc || !algo->xshm_block_free || + !algo->xshm_pool_same) return -EINVAL;

list_for_each_entry(tmp, &registered_algo, algo_node) @@ -638,15 +640,23 @@ int xshmem_register_algo(struct xshm_pool_algo *algo) } EXPORT_SYMBOL_GPL(xshmem_register_algo);

-static int empty_algo_pool_init(struct xshm_pool *xp) +static bool empty_algo_pool_same(struct xshm_pool *xp, struct xshm_reg_arg *arg) { - pr_info("pool_init_hook: algo:%s, pool_size: %d\n", xp->algo->name, xp->size); + pr_info("pool_same_hook: algo:%s, pool_size: %d\n", xp->algo->name, arg->pool_size); + return (unsigned long)xp->private == arg->pool_size; +} + +static int empty_algo_pool_init(struct xshm_pool *xp, struct xshm_reg_arg *arg) +{ + pr_info("pool_init_hook: algo:%s, pool_size: %d\n", xp->algo->name, arg->pool_size); + xp->private = (void *)(unsigned long)arg->pool_size; return 0; }

static int empty_algo_pool_free(struct xshm_pool *xp) { - pr_info("pool_free_hook: algo:%s, pool_size: %d, pool_id:%d\n", xp->algo->name, xp->size, xp->pool_id); + pr_info("pool_free_hook: algo:%s, pool_size: %ld, pool_id:%d\n", + xp->algo->name, (unsigned long)xp->private, xp->pool_id); return 0; }

@@ -666,6 +676,7 @@ static int empty_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) static struct xshm_pool_algo empty_algo = { .num = XSHMEM_ALGO_EMPTY, .name = "empty_algo", + .xshm_pool_same = empty_algo_pool_same, .xshm_pool_init = empty_algo_pool_init, .xshm_pool_free = empty_algo_pool_free, .xshm_block_alloc = empty_algo_block_alloc, diff --git a/lib/xshmem/xshmem_framework.h b/lib/xshmem/xshmem_framework.h index 3624841..f543c27 100644 --- a/lib/xshmem/xshmem_framework.h +++ b/lib/xshmem/xshmem_framework.h @@ -11,7 +11,6 @@

struct xshm_pool { int pool_id; - int size; atomic_t refcnt;

/* Used to protect the list of TASK attached */ @@ -47,7 +46,8 @@ struct xshm_pool_algo { int num; char name[ALGO_NAME_MAX]; struct list_head algo_node; - int (*xshm_pool_init)(struct xshm_pool *xp); + bool (*xshm_pool_same)(struct xshm_pool *xp, struct xshm_reg_arg *arg); + int (*xshm_pool_init)(struct xshm_pool *xp, struct xshm_reg_arg *arg); int (*xshm_pool_free)(struct xshm_pool *xp); int (*xshm_block_alloc)(struct xshm_pool *xp, struct xshm_block *blk, u32 size); int (*xshm_block_free)(struct xshm_pool *xp, struct xshm_block *blk); diff --git a/lib/xshmem/xshmem_fsc.c b/lib/xshmem/xshmem_fsc.c index 2b18a25..941918d 100644 --- a/lib/xshmem/xshmem_fsc.c +++ b/lib/xshmem/xshmem_fsc.c @@ -65,13 +65,20 @@ static void fsc_delete_block(struct fsc_ctrl *ctrl, struct fsc_block *block) clear_bit(type, ctrl->free_map); }

-static int fsc_algo_pool_init(struct xshm_pool *xp) +static bool fsc_algo_pool_same(struct xshm_pool *xp, struct xshm_reg_arg *arg) +{ + struct fsc_ctrl *ctrl = xp->private; + + return ctrl->total_size == arg->pool_size; +} + +static int fsc_algo_pool_init(struct xshm_pool *xp, struct xshm_reg_arg *arg) { int i; struct fsc_ctrl *ctrl; struct fsc_block *block;

- if (!IS_ALIGNED(xp->size, XSHMEM_FSC_ALIGN) || !xp->size) { + if (!IS_ALIGNED(arg->pool_size, XSHMEM_FSC_ALIGN) || !arg->pool_size) { pr_err("input xshm_pool size invalid\n"); return -EINVAL; } @@ -84,14 +91,14 @@ static int fsc_algo_pool_init(struct xshm_pool *xp) return -ENOMEM; }

- ctrl->total_size = xp->size; + ctrl->total_size = arg->pool_size; ctrl->free_size = 0; INIT_LIST_HEAD(&ctrl->block_head); for (i = 0; i < ARRAY_SIZE(ctrl->free_list); i++) INIT_LIST_HEAD(&ctrl->free_list[i]);

block->start = 0; - block->size = xp->size; + block->size = arg->pool_size; list_add_tail(&block->block_node, &ctrl->block_head); fsc_insert_block(ctrl, block);

@@ -254,6 +261,7 @@ static int fsc_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) struct xshm_pool_algo fsc_algo = { .num = XSHMEM_ALGO_FSC, .name = "fsc_algo", + .xshm_pool_same = fsc_algo_pool_same, .xshm_pool_init = fsc_algo_pool_init, .xshm_pool_free = fsc_algo_pool_free, .xshm_block_alloc = fsc_algo_block_alloc,

From: Wang Wensheng wangwensheng4@huawei.com

The return value of XSHMEM_BLOCK_ALLOC was stored in the input xshm_block_arg, since ioctl could not return long type.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- include/uapi/linux/xshmem_framework.h | 18 +++++++++--------- lib/xshmem/xshmem_blk.c | 6 +++--- lib/xshmem/xshmem_framework.c | 30 ++++++++++++++++++------------ lib/xshmem/xshmem_framework.h | 4 ++-- lib/xshmem/xshmem_fsc.c | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/include/uapi/linux/xshmem_framework.h b/include/uapi/linux/xshmem_framework.h index ffb3546..07181d7 100644 --- a/include/uapi/linux/xshmem_framework.h +++ b/include/uapi/linux/xshmem_framework.h @@ -10,7 +10,7 @@

struct xshm_reg_arg { int algo; - unsigned int pool_size; + unsigned long pool_size; unsigned int block_size; /* used for blk algorithm */ unsigned int key_len; char *key; @@ -19,17 +19,17 @@ struct xshm_reg_arg { struct xshm_block_arg { int pool_id; union { - int size; // for alloc - unsigned int offset; // for get and put + unsigned long size; /* for alloc input */ + unsigned long offset; /* for get, put, delete and alloc output */ }; };

-#define XSHMEM_POOL_REGISTER _IOW('X', 1, struct xshm_reg_arg) -#define XSHMEM_POOL_UNREGISTER _IO ('X', 2) -#define XSHMEM_BLOCK_ALLOC _IOW('X', 3, struct xshm_block_arg) -#define XSHMEM_BLOCK_GET _IOW('X', 4, struct xshm_block_arg) -#define XSHMEM_BLOCK_PUT _IOW('X', 5, struct xshm_block_arg) -#define XSHMEM_BLOCK_DELETE _IOW('X', 6, struct xshm_block_arg) +#define XSHMEM_POOL_REGISTER _IOW ('X', 1, struct xshm_reg_arg) +#define XSHMEM_POOL_UNREGISTER _IO ('X', 2) +#define XSHMEM_BLOCK_ALLOC _IOWR('X', 3, struct xshm_block_arg) +#define XSHMEM_BLOCK_GET _IOW ('X', 4, struct xshm_block_arg) +#define XSHMEM_BLOCK_PUT _IOW ('X', 5, struct xshm_block_arg) +#define XSHMEM_BLOCK_DELETE _IOW ('X', 6, struct xshm_block_arg)

#define XSHMEM_ALGO_EMPTY -1 #define XSHMEM_ALGO_BLOCK 0 diff --git a/lib/xshmem/xshmem_blk.c b/lib/xshmem/xshmem_blk.c index a3d93a3..81012e2 100644 --- a/lib/xshmem/xshmem_blk.c +++ b/lib/xshmem/xshmem_blk.c @@ -65,9 +65,9 @@ static int blk_algo_pool_free(struct xshm_pool *xp) return 0; }

-static int blk_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, u32 size) +static long blk_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size) { - int idx; + long idx; struct blk_ctrl *ctrl = xp->private;

if (size > ctrl->block_size) { @@ -97,7 +97,7 @@ static int blk_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, u3

static int blk_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) { - int idx = (long)blk->private; + long idx = (long)blk->private; struct blk_ctrl *ctrl = xp->private;

clear_bit(idx, ctrl->map); diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index 74bce3b..46fefb8 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -379,9 +379,10 @@ static struct xshm_block *xshmem_find_block(struct xshm_pool *xp, u32 offset) return NULL; }

-static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_node *node, int size) +static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_node *node, + struct xshm_block_arg *arg) { - int ret; + long ret; struct xshm_block *blk; struct xshm_task_block_cnt *cnt;

@@ -399,13 +400,14 @@ static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_ }

mutex_lock(&xp->xp_block_mutex); - ret = xp->algo->xshm_block_alloc(xp, blk, size); + ret = xp->algo->xshm_block_alloc(xp, blk, arg->size); if (ret < 0) { kfree(blk); kfree(cnt); } else { blk->refcnt = 1; blk->offset = ret; + arg->offset = ret; INIT_LIST_HEAD(&blk->list_head); list_add_tail(&blk->block_node, &xp->block_head);

@@ -417,7 +419,7 @@ static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_ } mutex_unlock(&xp->xp_block_mutex);

- return ret; + return ret < 0 ? ret : 0; }

static struct xshm_task_block_cnt * @@ -520,8 +522,9 @@ static int ioctl_xshmem_block_common(struct xshm_task *task, unsigned int cmd, u struct xshm_pool *xp; struct xshm_block_arg block_arg; struct xshm_task_pool_node *node; + struct xshm_block_arg *usr_arg = (struct xshm_block_arg __user *)arg;

- if (copy_from_user(&block_arg, (struct xshm_free_arg __user*)arg, sizeof(block_arg))) { + if (copy_from_user(&block_arg, usr_arg, sizeof(block_arg))) { pr_err("copy_from_user failed\n"); return -EFAULT; } @@ -540,7 +543,10 @@ static int ioctl_xshmem_block_common(struct xshm_task *task, unsigned int cmd, u

switch(cmd) { case XSHMEM_BLOCK_ALLOC: - ret = ioctl_xshmem_block_alloc(xp, node, block_arg.size); + ret = ioctl_xshmem_block_alloc(xp, node, &block_arg); + if (!ret) + /* should never failed */ + ret = put_user(block_arg.offset, &usr_arg->offset); break; case XSHMEM_BLOCK_GET: ret = ioctl_xshmem_block_get(xp, node, block_arg.offset); @@ -642,14 +648,14 @@ EXPORT_SYMBOL_GPL(xshmem_register_algo);

static bool empty_algo_pool_same(struct xshm_pool *xp, struct xshm_reg_arg *arg) { - pr_info("pool_same_hook: algo:%s, pool_size: %d\n", xp->algo->name, arg->pool_size); + pr_info("pool_same_hook: algo:%s, pool_size: %ld\n", xp->algo->name, arg->pool_size); return (unsigned long)xp->private == arg->pool_size; }

static int empty_algo_pool_init(struct xshm_pool *xp, struct xshm_reg_arg *arg) { - pr_info("pool_init_hook: algo:%s, pool_size: %d\n", xp->algo->name, arg->pool_size); - xp->private = (void *)(unsigned long)arg->pool_size; + pr_info("pool_init_hook: algo:%s, pool_size: %ld\n", xp->algo->name, arg->pool_size); + xp->private = (void *)arg->pool_size; return 0; }

@@ -660,15 +666,15 @@ static int empty_algo_pool_free(struct xshm_pool *xp) return 0; }

-static int empty_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, u32 size) +static long empty_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size) { - pr_info("block_alloc_hook:pool_id:%d, alloc_size:%d\n", xp->pool_id, size); + pr_info("block_alloc_hook:pool_id:%d, alloc_size:%lu\n", xp->pool_id, size); return 0; }

static int empty_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) { - pr_info("block_free_hook:pool_id:%d, offset:%d\n", xp->pool_id, blk->offset); + pr_info("block_free_hook:pool_id:%d, offset:%ld\n", xp->pool_id, blk->offset); return 0; }

diff --git a/lib/xshmem/xshmem_framework.h b/lib/xshmem/xshmem_framework.h index f543c27..a939a69 100644 --- a/lib/xshmem/xshmem_framework.h +++ b/lib/xshmem/xshmem_framework.h @@ -32,7 +32,7 @@ struct xshm_pool {

struct xshm_block { int refcnt; - u32 offset; /* the size of the block is not cared in the + unsigned long offset; /* the size of the block is not cared in the framework, the specified algorithm should manage it properly */ struct list_head block_node; @@ -49,7 +49,7 @@ struct xshm_pool_algo { bool (*xshm_pool_same)(struct xshm_pool *xp, struct xshm_reg_arg *arg); int (*xshm_pool_init)(struct xshm_pool *xp, struct xshm_reg_arg *arg); int (*xshm_pool_free)(struct xshm_pool *xp); - int (*xshm_block_alloc)(struct xshm_pool *xp, struct xshm_block *blk, u32 size); + long (*xshm_block_alloc)(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size); int (*xshm_block_free)(struct xshm_pool *xp, struct xshm_block *blk); };

diff --git a/lib/xshmem/xshmem_fsc.c b/lib/xshmem/xshmem_fsc.c index 941918d..fac8b12 100644 --- a/lib/xshmem/xshmem_fsc.c +++ b/lib/xshmem/xshmem_fsc.c @@ -173,7 +173,7 @@ static int split_new_block(struct fsc_ctrl *ctrl, struct fsc_block *block, u32 s } }

-static int fsc_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, u32 size) +static long fsc_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size) { int ret; u32 aligned_size;

From: Wang Wensheng wangwensheng4@huawei.com

Add /proc/xshmem directory that contains information of the POOLs and TASKs.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- lib/xshmem/xshmem_blk.c | 20 ++++- lib/xshmem/xshmem_framework.c | 174 +++++++++++++++++++++++++++++++++++------- lib/xshmem/xshmem_framework.h | 11 ++- lib/xshmem/xshmem_fsc.c | 18 ++++- 4 files changed, 185 insertions(+), 38 deletions(-)

diff --git a/lib/xshmem/xshmem_blk.c b/lib/xshmem/xshmem_blk.c index 81012e2..b8ce7df 100644 --- a/lib/xshmem/xshmem_blk.c +++ b/lib/xshmem/xshmem_blk.c @@ -19,6 +19,7 @@ struct blk_ctrl { int block_size;

int total_block; + int free_block; int next_free; unsigned long map[0]; }; @@ -53,6 +54,7 @@ static int blk_algo_pool_init(struct xshm_pool *xp, struct xshm_reg_arg *arg) ctrl->block_size = arg->block_size; ctrl->total_block = block_cnt; ctrl->next_free = 0; + ctrl->free_block = block_cnt;

xp->private = ctrl;

@@ -65,9 +67,18 @@ static int blk_algo_pool_free(struct xshm_pool *xp) return 0; }

-static long blk_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size) +static void blk_algo_pool_show(struct xshm_pool *xp, struct seq_file *seq) +{ + struct blk_ctrl *ctrl = xp->private; + + seq_printf(seq, " pool_size:%#x, block_size:%#x, free_block:%d\n", + ctrl->pool_size, ctrl->block_size, ctrl->free_block); +} + +static int blk_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk) { long idx; + unsigned long size = blk->alloc_size; struct blk_ctrl *ctrl = xp->private;

if (size > ctrl->block_size) { @@ -89,10 +100,13 @@ static long blk_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, u

set_bit(idx, ctrl->map); ctrl->next_free = (idx + 1) % ctrl->total_block; + ctrl->free_block--;

blk->private = (void *)(long)idx; + blk->offset = idx * ctrl->block_size; + blk->real_size = ctrl->block_size;

- return idx * ctrl->block_size; + return 0; }

static int blk_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) @@ -100,6 +114,7 @@ static int blk_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) long idx = (long)blk->private; struct blk_ctrl *ctrl = xp->private;

+ ctrl->free_block++; clear_bit(idx, ctrl->map); return 0; } @@ -110,6 +125,7 @@ struct xshm_pool_algo blk_algo = { .xshm_pool_same = blk_algo_pool_same, .xshm_pool_init = blk_algo_pool_init, .xshm_pool_free = blk_algo_pool_free, + .xshm_pool_show = blk_algo_pool_show, .xshm_block_alloc = blk_algo_block_alloc, .xshm_block_free = blk_algo_block_free, }; diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index 97c44e3..0d4e819 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -17,6 +17,8 @@ #include <linux/uaccess.h> #include <linux/jhash.h> #include <linux/rcupdate.h> +#include <linux/proc_fs.h> +#include <linux/seq_file.h>

#include "xshmem_framework.h"

@@ -59,27 +61,61 @@ struct xshm_task_block_cnt { struct xshm_task_pool_node *node; };

-static void task_block_cnt_destroy(struct xshm_task_block_cnt *cnt) -{ - list_del(&cnt->node_list); - list_del(&cnt->blk_list); - kfree(cnt); -} - struct xshm_task_pool_node { struct list_head task_node; // list node in TASK struct list_head pool_node; // list node in POOL struct xshm_task *task; struct xshm_pool *pool;

+ atomic64_t alloc_size; + atomic64_t real_alloc_size; /* list_head for all BLOCKs' refcnt in the POOL allocated by the TASK */ struct list_head list_head; };

+static void task_alloc_size_update(struct xshm_task_pool_node *node, + struct xshm_block *blk, bool inc) +{ + if (inc) { + atomic64_add(blk->alloc_size, &node->alloc_size); + atomic64_add(blk->real_size, &node->real_alloc_size); + } else { + atomic64_sub(blk->alloc_size, &node->alloc_size); + atomic64_sub(blk->real_size, &node->real_alloc_size); + } +} + +static void task_link_blk(struct xshm_task_pool_node *node, + struct xshm_block *blk, + struct xshm_task_block_cnt *cnt) +{ + cnt->refcnt = 1; + cnt->blk = blk; + cnt->node = node; + list_add_tail(&cnt->node_list, &node->list_head); + list_add_tail(&cnt->blk_list, &blk->list_head); + + task_alloc_size_update(node, blk, true); +} + +static void task_unlink_blk(struct xshm_task_block_cnt *cnt) +{ + list_del(&cnt->node_list); + list_del(&cnt->blk_list); + + task_alloc_size_update(cnt->node, cnt->blk, false); + + kfree(cnt); +} + +static DEFINE_MUTEX(task_mutex); +static LIST_HEAD(all_task); + struct xshm_task { pid_t pid; int pool_count; struct list_head list_head; + struct list_head list; struct mutex task_mutex; };

@@ -98,6 +134,10 @@ static int xshmem_open(struct inode *inode, struct file *file)

file->private_data = ptask;

+ mutex_lock(&task_mutex); + list_add_tail(&ptask->list, &all_task); + mutex_unlock(&task_mutex); + return 0; }

@@ -133,6 +173,8 @@ static int xshmem_pool_attach(struct xshm_task *task, struct xshm_pool *xp)

node->task = task; node->pool = xp; + atomic64_set(&node->alloc_size, 0); + atomic64_set(&node->real_alloc_size, 0); list_add_tail(&node->task_node, &task->list_head); INIT_LIST_HEAD(&node->list_head);

@@ -164,10 +206,11 @@ static void xshmem_task_clear_block(struct xshm_task_pool_node *node) struct xshm_block *blk = cnt->blk;

blk->refcnt -= cnt->refcnt; + + task_unlink_blk(cnt); + if (!blk->refcnt) xshmem_block_destroy(xp, blk); - - task_block_cnt_destroy(cnt); } mutex_unlock(&xp->xp_block_mutex); } @@ -384,7 +427,7 @@ static struct xshm_block *xshmem_find_block(struct xshm_pool *xp, u32 offset) static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_node *node, struct xshm_block_arg *arg) { - long ret; + int ret; struct xshm_block *blk; struct xshm_task_block_cnt *cnt;

@@ -401,27 +444,23 @@ static int ioctl_xshmem_block_alloc(struct xshm_pool *xp, struct xshm_task_pool_ return -ENOMEM; }

+ blk->alloc_size = arg->size; mutex_lock(&xp->xp_block_mutex); - ret = xp->algo->xshm_block_alloc(xp, blk, arg->size); + ret = xp->algo->xshm_block_alloc(xp, blk); if (ret < 0) { kfree(blk); kfree(cnt); } else { blk->refcnt = 1; - blk->offset = ret; - arg->offset = ret; + arg->offset = blk->offset; INIT_LIST_HEAD(&blk->list_head); list_add_tail(&blk->block_node, &xp->block_head);

- cnt->refcnt = 1; - cnt->blk = blk; - cnt->node = node; - list_add_tail(&cnt->node_list, &node->list_head); - list_add_tail(&cnt->blk_list, &blk->list_head); + task_link_blk(node, blk, cnt); } mutex_unlock(&xp->xp_block_mutex);

- return ret < 0 ? ret : 0; + return ret; }

static struct xshm_task_block_cnt * @@ -455,12 +494,7 @@ static int ioctl_xshmem_block_get(struct xshm_pool *xp, struct xshm_task_pool_no return -ENOMEM; }

- cnt->refcnt = 1; - cnt->blk = blk; - cnt->node = node; - - list_add_tail(&cnt->node_list, &node->list_head); - list_add_tail(&cnt->blk_list, &blk->list_head); + task_link_blk(node, blk, cnt); } else cnt->refcnt++;

@@ -495,7 +529,7 @@ static int ioctl_xshmem_block_put(struct xshm_pool *xp, struct xshm_task_pool_no if (force_del) { list_for_each_entry_safe(cnt, tmp, &blk->list_head, blk_list) { blk->refcnt -= cnt->refcnt; - task_block_cnt_destroy(cnt); + task_unlink_blk(cnt); }

WARN(blk->refcnt, "refcnt for BLOCK broken\n"); @@ -503,7 +537,7 @@ static int ioctl_xshmem_block_put(struct xshm_pool *xp, struct xshm_task_pool_no } else { cnt->refcnt--; if (!cnt->refcnt) - task_block_cnt_destroy(cnt); + task_unlink_blk(cnt);

blk->refcnt--; if (!blk->refcnt) @@ -600,6 +634,10 @@ static int xshmem_release(struct inode *inode, struct file *file) struct xshm_task_pool_node *node, *tmp; struct xshm_task *task = file->private_data;

+ mutex_lock(&task_mutex); + list_del(&task->list); + mutex_unlock(&task_mutex); + list_for_each_entry_safe(node, tmp, &task->list_head, task_node) { spin_lock(&node->pool->xp_task_spinlock); list_del(&node->pool_node); @@ -624,6 +662,79 @@ static struct file_operations xshmem_fops = { .unlocked_ioctl = xshmem_ioctl, };

+static void per_task_info_show(struct seq_file *seq, struct xshm_task *task) +{ + struct xshm_task_pool_node *node; + + mutex_lock(&task->task_mutex); + + seq_printf(seq, "TASK:%d\n", task->pid); + list_for_each_entry(node, &task->list_head, task_node) + seq_printf(seq, " POOL:%-4d %#-8llx %#-8llx\n", node->pool->pool_id, + (u64)atomic64_read(&node->alloc_size), + (u64)atomic64_read(&node->real_alloc_size)); + + mutex_unlock(&task->task_mutex); +} + +static int task_info_show(struct seq_file *seq, void *offset) +{ + struct xshm_task *task; + + mutex_lock(&task_mutex); + list_for_each_entry(task, &all_task, list) + per_task_info_show(seq, task); + mutex_unlock(&task_mutex); + + return 0; +} + +static int per_pool_info_show(int id, void *p, void *data) +{ + struct xshm_pool *xp = p; + struct seq_file *seq = data; + + seq_printf(seq, "POOL:%d\n key:%s, refcnt:%d, algo:%s\n", + id, xp->key, atomic_read(&xp->refcnt), xp->algo->name); + if (xp->algo->xshm_pool_show) { + mutex_lock(&xp->xp_block_mutex); + xp->algo->xshm_pool_show(xp, seq); + mutex_unlock(&xp->xp_block_mutex); + } + + return 0; +} + +static int pool_info_show(struct seq_file *seq, void *offset) +{ + int ret; + + mutex_lock(&xshmem_mutex); + ret = idr_for_each(&xshmem_idr, per_pool_info_show, seq); + mutex_unlock(&xshmem_mutex); + + return ret; +} + +static void __init xshmem_proc_fs_init(void) +{ + struct proc_dir_entry *entry; + + entry = proc_mkdir("xshmem", NULL); + if (!entry) { + pr_err("create proc dir failed\n"); + return; + } + + if (!proc_create_single("task_info", 0400, entry, task_info_show)) + pr_warn("task_info file create failed\n"); + + if (!proc_create_single("pool_info", 0400, entry, pool_info_show)) + pr_warn("pool_info file create failed\n"); + + return; +} + int xshmem_register_algo(struct xshm_pool_algo *algo) { struct xshm_pool_algo *tmp; @@ -668,9 +779,11 @@ static int empty_algo_pool_free(struct xshm_pool *xp) return 0; }

-static long empty_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size) +static int empty_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk) { - pr_info("block_alloc_hook:pool_id:%d, alloc_size:%lu\n", xp->pool_id, size); + pr_info("block_alloc_hook:pool_id:%d, alloc_size:%lu\n", xp->pool_id, blk->alloc_size); + blk->real_size = blk->alloc_size; + blk->offset = 0; return 0; }

@@ -712,12 +825,15 @@ static int __init xshmem_init(void) xshmem_register_algo(&blk_algo); xshmem_register_algo(&empty_algo);

+ xshmem_proc_fs_init(); + return 0; }; module_init(xshmem_init);

static void __exit xshmem_exit(void) { + remove_proc_subtree("xshmem", NULL); misc_deregister(&xshmem_dev); pr_info("module exit\n"); } diff --git a/lib/xshmem/xshmem_framework.h b/lib/xshmem/xshmem_framework.h index a939a69..6234047 100644 --- a/lib/xshmem/xshmem_framework.h +++ b/lib/xshmem/xshmem_framework.h @@ -8,6 +8,7 @@

#include <linux/types.h> #include <linux/xshmem_framework.h> +#include <linux/seq_file.h>

struct xshm_pool { int pool_id; @@ -32,9 +33,10 @@ struct xshm_pool {

struct xshm_block { int refcnt; - unsigned long offset; /* the size of the block is not cared in the - framework, the specified algorithm should - manage it properly */ + unsigned long offset; + unsigned long alloc_size; + unsigned long real_size; + struct list_head block_node; void *private;

@@ -49,7 +51,8 @@ struct xshm_pool_algo { bool (*xshm_pool_same)(struct xshm_pool *xp, struct xshm_reg_arg *arg); int (*xshm_pool_init)(struct xshm_pool *xp, struct xshm_reg_arg *arg); int (*xshm_pool_free)(struct xshm_pool *xp); - long (*xshm_block_alloc)(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size); + void (*xshm_pool_show)(struct xshm_pool *xp, struct seq_file *seq); + int (*xshm_block_alloc)(struct xshm_pool *xp, struct xshm_block *blk); int (*xshm_block_free)(struct xshm_pool *xp, struct xshm_block *blk); };

diff --git a/lib/xshmem/xshmem_fsc.c b/lib/xshmem/xshmem_fsc.c index fac8b12..5e52da3 100644 --- a/lib/xshmem/xshmem_fsc.c +++ b/lib/xshmem/xshmem_fsc.c @@ -124,6 +124,13 @@ static int fsc_algo_pool_free(struct xshm_pool *xp) return 0; }

+static void fsc_algo_pool_show(struct xshm_pool *xp, struct seq_file *seq) +{ + struct fsc_ctrl *ctrl = xp->private; + + seq_printf(seq, " total_size: %#x, free_size: %#x\n", ctrl->total_size, ctrl->free_size); +} + static struct fsc_block *find_free_block(struct fsc_ctrl *ctrl, u32 size) { u32 type; @@ -173,10 +180,10 @@ static int split_new_block(struct fsc_ctrl *ctrl, struct fsc_block *block, u32 s } }

-static long fsc_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, unsigned long size) +static int fsc_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk) { int ret; - u32 aligned_size; + u32 aligned_size, size = blk->alloc_size; struct fsc_block *block; struct fsc_ctrl *ctrl = xp->private;

@@ -196,9 +203,13 @@ static long fsc_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk, u return -ENOSPC;

ret = split_new_block(ctrl, block, aligned_size); - if (ret >= 0) + if (ret >= 0) { /* split success */ blk->private = block; + blk->offset = ret; + blk->real_size = block->size; + return 0; + }

return ret; } @@ -264,6 +275,7 @@ struct xshm_pool_algo fsc_algo = { .xshm_pool_same = fsc_algo_pool_same, .xshm_pool_init = fsc_algo_pool_init, .xshm_pool_free = fsc_algo_pool_free, + .xshm_pool_show = fsc_algo_pool_show, .xshm_block_alloc = fsc_algo_block_alloc, .xshm_block_free = fsc_algo_block_free, };

From: Wang Wensheng wangwensheng4@huawei.com

Split function xshmem_pool_create().

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- lib/xshmem/xshmem_framework.c | 50 ++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index 9966b48..392abaa 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -246,32 +246,20 @@ static int xshmem_pool_detach(struct xshm_task *task, struct xshm_pool *xp) }

/* - * get the POOL specified by key, create one if it not exist. + * The caller must hold xshmem_mutex and this function would unlock xshmem_mutex on failure. */ -static struct xshm_pool *xshmem_pool_create(struct xshm_pool_algo *algo, struct xshm_reg_arg *arg) +static struct xshm_pool *xshmem_pool_create(struct xshm_pool_algo *algo, struct xshm_reg_arg *arg, + struct hlist_head *bucket) { int id, ret; struct xshm_pool *xp; - struct hlist_head *bucket; char *key = arg->key; unsigned int key_len = arg->key_len;

- bucket = hash_bucket(key, key_len); - - mutex_lock(&xshmem_mutex); - hlist_for_each_entry(xp, bucket, hnode) - if (key_len == xp->key_len && !strncmp(key, xp->key, key_len)) { - if (xp->algo != algo || !algo->xshm_pool_same(xp, arg)) { - mutex_unlock(&xshmem_mutex); - pr_err("the pool reg arg check failed\n"); - return ERR_PTR(-EINVAL); - } else - goto exist; - } - xp = kmalloc(sizeof(*xp) + key_len + 1, GFP_KERNEL); if (unlikely(!xp)) { mutex_unlock(&xshmem_mutex); + pr_err("alloc pool memory failed\n"); return ERR_PTR(-ENOMEM); }

@@ -313,6 +301,34 @@ static struct xshm_pool *xshmem_pool_create(struct xshm_pool_algo *algo, struct

hlist_add_head(&xp->hnode, bucket);

+ return xp; +} + +/* + * get the POOL specified by key, create one if it not exist. + */ +static struct xshm_pool *xshmem_pool_register(struct xshm_pool_algo *algo, struct xshm_reg_arg *arg) +{ + struct xshm_pool *xp; + struct hlist_head *bucket; + + bucket = hash_bucket(arg->key, arg->key_len); + + mutex_lock(&xshmem_mutex); + hlist_for_each_entry(xp, bucket, hnode) + if (arg->key_len == xp->key_len && !strncmp(arg->key, xp->key, arg->key_len)) { + if (xp->algo != algo || !algo->xshm_pool_same(xp, arg)) { + mutex_unlock(&xshmem_mutex); + pr_err("the pool reg arg check failed\n"); + return ERR_PTR(-EINVAL); + } else + goto exist; + } + + xp = xshmem_pool_create(algo, arg, bucket); + if (IS_ERR(xp)) + return xp; + exist: /* * Here increase the POOL's refcnt by one, one for the convenience of fallback in @@ -390,7 +406,7 @@ static int ioctl_xshmem_pool_register(struct xshm_task *task, unsigned long arg) }

reg_arg.key = key; - xp = xshmem_pool_create(algo, &reg_arg); + xp = xshmem_pool_register(algo, &reg_arg); if (IS_ERR(xp)) return PTR_ERR(xp);

From: Wang Wensheng wangwensheng4@huawei.com

This algorithm is something alike to the algorithm that used for vm_area alloc in vmalloc and mmap.

Use the augmented rb_tree to organize all the free block.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com --- include/uapi/linux/xshmem_framework.h | 1 + lib/xshmem/Makefile | 2 +- lib/xshmem/xshmem_framework.c | 2 + lib/xshmem/xshmem_vma.c | 343 ++++++++++++++++++++++++++++++++++ 4 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 lib/xshmem/xshmem_vma.c

diff --git a/include/uapi/linux/xshmem_framework.h b/include/uapi/linux/xshmem_framework.h index 205c928..63b8c3b 100644 --- a/include/uapi/linux/xshmem_framework.h +++ b/include/uapi/linux/xshmem_framework.h @@ -34,6 +34,7 @@ struct xshm_block_arg { #define XSHMEM_ALGO_EMPTY -1 #define XSHMEM_ALGO_BLOCK 0 #define XSHMEM_ALGO_FSC 1 +#define XSHMEM_ALGO_VMA 2

#define XSHMEM_POOL_STATUS_DELETE 0 #define XSHMEM_POOL_STATUS_EXIST 1 diff --git a/lib/xshmem/Makefile b/lib/xshmem/Makefile index b7cf800..42637d4 100644 --- a/lib/xshmem/Makefile +++ b/lib/xshmem/Makefile @@ -1,2 +1,2 @@ obj-m+=xshmem.o -xshmem-objs+=xshmem_fsc.o xshmem_framework.o xshmem_blk.o +xshmem-objs+=xshmem_fsc.o xshmem_framework.o xshmem_blk.o xshmem_vma.o diff --git a/lib/xshmem/xshmem_framework.c b/lib/xshmem/xshmem_framework.c index cf8f91e..cd09aca 100644 --- a/lib/xshmem/xshmem_framework.c +++ b/lib/xshmem/xshmem_framework.c @@ -866,6 +866,7 @@ static struct xshm_pool_algo empty_algo = { .xshm_block_free = empty_algo_block_free, };

+extern struct xshm_pool_algo vma_algo; extern struct xshm_pool_algo fsc_algo; extern struct xshm_pool_algo blk_algo;

@@ -883,6 +884,7 @@ static int __init xshmem_init(void) return ret; }

+ xshmem_register_algo(&vma_algo); xshmem_register_algo(&fsc_algo); xshmem_register_algo(&blk_algo); xshmem_register_algo(&empty_algo); diff --git a/lib/xshmem/xshmem_vma.c b/lib/xshmem/xshmem_vma.c new file mode 100644 index 0000000..bb8de95 --- /dev/null +++ b/lib/xshmem/xshmem_vma.c @@ -0,0 +1,343 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2021. All rights reserved. + * Author: Huawei OS Kernel Lab + * Create: Mon Jul 19 07:01:42 2021 + */ +#include <asm-generic/ioctl.h> +#define pr_fmt(fmt) "XSHMEM_VMA: " fmt + +#include <linux/types.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/rbtree_augmented.h> + +#include "xshmem_framework.h" + +#define XSHMEM_VMA_ALIGN 16U + +struct vma_ctrl { + unsigned long total_size; + unsigned long free_size; + unsigned long alloc_block_cnt; + + struct rb_root free_block_root; +}; + +struct vma_block { + struct rb_node rb_node; + + unsigned long start; + unsigned long size; + unsigned long subtree_max_size; +}; + +static struct vma_block *to_vma_block(struct rb_node *node) +{ + return rb_entry(node, struct vma_block, rb_node); +} + +static unsigned long subtree_max_size(struct rb_node *node) +{ + return node ? to_vma_block(node)->subtree_max_size : 0; +} + +static unsigned long compute_subtree_max_size(struct vma_block *block) +{ + return max3(block->size, subtree_max_size(block->rb_node.rb_left), + subtree_max_size(block->rb_node.rb_right)); +} + +RB_DECLARE_CALLBACKS(static, vma_block_subtree_max_size_cb, struct vma_block, rb_node, + unsigned long, subtree_max_size, compute_subtree_max_size); + +static void block_update_subtree_max_size(struct vma_block *block) +{ + vma_block_subtree_max_size_cb_propagate(&block->rb_node, NULL); +} + +static bool vma_algo_pool_same(struct xshm_pool *xp, struct xshm_reg_arg *arg) +{ + struct vma_ctrl *ctrl = xp->private; + + return ctrl->total_size == arg->pool_size; +} + +static int vma_algo_pool_init(struct xshm_pool *xp, struct xshm_reg_arg *arg) +{ + struct vma_ctrl *ctrl = NULL; + struct vma_block *block = NULL; + + if (!IS_ALIGNED(arg->pool_size, XSHMEM_VMA_ALIGN) || !arg->pool_size) { + pr_err("input xshm_pool size invalid\n"); + return -EINVAL; + } + + ctrl = kmalloc(sizeof(*ctrl), GFP_KERNEL); + block = kmalloc(sizeof(*block), GFP_KERNEL); + if (unlikely(ctrl == NULL || block == NULL)) { + kfree(ctrl); + kfree(block); + return -ENOMEM; + } + + ctrl->total_size = arg->pool_size; + ctrl->free_size = arg->pool_size; + ctrl->alloc_block_cnt = 0; + ctrl->free_block_root.rb_node = NULL; + + block->start = 0; + block->size = arg->pool_size; + block->subtree_max_size = arg->pool_size; + rb_link_node(&block->rb_node, NULL, &ctrl->free_block_root.rb_node); + rb_insert_color(&block->rb_node, &ctrl->free_block_root); + + xp->private = ctrl; + + return 0; +} + +#ifdef DEBUG_XSHMEM_VMA +static void block_dump(struct vma_block *block) +{ + pr_info("block: start: %#lx, size: %#lx, subtree_max_size: %#lx\n", + block->start, block->size, block->subtree_max_size); +} + +static void pool_dump(struct vma_ctrl *ctrl) +{ + struct vma_block *block, *tmp; + + rbtree_postorder_for_each_entry_safe(block, tmp, &ctrl->free_block_root, rb_node) + block_dump(block); +} +#else +static void pool_dump(struct vma_ctrl *ctrl) {} +#endif + +static int vma_algo_pool_free(struct xshm_pool *xp) +{ + struct vma_ctrl *ctrl = xp->private; + struct vma_block *block = to_vma_block(ctrl->free_block_root.rb_node); + + WARN(ctrl->total_size != ctrl->free_size, "not all block freed\n"); + WARN(ctrl->total_size != block->size, "not all block freed\n"); + WARN(ctrl->total_size != block->subtree_max_size, "not all block freed\n"); + + pool_dump(ctrl); + + kfree(block); + kfree(ctrl); + + return 0; +} + +static struct vma_block *find_first_proper_block(struct rb_root *root, unsigned long size) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct vma_block *block = to_vma_block(node); + + if (subtree_max_size(block->rb_node.rb_left) >= size) { + node = block->rb_node.rb_left; + continue; + } + + if (block->size >= size) + return block; + + if (subtree_max_size(block->rb_node.rb_right) >= size) + node = block->rb_node.rb_right; + else + return NULL; + } + + return NULL; +} + +static long vma_alloc_block(struct rb_root *root, unsigned long size) +{ + long start; + struct vma_block *block = NULL; + + block = find_first_proper_block(root, size); + if (block == NULL) + return -ENOSPC; + + start = block->start; + if (block->size == size) { + rb_erase_augmented(&block->rb_node, root, &vma_block_subtree_max_size_cb); + kfree(block); + } else { + block->start += size; + block->size -= size; + block_update_subtree_max_size(block); + } + + return start; +} + +static int vma_algo_block_alloc(struct xshm_pool *xp, struct xshm_block *blk) +{ + long offset; + unsigned long real_size; + struct vma_ctrl *ctrl = xp->private; + + if (!blk->alloc_size) { + pr_err("alloc size invalid\n"); + return -EINVAL; + } + + real_size = ALIGN(blk->alloc_size, XSHMEM_VMA_ALIGN); + if (real_size < blk->alloc_size) + return -EOVERFLOW; + + if (real_size > ctrl->free_size) + return -ENOSPC; + + offset = vma_alloc_block(&ctrl->free_block_root, real_size); + if (offset < 0) { + pr_err("vma_alloc_block failed, %ld\n", offset); + return (int)offset; + } + + ctrl->free_size -= real_size; + ctrl->alloc_block_cnt++; + + blk->real_size = real_size; + blk->offset = offset; + + return 0; +} + +static struct vma_block *prev_block(struct rb_node *parent, struct rb_node **link) +{ + if (parent == NULL) + return NULL; + + if (link == &parent->rb_right) + return to_vma_block(parent); + else + /* the prev node should always exist */ + return to_vma_block(rb_prev(parent)); +} + +static struct vma_block *next_block(struct rb_node *parent, struct rb_node **link) +{ + if (parent == NULL) + return NULL; + + if (link == &parent->rb_left) + return to_vma_block(parent); + else + /* the next node should always exist */ + return to_vma_block(rb_next(parent)); +} + +/* two intervals overlapped ? [l1, h1), [l2, h2) */ +#define is_overlap(l1, h1, l2, h2) (((l1) < (h2)) && ((l2) < (h1))) + +static int vma_free_block(struct rb_root *root, unsigned long start, unsigned long size) +{ + bool merged = false; + struct vma_block *block = NULL, *prev = NULL; + struct rb_node *parent = NULL; + struct rb_node **link = &root->rb_node; + + while (*link) { + parent = *link; + block = to_vma_block(*link); + + if (start < block->start) + link = &parent->rb_left; + else if (start > block->start) + link = &parent->rb_right; + else { + pr_err("double free a block\n"); + return -EFAULT; + } + } + + prev = prev_block(parent, link); + if (prev) { + if (prev->start + prev->size == start) { + prev->size += size; + block_update_subtree_max_size(prev); + merged = true; + } else if (is_overlap(start, start + size, prev->start, prev->start + prev->size)) { + pr_err("prev-node overlap detected\n"); + return -EINVAL; + } + } + + block = next_block(parent, link); + if (block) { + if (start + size == block->start) { + if (merged) { + rb_erase_augmented(&block->rb_node, root, &vma_block_subtree_max_size_cb); + prev->size += block->size; + kfree(block); + block = prev; + } else { + block->start -= size; + block->size += size; + } + + block_update_subtree_max_size(block); + merged = true; + } else if (is_overlap(start, start + size, block->start, block->start + block->size)) { + pr_err("next-node overlap detected\n"); + return -EINVAL; + } + } + + if (!merged) { + block = kmalloc(sizeof(*block), GFP_KERNEL); + if (unlikely(block == NULL)) { + pr_err("alloc new free block memroy failed\n"); + return -ENOMEM; + } + + block->start = start; + block->size = size; + block->subtree_max_size = size; + + rb_link_node(&block->rb_node, parent, link); + rb_insert_augmented(&block->rb_node, root, &vma_block_subtree_max_size_cb); + } + + return 0; +} + +static int vma_algo_block_free(struct xshm_pool *xp, struct xshm_block *blk) +{ + int ret; + struct vma_ctrl *ctrl = xp->private; + + ret = vma_free_block(&ctrl->free_block_root, blk->offset, blk->real_size); + + ctrl->alloc_block_cnt--; + ctrl->free_size += blk->real_size; + + return ret; +} + +static void vma_algo_pool_show(struct xshm_pool *xp, struct seq_file *seq) +{ + struct vma_ctrl *ctrl = xp->private; + + seq_printf(seq, " total_size: %#lx, free_size: %#lx, alloc_block_cnt: %ld\n", + ctrl->total_size, ctrl->free_size, ctrl->alloc_block_cnt); +} + +struct xshm_pool_algo vma_algo = { + .num = XSHMEM_ALGO_VMA, + .name = "vma_algo", + .xshm_pool_same = vma_algo_pool_same, + .xshm_pool_init = vma_algo_pool_init, + .xshm_pool_free = vma_algo_pool_free, + .xshm_pool_show = vma_algo_pool_show, + .xshm_block_alloc = vma_algo_block_alloc, + .xshm_block_free = vma_algo_block_free, +};