From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add vstream related data structures: - vstream_info. Add vstream related context and entity data structures: - xsched_entity - xsched_context Add xsched_init() implementation. Add vstream_alloc() and xcu_alloc() implementation. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- drivers/xcu/xcu_group.c | 47 ++++- include/linux/vstream.h | 45 +++++ include/linux/xcu_group.h | 16 ++ include/linux/xsched.h | 72 ++++++++ kernel/xsched/core.c | 77 ++++++++ kernel/xsched/vstream.c | 359 +++++++++++++++++++++++++++++++++++++- 6 files changed, 614 insertions(+), 2 deletions(-) diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index a2419db384ac..b70e9e7392e3 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -193,7 +193,52 @@ int xcu_finish(struct xcu_op_handler_params *params) */ int xcu_alloc(struct xcu_op_handler_params *params) { - return 0; + int ret = 0; + + if (params->group->opt && params->group->opt->alloc) + ret = params->group->opt->alloc(params); + else + XSCHED_DEBUG("No function [alloc] called.\n"); + + return ret; +} + +/* This function runs a "logic_alloc" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement allocation + * and registering memory of logic CQ buffer. + */ +int xcu_logic_alloc(struct xcu_op_handler_params *params) +{ + int ret = 0; + + if (params->group->opt && params->group->opt->logic_alloc) + ret = params->group->opt->logic_alloc(params); + else + XSCHED_DEBUG("No function [logic_alloc] called.\n"); + + return ret; +} + +/* This function runs a "logic_free" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement deallocation + * and unregistering memory of a logic CQ buffer. + */ +int xcu_logic_free(struct xcu_op_handler_params *params) +{ + int ret = 0; + + if (params->group->opt && params->group->opt->logic_free) + ret = params->group->opt->logic_free(params); + else + XSCHED_DEBUG("No function [logic_free] called.\n"); + + return ret; } static struct xcu_group __xcu_group_root = { diff --git a/include/linux/vstream.h b/include/linux/vstream.h index 627f754f83c5..39cc6a26db50 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -4,6 +4,51 @@ #include <uapi/linux/xcu_vstream.h> +typedef struct vstream_info { + uint32_t user_streamId; + uint32_t id; + uint32_t vcqId; + uint32_t logic_vcqId; + uint32_t devId; + uint32_t channel_id; + uint32_t fd; + uint32_t task_type; + int tgid; + int sqcq_type; + + void *drv_ctx; + + /* Pointer to corresponding context. */ + struct xsched_context *ctx; + + /* List node in context's vstream list. */ + struct list_head ctx_node; + + /* Pointer to an CU object on which this + * vstream is currently being processed. + * NULL if vstream is not being processed. + */ + struct xsched_cu *xcu; + + /* List node in an CU list of vstreams that + * are currently being processed by this specific CU. + */ + struct list_head xcu_node; + + /* Private vstream data. */ + void *data; + + spinlock_t stream_lock; + + uint32_t kicks_count; + + /* List of metadata a.k.a. all recorded unprocesed + * kicks for this exact vstream. + */ + struct list_head metadata_list; + +} vstream_info_t; + typedef int vstream_manage_t(struct vstream_args *arg); #endif /* _LINUX_VSTREAM_H */ diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h index 55facb0e5760..fc4f23fce726 100644 --- a/include/linux/xcu_group.h +++ b/include/linux/xcu_group.h @@ -23,6 +23,20 @@ enum xcu_version { }; struct xcu_op_handler_params { + int fd; + struct xcu_group *group; + void *payload; + union { + struct { + void *param_1; + void *param_2; + void *param_3; + void *param_4; + void *param_5; + void *param_6; + void *param_7; + }; + }; }; typedef int (*xcu_op_handler_fn_t)(struct xcu_op_handler_params *params); @@ -33,6 +47,8 @@ struct xcu_operation { xcu_op_handler_fn_t wait; xcu_op_handler_fn_t complete; xcu_op_handler_fn_t alloc; + xcu_op_handler_fn_t logic_alloc; + xcu_op_handler_fn_t logic_free; }; struct xcu_group { diff --git a/include/linux/xsched.h b/include/linux/xsched.h index bdac1653ae36..dc930d7692ba 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -3,6 +3,8 @@ #define __LINUX_XSCHED_H__ #include <linux/xcu_group.h> +#include <linux/kref.h> +#include <linux/vstream.h> #ifndef pr_fmt #define pr_fmt(fmt) fmt #endif @@ -88,5 +90,75 @@ struct xsched_cu { wait_queue_head_t wq_xcore_running; }; +struct xsched_entity { + uint32_t task_type; + + bool on_rq; + + pid_t owner_pid; + pid_t tgid; + + /* File descriptor coming from an associated context + * used for identifying a given xsched entity in + * info and error prints. + */ + uint32_t fd; + + /* Xsched class for this xse. */ + const struct xsched_class *class; + + /* Pointer to context object. */ + struct xsched_context *ctx; + + /* Pointer to an XCU object that represents an XCU + * on which this xse is to be processed or is being + * processed currently. + */ + struct xsched_cu *xcu; + + /* General purpose xse lock. */ + spinlock_t xse_lock; +}; + +struct xsched_context { + uint32_t fd; + uint32_t devId; + pid_t tgid; + + struct list_head vstream_list; + struct list_head ctx_node; + + struct xsched_entity xse; + + spinlock_t ctx_lock; + struct mutex ctx_mutex; + struct kref kref; +}; + +extern struct list_head xsched_ctx_list; +extern struct mutex xsched_ctx_list_mutex; + +/* Returns a pointer to xsched_context object corresponding to a given + * device file descriptor provided by fd argument. + */ +static inline struct xsched_context *find_ctx_by_tgid(pid_t tgid) +{ + struct xsched_context *ctx; + struct xsched_context *ret = NULL; + + list_for_each_entry(ctx, &xsched_ctx_list, ctx_node) { + if (ctx->tgid == tgid) { + ret = ctx; + break; + } + } + + return ret; +} + int xsched_register_xcu(struct xcu_group *group); +int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs); +int bind_ctx_to_xcu(vstream_info_t *vstream_info, struct xsched_context *ctx); +int bind_vstream_to_xcu(vstream_info_t *vstream_info); +struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 96a814275963..52a600b945f4 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -32,6 +32,73 @@ spinlock_t xcu_mgr_lock; DECLARE_BITMAP(xcu_online_mask, XSCHED_NR_CUS); struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; +/* Storage list for contexts. */ +struct list_head xsched_ctx_list; +DEFINE_MUTEX(xsched_ctx_list_mutex); + +int bind_vstream_to_xcu(vstream_info_t *vstream_info) +{ + struct xsched_cu *xcu_found = NULL; + __u32 type = XCU_TYPE_NPU; + + xcu_found = xcu_find(&type, vstream_info->devId, vstream_info->channel_id); + + if (!xcu_found) + return -1; + + /* Bind vstream to a xcu. */ + vstream_info->xcu = xcu_found; + + XSCHED_INFO("XCU bound to a vstream: type=%u, dev_id=%u, chan_id=%u.\n", + type, vstream_info->devId, vstream_info->channel_id); + + return 0; +} + +struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id) +{ + struct xcu_group *group = NULL; + __u32 local_type = *type; + + /* Find xcu by type. */ + group = xcu_group_find_noalloc(xcu_group_root, local_type); + if (group == NULL) { + XSCHED_INFO("Find XCU group with real device is failed.\n"); + + local_type = XCU_TYPE_NPU; + group = xcu_group_find_noalloc(xcu_group_root, local_type); + if (group == NULL) { + XSCHED_ERR("Find XCU with qemu device is failed.\n"); + return NULL; + } + } + + /* Find by device id group. */ + group = xcu_group_find_noalloc(group, devId); + if (group == NULL) { + XSCHED_ERR("Find device group is failed.\n"); + return NULL; + } + /* Find channel id group. */ + group = xcu_group_find_noalloc(group, channel_id); + if (group == NULL) { + XSCHED_ERR("Find channel group is failed.\n"); + return NULL; + } + + *type = local_type; + + XSCHED_INFO("XCU found: type=%u, dev_id=%u, chan_id=%u.\n", local_type, + devId, channel_id); + + return group->xcu; +} + +int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) +{ + return 0; +} + static int xsched_schedule(void *input_xcu) { return 0; @@ -109,3 +176,13 @@ int xsched_register_xcu(struct xcu_group *group) return 0; } EXPORT_SYMBOL(xsched_register_xcu); + +int __init xsched_init(void) +{ + /* Initializing global Xsched context list. */ + INIT_LIST_HEAD(&xsched_ctx_list); + + return 0; +} + +late_initcall(xsched_init); diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 5723c359e0f2..73017da55ff2 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -18,12 +18,369 @@ */ #include <linux/syscalls.h> #include <linux/vstream.h> +#include <linux/xsched.h> #ifdef CONFIG_XCU_VSTREAM +#define MAX_VSTREAM_NUM (512 - 1) -int vstream_alloc(struct vstream_args *arg) +static DEFINE_MUTEX(vs_mutex); +static vstream_info_t *vstream_array[MAX_VSTREAM_NUM]; +static void init_xsched_ctx(struct xsched_context *ctx, + const struct vstream_info *vs) +{ + ctx->tgid = vs->tgid; + ctx->fd = vs->fd; + ctx->devId = vs->devId; + kref_init(&ctx->kref); + + INIT_LIST_HEAD(&ctx->vstream_list); + INIT_LIST_HEAD(&ctx->ctx_node); + + spin_lock_init(&ctx->ctx_lock); + mutex_init(&ctx->ctx_mutex); +} + +/* Allocates a new xsched_context if a new vstream_info is bound + * to a device that no other vstream that is currently present + * is bound to. + */ +static int alloc_ctx_from_vstream(struct vstream_info *vstream_info, + struct xsched_context **ctx) +{ + int err = 0; + + XSCHED_CALL_STUB(); + + *ctx = find_ctx_by_tgid(vstream_info->tgid); + if (*ctx) { + XSCHED_INFO("Ctx %d found @ %s\n", + vstream_info->tgid, __func__); + goto out_err; + } + + *ctx = kzalloc(sizeof(struct xsched_context), GFP_KERNEL); + + if (!*ctx) { + XSCHED_ERR("Could not allocate xsched context (tgid=%d) @ %s\n", + vstream_info->tgid, __func__); + err = -ENOMEM; + goto out_err; + } + + init_xsched_ctx(*ctx, vstream_info); + + err = xsched_ctx_init_xse(*ctx, vstream_info); + + if (err) { + XSCHED_ERR("Failed to initialize XSE for context @ %s\n", + __func__); + kfree(*ctx); + err = -EINVAL; + goto out_err; + } + + list_add(&(*ctx)->ctx_node, &xsched_ctx_list); + +out_err: + XSCHED_EXIT_STUB(); + + return err; +} + +/* Bounds a new vstream_info object to a corresponding xsched context. */ +static int vstream_bind_to_ctx(struct vstream_info *vs) +{ + struct xsched_context *ctx = NULL; + int alloc_err = 0; + + XSCHED_CALL_STUB(); + + XSCHED_INFO("Ctx list mutext taken @ %s\n", __func__); + mutex_lock(&xsched_ctx_list_mutex); + + ctx = find_ctx_by_tgid(vs->tgid); + if (ctx) { + XSCHED_INFO("Ctx %d found @ %s\n", vs->tgid, __func__); + kref_get(&ctx->kref); + } else { + alloc_err = alloc_ctx_from_vstream(vs, &ctx); + if (alloc_err) + goto out_err; + } + + XSCHED_INFO("Ctx = %p @ %s\n", ctx, __func__); + + vs->ctx = ctx; + vs->xcu = ctx->xse.xcu; + ctx->devId = vs->devId; + list_add(&vs->ctx_node, &vs->ctx->vstream_list); + +out_err: + mutex_unlock(&xsched_ctx_list_mutex); + XSCHED_INFO("Ctx list mutex released @ %s\n", __func__); + + XSCHED_EXIT_STUB(); + + return alloc_err; +} + +static vstream_info_t *vstream_create_info(struct vstream_args *arg) +{ + struct vstream_info *vstream; + + XSCHED_CALL_STUB(); + + vstream = kzalloc(sizeof(vstream_info_t), GFP_KERNEL); + if (!vstream) { + XSCHED_ERR("Failed to allocate vstream.\n"); + vstream = NULL; + goto out_err; + } + + vstream->devId = arg->devid; + vstream->channel_id = arg->channel_id; + + INIT_LIST_HEAD(&vstream->ctx_node); + INIT_LIST_HEAD(&vstream->xcu_node); + INIT_LIST_HEAD(&vstream->metadata_list); + + spin_lock_init(&vstream->stream_lock); + + vstream->kicks_count = 0; + + vstream->xcu = NULL; + +out_err: + XSCHED_EXIT_STUB(); + + return vstream; +} + +static int vstream_add(vstream_info_t *vstream, uint32_t id) +{ + XSCHED_CALL_STUB(); + XSCHED_INFO("Adding vstream %u @ %s\n", id, __func__); + + if (id >= MAX_VSTREAM_NUM) { + XSCHED_ERR("vstreamId out of range.\n"); + return -EINVAL; + } + + mutex_lock(&vs_mutex); + if (vstream_array[id] != NULL) { + mutex_unlock(&vs_mutex); + XSCHED_ERR("VstreamId=%u cell is busy.\n", id); + return -EINVAL; + } + vstream_array[id] = vstream; + mutex_unlock(&vs_mutex); + + return 0; +} + +static vstream_info_t * +vstream_get_by_user_stream_id(uint32_t user_streamId) +{ + int id; + + for (id = 0; id < MAX_VSTREAM_NUM; id++) { + if (vstream_array[id] != NULL) + if (vstream_array[id]->user_streamId == + user_streamId) + return vstream_array[id]; + } + return NULL; +} + +static int sqcq_alloc(struct vstream_args *arg) { + vstream_alloc_args_t *va_args = &arg->va_args; + struct xsched_context *ctx = NULL; + struct xcu_op_handler_params params; + uint32_t logic_cq_id = 0; + vstream_info_t *vstream; + int err = -EINVAL; + uint32_t tgid = 0; + uint32_t cq_id = 0; + uint32_t sq_id = 0; + + XSCHED_CALL_STUB(); + + vstream = vstream_create_info(arg); + if (!vstream) { + XSCHED_ERR("vstream create failed.\n"); + err = -ENOSPC; + goto out_err; + } + vstream->fd = arg->fd; + vstream->task_type = arg->task_type; + + err = bind_vstream_to_xcu(vstream); + if (err) { + XSCHED_ERR( + "Couldn't find valid xcu for vstream dev_id=%u chan_id=%u @ %s\n", + vstream->devId, vstream->channel_id, __func__); + err = -EINVAL; + goto out_err_vstream_free; + } + + /* Allocates vstream's SQ and CQ memory on a XCU for processing. */ + params.group = vstream->xcu->group; + params.fd = arg->fd; + params.payload = arg->payload; + params.param_1 = &tgid; + params.param_2 = &sq_id; + params.param_3 = &cq_id; + params.param_4 = &logic_cq_id; + err = xcu_alloc(¶ms); + if (err) { + XSCHED_ERR( + "Failed to allocate SQ and CQ memory to an app stream.\n"); + goto out_err_vstream_free; + } + vstream->drv_ctx = params.param_5; + vstream->id = sq_id; + vstream->vcqId = cq_id; + vstream->logic_vcqId = logic_cq_id; + XSCHED_INFO("New vstream id %u, cq_id=%u, logic_cqid=%u @ %s\n", + vstream->id, vstream->vcqId, vstream->logic_vcqId, __func__); + + vstream->user_streamId = va_args->user_stream_id; + vstream->tgid = tgid; + vstream->sqcq_type = va_args->type; + + + err = vstream_bind_to_ctx(vstream); + if (err) { + XSCHED_ERR( + "Failed to bind vstream %u to an app context.\n", + vstream->id); + goto out_err_vstream_free; + } + + ctx = vstream->ctx; + + err = vstream_add(vstream, vstream->id); + if (err) { + XSCHED_ERR( + "Failed to add vstream id=%u to vstream_array.\n", + vstream->id); + goto out_err_vstream_free; + } + + XSCHED_INFO( + "vstream allocation success: user_streamId=%u, sqid=%u, cqid=%u, ctx=%p.\n", + vstream->user_streamId, vstream->id, vstream->vcqId, ctx); + + XSCHED_EXIT_STUB(); + return 0; + +out_err_vstream_free: + kfree(vstream); + +out_err: + XSCHED_INFO( + "Exit %s with error, current_pid=%d, err=%d.\n", + __func__, current->pid, err); + + return err; +} + +static int logic_cq_alloc(struct vstream_args *arg) +{ + int err = 0; + struct xcu_op_handler_params params; + vstream_info_t *vstream = NULL; + vstream_alloc_args_t *logic_cq_alloc_para = &arg->va_args; + uint32_t logic_cq_id = 0; + + XSCHED_CALL_STUB(); + + XSCHED_INFO( + "Enter %s, current_pid=%d, fd=%u.\n", + __func__, current->pid, arg->fd); + + vstream = vstream_get_by_user_stream_id( + logic_cq_alloc_para->user_stream_id); + if (!vstream) { + struct xsched_cu *xcu_found = NULL; + __u32 type = XCU_TYPE_NPU; + + xcu_found = xcu_find(&type, arg->devid, arg->channel_id); + if (xcu_found == NULL) { + XSCHED_ERR( + "Couldn't find valid xcu for control vstream dev_id=%u chan_id=%u @ %s\n", + arg->devid, arg->channel_id, __func__); + err = -EINVAL; + goto out_err; + } + + /* Allocates control vstream's SQ and CQ memory on a XCU for processing. */ + params.group = xcu_found->group; + params.fd = arg->fd; + params.payload = arg->payload; + params.param_1 = &logic_cq_id; + + err = xcu_logic_alloc(¶ms); + if (err) { + XSCHED_ERR( + "Failed to allocate logic CQ memory to a vstream.\n"); + goto out_err; + } + + XSCHED_INFO( + "vstream logic CQ memory allocation success: logic_cq_id=%u.\n", + logic_cq_id); + + return 0; + } + + params.group = vstream->xcu->group; + params.fd = arg->fd; + params.payload = arg->payload; + params.param_1 = &logic_cq_id; + + err = xcu_logic_alloc(¶ms); + if (err) { + XSCHED_ERR( + "Failed to allocate logic CQ memory to an app stream.\n"); + goto out_err; + } + + vstream->logic_vcqId = logic_cq_id; + + XSCHED_INFO( + "Vstream logic CQ memory allocation success: user_streamId=%u, logic_cqid=%u.\n", + vstream->user_streamId, vstream->logic_vcqId); + + return 0; + +out_err: + XSCHED_INFO( + "Exit %s with error, current_pid=%d, err=%d.\n", + __func__, current->pid, err); + + return err; +} + +int vstream_alloc(struct vstream_args *arg) +{ + vstream_alloc_args_t *va_args = &arg->va_args; + int ret; + + XSCHED_CALL_STUB(); + XSCHED_INFO( + "Enter %s, current_pid=%d, fd=%u, type=%d.\n", + __func__, current->pid, arg->fd, va_args->type); + + if (!va_args->type) + ret = sqcq_alloc(arg); + else + ret = logic_cq_alloc(arg); + + XSCHED_EXIT_STUB(); + return ret; } int vstream_free(struct vstream_args *arg) -- 2.34.1