From: Liu Kai <liukai284@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Implement multi-device support allowing xsched to enable simultaneously on multiple hardware devices. Signed-off-by: Dashchynski Aliaksandr <dashchynski.aliaksandr1@h-partners.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- include/linux/xsched.h | 29 ++++-- kernel/xsched/cfs_quota.c | 1 - kernel/xsched/core.c | 61 +++++++----- kernel/xsched/vstream.c | 193 ++++++++++++++++++++------------------ 4 files changed, 158 insertions(+), 126 deletions(-) diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 825ff2dc0c8e..3a6007f33387 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -7,10 +7,15 @@ #include <linux/xcu_group.h> #include <linux/cgroup.h> #include <linux/vstream.h> + #ifndef pr_fmt #define pr_fmt(fmt) fmt #endif +#ifdef CONFIG_XCU_VSTREAM +#define MAX_VSTREAM_NUM (512) +#endif + #define XSCHED_ERR_PREFIX "XSched [ERROR]: " #define XSCHED_ERR(fmt, ...) \ pr_err(pr_fmt(XSCHED_ERR_PREFIX fmt), ##__VA_ARGS__) @@ -92,18 +97,14 @@ enum xse_flag { XSE_TIF_BALANCE, /* Unused so far */ }; - extern const struct xsched_class rt_xsched_class; extern const struct xsched_class fair_xsched_class; #define xsched_first_class (&rt_xsched_class) - #define for_each_xsched_class(class) \ for (class = xsched_first_class; class; class = class->next) - #define for_each_xse_prio(prio) \ for (prio = XSE_PRIO_LOW; prio < NR_XSE_PRIO; prio++) - #define for_each_vstream_in_ctx(vs, ctx) \ list_for_each_entry((vs), &((ctx)->vstream_list), ctx_node) @@ -179,6 +180,16 @@ struct xsched_cu { struct task_struct *worker; + /* Storage list for contexts associated with this xcu */ + uint32_t nr_ctx; + struct list_head ctx_list; + struct mutex ctx_list_lock; + +#ifdef CONFIG_XCU_VSTREAM + vstream_info_t *vs_array[MAX_VSTREAM_NUM]; + struct mutex vs_array_lock; +#endif + struct xsched_rq xrq; struct list_head vsm_list; @@ -531,20 +542,20 @@ extern struct list_head xsched_ctx_list; extern struct mutex xsched_ctx_list_mutex; /* Returns a pointer to xsched_context object corresponding to a given - * device file descriptor provided by fd argument. + * tgid and xcu. */ -static inline struct xsched_context *ctx_find_by_tgid(pid_t tgid) +static inline struct xsched_context * +ctx_find_by_tgid_and_xcu(pid_t tgid, struct xsched_cu *xcu) { struct xsched_context *ctx; struct xsched_context *ret = NULL; - list_for_each_entry(ctx, &xsched_ctx_list, ctx_node) { + list_for_each_entry(ctx, &xcu->ctx_list, ctx_node) { if (ctx->tgid == tgid) { ret = ctx; break; } } - return ret; } @@ -586,7 +597,7 @@ static inline void xsched_init_vsm(struct vstream_metadata *vsm, INIT_LIST_HEAD(&vsm->node); } -int xsched_xcu_register(struct xcu_group *group); +int xsched_xcu_register(struct xcu_group *group, int phys_id); void xsched_task_free(struct kref *kref); int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs); int ctx_bind_to_xcu(vstream_info_t *vstream_info, struct xsched_context *ctx); diff --git a/kernel/xsched/cfs_quota.c b/kernel/xsched/cfs_quota.c index 56f85ff9b31f..cb05d812abae 100644 --- a/kernel/xsched/cfs_quota.c +++ b/kernel/xsched/cfs_quota.c @@ -33,7 +33,6 @@ void xsched_quota_refill(struct work_struct *work) spin_unlock(&xg->lock); for_each_active_xcu(xcu, id) { - xcu = xsched_cu_mgr[id]; mutex_lock(&xcu->xcu_lock); if (!READ_ONCE(xg->perxcu_priv[id].xse.on_rq)) { enqueue_ctx(&xg->perxcu_priv[id].xse, xcu); diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index bce720c89e1b..262ec672d249 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -32,10 +32,6 @@ extern struct xsched_group *root_xcg; DECLARE_BITMAP(xcu_online_mask, XSCHED_NR_CUS); struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; -/* Storage list for contexts. */ -struct list_head xsched_ctx_list; -DEFINE_MUTEX(xsched_ctx_list_mutex); - static DEFINE_MUTEX(revmap_mutex); static DEFINE_HASHTABLE(ctx_revmap, XCU_HASH_ORDER); @@ -216,14 +212,16 @@ void xsched_task_free(struct kref *kref) { struct xsched_context *ctx; vstream_info_t *vs, *tmp; + struct xsched_cu *xcu; ctx = container_of(kref, struct xsched_context, kref); + xcu = ctx->xse.xcu; /* Wait till xse dequeues */ while (READ_ONCE(ctx->xse.on_rq)) usleep_range(100, 200); - mutex_lock(&xsched_ctx_list_mutex); + mutex_lock(&xcu->ctx_list_lock); list_for_each_entry_safe(vs, tmp, &ctx->vstream_list, ctx_node) { list_del(&vs->ctx_node); kfree(vs->data); @@ -232,7 +230,8 @@ void xsched_task_free(struct kref *kref) delete_ctx(ctx); list_del(&ctx->ctx_node); - mutex_unlock(&xsched_ctx_list_mutex); + --xcu->nr_ctx; + mutex_unlock(&xcu->ctx_list_lock); kfree(ctx); } @@ -287,6 +286,7 @@ int vstream_bind_to_xcu(vstream_info_t *vstream_info) /* Bind vstream to a xcu. */ vstream_info->xcu = xcu_found; + vstream_info->dev_id = xcu_found->id; XSCHED_DEBUG("XCU bound to a vstream: type=%u, dev_id=%u, chan_id=%u.\n", type, vstream_info->dev_id, vstream_info->channel_id); @@ -371,8 +371,8 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) } xse->ctx = ctx; - if (likely(vs->xcu != NULL)) - xse->xcu = vs->xcu; + BUG_ON(vs->xcu == NULL); + xse->xcu = vs->xcu; err = xsched_xse_set_class(xse); if (err) { @@ -602,13 +602,20 @@ static void xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, xcu->id = xcu_id; xcu->state = XSCHED_XCU_NONE; xcu->group = group; + xcu->nr_ctx = 0; atomic_set(&xcu->pending_kicks_rt, 0); atomic_set(&xcu->pending_kicks_cfs, 0); INIT_LIST_HEAD(&xcu->vsm_list); + INIT_LIST_HEAD(&xcu->ctx_list); init_waitqueue_head(&xcu->wq_xcu_idle); mutex_init(&xcu->xcu_lock); + mutex_init(&xcu->ctx_list_lock); + +#ifdef CONFIG_XCU_VSTREAM + mutex_init(&xcu->vs_array_lock); +#endif /* Mark current XCU in a mask inside XCU root group. */ set_bit(xcu->id, xcu_group_root->xcu_mask); @@ -624,22 +631,22 @@ static void xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, xcu->worker = kthread_run(xsched_schedule, xcu, "xcu_%u", xcu->id); } -/* Allocates xcu id in xcu_manager array. */ -static int alloc_xcu_id(void) +/* Increment xcu id */ +static int nr_active_cu_inc(void) { - int xcu_id = -1; + int cur_num = -1; spin_lock(&xcu_mgr_lock); if (num_active_xcu >= XSCHED_NR_CUS) goto out_unlock; - xcu_id = num_active_xcu; + cur_num = num_active_xcu; num_active_xcu++; - XSCHED_DEBUG("Number of active xcu: %d.\n", num_active_xcu); + XSCHED_DEBUG("Number of active xcus: %d.\n", num_active_xcu); out_unlock: spin_unlock(&xcu_mgr_lock); - return xcu_id; + return cur_num; } /* Adds vstream_metadata object to a specified vstream. */ @@ -701,14 +708,24 @@ struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs) /* * Initialize and register xcu in xcu_manager array. */ -int xsched_xcu_register(struct xcu_group *group) +int xsched_xcu_register(struct xcu_group *group, int phys_id) { - int xcu_id; + int xcu_cur_num; struct xsched_cu *xcu; - xcu_id = alloc_xcu_id(); - if (xcu_id < 0) { - XSCHED_ERR("Fail to alloc xcu id.\n"); + /* Can be refactored in future because it's possible that + * device contains more than 1 hardware task scheduler. + */ + if (phys_id >= XSCHED_NR_CUS) { + XSCHED_ERR("phys_id (%d) >= XSCHED_NR_CUS (%d).\n", + phys_id, XSCHED_NR_CUS); + return -EINVAL; + } + + xcu_cur_num = nr_active_cu_inc(); + if (xcu_cur_num < 0) { + XSCHED_ERR("Number of present XCU's exceeds %d: %d.\n", + XSCHED_NR_CUS, num_active_xcu); return -ENOSPC; }; @@ -719,18 +736,16 @@ int xsched_xcu_register(struct xcu_group *group) }; group->xcu = xcu; - xsched_cu_mgr[xcu_id] = xcu; + xsched_cu_mgr[phys_id] = xcu; /* Init xcu's internals. */ - xsched_xcu_init(xcu, group, xcu_id); + xsched_xcu_init(xcu, group, phys_id); return 0; } EXPORT_SYMBOL(xsched_xcu_register); int __init xsched_init(void) { - /* Initializing global XSched context list. */ - INIT_LIST_HEAD(&xsched_ctx_list); xcu_cg_init_common(root_xcg); return 0; } diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 869b3c56aad0..9ad065dd2422 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -21,12 +21,7 @@ #include <linux/delay.h> #ifdef CONFIG_XCU_VSTREAM -#define MAX_VSTREAM_NUM 512 - -static DEFINE_MUTEX(vs_mutex); -static vstream_info_t *vstream_array[MAX_VSTREAM_NUM]; - -static int vstream_del(uint32_t vstream_id); +static int vstream_del(vstream_info_t *vstream, uint32_t vstream_id); static int vstream_file_release(struct inode *inode, struct file *file); static const struct file_operations vstreamfd_fops = { .release = vstream_file_release, @@ -44,13 +39,8 @@ static inline void vstream_file_put(struct file *vstream_file) static int vstream_file_create(struct vstream_info *vs) { - int err = anon_inode_getfd("[vstreamfd]", - &vstreamfd_fops, vs, O_RDWR | O_CLOEXEC | O_NONBLOCK); - if (err < 0) - XSCHED_ERR("Fail to alloc anon inode vs %u @ %s\n", - vs->id, __func__); - - return err; + return anon_inode_getfd("[vstreamfd]", &vstreamfd_fops, vs, + O_RDWR | O_CLOEXEC | O_NONBLOCK); } static int vstream_destroy(vstream_info_t *vstream) @@ -59,7 +49,7 @@ static int vstream_destroy(vstream_info_t *vstream) struct xsched_context *ctx = NULL; struct xsched_entity *xse = NULL; - err = vstream_del(vstream->id); + err = vstream_del(vstream, vstream->id); if (err) return err; @@ -104,7 +94,9 @@ static void init_xsched_ctx(struct xsched_context *ctx, static int alloc_ctx_from_vstream(struct vstream_info *vstream_info, struct xsched_context **ctx) { - *ctx = ctx_find_by_tgid(vstream_info->tgid); + struct xsched_cu *xcu = vstream_info->xcu; + + *ctx = ctx_find_by_tgid_and_xcu(vstream_info->tgid, xcu); if (*ctx) return 0; @@ -123,8 +115,8 @@ static int alloc_ctx_from_vstream(struct vstream_info *vstream_info, kfree(*ctx); return -EINVAL; } - - list_add(&(*ctx)->ctx_node, &xsched_ctx_list); + list_add(&(*ctx)->ctx_node, &xcu->ctx_list); + ++xcu->nr_ctx; return 0; } @@ -133,27 +125,26 @@ static int alloc_ctx_from_vstream(struct vstream_info *vstream_info, static int vstream_bind_to_ctx(struct vstream_info *vs) { struct xsched_context *ctx = NULL; - int alloc_err = 0; + struct xsched_cu *xcu = vs->xcu; + int err = 0; - mutex_lock(&xsched_ctx_list_mutex); - ctx = ctx_find_by_tgid(vs->tgid); + mutex_lock(&xcu->ctx_list_lock); + ctx = ctx_find_by_tgid_and_xcu(vs->tgid, xcu); if (ctx) { XSCHED_DEBUG("Ctx %d found @ %s\n", vs->tgid, __func__); kref_get(&ctx->kref); } else { - alloc_err = alloc_ctx_from_vstream(vs, &ctx); - if (alloc_err) + err = alloc_ctx_from_vstream(vs, &ctx); + if (err) goto out_err; } vs->ctx = ctx; - vs->xcu = ctx->xse.xcu; - ctx->dev_id = vs->dev_id; list_add(&vs->ctx_node, &vs->ctx->vstream_list); out_err: - mutex_unlock(&xsched_ctx_list_mutex); - return alloc_err; + mutex_unlock(&xcu->ctx_list_lock); + return err; } static vstream_info_t *vstream_create(struct vstream_args *arg) @@ -166,7 +157,6 @@ static vstream_info_t *vstream_create(struct vstream_args *arg) return NULL; } - vstream->inode_fd = vstream_file_create(vstream); vstream->dev_id = arg->dev_id; vstream->channel_id = arg->channel_id; vstream->kicks_count = 0; @@ -183,63 +173,77 @@ static vstream_info_t *vstream_create(struct vstream_args *arg) static int vstream_add(vstream_info_t *vstream, uint32_t id) { + int err = 0; + struct xsched_cu *xcu = vstream->xcu; + if (id >= MAX_VSTREAM_NUM) { - XSCHED_ERR("vstream id out of range.\n"); + XSCHED_ERR("Vstream id=%u out of range @ %s.\n", + id, __func__); return -EINVAL; } - mutex_lock(&vs_mutex); - if (vstream_array[id] != NULL) { - mutex_unlock(&vs_mutex); + mutex_lock(&xcu->vs_array_lock); + if (xcu->vs_array[id] != NULL) { XSCHED_ERR("Vstream id=%u cell is busy.\n", id); - return -EINVAL; + err = -EINVAL; + goto out_err; } - vstream_array[id] = vstream; - mutex_unlock(&vs_mutex); + xcu->vs_array[id] = vstream; - return 0; +out_err: + mutex_unlock(&xcu->vs_array_lock); + return err; } -static int vstream_del(uint32_t vstream_id) +static int vstream_del(vstream_info_t *vstream, uint32_t vstream_id) { + struct xsched_cu *xcu = vstream->xcu; + if (vstream_id >= MAX_VSTREAM_NUM) { - XSCHED_ERR("Vstream id=%u out of range.\n", vstream_id); + XSCHED_ERR("Vstream id=%u out of range @ %s.\n", + vstream_id, __func__); return -EINVAL; } - mutex_lock(&vs_mutex); - vstream_array[vstream_id] = NULL; - mutex_unlock(&vs_mutex); + mutex_lock(&xcu->vs_array_lock); + xcu->vs_array[vstream_id] = NULL; + mutex_unlock(&xcu->vs_array_lock); return 0; } -static vstream_info_t *vstream_get(uint32_t vstream_id) +static vstream_info_t *vstream_get(struct xsched_cu *xcu, uint32_t vstream_id) { vstream_info_t *vstream = NULL; if (vstream_id >= MAX_VSTREAM_NUM) { - XSCHED_ERR("Vstream id=%u out of range.\n", vstream_id); + XSCHED_ERR("Vstream id=%u out of range @ %s.\n", + vstream_id, __func__); return NULL; } - mutex_lock(&vs_mutex); - vstream = vstream_array[vstream_id]; - mutex_unlock(&vs_mutex); + mutex_lock(&xcu->vs_array_lock); + vstream = xcu->vs_array[vstream_id]; + mutex_unlock(&xcu->vs_array_lock); return vstream; } static vstream_info_t * -vstream_get_by_user_stream_id(uint32_t user_stream_id) +vstream_get_by_user_stream_id(struct xsched_cu *xcu, uint32_t user_stream_id) { int id; + static vstream_info_t *ret; + mutex_lock(&xcu->vs_array_lock); for (id = 0; id < MAX_VSTREAM_NUM; id++) { - if (vstream_array[id] != NULL && - vstream_array[id]->user_stream_id == user_stream_id) - return vstream_array[id]; + if (xcu->vs_array[id] != NULL && + xcu->vs_array[id]->user_stream_id == user_stream_id) { + ret = xcu->vs_array[id]; + break; + } } - return NULL; + mutex_unlock(&xcu->vs_array_lock); + return ret; } static int sqcq_alloc(struct vstream_args *arg) @@ -262,7 +266,7 @@ static int sqcq_alloc(struct vstream_args *arg) vstream->task_type = arg->task_type; ret = vstream_bind_to_xcu(vstream); - if (ret < 0) { + if (ret) { ret = -EINVAL; goto out_err_vstream_free; } @@ -290,23 +294,27 @@ static int sqcq_alloc(struct vstream_args *arg) vstream->sqcq_type = va_args->type; ret = vstream_bind_to_ctx(vstream); - if (ret < 0) + if (ret) goto out_err_vstream_free; ctx = vstream->ctx; + ret = vstream_file_create(vstream); + if (ret < 0) { + XSCHED_ERR("Fail to alloc anon inode for vstream %u @ %s\n", + vstream->id, __func__); + goto out_err_vstream_free; + } + vstream->inode_fd = ret; /* Add new vstream to array after allocating inode */ ret = vstream_add(vstream, vstream->id); - if (ret < 0) + if (ret) goto out_err_vstream_free; return 0; out_err_vstream_free: kfree(vstream); - XSCHED_ERR("Exit %s with error, current_pid=%d, err=%d.\n", - __func__, current->pid, ret); - return ret; } @@ -317,20 +325,17 @@ static int logic_cq_alloc(struct vstream_args *arg) vstream_info_t *vstream = NULL; vstream_alloc_args_t *logic_cq_alloc_para = &arg->va_args; struct xsched_cu *xcu_found = NULL; - uint32_t logic_cq_id = 0, type = XCU_TYPE_XPU; + uint32_t logic_cq_id = 0; + uint32_t type = XCU_TYPE_XPU; + + xcu_found = xcu_find(&type, arg->dev_id, arg->channel_id); + if (!xcu_found) + return -EINVAL; - vstream = vstream_get_by_user_stream_id( + vstream = vstream_get_by_user_stream_id(xcu_found, logic_cq_alloc_para->user_stream_id); - if (!vstream) { - xcu_found = xcu_find(&type, arg->dev_id, arg->channel_id); - if (!xcu_found) { - err = -EINVAL; - goto out_err; - } - } else { + if (vstream) xcu_found = vstream->xcu; - } - params.group = xcu_found->group; params.fd = arg->fd; params.payload = arg->payload; @@ -338,21 +343,12 @@ static int logic_cq_alloc(struct vstream_args *arg) err = xcu_logic_alloc(¶ms); if (err) { XSCHED_ERR("Fail to alloc logic CQ memory to a vstream.\n"); - goto out_err; + return err; } + if (vstream) + vstream->logic_vcq_id = logic_cq_id; - vstream->logic_vcq_id = logic_cq_id; - XSCHED_DEBUG( - "Vstream logic CQ: dev_id=%u, stream_id=%u, logic_cqid=%u @ %s\n", - vstream->dev_id, vstream->user_stream_id, - vstream->logic_vcq_id, __func__); return 0; - -out_err: - XSCHED_ERR( - "Exit %s with error, current_pid=%d, err=%d.\n", - __func__, current->pid, err); - return err; } int vstream_alloc(struct vstream_args *arg) @@ -372,11 +368,17 @@ int vstream_free(struct vstream_args *arg) { struct file *vs_file; struct xcu_op_handler_params params; + struct xsched_cu *xcu_found; uint32_t vstream_id = arg->sq_id; + uint32_t type = XCU_TYPE_XPU; vstream_info_t *vstream = NULL; int err = 0; - vstream = vstream_get(vstream_id); + xcu_found = xcu_find(&type, arg->dev_id, arg->channel_id); + if (!xcu_found) + return -EINVAL; + + vstream = vstream_get(xcu_found, vstream_id); if (!vstream) { XSCHED_ERR("Fail to free NULL vstream, vstream id=%u\n", vstream_id); return -EINVAL; @@ -387,10 +389,13 @@ int vstream_free(struct vstream_args *arg) params.payload = arg->payload; vs_file = vstream_file_get(vstream->inode_fd); - vstream_destroy(vstream); - vs_file->private_data = NULL; - vstream_file_put(vs_file); + if (vs_file) { + vs_file->private_data = NULL; + vstream_file_put(vs_file); + } + /* After vstream_get(), destroying the vstream may not fail */ + vstream_destroy(vstream); err = xcu_finish(¶ms); if (err) XSCHED_ERR("Fail to free vstream sqId=%u, cqId=%u.\n", @@ -402,23 +407,25 @@ int vstream_free(struct vstream_args *arg) int vstream_kick(struct vstream_args *arg) { vstream_info_t *vstream; - int vstream_id = arg->sq_id; + struct xsched_cu *xcu = NULL; struct xsched_entity *xse; int err = 0; + uint32_t vstream_id = arg->sq_id; + uint32_t type = XCU_TYPE_XPU; - struct xsched_cu *xcu = NULL; - - XSCHED_CALL_STUB(); + xcu = xcu_find(&type, arg->dev_id, arg->channel_id); + if (!xcu) + return -EINVAL; /* Get vstream. */ - vstream = vstream_get(vstream_id); + vstream = vstream_get(xcu, vstream_id); if (!vstream || !vstream->ctx) { - XSCHED_ERR("Vstream NULL or doesn't have a context.\n"); + XSCHED_ERR("Vstream NULL or doesn't have a context. " + "vstream_id=%u, dev_id=%u\n", vstream_id, arg->dev_id); return -EINVAL; } xse = &vstream->ctx->xse; - xcu = vstream->xcu; XSCHED_DEBUG("New kick on xse %d @ %s\n", xse->tgid, __func__); do { @@ -474,17 +481,17 @@ SYSCALL_DEFINE2(vstream_manage, struct vstream_args __user *, arg, int, cmd) struct vstream_args vstream_arg; if (copy_from_user(&vstream_arg, arg, sizeof(struct vstream_args))) { - pr_err("copy_from_user failed\n"); + XSCHED_ERR("copy_from_user failed\n"); return -EFAULT; } res = vstream_command_table[cmd](&vstream_arg); if (copy_to_user(arg, &vstream_arg, sizeof(struct vstream_args))) { - pr_err("copy_to_user failed\n"); + XSCHED_ERR("copy_to_user failed\n"); return -EFAULT; } - pr_debug("vstream_manage: cmd %d\n", cmd); + XSCHED_DEBUG("vstream_manage: cmd %d\n", cmd); return res; } #else -- 2.34.1