From: Dmitriy Alekseev <alekseev.dmitry@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- The xpu device should be exclusive in sched slice. So add xcu_wait() implementation which return after xpu kernels executed completed. It is called after submit_kick(). Signed-off-by: Dmitriy Alekseev <alekseev.dmitry@huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> --- drivers/xcu/xcu_group.c | 24 +++- include/linux/vstream.h | 7 +- include/linux/xcu_group.h | 8 ++ include/linux/xsched.h | 15 +-- include/uapi/linux/xcu_vstream.h | 4 +- kernel/xsched/core.c | 181 +++++++++++++++++-------------- kernel/xsched/rt.c | 6 +- 7 files changed, 145 insertions(+), 100 deletions(-) diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index 0d4b0de482fc..45725758683e 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -167,7 +167,11 @@ int xcu_run(struct xcu_op_handler_params *params) */ int xcu_wait(struct xcu_op_handler_params *params) { - return 0; + if (!params->group->opt || !params->group->opt->wait) { + XSCHED_DEBUG("No function [wait] called.\n"); + return 0; + } + return params->group->opt->wait(params); } /* This function runs "complete" callback for a given xcu_group @@ -252,6 +256,24 @@ int xcu_logic_free(struct xcu_op_handler_params *params) return ret; } +/* This function runs a "sqe_op" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to set or get sqe info. + */ +int xcu_sqe_op(struct xcu_op_handler_params *params) +{ + int ret = 0; + + if (params->group->opt && params->group->opt->sqe_op) + ret = params->group->opt->sqe_op(params); + else + XSCHED_DEBUG("No function [sqe_op] called.\n"); + + return ret; +} + static struct xcu_group __xcu_group_root = { .id = 0, .type = XCU_TYPE_ROOT, diff --git a/include/linux/vstream.h b/include/linux/vstream.h index 8e7bf5992898..3c35b9466832 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -12,15 +12,18 @@ * to be processed by a driver. */ typedef struct vstream_metadata { - uint32_t exec_time; /* A value of SQ tail that has been passed with the * kick that is described by this exact metadata object. */ uint32_t sq_tail; uint32_t sqe_num; uint32_t sq_id; + uint8_t sqe[XCU_SQE_SIZE_MAX]; + + /* Report buffer for fake read. */ + int8_t cqe[XCU_CQE_BUF_SIZE]; + uint32_t cqe_num; int32_t timeout; - int8_t sqe[XCU_SQE_SIZE_MAX]; /* A node for metadata list */ struct list_head node; diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h index a2ecfbe87dc8..c74387c760c3 100644 --- a/include/linux/xcu_group.h +++ b/include/linux/xcu_group.h @@ -22,6 +22,11 @@ enum xcu_version { XCU_HW_V2 }; +enum xcu_sqe_op_type { + SQE_SET_NOTIFY, + SQE_IS_NOTIFY +}; + /** * @group: value for this entry. * @hash_node: hash node list. @@ -46,6 +51,7 @@ struct xcu_op_handler_params { void *param_5; void *param_6; void *param_7; + void *param_8; }; }; }; @@ -60,6 +66,7 @@ struct xcu_operation { xcu_op_handler_fn_t alloc; xcu_op_handler_fn_t logic_alloc; xcu_op_handler_fn_t logic_free; + xcu_op_handler_fn_t sqe_op; }; struct xcu_group { @@ -110,6 +117,7 @@ extern int xcu_finish(struct xcu_op_handler_params *params); extern int xcu_alloc(struct xcu_op_handler_params *params); extern int xcu_logic_alloc(struct xcu_op_handler_params *params); extern int xcu_logic_free(struct xcu_op_handler_params *params); +extern int xcu_sqe_op(struct xcu_op_handler_params *params); #endif /* CONFIG_XCU_SCHEDULER */ #endif /* !__XSCHED_XCU_GROUP_H__ */ diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 5de53d9c231b..7c8a02c862d1 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -463,12 +463,10 @@ static inline int xsched_inc_pending_kicks_xse(struct xsched_entity *xse) /* Incrementing prio based pending kicks counter for RT class */ if (xse_is_rt(xse)) { atomic_inc(&xse->xcu->xrq.rt.prio_nr_kicks[xse->rt.prio]); - XSCHED_INFO("xcu increased pending kicks @ %s\n", __func__); - } else { - XSCHED_INFO("xse %u isn't rt class @ %s\n", xse->tgid, - __func__); } + XSCHED_INFO("xcu increased pending kicks @ %s\n", __func__); + return 0; } @@ -511,12 +509,10 @@ static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) } atomic_dec(kicks_prio_rt); - XSCHED_INFO("xcu decreased pending kicks @ %s\n", __func__); - } else { - XSCHED_INFO("xse %u isn't rt class @ %s\n", xse->tgid, - __func__); } + XSCHED_INFO("xcu decreased pending kicks @ %s\n", __func__); + return 0; } @@ -623,7 +619,6 @@ static inline void xsched_init_vsm(struct vstream_metadata *vsm, vstream_args_t *arg) { vsm->sq_id = arg->sq_id; - vsm->exec_time = arg->vk_args.exec_time; vsm->sqe_num = arg->vk_args.sqe_num; vsm->timeout = arg->vk_args.timeout; memcpy(vsm->sqe, arg->vk_args.sqe, XCU_SQE_SIZE_MAX); @@ -641,8 +636,6 @@ struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id); /* Vstream metadata proccesing functions.*/ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); -void submit_kick(struct vstream_info *vs, struct xcu_op_handler_params *params, - struct vstream_metadata *vsm); /* Xsched group manage functions */ int xsched_group_inherit(struct task_struct *tsk, struct xsched_entity *xse); void xcu_cg_init_common(struct xsched_group *xcg); diff --git a/include/uapi/linux/xcu_vstream.h b/include/uapi/linux/xcu_vstream.h index 1024f5af5c2d..0c19421e1650 100644 --- a/include/uapi/linux/xcu_vstream.h +++ b/include/uapi/linux/xcu_vstream.h @@ -6,6 +6,9 @@ #define PAYLOAD_SIZE_MAX 512 #define XCU_SQE_SIZE_MAX 64 +#define XCU_CQE_SIZE_MAX 32 +#define XCU_CQE_REPORT_NUM 4 +#define XCU_CQE_BUF_SIZE (XCU_CQE_REPORT_NUM * XCU_CQE_SIZE_MAX) /* * VSTREAM_ALLOC: alloc a vstream, buffer for tasks @@ -28,7 +31,6 @@ typedef struct vstream_free_args { } vstream_free_args_t; typedef struct vstream_kick_args { __u32 sqe_num; - __u32 exec_time; __s32 timeout; __s8 sqe[XCU_SQE_SIZE_MAX]; } vstream_kick_args_t; diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 98f4965d97e5..288cad6a5652 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -62,8 +62,7 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) { uint32_t kick_count; struct vstream_info *vs; - unsigned int sum_exec_time = 0; - size_t kicks_submitted = 0; + size_t kicks_scheduled = 0; struct vstream_metadata *vsm; size_t not_empty; @@ -85,30 +84,46 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) spin_unlock(&vs->stream_lock); if (vsm) { list_add_tail(&vsm->node, &xcu->vsm_list); - - sum_exec_time += vsm->exec_time; - kicks_submitted++; + kicks_scheduled++; xsched_dec_pending_kicks_xse(xse); XSCHED_INFO( - "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", - vs->id, vsm->exec_time, vsm->sq_tail, - vsm->sqe_num, vsm->sq_id, __func__); + "vs id = %u sq_tail %u @ %s\n", + vs->id, vsm->sq_tail, __func__); not_empty++; } } - } while ((sum_exec_time < XSCHED_CFS_MIN_TIMESLICE) && (not_empty)); + } while ((kicks_scheduled < 10) && (not_empty)); + + /* + * Iterate over all vstreams in context: + * Set wr_cqe bit in last computing task in vsm_list + */ + for_each_vstream_in_ctx(vs, xse->ctx) { + list_for_each_entry_reverse(vsm, &xcu->vsm_list, node) { + if (vsm->parent == vs) { + struct xcu_op_handler_params params; + int op_type = SQE_SET_NOTIFY; + + params.group = vsm->parent->xcu->group; + params.param_1 = &op_type; + params.param_2 = &vsm->sqe; + xcu_sqe_op(¶ms); + break; + } + } + } kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", kick_count, __func__); - xse->kicks_submitted += kicks_submitted; + xse->kicks_submitted += kicks_scheduled; - XSCHED_INFO("xse %d kicks_submitted = %lu @ %s\n", + XSCHED_INFO("xse %d kicks_scheduled = %lu @ %s\n", xse->tgid, xse->kicks_submitted, __func__); out_err: - return kicks_submitted; + return kicks_scheduled; } static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) @@ -208,10 +223,13 @@ static int delete_ctx(struct xsched_context *ctx) } mutex_lock(&xcu->xcu_lock); + XSCHED_INFO("%s: Xcu lock taken\n", __func__); if (curr_xse == xse) xcu->xrq.curr_xse = NULL; dequeue_ctx(xse, xcu); + + XSCHED_INFO("%s: Xcu lock released\n", __func__); mutex_unlock(&xcu->xcu_lock); XSCHED_INFO("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, @@ -491,71 +509,99 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) return err; } -/* - * A function for submitting stream's commands (sending commands to a XCU). - */ -static int xsched_proc(struct xsched_cu *xcu, struct vstream_info *vs, - struct vstream_metadata *vsm) +static void submit_kick(struct vstream_metadata *vsm) { + struct vstream_info *vs = vsm->parent; struct xcu_op_handler_params params; - struct xsched_entity *xse; - - XSCHED_CALL_STUB(); + params.group = vs->xcu->group; + params.fd = vs->fd; + params.param_1 = &vs->id; + params.param_2 = &vs->channel_id; + params.param_3 = vsm->sqe; + params.param_4 = &vsm->sqe_num; + params.param_5 = &vsm->timeout; + params.param_6 = &vs->sqcq_type; + params.param_7 = vs->drv_ctx; + params.param_8 = &vs->logic_vcqId; - xse = &vs->ctx->xse; - - /* Init input parameters for xcu_run and xcu_wait callbacks. */ - params.group = xcu->group; - - /* Increase process time by abstract kick handling time. */ - xse->last_process_time += vsm->exec_time; - - XSCHED_INFO("Process vsm sq_tail %d exec_time %u sqe_num %d sq_id %d@ %s\n", - vsm->sq_tail, vsm->exec_time, vsm->sqe_num, vsm->sq_id, __func__); - submit_kick(vs, ¶ms, vsm); + /* Send vstream on a device for processing. */ + if (xcu_run(¶ms) != 0) { + XSCHED_ERR( + "Failed to send vstream tasks vstreamId=%u to a device for processing.\n", + vs->id); + } - xse->kicks_processed++; + XSCHED_INFO("Vstream_id %u submit vsm: sq_tail %u\n", vs->id, vsm->sq_tail); +} - XSCHED_INFO("xse %d kicks_processed = %lu @ %s\n", - xse->tgid, xse->kicks_processed, __func__); +static void submit_wait(struct vstream_metadata *vsm) +{ + struct vstream_info *vs = vsm->parent; + struct xcu_op_handler_params params; + /* Wait timeout in ms. */ + int32_t timeout = 500; + + params.group = vs->xcu->group; + params.param_1 = &vs->channel_id; + params.param_2 = &vs->logic_vcqId; + params.param_3 = &vs->user_streamId; + params.param_4 = &vsm->sqe; + params.param_5 = vsm->cqe; + params.param_6 = vs->drv_ctx; + params.param_7 = &timeout; + + /* Wait for a device to complete processing. */ + if (xcu_wait(¶ms) != 0) { + XSCHED_ERR( + "Failed to wait vstream tasks vstreamId=%u, cq_logic=%u.\n", + vs->id, vs->logic_vcqId); + } - XSCHED_EXIT_STUB(); - return 0; + XSCHED_INFO("Vstream_id %u wait finish, cq_logic %u\n", vs->id, vs->logic_vcqId); } static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) { int err = 0; struct vstream_metadata *vsm, *tmp; - unsigned int submit_exec_time = 0; size_t kicks_submitted = 0; - unsigned long wait_us; + ktime_t submit_exec_time = 0; + ktime_t t_start = 0; XSCHED_CALL_STUB(); - XSCHED_INFO("%s called for xse %d on xcu %u\n", __func__, xse->tgid, xcu->id); list_for_each_entry_safe(vsm, tmp, &xcu->vsm_list, node) { - xsched_proc(xcu, vsm->parent, vsm); - submit_exec_time += vsm->exec_time; + struct xcu_op_handler_params params; + int op_type = SQE_IS_NOTIFY; + + submit_kick(vsm); + + params.group = vsm->parent->xcu->group; + params.param_1 = &op_type; + params.param_2 = &vsm->sqe; + if (xcu_sqe_op(¶ms)) { + XSCHED_INFO("%s: Xcu lock released\n", __func__); + mutex_unlock(&xcu->xcu_lock); + t_start = ktime_get(); + submit_wait(vsm); + submit_exec_time += ktime_get() - t_start; + mutex_lock(&xcu->xcu_lock); + XSCHED_INFO("%s: Xcu lock taken\n", __func__); + } kicks_submitted++; + list_del(&vsm->node); + kfree(vsm); } - INIT_LIST_HEAD(&xcu->vsm_list); - - mutex_unlock(&xcu->xcu_lock); - - wait_us = div_u64(submit_exec_time, NSEC_PER_USEC); - XSCHED_INFO("XCU kicks_submitted=%lu wait_us=%lu @ %s\n", - kicks_submitted, wait_us, __func__); - - if (wait_us > 0) { - /* Sleep shift not larger than 12.5% */ - usleep_range(wait_us, wait_us + (wait_us >> 3)); - } + xse->last_process_time += submit_exec_time; + xse->kicks_processed += kicks_submitted; + XSCHED_INFO("XCU kicks submitted=%lu total=%lu exec_time = %lld @ %s\n", + kicks_submitted, xse->kicks_processed, submit_exec_time, + __func__); - mutex_lock(&xcu->xcu_lock); + INIT_LIST_HEAD(&xcu->vsm_list); XSCHED_EXIT_STUB(); @@ -640,33 +686,6 @@ static int xsched_schedule(void *input_xcu) return err; } -void submit_kick(struct vstream_info *vs, - struct xcu_op_handler_params *params, - struct vstream_metadata *vsm) -{ - int ret; - - params->fd = vs->fd; - params->param_1 = &vs->id; - params->param_2 = &vs->channel_id; - params->param_3 = vsm->sqe; - params->param_4 = &vsm->sqe_num; - params->param_5 = &vsm->timeout; - params->param_6 = &vs->sqcq_type; - params->param_7 = vs->drv_ctx; - /* Send vstream on a device for processing. */ - ret = xcu_run(params); - if (ret) { - XSCHED_ERR( - "Failed to send vstream tasks vstreamId=%d to a device for processing.\n", - vs->id); - } - - XSCHED_INFO("Vstream_id %d submit vsm: sq_tail %d\n", vs->id, vsm->sq_tail); - - kfree(vsm); -} - /* Initialize xsched rt runqueue during kernel init. * Should only be called from xsched_init function. */ diff --git a/kernel/xsched/rt.c b/kernel/xsched/rt.c index 8bd3f7c7f403..60845981114c 100644 --- a/kernel/xsched/rt.c +++ b/kernel/xsched/rt.c @@ -208,13 +208,11 @@ static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) spin_lock(&vs->stream_lock); while ((vsm = xsched_vsm_fetch_first(vs))) { list_add_tail(&vsm->node, &xcu->vsm_list); - vsm->exec_time = 0; kicks_submitted++; xsched_dec_pending_kicks_xse(xse); XSCHED_INFO( - "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", - vs->id, vsm->exec_time, vsm->sq_tail, - vsm->sqe_num, vsm->sq_id, __func__); + "vs id = %u Kick submit sq_tail %u sqe_num %u sq_id %u @ %s\n", + vs->id, vsm->sq_tail, vsm->sqe_num, vsm->sq_id, __func__); } spin_unlock(&vs->stream_lock); } -- 2.34.1