From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add rt class callbacks implementation: - dequeue_ctx - enqueue_ctx - pick_next_ctx - put_prev_ctx - submit_prepare_ctx - select_work - check_preempt Add RT class callbacks support in core.c. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- include/linux/xsched.h | 3 + kernel/xsched/core.c | 150 ++++++++++++++++++++++++++++++- kernel/xsched/rt.c | 198 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 342 insertions(+), 9 deletions(-) diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 64436db90647..3f52cabbb99b 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -217,6 +217,9 @@ struct xsched_entity { /* Amount of submitted kicks context, used for resched decision. */ atomic_t kicks_submited; + size_t kicks_submitted; + size_t kicks_processed; + /* File descriptor coming from an associated context * used for identifying a given xsched entity in * info and error prints. diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 28ebf1791930..3fd8fe9cc7e1 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -42,19 +42,134 @@ static DEFINE_HASHTABLE(ctx_revmap, XCU_HASH_ORDER); static void put_prev_ctx(struct xsched_entity *xse) { + struct xsched_cu *xcu = xse->xcu; + + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + xse->class->put_prev_ctx(xse); + + xse->last_process_time = 0; + + XSCHED_EXIT_STUB(); +} + +static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) +{ + uint32_t kick_count; + struct vstream_info *vs; + unsigned int sum_exec_time = 0; + size_t kicks_submitted = 0; + struct vstream_metadata *vsm; + size_t not_empty; + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("Before decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + if (kick_count == 0) { + XSCHED_ERR("Tried to submit xse that has 0 kicks @ %s\n", + __func__); + goto out_err; + } + + do { + not_empty = 0; + for_each_vstream_in_ctx(vs, xse->ctx) { + spin_lock(&vs->stream_lock); + vsm = xsched_vsm_fetch_first(vs); + spin_unlock(&vs->stream_lock); + if (vsm) { + list_add_tail(&vsm->node, &xcu->vsm_list); + + sum_exec_time += vsm->exec_time; + kicks_submitted++; + xsched_dec_pending_kicks_xse(xse); + XSCHED_INFO( + "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", + vs->id, vsm->exec_time, vsm->sq_tail, + vsm->sqe_num, vsm->sq_id, __func__); + not_empty++; + } + } + } while (not_empty); + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + xse->kicks_submitted += kicks_submitted; + + XSCHED_INFO("xse %d kicks_submitted = %lu @ %s\n", + xse->tgid, xse->kicks_submitted, __func__); + +out_err: + return kicks_submitted; } static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) { - return NULL; + const struct xsched_class *class; + struct xsched_entity *next = NULL; + + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + for_each_xsched_class(class) { + next = class->pick_next_ctx(xcu); + if (next) { + if (class->select_work) + class->select_work(xcu, next); + else + select_work_def(xcu, next); + break; + } + } + + XSCHED_EXIT_STUB(); + return next; } void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) { + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + if (!xse_integrity_check(xse)) { + XSCHED_ERR("Failed xse integrity check @ %s\n", __func__); + return; + } + + if (!xse->on_rq) { + xse->on_rq = true; + xse->class->enqueue_ctx(xse, xcu); + __XSCHED_TRACE("Enqueue xse %d @ %s\n", xse->tgid, __func__); + } + + XSCHED_EXIT_STUB(); } void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) { + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + if (!xse_integrity_check(xse)) { + XSCHED_ERR("Failed xse integrity check @ %s\n", __func__); + return; + } + + if (xse->on_rq) { + xse->class->dequeue_ctx(xse); + xse->on_rq = false; + __XSCHED_TRACE("Dequeue xse %d @ %s\n", xse->tgid, __func__); + } + + XSCHED_EXIT_STUB(); } static int delete_ctx(struct xsched_context *ctx) @@ -288,6 +403,15 @@ struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id) int xsched_xse_set_class(struct xsched_entity *xse) { + switch (xse->task_type) { + case XSCHED_TYPE_RT: + xse->class = &rt_xsched_class; + XSCHED_INFO("Context is in RT class %s\n", __func__); + break; + default: + XSCHED_ERR("Xse has incorrect class @ %s\n", __func__); + return -EINVAL; + } return 0; } @@ -326,6 +450,25 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) goto out_err; } + if (xse_is_rt(xse)) { + xse->rt.state = XSE_PREPARE; + xse->rt.flag = XSE_TIF_NONE; + xse->rt.prio = GET_VS_TASK_PRIO_RT(vs); + xse->rt.kick_slice = XSCHED_RT_KICK_SLICE; + + /* XSE priority is being decreased by 1 here because + * in libucc priority counter starts from 1 while in the + * kernel counter starts with 0. + * + * This inconsistency has to be solve in libucc in the + * future rather that having this confusing decrement to + * priority inside the kernel. + */ + if (xse->rt.prio > 0) + xse->rt.prio -= 1; + + INIT_LIST_HEAD(&xse->rt.list_node); + } WRITE_ONCE(xse->on_rq, false); spin_lock_init(&xse->xse_lock); @@ -340,6 +483,11 @@ static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) return 0; } +static inline bool should_preempt(struct xsched_entity *xse) +{ + return xse->class->check_preempt(xse); +} + static int xsched_schedule(void *input_xcu) { struct xsched_cu *xcu = input_xcu; diff --git a/kernel/xsched/rt.c b/kernel/xsched/rt.c index 12b188dce567..da9959778366 100644 --- a/kernel/xsched/rt.c +++ b/kernel/xsched/rt.c @@ -23,25 +23,162 @@ #include <linux/xsched.h> #include <linux/vstream.h> -static void dequeue_ctx_rt(struct xsched_entity *xse) {} +/* Add xsched entitiy to a run list based on priority, set on_cu flag + * and set a corresponding curr_prios bit if necessary. + */ +static inline void +xse_rt_add(struct xsched_entity *xse, struct xsched_cu *xcu) +{ + list_add_tail(&xse->rt.list_node, &xcu->xrq.rt.rq[xse->rt.prio]); + __set_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +/* Delete xsched entitiy from a run list, unset on_cu flag and + * unset corresponding curr_prios bit if necessary. + */ +static inline void xse_rt_del(struct xsched_entity *xse) +{ + struct xsched_cu *xcu = xse->xcu; + + list_del_init(&xse->rt.list_node); + + if (list_empty(&xcu->xrq.rt.rq[xse->rt.prio])) + __clear_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +static inline void xse_rt_move_tail(struct xsched_entity *xse) +{ + struct xsched_cu *xcu = xse->xcu; + + list_move_tail(&xse->rt.list_node, &xcu->xrq.rt.rq[xse->rt.prio]); +} + +/* Increase RT runqueue total and per prio nr_running stat. */ +static inline void xrq_inc_nr_running(struct xsched_entity *xse, + struct xsched_cu *xcu) +{ + xcu->xrq.nr_running++; + xcu->xrq.rt.prio_nr_running[xse->rt.prio]++; + set_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +/* Decrease RT runqueue total and per prio nr_running stat + * and raise a bug if nr_running decrease beyond zero. + */ +static inline void xrq_dec_nr_running(struct xsched_entity *xse) +{ + struct xsched_cu *xcu = xse->xcu; + + xcu->xrq.nr_running--; + xcu->xrq.rt.prio_nr_running[xse->rt.prio]--; + + if (!xcu->xrq.rt.prio_nr_running[xse->rt.prio]) + clear_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +static void dequeue_ctx_rt(struct xsched_entity *xse) +{ + xse_rt_del(xse); + xrq_dec_nr_running(xse); +} + +static void enqueue_ctx_rt(struct xsched_entity *xse, struct xsched_cu *xcu) +{ + xse_rt_add(xse, xcu); + xrq_inc_nr_running(xse, xcu); +} + +static inline struct xsched_entity *xrq_next_xse(struct xsched_cu *xcu, + int prio) +{ + return list_first_entry(&xcu->xrq.rt.rq[prio], struct xsched_entity, + rt.list_node); +} + +/* Returns the next priority for pick_next_ctx taking into + * account if there are pending kicks on certain priority. + */ +static inline uint32_t get_next_prio_rt(struct xsched_rq *xrq) +{ + int32_t curr_prio; + bool bit_val; + unsigned long *prios = xrq->rt.curr_prios; + atomic_t *prio_nr_kicks = xrq->rt.prio_nr_kicks; -static void enqueue_ctx_rt(struct xsched_entity *xse, struct xsched_cu *xcu) {} + /* Using generic for loop instead of for_each_set_bit + * because it will be faster than for_each_set_bit. + */ + for (curr_prio = NR_XSE_PRIO - 1; curr_prio >= 0; curr_prio--) { + bit_val = test_bit(curr_prio, prios); + if (!bit_val && atomic_read(&prio_nr_kicks[curr_prio])) { + XSCHED_ERR( + "kicks > 0 on RT priority with the priority bit unset\n"); + WARN_ON_ONCE(1); + return NR_XSE_PRIO; + } + + if (bit_val && atomic_read(&prio_nr_kicks[curr_prio])) + return curr_prio; + } + return NR_XSE_PRIO; +} static struct xsched_entity *pick_next_ctx_rt(struct xsched_cu *xcu) { - return NULL; + struct xsched_entity *result; + int next_prio; + + next_prio = get_next_prio_rt(&xcu->xrq); + if (next_prio >= NR_XSE_PRIO) { + XSCHED_INFO("No pending kicks in RT class @ %s\n", __func__); + return NULL; + } + + if (!xcu->xrq.rt.prio_nr_running[next_prio]) { + XSCHED_ERR( + "In RT runqueue nr_running is zero while there are pending kicks for %u prio\n", + next_prio); + return NULL; + } + + result = xrq_next_xse(xcu, next_prio); + if (!result) + XSCHED_ERR("Next XSE not found @ %s\n", __func__); + + return result; } -static void put_prev_ctx_rt(struct xsched_entity *xse) {} +static void put_prev_ctx_rt(struct xsched_entity *xse) +{ + xse->rt.kick_slice -= atomic_read(&xse->kicks_submited); + XSCHED_INFO( + "Update XSE=%d kick_slice=%lld, XSE kicks_submited=%d in RT class @ %s\n", + xse->tgid, xse->rt.kick_slice, + atomic_read(&xse->kicks_submited), __func__); + + if (xse->rt.kick_slice <= 0) { + xse->rt.kick_slice = XSCHED_RT_KICK_SLICE; + XSCHED_INFO("Refill XSE=%d kick_slice=%lld in RT class @ %s\n", + xse->tgid, xse->rt.kick_slice, __func__); + xse_rt_move_tail(xse); + } + + atomic_set(&xse->kicks_submited, 0); +} static int submit_prepare_ctx_rt(struct xsched_entity *xse, struct xsched_cu *xcu) { - return 0; -} + if (!atomic_read(&xse->kicks_pending_ctx_cnt)) { + XSCHED_INFO("xse %d doesn't have pending kicks @ %s\n", + xse->tgid, __func__); + xse->rt.state = XSE_READY; + xse->rt.kick_slice = 0; + return -EAGAIN; + } + + xse->rt.state = XSE_RUNNING; -static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) -{ return 0; } @@ -50,6 +187,51 @@ static bool check_preempt_ctx_rt(struct xsched_entity *xse) return true; } +static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) +{ + uint32_t kick_count; + struct vstream_info *vs; + size_t kicks_submitted = 0; + struct vstream_metadata *vsm; + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("Before decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + if (kick_count == 0) { + XSCHED_ERR("Tried to submit xse that has 0 kicks @ %s\n", + __func__); + goto out_err; + } + + for_each_vstream_in_ctx(vs, xse->ctx) { + spin_lock(&vs->stream_lock); + while ((vsm = xsched_vsm_fetch_first(vs))) { + list_add_tail(&vsm->node, &xcu->vsm_list); + vsm->exec_time = 0; + kicks_submitted++; + xsched_dec_pending_kicks_xse(xse); + XSCHED_INFO( + "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", + vs->id, vsm->exec_time, vsm->sq_tail, + vsm->sqe_num, vsm->sq_id, __func__); + } + spin_unlock(&vs->stream_lock); + } + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + xse->kicks_submitted += kicks_submitted; + + XSCHED_INFO("xse %d kicks_submitted = %lu @ %s\n", + xse->tgid, xse->kicks_submitted, __func__); + +out_err: + return kicks_submitted; +} + const struct xsched_class rt_xsched_class = { .next = NULL, .dequeue_ctx = dequeue_ctx_rt, -- 2.34.1