From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add base xcu_group structure, xcu_type enum, xcu_operation struct Add build support in Makefiles. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- drivers/Makefile | 1 + drivers/xcu/Makefile | 2 + drivers/xcu/xcu_group.c | 79 +++++++++++++++++++++++++++++++++++++++ include/linux/xcu_group.h | 56 +++++++++++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 drivers/xcu/Makefile create mode 100644 drivers/xcu/xcu_group.c create mode 100644 include/linux/xcu_group.h diff --git a/drivers/Makefile b/drivers/Makefile index 3955e605df14..b06192df4c3c 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -195,6 +195,7 @@ obj-$(CONFIG_GNSS) += gnss/ obj-$(CONFIG_INTERCONNECT) += interconnect/ obj-$(CONFIG_COUNTER) += counter/ obj-$(CONFIG_MOST) += most/ +obj-$(CONFIG_XCU_SCHEDULER) += xcu/ obj-$(CONFIG_PECI) += peci/ obj-$(CONFIG_HTE) += hte/ obj-$(CONFIG_DRM_ACCEL) += accel/ diff --git a/drivers/xcu/Makefile b/drivers/xcu/Makefile new file mode 100644 index 000000000000..575115b148ec --- /dev/null +++ b/drivers/xcu/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_XCU_SCHEDULER) += xcu_group.o diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c new file mode 100644 index 000000000000..11bf0e54aaaa --- /dev/null +++ b/drivers/xcu/xcu_group.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Code for NPU driver support + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/xcu_group.h> + +/* This function runs "run" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object + */ +int xcu_run(struct xcu_op_handler_params *params) +{ + return 0; +} + +/* This function runs "wait" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object + */ +int xcu_wait(struct xcu_op_handler_params *params) +{ + return 0; +} + +/* This function runs "complete" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + */ +int xcu_complete(struct xcu_op_handler_params *params) +{ + return 0; +} + +/* This function runs "finish" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement deallocation + * and freeing memory for SQ and CQ buffers. + */ +int xcu_finish(struct xcu_op_handler_params *params) +{ + return 0; +} + +/* This function runs a "alloc" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement allocation + * and registering memory for SQ and CQ buffers. + */ +int xcu_alloc(struct xcu_op_handler_params *params) +{ + return 0; +} + +static struct xcu_group __xcu_group_root = { + .id = 0, + .type = XCU_TYPE_ROOT, + .next_layer = IDR_INIT(next_layer), +}; + +struct xcu_group *xcu_group_root = &__xcu_group_root; +EXPORT_SYMBOL(xcu_group_root); diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h new file mode 100644 index 000000000000..d1a6af4c8c47 --- /dev/null +++ b/include/linux/xcu_group.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __XSCHED_XCU_GROUP_H__ +#define __XSCHED_XCU_GROUP_H__ + +#include <linux/idr.h> +#include <uapi/linux/xcu_vstream.h> + +extern struct xcu_group *xcu_group_root; + +enum xcu_type { + XCU_TYPE_ROOT, + XCU_TYPE_XPU, +}; + +struct xcu_op_handler_params { +}; + +typedef int (*xcu_op_handler_fn_t)(struct xcu_op_handler_params *params); + +struct xcu_operation { + xcu_op_handler_fn_t run; + xcu_op_handler_fn_t finish; + xcu_op_handler_fn_t wait; + xcu_op_handler_fn_t complete; + xcu_op_handler_fn_t alloc; +}; + +struct xcu_group { + /* sq id. */ + uint32_t id; + + /* Type of XCU group. */ + enum xcu_type type; + + /* IDR for the next layer of XCU group tree. */ + struct idr next_layer; +}; + +#ifdef CONFIG_XCU_SCHEDULER +int xcu_group_attach(struct xcu_group *new_group, + struct xcu_group *previous_group); +void xcu_group_detach(struct xcu_group *group); +struct xcu_group *xcu_group_find(struct xcu_group *group, int id); +struct xcu_group *xcu_group_init(int id); +void xcu_group_free(struct xcu_group *group); + +extern int xcu_run(struct xcu_op_handler_params *params); +extern int xcu_wait(struct xcu_op_handler_params *params); +extern int xcu_complete(struct xcu_op_handler_params *params); +extern int xcu_finish(struct xcu_op_handler_params *params); +extern int xcu_alloc(struct xcu_op_handler_params *params); +extern int xcu_logic_alloc(struct xcu_op_handler_params *params); +extern int xcu_logic_free(struct xcu_op_handler_params *params); +#endif /* !CONFIG_XCU_SCHEDULER */ + +#endif /* __XSCHED_XCU_GROUP_H__ */ -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add xcu group alloc/find/attach/detach funcs implementation. Add xsched_cu data structures, all related enumerators. Add xsched_xcu_register() for driver to register xcu. Add XSCHED_NR_CUS config parameters. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- drivers/xcu/xcu_group.c | 99 ++++++++++++++++++++++++++++++++++ include/linux/xcu_group.h | 21 ++++++++ include/linux/xsched.h | 40 ++++++++++++++ kernel/xsched/Kconfig | 9 ++++ kernel/xsched/Makefile | 1 + kernel/xsched/core.c | 110 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 280 insertions(+) create mode 100644 kernel/xsched/core.c diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index 11bf0e54aaaa..fef05b9787f3 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -16,7 +16,106 @@ * more details. * */ +#include <linux/rwsem.h> +#include <linux/slab.h> #include <linux/xcu_group.h> +#include <linux/xsched.h> + +static DECLARE_RWSEM(xcu_group_rwsem); + +struct xcu_group *xcu_group_init(int id) +{ + struct xcu_group *node = kzalloc(sizeof(*node), GFP_KERNEL); + + if (!node) + return NULL; + + node->id = id; + node->type = XCU_TYPE_XPU; + idr_init(&node->next_layer); + + return node; +} +EXPORT_SYMBOL(xcu_group_init); + +int __xcu_group_attach(struct xcu_group *new_group, + struct xcu_group *previous_group) +{ + int id = new_group->id; + + if (id == -1) + id = idr_alloc(&previous_group->next_layer, new_group, 0, + INT_MAX, GFP_KERNEL); + else + id = idr_alloc(&previous_group->next_layer, new_group, id, + id + 1, GFP_KERNEL); + + if (id < 0) { + XSCHED_ERR("Fail to attach xcu_group: id conflict @ %s\n", + __func__); + return -EEXIST; + } + new_group->id = id; + new_group->previous_layer = previous_group; + + return 0; +} + +int xcu_group_attach(struct xcu_group *new_group, + struct xcu_group *previous_group) +{ + int ret; + + down_write(&xcu_group_rwsem); + ret = __xcu_group_attach(new_group, previous_group); + up_write(&xcu_group_rwsem); + + return ret; +} +EXPORT_SYMBOL(xcu_group_attach); + +static inline void __xcu_group_detach(struct xcu_group *group) +{ + if (!group || !group->previous_layer) + return; + + idr_remove(&group->previous_layer->next_layer, group->id); + group->previous_layer = NULL; +} + +void xcu_group_detach(struct xcu_group *group) +{ + down_write(&xcu_group_rwsem); + __xcu_group_detach(group); + up_write(&xcu_group_rwsem); +} +EXPORT_SYMBOL(xcu_group_detach); + +void xcu_group_free(struct xcu_group *group) +{ + idr_destroy(&group->next_layer); + if (group != xcu_group_root) + kfree(group); +} +EXPORT_SYMBOL(xcu_group_free); + +static struct xcu_group *__xcu_group_find_nolock(struct xcu_group *group, + int id) +{ + return idr_find(&group->next_layer, id); +} + +struct xcu_group *xcu_group_find(struct xcu_group *group, int id) +{ + struct xcu_group *result; + + down_read(&xcu_group_rwsem); + result = __xcu_group_find_nolock(group, id); + up_read(&xcu_group_rwsem); + + return result; +} +EXPORT_SYMBOL(xcu_group_find); /* This function runs "run" callback for a given xcu_group * and a given vstream that are passed within diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h index d1a6af4c8c47..ef11886c18a1 100644 --- a/include/linux/xcu_group.h +++ b/include/linux/xcu_group.h @@ -5,6 +5,11 @@ #include <linux/idr.h> #include <uapi/linux/xcu_vstream.h> +#ifndef CONFIG_XSCHED_NR_CUS +#define CONFIG_XSCHED_NR_CUS 1 +#endif /* !CONFIG_XSCHED_NR_CUS */ +#define XSCHED_NR_CUS CONFIG_XSCHED_NR_CUS + extern struct xcu_group *xcu_group_root; enum xcu_type { @@ -34,6 +39,22 @@ struct xcu_group { /* IDR for the next layer of XCU group tree. */ struct idr next_layer; + + /* Pointer to the previous XCU group in the XCU group tree. */ + struct xcu_group *previous_layer; + + /* Pointer to operation fn pointers object describing + * this XCU group's callbacks. + */ + struct xcu_operation *opt; + + /* Pointer to the XCU related to this XCU group. */ + struct xsched_cu *xcu; + + /* Mask of XCU ids associated with this XCU group + * and this group's children's XCUs. + */ + DECLARE_BITMAP(xcu_mask, XSCHED_NR_CUS); }; #ifdef CONFIG_XCU_SCHEDULER diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 628a2201b3d8..011f0e9bc227 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -2,6 +2,7 @@ #ifndef __LINUX_XSCHED_H__ #define __LINUX_XSCHED_H__ +#include <linux/xcu_group.h> #ifndef pr_fmt #define pr_fmt(fmt) fmt #endif @@ -28,4 +29,43 @@ #define XSCHED_EXIT_STUB() \ XSCHED_DEBUG(" -----* %s @ %s exited *-----\n", __func__, __FILE__) +enum xcu_state { + XCU_INACTIVE, + XCU_IDLE, + XCU_BUSY, + XCU_SUBMIT, +}; + +enum xsched_cu_status { + /* Worker not initialized. */ + XSCHED_XCU_NONE, + + /* Worker is sleeping in idle state. */ + XSCHED_XCU_WAIT_IDLE, + + /* Worker is sleeping in running state. */ + XSCHED_XCU_WAIT_RUNNING, + + /* Worker is active but not processing anything. */ + XSCHED_XCU_ACTIVE, + + NR_XSCHED_XCU_STATUS, +}; + +/* This is the abstraction object of the xcu computing unit. */ +struct xsched_cu { + uint32_t id; + uint32_t state; + + struct task_struct *worker; + + struct xcu_group *group; + + struct mutex xcu_lock; + + wait_queue_head_t wq_xcu_idle; + wait_queue_head_t wq_xcu_running; +}; + +int xsched_xcu_register(struct xcu_group *group); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig index 47a5361f2ad3..fc5d7767d8e0 100644 --- a/kernel/xsched/Kconfig +++ b/kernel/xsched/Kconfig @@ -25,3 +25,12 @@ config XCU_VSTREAM Requires XCU_SCHEDULER to be enabled. May increase kernel memory usage. Recommended for high-throughput XPU workloads. If unsure, say N. + +config XSCHED_NR_CUS + int "Number of CUs (a.k.a. XCUs) available to XSched mechanism" + default 8 + depends on XCU_SCHEDULER + help + This option defines the maximum number of Compute Units (CUs) that can be + managed by the XSched scheduler, consider changing this value proportionally + to the number of available XCU cores. diff --git a/kernel/xsched/Makefile b/kernel/xsched/Makefile index e972cd93b607..62e58e4151b0 100644 --- a/kernel/xsched/Makefile +++ b/kernel/xsched/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += vstream.o +obj-$(CONFIG_XCU_SCHEDULER) += core.o diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c new file mode 100644 index 000000000000..bbd125044c88 --- /dev/null +++ b/kernel/xsched/core.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Core kernel scheduler code for XPU device + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/kthread.h> +#include <linux/slab.h> +#include <linux/spinlock_types.h> +#include <linux/types.h> +#include <linux/xsched.h> +#include <uapi/linux/sched/types.h> + +int num_active_xcu; +spinlock_t xcu_mgr_lock; + +/* Xsched XCU array and bitmask that represents which XCUs + * are present and online. + */ +DECLARE_BITMAP(xcu_online_mask, XSCHED_NR_CUS); +struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; + +static int xsched_schedule(void *input_xcu) +{ + return 0; +} + +/* Initializes all xsched XCU objects. + * Should only be called from xsched_xcu_register function. + */ +static void xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, + int xcu_id) +{ + bitmap_clear(xcu_group_root->xcu_mask, 0, XSCHED_NR_CUS); + + xcu->id = xcu_id; + xcu->state = XSCHED_XCU_NONE; + xcu->group = group; + + mutex_init(&xcu->xcu_lock); + + /* Mark current XCU in a mask inside XCU root group. */ + set_bit(xcu->id, xcu_group_root->xcu_mask); + + /* This worker should set XCU to XSCHED_XCU_WAIT_IDLE. + * If after initialization XCU still has XSCHED_XCU_NONE + * status then we can assume that there was a problem + * with XCU kthread job. + */ + xcu->worker = kthread_run(xsched_schedule, xcu, "xcu_%u", xcu->id); +} + +/* Allocates xcu id in xcu_manager array. */ +static int alloc_xcu_id(void) +{ + int xcu_id = -1; + + spin_lock(&xcu_mgr_lock); + if (num_active_xcu >= XSCHED_NR_CUS) + goto out_unlock; + + xcu_id = num_active_xcu; + num_active_xcu++; + XSCHED_DEBUG("Number of active xcu: %d.\n", num_active_xcu); + +out_unlock: + spin_unlock(&xcu_mgr_lock); + return xcu_id; +} + +/* + * Initialize and register xcu in xcu_manager array. + */ +int xsched_xcu_register(struct xcu_group *group) +{ + int xcu_id; + struct xsched_cu *xcu; + + xcu_id = alloc_xcu_id(); + if (xcu_id < 0) { + XSCHED_ERR("Fail to alloc xcu id.\n"); + return -ENOSPC; + }; + + xcu = kzalloc(sizeof(struct xsched_cu), GFP_KERNEL); + if (!xcu) { + XSCHED_ERR("Fail to alloc xcu.\n"); + return -ENOMEM; + }; + + group->xcu = xcu; + xsched_cu_mgr[xcu_id] = xcu; + + /* Init xcu's internals. */ + xsched_xcu_init(xcu, group, xcu_id); + return 0; +} +EXPORT_SYMBOL(xsched_xcu_register); -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add vstream_metadata data structures. Add vstream_kick() and xcu_run() implementation. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- include/linux/vstream.h | 28 ++++++++++++++++++++++++ include/linux/xsched.h | 15 +++++++++++++ kernel/xsched/core.c | 25 +++++++++++++++++++++ kernel/xsched/vstream.c | 48 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/include/linux/vstream.h b/include/linux/vstream.h index ca956ac2cf4f..58ee4c235a07 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -3,6 +3,34 @@ #define _LINUX_VSTREAM_H #include <uapi/linux/xcu_vstream.h> +#include <linux/ktime.h> + +#define MAX_VSTREAM_SIZE 2048 + +/* Vstream metadata describes each incoming kick + * that gets stored into a list of pending kicks + * inside a vstream to keep track of what is left + * to be processed by a driver. + */ +typedef struct vstream_metadata { + uint32_t exec_time; + /* A value of SQ tail that has been passed with the + * kick that is described by this exact metadata object. + */ + uint32_t sq_tail; + uint32_t sqe_num; + uint32_t sq_id; + int32_t timeout; + int8_t sqe[XCU_SQE_SIZE_MAX]; + + /* A node for metadata list */ + struct list_head node; + + struct vstream_info *parent; + + /* Time of list insertion */ + ktime_t add_time; +} vstream_metadata_t; typedef struct vstream_info { uint32_t user_stream_id; diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 702483b85621..7a25d2a7455b 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -135,6 +135,18 @@ static inline struct xsched_context *ctx_find_by_tgid(pid_t tgid) return ret; } + +static inline void xsched_init_vsm(struct vstream_metadata *vsm, + struct vstream_info *vs, vstream_args_t *arg) +{ + vsm->sq_id = arg->sq_id; + vsm->sqe_num = arg->vk_args.sqe_num; + vsm->timeout = arg->vk_args.timeout; + memcpy(vsm->sqe, arg->vk_args.sqe, XCU_SQE_SIZE_MAX); + vsm->parent = vs; + INIT_LIST_HEAD(&vsm->node); +} + int xsched_xcu_register(struct xcu_group *group); void xsched_task_free(struct kref *kref); int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs); @@ -142,4 +154,7 @@ int ctx_bind_to_xcu(vstream_info_t *vstream_info, struct xsched_context *ctx); int vstream_bind_to_xcu(vstream_info_t *vstream_info); struct xsched_cu *xcu_find(uint32_t *type, uint32_t dev_id, uint32_t channel_id); + +/* Vstream metadata proccesing functions.*/ +int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 018c73101a70..5064ecbbd179 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -163,6 +163,31 @@ static int alloc_xcu_id(void) return xcu_id; } +/* Adds vstream_metadata object to a specified vstream. */ +int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) +{ + struct vstream_metadata *new_vsm; + + new_vsm = kmalloc(sizeof(struct vstream_metadata), GFP_KERNEL); + if (!new_vsm) { + XSCHED_ERR("Failed to alloc kick metadata for vs %u @ %s\n", + vs->id, __func__); + return -ENOMEM; + } + + if (vs->kicks_count > MAX_VSTREAM_SIZE) { + kfree(new_vsm); + return -EBUSY; + } + + xsched_init_vsm(new_vsm, vs, arg); + list_add_tail(&new_vsm->node, &vs->metadata_list); + new_vsm->add_time = ktime_get(); + vs->kicks_count += 1; + + return 0; +} + /* * Initialize and register xcu in xcu_manager array. */ diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 127ff96ce48c..d4f916415682 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -403,7 +403,53 @@ int vstream_free(struct vstream_args *arg) int vstream_kick(struct vstream_args *arg) { - return 0; + vstream_info_t *vstream; + int vstream_id = arg->sq_id; + struct xsched_entity *xse; + int err = 0; + + struct xsched_cu *xcu = NULL; + + XSCHED_CALL_STUB(); + + /* Get vstream. */ + vstream = vstream_get(vstream_id); + if (!vstream || !vstream->ctx) { + XSCHED_ERR("Vstream NULL or doesn't have a context.\n"); + return -EINVAL; + } + + xse = &vstream->ctx->xse; + xcu = vstream->xcu; + XSCHED_DEBUG("New kick on xse %d @ %s\n", xse->tgid, __func__); + + do { + mutex_lock(&xcu->xcu_lock); + spin_lock(&vstream->stream_lock); + + /* Adding kick metadata. */ + err = xsched_vsm_add_tail(vstream, arg); + if (err == -EBUSY) { + spin_unlock(&vstream->stream_lock); + mutex_unlock(&xcu->xcu_lock); + + /* Retry after a while */ + usleep_range(100, 200); + continue; + } + + /* Don't forget to unlock */ + if (err) { + XSCHED_ERR("Fail to add kick metadata to vs %u @ %s\n", + vstream->id, __func__); + break; + } + } while (err == -EBUSY); + + spin_unlock(&vstream->stream_lock); + mutex_unlock(&xcu->xcu_lock); + + return err; } /* -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add XCU xsched_schedule() implementation. Add xsched_rq data structures and related process. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- include/linux/xsched.h | 79 +++++++++++++++++++- kernel/xsched/core.c | 161 +++++++++++++++++++++++++++++++++++++++- kernel/xsched/vstream.c | 8 ++ 3 files changed, 246 insertions(+), 2 deletions(-) diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 47f0a43a72dc..efe5d92a5acd 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -39,6 +39,28 @@ #define GET_VS_TASK_TYPE(vs_ptr) __GET_VS_TASK_TYPE((vs_ptr)->task_type) +enum xsched_rq_state { + XRQ_STATE_INACTIVE = 0x00, + XRQ_STATE_IDLE = 0x01, + XRQ_STATE_BUSY = 0x02, + XRQ_STATE_SUBMIT = 0x04, + XRQ_STATE_WAIT_RUNNING = 0x08, +}; + +#define for_each_vstream_in_ctx(vs, ctx) \ + list_for_each_entry((vs), &((ctx)->vstream_list), ctx_node) + + +/* Base XSched runqueue object structure that contains both mutual and + * individual parameters for different scheduling classes. + */ +struct xsched_rq { + struct xsched_entity *curr_xse; + + int state; + int nr_running; +}; + enum xcu_state { XCU_INACTIVE, XCU_IDLE, @@ -69,10 +91,15 @@ struct xsched_cu { struct task_struct *worker; + struct xsched_rq xrq; + struct list_head vsm_list; + struct xcu_group *group; struct mutex xcu_lock; + atomic_t has_active; + wait_queue_head_t wq_xcu_idle; wait_queue_head_t wq_xcu_running; }; @@ -113,6 +140,53 @@ struct xsched_entity { spinlock_t xse_lock; }; +/* Increments pending kicks counter for an XCU that the given + * xsched entity is attached to and for xsched entity's xsched + * class. + */ +static inline int xsched_inc_pending_kicks_xse(struct xsched_entity *xse) +{ + /* Icrement pending kicks for current XSE. */ + atomic_inc(&xse->kicks_pending_ctx_cnt); + + return 0; +} + +/* Decrements pending kicks counter for an XCU that the given + * xsched entity is attached to and for XSched entity's sched + * class. + */ +static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) +{ + /* Decrementing pending kicks for current XSE. */ + atomic_dec(&xse->kicks_pending_ctx_cnt); + + return 0; +} + +/* Checks if there are pending kicks left on a given XCU for all + * xsched classes. + */ +static inline bool xsched_check_pending_kicks_xcu(struct xsched_cu *xcu) +{ + return 0; +} + +static inline int xse_integrity_check(const struct xsched_entity *xse) +{ + if (!xse) { + XSCHED_ERR("xse is null @ %s\n", __func__); + return -EINVAL; + } + + if (!xse->class) { + XSCHED_ERR("xse->class is null @ %s\n", __func__); + return -EINVAL; + } + + return 0; +} + struct xsched_context { uint32_t fd; uint32_t dev_id; @@ -170,4 +244,7 @@ struct xsched_cu *xcu_find(uint32_t *type, /* Vstream metadata proccesing functions.*/ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); -#endif /* !__LINUX_XSCHED_H__ */ +struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); +void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); +void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); +#endif /* __LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 3c20a493629e..d21e90bc9826 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -16,6 +16,7 @@ * more details. * */ +#include <linux/delay.h> #include <linux/kthread.h> #include <linux/slab.h> #include <linux/spinlock_types.h> @@ -39,6 +40,66 @@ DEFINE_MUTEX(xsched_ctx_list_mutex); static DEFINE_MUTEX(revmap_mutex); static DEFINE_HASHTABLE(ctx_revmap, XCU_HASH_ORDER); +static void put_prev_ctx(struct xsched_entity *xse) +{ +} + +static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) +{ + return NULL; +} + +void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) +{ +} + +void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) +{ +} + +static int delete_ctx(struct xsched_context *ctx) +{ + struct xsched_cu *xcu = ctx->xse.xcu; + struct xsched_entity *curr_xse = xcu->xrq.curr_xse; + struct xsched_entity *xse = &ctx->xse; + + if (xse_integrity_check(xse)) { + XSCHED_ERR("Fail to check xse integrity @ %s\n", __func__); + return -EINVAL; + } + + if (!xse->xcu) { + XSCHED_ERR("Try to delete ctx that is not attached to xcu @ %s\n", + __func__); + return -EINVAL; + } + + /* Wait till context has been submitted. */ + while (atomic_read(&xse->kicks_pending_ctx_cnt)) { + XSCHED_DEBUG("Deleting ctx %d, xse->kicks_pending_ctx_cnt=%d @ %s\n", + xse->tgid, atomic_read(&xse->kicks_pending_ctx_cnt), + __func__); + usleep_range(100, 200); + } + + if (atomic_read(&xse->kicks_pending_ctx_cnt)) { + XSCHED_ERR("Deleting ctx %d that has pending kicks left @ %s\n", + xse->tgid, __func__); + return -EINVAL; + } + + mutex_lock(&xcu->xcu_lock); + if (curr_xse == xse) + xcu->xrq.curr_xse = NULL; + + dequeue_ctx(xse, xcu); + mutex_unlock(&xcu->xcu_lock); + XSCHED_DEBUG("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, + atomic_read(&xse->kicks_pending_ctx_cnt), __func__); + + return 0; +} + /* Frees a given vstream and also frees and dequeues it's context * if a given vstream is the last and only vstream attached to it's * corresponding context object. @@ -50,6 +111,10 @@ void xsched_task_free(struct kref *kref) ctx = container_of(kref, struct xsched_context, kref); + /* Wait till xse dequeues */ + while (READ_ONCE(ctx->xse.on_rq)) + usleep_range(100, 200); + mutex_lock(&xsched_ctx_list_mutex); list_for_each_entry_safe(vs, tmp, &ctx->vstream_list, ctx_node) { list_del(&vs->ctx_node); @@ -57,6 +122,7 @@ void xsched_task_free(struct kref *kref) kfree(vs); } + delete_ctx(ctx); list_del(&ctx->ctx_node); mutex_unlock(&xsched_ctx_list_mutex); @@ -192,11 +258,67 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) return err; } -static int xsched_schedule(void *input_xcu) +static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) { return 0; } +static int xsched_schedule(void *input_xcu) +{ + struct xsched_cu *xcu = input_xcu; + int err = 0; + struct xsched_entity *curr_xse = NULL; + struct xsched_entity *next_xse = NULL; + + while (!kthread_should_stop()) { + mutex_unlock(&xcu->xcu_lock); + wait_event_interruptible(xcu->wq_xcu_idle, + atomic_read(&xcu->has_active) || xcu->xrq.nr_running); + + XSCHED_DEBUG("%s: rt_nr_running = %d, has_active = %d\n", + __func__, xcu->xrq.nr_running, atomic_read(&xcu->has_active)); + + mutex_lock(&xcu->xcu_lock); + if (!xsched_check_pending_kicks_xcu(xcu)) { + XSCHED_WARN("%s: No pending kicks on xcu %u\n", __func__, xcu->id); + continue; + } + + next_xse = __raw_pick_next_ctx(xcu); + if (!next_xse) { + XSCHED_WARN("%s: Couldn't find next xse on xcu %u\n", __func__, xcu->id); + continue; + } + + xcu->xrq.curr_xse = next_xse; + + if (__xsched_submit(xcu, next_xse) == 0) + continue; + + curr_xse = xcu->xrq.curr_xse; + if (curr_xse) { /* if not deleted yet */ + put_prev_ctx(curr_xse); + if (!atomic_read(&curr_xse->kicks_pending_ctx_cnt)) { + dequeue_ctx(curr_xse, xcu); + XSCHED_DEBUG( + "%s: Dequeue xse %d due to zero kicks on xcu %u\n", + __func__, curr_xse->tgid, xcu->id); + curr_xse = xcu->xrq.curr_xse = NULL; + } + } + } + + return err; +} + +/* Initialize xsched classes' runqueues. */ +static inline void xsched_rq_init(struct xsched_cu *xcu) +{ + xcu->xrq.nr_running = 0; + xcu->xrq.curr_xse = NULL; + xcu->xrq.state = XRQ_STATE_IDLE; +} + /* Initializes all xsched XCU objects. * Should only be called from xsched_xcu_register function. */ @@ -209,6 +331,12 @@ static void xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, xcu->state = XSCHED_XCU_NONE; xcu->group = group; + atomic_set(&xcu->has_active, 0); + + INIT_LIST_HEAD(&xcu->vsm_list); + + init_waitqueue_head(&xcu->wq_xcu_idle); + mutex_init(&xcu->xcu_lock); /* Mark current XCU in a mask inside XCU root group. */ @@ -265,6 +393,37 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) return 0; } +/* Fetch the first vstream metadata from vstream metadata list + * and removes it from that list. Returned vstream metadata pointer + * to be freed after. + */ +struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs) +{ + struct vstream_metadata *vsm; + + if (list_empty(&vs->metadata_list)) { + XSCHED_DEBUG("No metadata to fetch from vs %u @ %s\n", + vs->id, __func__); + return NULL; + } + + vsm = list_first_entry(&vs->metadata_list, struct vstream_metadata, node); + if (!vsm) { + XSCHED_ERR("Corrupted metadata list in vs %u @ %s\n", + vs->id, __func__); + return NULL; + } + + list_del(&vsm->node); + if (vs->kicks_count == 0) + XSCHED_WARN("kicks_count underflow in vs %u @ %s\n", + vs->id, __func__); + else + vs->kicks_count -= 1; + + return vsm; +} + /* * Initialize and register xcu in xcu_manager array. */ diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index d4f916415682..78ead98d1a66 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -444,10 +444,18 @@ int vstream_kick(struct vstream_args *arg) vstream->id, __func__); break; } + + enqueue_ctx(xse, xcu); + /* Increasing a total amount of kicks on an CU to which this + * context is attached to based on sched_class. + */ + xsched_inc_pending_kicks_xse(&vstream->ctx->xse); } while (err == -EBUSY); spin_unlock(&vstream->stream_lock); mutex_unlock(&xcu->xcu_lock); + if (!err) + wake_up_interruptible(&xcu->wq_xcu_idle); return err; } -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add __xsched_submit() implementation: - Add xsched_proc() implementation. - Add submit_kick() implementation. Add xcu_run() implementation. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- drivers/xcu/xcu_group.c | 7 +++- include/linux/xsched.h | 2 + kernel/xsched/core.c | 92 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index 86b935af00b6..605c023a2cfe 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -123,7 +123,12 @@ EXPORT_SYMBOL(xcu_group_find); */ int xcu_run(struct xcu_op_handler_params *params) { - return 0; + if (!params->group->opt || !params->group->opt->run) { + XSCHED_ERR("No function [run] called.\n"); + return -EINVAL; + } + + return params->group->opt->run(params); } /* This function runs "wait" callback for a given xcu_group diff --git a/include/linux/xsched.h b/include/linux/xsched.h index ba2c2e903f59..18e62f265199 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -471,6 +471,8 @@ struct xsched_cu *xcu_find(uint32_t *type, /* Vstream metadata proccesing functions.*/ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); +void submit_kick(struct vstream_info *vs, struct xcu_op_handler_params *params, + struct vstream_metadata *vsm); void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); #endif /* __LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 97b073d804bc..543967335b17 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -387,11 +387,72 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) return err; } -static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) +/* + * A function for submitting stream's commands (sending commands to a XCU). + */ +static int xsched_proc(struct xsched_cu *xcu, struct vstream_info *vs, + struct vstream_metadata *vsm) { + struct xcu_op_handler_params params; + struct xsched_entity *xse; + + XSCHED_CALL_STUB(); + + xse = &vs->ctx->xse; + + /* Init input parameters for xcu_run and xcu_wait callbacks. */ + params.group = xcu->group; + + /* Increase process time by abstract kick handling time. */ + xse->last_exec_runtime += vsm->exec_time; + + XSCHED_DEBUG("Process vsm sq_tail %d exec_time %u sqe_num %d sq_id %d@ %s\n", + vsm->sq_tail, vsm->exec_time, vsm->sqe_num, vsm->sq_id, __func__); + submit_kick(vs, ¶ms, vsm); + + xse->total_submitted++; + + XSCHED_DEBUG("xse %d total_submitted = %lu @ %s\n", + xse->tgid, xse->total_submitted, __func__); + + XSCHED_EXIT_STUB(); return 0; } +static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) +{ + struct vstream_metadata *vsm, *tmp; + unsigned int submit_exec_time = 0; + size_t kicks_submitted = 0; + unsigned long wait_us; + + XSCHED_DEBUG("%s called for xse %d on xcu %u\n", + __func__, xse->tgid, xcu->id); + + list_for_each_entry_safe(vsm, tmp, &xcu->vsm_list, node) { + xsched_proc(xcu, vsm->parent, vsm); + submit_exec_time += vsm->exec_time; + kicks_submitted++; + } + + INIT_LIST_HEAD(&xcu->vsm_list); + + mutex_unlock(&xcu->xcu_lock); + + wait_us = div_u64(submit_exec_time, NSEC_PER_USEC); + XSCHED_DEBUG("XCU kicks_submitted=%lu wait_us=%lu @ %s\n", + kicks_submitted, wait_us, __func__); + + if (wait_us > 0) { + /* Sleep shift not larger than 12.5% */ + usleep_range(wait_us, wait_us + (wait_us >> 3)); + } + + mutex_lock(&xcu->xcu_lock); + + return kicks_submitted; +} + static inline bool should_preempt(struct xsched_entity *xse) { return xse->class->check_preempt(xse); @@ -445,6 +506,35 @@ static int xsched_schedule(void *input_xcu) return err; } +void submit_kick(struct vstream_info *vs, + struct xcu_op_handler_params *params, + struct vstream_metadata *vsm) +{ + int ret; + + params->fd = vs->fd; + params->param_1 = &vs->id; + params->param_2 = &vs->channel_id; + params->param_3 = vsm->sqe; + params->param_4 = &vsm->sqe_num; + params->param_5 = &vsm->timeout; + params->param_6 = &vs->sqcq_type; + params->param_7 = vs->drv_ctx; + /* Send vstream on a device for processing. */ + ret = xcu_run(params); + if (ret) { + XSCHED_ERR( + "Failed to send vstream tasks vstreamId=%d to a device for processing.\n", + vs->id); + } + + XSCHED_DEBUG("Vstream_id %d submit vsm: sq_tail %d\n", vs->id, vsm->sq_tail); + + kfree(vsm); + + return; +} + /* Initialize xsched rt runqueue during kernel init. * Should only be called from xsched_init function. */ -- 2.34.1

From: Alekseev Dmitry <alekseev.dmitry@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add cgroup initialization inculing root cgroup. Add xcu cgroup callbacks: alloc, free, attach, detach, etc. Add xsched_group cgroup management files and methods for: - sched type - shares Add xcu cgroup subsys and option CONFIG_CGROUP_XCU Add cgroup.c in /kernel/xsched Makefile. Signed-off-by: Alekseev Dmitry <alekseev.dmitry@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- include/linux/cgroup_subsys.h | 4 + include/linux/xsched.h | 112 ++++++- kernel/cgroup/cgroup.c | 2 +- kernel/xsched/Kconfig | 15 + kernel/xsched/Makefile | 1 + kernel/xsched/cfs.c | 73 +++- kernel/xsched/cgroup.c | 616 ++++++++++++++++++++++++++++++++++ kernel/xsched/core.c | 16 +- 8 files changed, 812 insertions(+), 27 deletions(-) create mode 100644 kernel/xsched/cgroup.c diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h index 85fa78049bd0..e65ae90946c2 100644 --- a/include/linux/cgroup_subsys.h +++ b/include/linux/cgroup_subsys.h @@ -61,6 +61,10 @@ SUBSYS(pids) SUBSYS(rdma) #endif +#if IS_ENABLED(CONFIG_CGROUP_XCU) +SUBSYS(xcu) +#endif + #if IS_ENABLED(CONFIG_CGROUP_MISC) SUBSYS(misc) #endif diff --git a/include/linux/xsched.h b/include/linux/xsched.h index b4b6274b4c06..e59e4fe5e4b4 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -5,7 +5,7 @@ #include <linux/hash.h> #include <linux/hashtable.h> #include <linux/xcu_group.h> -#include <linux/kref.h> +#include <linux/cgroup.h> #include <linux/vstream.h> #ifndef pr_fmt #define pr_fmt(fmt) fmt @@ -39,13 +39,11 @@ #define XSCHED_TIME_INF RUNTIME_INF #define XSCHED_CFS_ENTITY_WEIGHT_DFLT 1 #define XSCHED_CFS_MIN_TIMESLICE (10*NSEC_PER_MSEC) +#define XSCHED_CFG_SHARE_DFLT 1024 #define __GET_VS_TASK_TYPE(t) ((t)&0xFF) - #define __GET_VS_TASK_PRIO_RT(t) (((t) >> 8) & 0xFF) - #define GET_VS_TASK_TYPE(vs_ptr) __GET_VS_TASK_TYPE((vs_ptr)->task_type) - #define GET_VS_TASK_PRIO_RT(vs_ptr) __GET_VS_TASK_PRIO_RT((vs_ptr)->task_type) /* @@ -57,6 +55,8 @@ */ #define XSCHED_CFS_KICK_SLICE 10 +extern struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; + enum xcu_sched_type { XSCHED_TYPE_RT, XSCHED_TYPE_DFLT = XSCHED_TYPE_RT, @@ -186,12 +186,15 @@ struct xsched_cu { struct mutex xcu_lock; - atomic_t has_active; - wait_queue_head_t wq_xcu_idle; wait_queue_head_t wq_xcu_running; }; +extern int num_active_xcu; +#define for_each_active_xcu(xcu, id) \ + for ((id) = 0, xcu = xsched_cu_mgr[(id)]; \ + (id) < num_active_xcu && (xcu = xsched_cu_mgr[(id)]); (id)++) + struct xsched_entity_rt { struct list_head list_node; enum xse_state state; @@ -260,6 +263,11 @@ struct xsched_entity { */ struct xsched_cu *xcu; + /* Link to list of xsched_group items */ + struct list_head group_node; + struct xsched_group *parent_grp; + bool is_group; + /* General purpose xse lock. */ spinlock_t xse_lock; }; @@ -274,6 +282,93 @@ static inline bool xse_is_cfs(const struct xsched_entity *xse) return xse && xse->class == &fair_xsched_class; } +/* xsched_group's xcu related stuff */ +struct xsched_group_xcu_priv { + /* Owner of this group */ + struct xsched_group *self; + + /* xcu id */ + int xcu_id; + + /* Link to scheduler */ + struct xsched_entity xse; /* xse of this group on runqueue */ + struct xsched_rq_cfs *cfs_rq; /* cfs runqueue "owned" by this group */ + struct xsched_rq_rt *rt_rq; /* rt runqueue "owned" by this group */ + + /* Statistics */ + int nr_throttled; + u64 throttled_time; + u64 overrun_time; +}; + +/* Xsched scheduling control group */ +struct xsched_group { + /* Cgroups controller structure */ + struct cgroup_subsys_state css; + + /* Control group settings: */ + int sched_type; + int prio; + + /* Bandwidth setting: shares value set by user */ + u64 shares_cfg; + u64 shares_cfg_red; + u32 weight; + u64 children_shares_sum; + + /* Bandwidth setting: maximal quota in period */ + s64 quota; + /* record the runtime of operators during the period */ + s64 runtime; + s64 period; + struct hrtimer quota_timeout; + struct work_struct refill_work; + u64 qoslevel; + + struct xsched_group_xcu_priv perxcu_priv[XSCHED_NR_CUS]; + + /* Groups hierarchcy */ + struct xsched_group *parent; + struct list_head children_groups; + struct list_head group_node; + + spinlock_t lock; + + /* for XSE to move in perxcu */ + struct list_head members; +}; + +#define XSCHED_RQ_OF(xse) \ + (container_of(((xse)->cfs.cfs_rq), struct xsched_rq, cfs)) + +#define XSCHED_RQ_OF_CFS_XSE(cfs_xse) \ + (container_of(((cfs_xse)->cfs_rq), struct xsched_rq, cfs)) + +#define XSCHED_SE_OF(cfs_xse) \ + (container_of((cfs_xse), struct xsched_entity, cfs)) + +#define xcg_parent_grp_xcu(xcg) \ + ((xcg)->self->parent->perxcu_priv[(xcg)->xcu_id]) + +#define xse_parent_grp_xcu(xse_cfs) \ + (&((XSCHED_SE_OF(xse_cfs) \ + ->parent_grp->perxcu_priv[(XSCHED_SE_OF(xse_cfs))->xcu->id]))) + +static inline struct xsched_group_xcu_priv * +xse_this_grp_xcu(struct xsched_entity_cfs *xse_cfs) +{ + struct xsched_entity *xse; + + xse = xse_cfs ? container_of(xse_cfs, struct xsched_entity, cfs) : NULL; + return xse ? container_of(xse, struct xsched_group_xcu_priv, xse) : NULL; +} + +static inline struct xsched_group * +xse_this_grp(struct xsched_entity_cfs *xse_cfs) +{ + return xse_cfs ? xse_this_grp_xcu(xse_cfs)->self : NULL; +} + /* Returns a pointer to an atomic_t variable representing a counter * of currently pending vstream kicks on a given XCU and for a * given xsched class. @@ -515,6 +610,11 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); void submit_kick(struct vstream_info *vs, struct xcu_op_handler_params *params, struct vstream_metadata *vsm); +/* Xsched group manage functions */ +int xsched_group_inherit(struct task_struct *tsk, struct xsched_entity *xse); +void xcu_cg_init_common(struct xsched_group *xcg); +void xcu_grp_shares_update(struct xsched_group *xg); +void xsched_group_xse_detach(struct xsched_entity *xse); void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); #endif /* __LINUX_XSCHED_H__ */ diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index c26a9b3a3576..b632590eae0f 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -6190,7 +6190,7 @@ int __init cgroup_init(void) struct cgroup_subsys *ss; int ssid; - BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16); + BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 17); BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files)); BUG_ON(cgroup_init_cftypes(NULL, cgroup_psi_files)); BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files)); diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig index fc5d7767d8e0..776c3dfbdaa4 100644 --- a/kernel/xsched/Kconfig +++ b/kernel/xsched/Kconfig @@ -34,3 +34,18 @@ config XSCHED_NR_CUS This option defines the maximum number of Compute Units (CUs) that can be managed by the XSched scheduler, consider changing this value proportionally to the number of available XCU cores. + +config CGROUP_XCU + bool "XCU bandwidth control and group scheduling for xsched_cfs" + default n + depends on XCU_SCHEDULER + help + This option enables the extended Compute Unit (XCU) resource controller for + CFS task groups, providing hierarchical scheduling and fine-grained bandwidth + allocation capabilities. Key features include: + - Proportional XCU time distribution across cgroups based on shares/quotas + - Nested group scheduling with latency isolation + - Integration with xsched_cfs for fair CPU resource management + + Required for systems requiring fine-grained resource control in cgroups. + If unsure, say N. diff --git a/kernel/xsched/Makefile b/kernel/xsched/Makefile index fe212f228cf6..c4c06b6038ff 100644 --- a/kernel/xsched/Makefile +++ b/kernel/xsched/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += vstream.o obj-$(CONFIG_XCU_SCHEDULER) += core.o rt.o cfs.o +obj-$(CONFIG_CGROUP_XCU) += cgroup.o diff --git a/kernel/xsched/cfs.c b/kernel/xsched/cfs.c index dadd16a5a90b..1313c7e73a11 100644 --- a/kernel/xsched/cfs.c +++ b/kernel/xsched/cfs.c @@ -21,6 +21,8 @@ #define CFS_INNER_RQ_EMPTY(cfs_xse) \ ((cfs_xse)->xruntime == XSCHED_TIME_INF) +extern struct xsched_group *root_xcg; + void xs_rq_add(struct xsched_entity_cfs *xse) { struct xsched_rq_cfs *cfs_rq = xse->cfs_rq; @@ -81,10 +83,46 @@ xs_pick_first(struct xsched_rq_cfs *cfs_rq) */ static void xs_update(struct xsched_entity_cfs *xse_cfs, u64 delta) { - u64 new_xrt = xse_cfs->xruntime + delta * xse_cfs->weight; + struct xsched_group_xcu_priv *xg = xse_parent_grp_xcu(xse_cfs); + + for (; xg; xse_cfs = &xg->xse.cfs, xg = &xcg_parent_grp_xcu(xg)) { + u64 new_xrt = xse_cfs->xruntime + delta * xse_cfs->weight; + + xs_cfs_rq_update(xse_cfs, new_xrt); + xse_cfs->sum_exec_runtime += delta; - xs_cfs_rq_update(xse_cfs, new_xrt); - xse_cfs->sum_exec_runtime += delta; + if (xg->self->parent == NULL) + break; + } +} + +/** + * xg_update() - Update container group's xruntime + * @gxcu: Descendant xsched group's private xcu control structure + * + * No locks required to access xsched_group_xcu_priv members, + * because only one worker thread works for one XCU. + */ +static void xg_update(struct xsched_group_xcu_priv *xg, int task_delta) +{ + u64 new_xrt; + struct xsched_entity_cfs *entry; + + for (; xg; xg = &xcg_parent_grp_xcu(xg)) { + xg->cfs_rq->nr_running += task_delta; + entry = xs_pick_first(xg->cfs_rq); + new_xrt = entry ? entry->xruntime * xg->xse.cfs.weight : XSCHED_TIME_INF; + + xg->cfs_rq->min_xruntime = new_xrt; + xg->xse.cfs.xruntime = new_xrt; + + if (!xg->xse.on_rq) + break; + if (!xg->self->parent) + break; + + xs_cfs_rq_update(&xg->xse.cfs, new_xrt); + } } /* @@ -94,19 +132,19 @@ static void xs_update(struct xsched_entity_cfs *xse_cfs, u64 delta) */ static void dequeue_ctx_fair(struct xsched_entity *xse) { + int task_delta; struct xsched_cu *xcu = xse->xcu; struct xsched_entity_cfs *first; struct xsched_entity_cfs *xse_cfs = &xse->cfs; + task_delta = + (xse->is_group) ? -(xse_this_grp_xcu(xse_cfs)->cfs_rq->nr_running) : -1; + xs_rq_remove(xse_cfs); + xg_update(xse_parent_grp_xcu(xse_cfs), task_delta); first = xs_pick_first(&xcu->xrq.cfs); xcu->xrq.cfs.min_xruntime = (first) ? first->xruntime : XSCHED_TIME_INF; - - if (xcu->xrq.cfs.min_xruntime == XSCHED_TIME_INF) { - atomic_set(&xcu->has_active, 0); - XSCHED_DEBUG("%s: set has_active to 0\n", __func__); - } } /** @@ -120,28 +158,27 @@ static void dequeue_ctx_fair(struct xsched_entity *xse) */ static void enqueue_ctx_fair(struct xsched_entity *xse, struct xsched_cu *xcu) { + int task_delta; struct xsched_entity_cfs *first; struct xsched_rq_cfs *rq; struct xsched_entity_cfs *xse_cfs = &xse->cfs; xse_cfs->weight = XSCHED_CFS_ENTITY_WEIGHT_DFLT; - rq = xse_cfs->cfs_rq = &xcu->xrq.cfs; + rq = xse_cfs->cfs_rq = xse_parent_grp_xcu(xse_cfs)->cfs_rq; + task_delta = + (xse->is_group) ? xse_this_grp_xcu(xse_cfs)->cfs_rq->nr_running : 1; - /* If no XSE of only empty groups */ + /* If no XSE or only empty groups */ if (xs_pick_first(rq) == NULL || rq->min_xruntime == XSCHED_TIME_INF) rq->min_xruntime = xse_cfs->xruntime; else xse_cfs->xruntime = max(xse_cfs->xruntime, rq->min_xruntime); xs_rq_add(xse_cfs); + xg_update(xse_parent_grp_xcu(xse_cfs), task_delta); first = xs_pick_first(&xcu->xrq.cfs); xcu->xrq.cfs.min_xruntime = (first) ? first->xruntime : XSCHED_TIME_INF; - - if (xcu->xrq.cfs.min_xruntime != XSCHED_TIME_INF) { - atomic_set(&xcu->has_active, 1); - XSCHED_DEBUG("%s: set has_active to 1\n", __func__); - } } static struct xsched_entity *pick_next_ctx_fair(struct xsched_cu *xcu) @@ -153,6 +190,12 @@ static struct xsched_entity *pick_next_ctx_fair(struct xsched_cu *xcu) if (!xse) return NULL; + for (; XSCHED_SE_OF(xse)->is_group; xse = xs_pick_first(rq)) { + if (!xse || CFS_INNER_RQ_EMPTY(xse)) + return NULL; + rq = xse_this_grp_xcu(xse)->cfs_rq; + } + return container_of(xse, struct xsched_entity, cfs); } diff --git a/kernel/xsched/cgroup.c b/kernel/xsched/cgroup.c new file mode 100644 index 000000000000..8ae17069e031 --- /dev/null +++ b/kernel/xsched/cgroup.c @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Support cgroup for xpu device + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/err.h> +#include <linux/cgroup.h> +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/xsched.h> + +enum xcu_file_type { + XCU_FILE_PERIOD_MS, + XCU_FILE_QUOTA_MS, + XCU_FILE_SHARES, +}; + +static struct xsched_group root_xsched_group; +struct xsched_group *root_xcg = &root_xsched_group; +static bool root_cg_inited; + +static struct xsched_group *old_xcg; +static DECLARE_WAIT_QUEUE_HEAD(xcg_attach_wq); +static bool attach_in_progress; +static DEFINE_MUTEX(xcg_mutex); + +static const char xcu_sched_name[XSCHED_TYPE_NUM][4] = { + [XSCHED_TYPE_RT] = "rt", + [XSCHED_TYPE_CFS] = "cfs" +}; + +void xcu_cg_init_common(struct xsched_group *xcg) +{ + spin_lock_init(&xcg->lock); + INIT_LIST_HEAD(&xcg->members); + INIT_LIST_HEAD(&xcg->children_groups); +} + +static void xcu_cfs_root_cg_init(void) +{ + int id; + struct xsched_cu *xcu; + + for_each_active_xcu(xcu, id) { + root_xcg->perxcu_priv[id].xcu_id = id; + root_xcg->perxcu_priv[id].self = root_xcg; + root_xcg->perxcu_priv[id].cfs_rq = &xcu->xrq.cfs; + root_xcg->perxcu_priv[id].xse.cfs.weight = 1; + } + + root_xcg->sched_type = XSCHED_TYPE_DFLT; +} + +/** + * xcu_cfs_cg_init() - Initialize xsched_group cfs runqueues and bw control. + * @xcg: new xsched_cgroup + * @parent_xg: parent's group + * + * One xsched_group can host many processes with contexts on different devices. + * Function creates xsched_entity for every XCU, and places it in runqueue + * of parent group. Create new cfs rq for xse inside group. + */ +static int xcu_cfs_cg_init(struct xsched_group *xcg, + struct xsched_group *parent_xg) +{ + int id = 0, err, i; + struct xsched_cu *xcu; + struct xsched_rq_cfs *sub_cfs_rq; + + if (unlikely(!root_cg_inited)) { + xcu_cfs_root_cg_init(); + root_cg_inited = true; + } + + for_each_active_xcu(xcu, id) { + xcg->perxcu_priv[id].xcu_id = id; + xcg->perxcu_priv[id].self = xcg; + + sub_cfs_rq = kzalloc(sizeof(struct xsched_rq_cfs), GFP_KERNEL); + if (!sub_cfs_rq) { + XSCHED_ERR("Fail to alloc cfs runqueue on xcu %d\n", id); + err = -ENOMEM; + goto alloc_error; + } + xcg->perxcu_priv[id].cfs_rq = sub_cfs_rq; + xcg->perxcu_priv[id].cfs_rq->ctx_timeline = RB_ROOT_CACHED; + + xcg->perxcu_priv[id].xse.is_group = true; + xcg->perxcu_priv[id].xse.xcu = xcu; + xcg->perxcu_priv[id].xse.class = &fair_xsched_class; + + /* Put new empty groups to the right in parent's rbtree: */ + xcg->perxcu_priv[id].xse.cfs.xruntime = XSCHED_TIME_INF; + xcg->perxcu_priv[id].xse.cfs.weight = + XSCHED_CFS_ENTITY_WEIGHT_DFLT; + xcg->perxcu_priv[id].xse.parent_grp = parent_xg; + + mutex_lock(&xcu->xcu_lock); + enqueue_ctx(&xcg->perxcu_priv[id].xse, xcu); + mutex_unlock(&xcu->xcu_lock); + } + + xcg->shares_cfg = XSCHED_CFG_SHARE_DFLT; + xcu_grp_shares_update(parent_xg); + + return 0; + +alloc_error: + for (i = 0; i < id; i++) + kfree(xcg->perxcu_priv[i].cfs_rq); + return err; +} + +static void xcu_cfs_cg_deinit(struct xsched_group *xcg) +{ + uint32_t id; + struct xsched_cu *xcu; + + for_each_active_xcu(xcu, id) { + mutex_lock(&xcu->xcu_lock); + dequeue_ctx(&xcg->perxcu_priv[id].xse, xcu); + mutex_unlock(&xcu->xcu_lock); + kfree(xcg->perxcu_priv[id].cfs_rq); + } + xcu_grp_shares_update(xcg->parent); +} + +/** + * xcu_cg_init() - Initialize non-root xsched_group structure. + * @xcg: new xsched_cgroup + * @parent_xg: parent's group + */ +static int xcu_cg_init(struct xsched_group *xcg, + struct xsched_group *parent_xg) +{ + xcu_cg_init_common(xcg); + xcg->parent = parent_xg; + list_add_tail(&xcg->group_node, &parent_xg->children_groups); + xcg->sched_type = parent_xg->sched_type; + + switch (xcg->sched_type) { + case XSCHED_TYPE_CFS: + return xcu_cfs_cg_init(xcg, parent_xg); + default: + pr_info("xcu_cgroup: init RT group css=0x%lx\n", + (uintptr_t)&xcg->css); + break; + } + + return 0; +} + +inline struct xsched_group *xcu_cg_from_css(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct xsched_group, css) : NULL; +} + +/** + * xcu_css_alloc() - Allocate and init xcu cgroup. + * @parent_css: css of parent xcu cgroup + * + * Called from kernel/cgroup.c with cgroup_lock() held. + * First called in subsys initialization to create root xcu cgroup, when + * XCUs haven't been initialized yet. Func used on every new cgroup creation, + * on second call to set root xsched_group runqueue. + * + * Return: pointer of new xcu cgroup css on success, -ENOMEM otherwise. + */ +static struct cgroup_subsys_state * +xcu_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct xsched_group *parent_xg; + struct xsched_group *xg; + int err; + + if (!parent_css) + return &root_xsched_group.css; + + xg = kzalloc(sizeof(*xg), GFP_KERNEL); + if (!xg) + return ERR_PTR(-ENOMEM); + + mutex_lock(&xcg_mutex); + parent_xg = xcu_cg_from_css(parent_css); + err = xcu_cg_init(xg, parent_xg); + mutex_unlock(&xcg_mutex); + if (err) { + kfree(xg); + XSCHED_ERR("Fail to alloc new xcu group %s\n", __func__); + return ERR_PTR(err); + } + + return &xg->css; +} + +static void xcu_css_free(struct cgroup_subsys_state *css) +{ + struct xsched_group *xcg; + + mutex_lock(&xcg_mutex); + xcg = xcu_cg_from_css(css); + if (xcg->parent != NULL) { + switch (xcg->sched_type) { + case XSCHED_TYPE_CFS: + xcu_cfs_cg_deinit(xcg); + break; + default: + pr_info("xcu_cgroup: deinit RT group css=0x%lx\n", + (uintptr_t)&xcg->css); + break; + } + } + list_del(&xcg->group_node); + mutex_unlock(&xcg_mutex); + + kfree(xcg); +} + +int xcu_css_online(struct cgroup_subsys_state *css) +{ + return 0; +} + +static void xcu_css_offline(struct cgroup_subsys_state *css) +{ + ; +} + +static void xsched_group_xse_attach(struct xsched_group *xg, + struct xsched_entity *xse) +{ + spin_lock(&xg->lock); + list_add_tail(&xse->group_node, &xg->members); + spin_unlock(&xg->lock); + xse->parent_grp = xg; +} + +void xsched_group_xse_detach(struct xsched_entity *xse) +{ + struct xsched_group *xcg = xse->parent_grp; + + spin_lock(&xcg->lock); + list_del(&xse->group_node); + spin_unlock(&xcg->lock); +} + +static int xcu_task_can_attach(struct task_struct *task, + struct xsched_group *old, struct xsched_group *dst) +{ + struct xsched_entity *xse; + bool has_xse = false; + + spin_lock(&old->lock); + list_for_each_entry(xse, &old->members, group_node) { + if (xse->owner_pid == task_pid_nr(task)) { + has_xse = true; + break; + } + } + spin_unlock(&old->lock); + + return has_xse ? -EINVAL : 0; +} + +static int xcu_can_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *dst_css, *old_css; + struct xsched_group *dst_xcg; + int ret = 0; + + mutex_lock(&xcg_mutex); + cgroup_taskset_for_each(task, dst_css, tset) { + old_css = task_css(task, xcu_cgrp_id); + dst_xcg = xcu_cg_from_css(dst_css); + old_xcg = xcu_cg_from_css(old_css); + ret = xcu_task_can_attach(task, old_xcg, dst_xcg); + if (ret) + break; + } + if (!ret) + attach_in_progress = true; + mutex_unlock(&xcg_mutex); + return ret; +} + +static void xcu_cancel_attach(struct cgroup_taskset *tset) +{ + mutex_lock(&xcg_mutex); + attach_in_progress = false; + wake_up(&xcg_attach_wq); + mutex_unlock(&xcg_mutex); +} + +void xcu_move_task(struct task_struct *task, struct xsched_group *old_xcg, + struct xsched_group *new_xcg) +{ + struct xsched_entity *xse, *tmp; + struct xsched_cu *xcu; + + spin_lock(&old_xcg->lock); + list_for_each_entry_safe(xse, tmp, &old_xcg->members, group_node) { + if (xse->owner_pid != task_pid_nr(task)) + continue; + + xcu = xse->xcu; + BUG_ON(old_xcg != xse->parent_grp); + + /* delete from the old_xcg */ + list_del(&xse->group_node); + + mutex_lock(&xcu->xcu_lock); + /* dequeue from the current runqueue */ + dequeue_ctx(xse, xcu); + /* attach to the new_xcg */ + xsched_group_xse_attach(new_xcg, xse); + /* enqueue to the runqueue in new_xcg */ + enqueue_ctx(xse, xcu); + mutex_unlock(&xcu->xcu_lock); + } + spin_unlock(&old_xcg->lock); +} + +static void xcu_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + + mutex_lock(&xcg_mutex); + cgroup_taskset_for_each(task, css, tset) { + xcu_move_task(task, old_xcg, xcu_cg_from_css(css)); + } + attach_in_progress = false; + wake_up(&xcg_attach_wq); + mutex_unlock(&xcg_mutex); +} + +/** + * xsched_group_inherit() - Attach new entity to task's xsched_group. + * @task: task_struct + * @xse: xsched entity + * + * Called in xsched context initialization to attach xse to task's group + * and inherit its xse scheduling class and bandwidth control policy. + * + * Return: Zero on success. + */ +int xsched_group_inherit(struct task_struct *task, struct xsched_entity *xse) +{ + struct cgroup_subsys_state *css; + struct xsched_group *xg; + +retry: + wait_event(xcg_attach_wq, !attach_in_progress); + + mutex_lock(&xcg_mutex); + if (attach_in_progress) { + mutex_unlock(&xcg_mutex); + goto retry; + } + xse->owner_pid = task_pid_nr(task); + css = task_get_css(task, xcu_cgrp_id); + xg = xcu_cg_from_css(css); + xsched_group_xse_attach(xg, xse); + css_put(css); + mutex_unlock(&xcg_mutex); + + return 0; +} + +static int xcu_sched_show(struct seq_file *sf, void *v) +{ + struct cgroup_subsys_state *css = seq_css(sf); + struct xsched_group *xg = xcu_cg_from_css(css); + + seq_printf(sf, "%s\n", xcu_sched_name[xg->sched_type]); + return 0; +} + +/** + * xcu_cg_set_sched() - Set scheduling type for group. + * @xg: xsched group + * @type: scheduler type + * + * Scheduler type can be changed if task is child of root group + * and haven't got scheduling entities. + * + * Return: Zero on success or -EINVAL + */ +int xcu_cg_set_sched(struct xsched_group *xg, int type) +{ + if (type == xg->sched_type) + return 0; + + if (xg->parent != root_xcg) + return -EINVAL; + + if (!list_empty(&xg->members)) + return -EBUSY; + + if (xg->sched_type == XSCHED_TYPE_CFS) + xcu_cfs_cg_deinit(xg); + + xg->sched_type = type; + if (type != XSCHED_TYPE_CFS) + return 0; + + /* type is XSCHED_TYPE_CFS */ + return xcu_cfs_cg_init(xg, xg->parent); +} + +static ssize_t xcu_sched_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct cgroup_subsys_state *css = of_css(of); + struct xsched_group *xg = xcu_cg_from_css(css); + char type_name[4]; + int type = -1; + + ssize_t ret = sscanf(buf, "%3s", type_name); + + if (ret < 1) + return -EINVAL; + + for (type = 0; type < XSCHED_TYPE_NUM; type++) { + if (!strcmp(type_name, xcu_sched_name[type])) + break; + } + + if (type == XSCHED_TYPE_NUM) + return -EINVAL; + + if (!list_empty(&css->children)) + return -EBUSY; + + mutex_lock(&xcg_mutex); + ret = xcu_cg_set_sched(xg, type); + mutex_unlock(&xcg_mutex); + + return (ret) ? ret : nbytes; +} + +static s64 xcu_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) +{ + s64 ret = 0; + struct xsched_group *xcucg = xcu_cg_from_css(css); + + spin_lock(&xcucg->lock); + switch (cft->private) { + case XCU_FILE_SHARES: + ret = xcucg->shares_cfg; + break; + default: + break; + } + spin_unlock(&xcucg->lock); + return ret; +} + +static inline u64 gcd(u64 a, u64 b) +{ + while (a != 0 && b != 0) { + if (a > b) + a %= b; + else + b %= a; + } + return (a) ? a : b; +} + +void xcu_grp_shares_update(struct xsched_group *xg) +{ + int id; + struct xsched_cu *xcu; + struct xsched_group *xgi, *parent = xg; + u64 sh_sum = 0, sh_gcd = 0, w_gcd = 0, sh_prod_red = 1; + + spin_lock(&parent->lock); + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) + sh_gcd = gcd(sh_gcd, xgi->shares_cfg); + } + + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) { + sh_sum += xgi->shares_cfg; + xgi->shares_cfg_red = div_u64(xgi->shares_cfg, sh_gcd); + + if ((sh_prod_red % xgi->shares_cfg_red) != 0) + sh_prod_red *= xgi->shares_cfg_red; + } + } + + parent->children_shares_sum = sh_sum; + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) { + xgi->weight = div_u64(sh_prod_red, xgi->shares_cfg_red); + w_gcd = gcd(w_gcd, xgi->weight); + } + } + + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) { + xgi->weight = div_u64(xgi->weight, w_gcd); + for_each_active_xcu(xcu, id) { + mutex_lock(&xcu->xcu_lock); + xgi->perxcu_priv[id].xse.cfs.weight = xgi->weight; + mutex_unlock(&xcu->xcu_lock); + } + } + } + spin_unlock(&parent->lock); +} + +static int xcu_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, + s64 val) +{ + int ret = 0; + struct xsched_group *xcucg = xcu_cg_from_css(css); + + spin_lock(&xcucg->lock); + switch (cft->private) { + case XCU_FILE_SHARES: + if (val <= 0) { + ret = -EINVAL; + break; + } + xcucg->shares_cfg = val; + xcu_grp_shares_update(xcucg->parent); + break; + default: + ret = -EINVAL; + break; + } + spin_unlock(&xcucg->lock); + + return ret; +} + +static int xcu_stat(struct seq_file *sf, void *v) +{ + struct cgroup_subsys_state *css = seq_css(sf); + struct xsched_group *xcucg = xcu_cg_from_css(css); + + u64 nr_throttled = 0; + u64 throttled_time = 0; + u64 exec_runtime = 0; + + int xcu_id; + struct xsched_cu *xcu; + + if (xcucg->sched_type == XSCHED_TYPE_RT) { + seq_printf(sf, "RT group stat is not supported\n"); + return 0; + } + + for_each_active_xcu(xcu, xcu_id) { + nr_throttled += xcucg->perxcu_priv[xcu_id].nr_throttled; + throttled_time += xcucg->perxcu_priv[xcu_id].throttled_time; + exec_runtime += + xcucg->perxcu_priv[xcu_id].xse.cfs.sum_exec_runtime; + } + + seq_printf(sf, "exec_runtime: %llu\n", exec_runtime); + seq_printf(sf, "shares cfg: %llu/%llu x%u\n", xcucg->shares_cfg, + xcucg->parent->children_shares_sum, xcucg->weight); + + return 0; +} + +static struct cftype xcu_cg_files[] = { + { + .name = "shares", + .flags = CFTYPE_NOT_ON_ROOT, + .read_s64 = xcu_read_s64, + .write_s64 = xcu_write_s64, + .private = XCU_FILE_SHARES, + }, + { + .name = "stat", + .seq_show = xcu_stat, + }, + { + .name = "sched", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = xcu_sched_show, + .write = xcu_sched_write, + }, + {} /* terminate */ +}; + +struct cgroup_subsys xcu_cgrp_subsys = { + .css_alloc = xcu_css_alloc, + .css_online = xcu_css_online, + .css_offline = xcu_css_offline, + .css_free = xcu_css_free, + .can_attach = xcu_can_attach, + .cancel_attach = xcu_cancel_attach, + .attach = xcu_attach, + .dfl_cftypes = xcu_cg_files, + .legacy_cftypes = xcu_cg_files, + .early_init = false, +}; diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 378dd4e66476..64f2cbafb8cd 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -26,6 +26,7 @@ int num_active_xcu; spinlock_t xcu_mgr_lock; +extern struct xsched_group *root_xcg; /* Xsched XCU array and bitmask that represents which XCUs * are present and online. @@ -49,6 +50,7 @@ static void put_prev_ctx(struct xsched_entity *xse) xse->class->put_prev_ctx(xse); xse->last_exec_runtime = 0; atomic_set(&xse->submitted_one_kick, 0); + XSCHED_DEBUG("Put current xse %d @ %s\n", xse->tgid, __func__); } static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) @@ -194,6 +196,8 @@ static int delete_ctx(struct xsched_context *ctx) XSCHED_DEBUG("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, atomic_read(&xse->kicks_pending_ctx_cnt), __func__); + xsched_group_xse_detach(xse); + return 0; } @@ -317,7 +321,10 @@ struct xsched_cu *xcu_find(uint32_t *type, int xsched_xse_set_class(struct xsched_entity *xse) { - switch (xse->task_type) { +#ifdef CONFIG_CGROUP_XCU + xsched_group_inherit(current, xse); +#endif + switch (xse->parent_grp->sched_type) { case XSCHED_TYPE_RT: xse->class = &rt_xsched_class; XSCHED_DEBUG("Context is in RT class %s\n", __func__); @@ -344,7 +351,7 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) xse->total_scheduled = 0; xse->total_submitted = 0; xse->last_exec_runtime = 0; - xse->task_type = XSCHED_TYPE_RT; + xse->task_type = GET_VS_TASK_TYPE(vs); xse->fd = ctx->fd; xse->tgid = ctx->tgid; @@ -715,10 +722,9 @@ EXPORT_SYMBOL(xsched_xcu_register); int __init xsched_init(void) { - /* Initializing global Xsched context list. */ + /* Initializing global XSched context list. */ INIT_LIST_HEAD(&xsched_ctx_list); - + xcu_cg_init_common(root_xcg); return 0; } - late_initcall(xsched_init); -- 2.34.1

From: Alekseev Dmitry <alekseev.dmitry@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- The xpu device should be exclusive in sched slice. So add xcu_wait() implementation which return after xpu kernels executed completed. It is called after submit_kick(). Signed-off-by: Alekseev Dmitry <alekseev.dmitry@huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- drivers/xcu/xcu_group.c | 23 +++- include/linux/vstream.h | 7 +- include/linux/xcu_group.h | 8 ++ include/linux/xsched.h | 19 +-- include/uapi/linux/xcu_vstream.h | 4 +- kernel/xsched/core.c | 200 ++++++++++++++++--------------- kernel/xsched/rt.c | 17 +++ 7 files changed, 161 insertions(+), 117 deletions(-) diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index 605c023a2cfe..891ce70bb36f 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -137,7 +137,12 @@ int xcu_run(struct xcu_op_handler_params *params) */ int xcu_wait(struct xcu_op_handler_params *params) { - return 0; + if (!params->group->opt || !params->group->opt->wait) { + XSCHED_ERR("No function [wait] called.\n"); + return -EINVAL; + } + + return params->group->opt->wait(params); } /* This function runs "complete" callback for a given xcu_group @@ -217,6 +222,22 @@ int xcu_logic_free(struct xcu_op_handler_params *params) return params->group->opt->logic_free(params); } +/* This function runs a "sqe_op" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to set or get sqe info. + */ +int xcu_sqe_op(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->sqe_op) { + XSCHED_ERR("No function [sqe_op] called.\n"); + return -EINVAL; + } + + return params->group->opt->sqe_op(params); +} + static struct xcu_group __xcu_group_root = { .id = 0, .type = XCU_TYPE_ROOT, diff --git a/include/linux/vstream.h b/include/linux/vstream.h index 58ee4c235a07..f0c290dc184c 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -13,15 +13,18 @@ * to be processed by a driver. */ typedef struct vstream_metadata { - uint32_t exec_time; /* A value of SQ tail that has been passed with the * kick that is described by this exact metadata object. */ uint32_t sq_tail; uint32_t sqe_num; uint32_t sq_id; + uint8_t sqe[XCU_SQE_SIZE_MAX]; + + /* Report buffer for fake read. */ + int8_t cqe[XCU_CQE_BUF_SIZE]; + uint32_t cqe_num; int32_t timeout; - int8_t sqe[XCU_SQE_SIZE_MAX]; /* A node for metadata list */ struct list_head node; diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h index 93f732f84694..e73c64f6c520 100644 --- a/include/linux/xcu_group.h +++ b/include/linux/xcu_group.h @@ -17,6 +17,11 @@ enum xcu_type { XCU_TYPE_XPU, }; +enum xcu_sqe_op_type { + SQE_SET_NOTIFY, + SQE_IS_NOTIFY, +}; + /** * @group: value for this entry. * @hash_node: hash node list. @@ -41,6 +46,7 @@ struct xcu_op_handler_params { void *param_5; void *param_6; void *param_7; + void *param_8; }; }; }; @@ -55,6 +61,7 @@ struct xcu_operation { xcu_op_handler_fn_t alloc; xcu_op_handler_fn_t logic_alloc; xcu_op_handler_fn_t logic_free; + xcu_op_handler_fn_t sqe_op; }; struct xcu_group { @@ -99,6 +106,7 @@ extern int xcu_finish(struct xcu_op_handler_params *params); extern int xcu_alloc(struct xcu_op_handler_params *params); extern int xcu_logic_alloc(struct xcu_op_handler_params *params); extern int xcu_logic_free(struct xcu_op_handler_params *params); +extern int xcu_sqe_op(struct xcu_op_handler_params *params); #endif /* !CONFIG_XCU_SCHEDULER */ #endif /* __XSCHED_XCU_GROUP_H__ */ diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 5ffaffc5afdb..825ff2dc0c8e 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -38,7 +38,6 @@ #define RUNTIME_INF ((u64)~0ULL) #define XSCHED_TIME_INF RUNTIME_INF #define XSCHED_CFS_ENTITY_WEIGHT_DFLT 1 -#define XSCHED_CFS_MIN_TIMESLICE (10 * NSEC_PER_MSEC) #define XSCHED_CFS_QUOTA_PERIOD_MS (100 * NSEC_PER_MSEC) #define XSCHED_CFG_SHARE_DFLT 1024 @@ -438,13 +437,8 @@ static inline int xsched_inc_pending_kicks_xse(struct xsched_entity *xse) atomic_inc(&xse->kicks_pending_ctx_cnt); /* Incrementing prio based pending kicks counter for RT class */ - if (xse_is_rt(xse)) { + if (xse_is_rt(xse)) atomic_inc(&xse->xcu->xrq.rt.prio_nr_kicks[xse->rt.prio]); - XSCHED_DEBUG("xcu increased pending kicks @ %s\n", __func__); - } else { - XSCHED_DEBUG("xse %u isn't rt class @ %s\n", xse->tgid, - __func__); - } return 0; } @@ -476,18 +470,12 @@ static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) /* Decrementing prio based pending kicks counter for RT class. */ if (xse_is_rt(xse)) { kicks_prio_rt = &xse->xcu->xrq.rt.prio_nr_kicks[xse->rt.prio]; - if (!atomic_read(kicks_prio_rt)) { XSCHED_ERR( - "Tried to decrement prio pending kicks beyond 0!\n"); + "Try to decrement prio pending kicks beyond 0!\n"); return -EINVAL; } - atomic_dec(kicks_prio_rt); - XSCHED_DEBUG("xcu decreased pending kicks @ %s\n", __func__); - } else { - XSCHED_DEBUG("xse %u isn't rt class @ %s\n", xse->tgid, - __func__); } return 0; @@ -591,7 +579,6 @@ static inline void xsched_init_vsm(struct vstream_metadata *vsm, struct vstream_info *vs, vstream_args_t *arg) { vsm->sq_id = arg->sq_id; - vsm->exec_time = arg->vk_args.exec_time; vsm->sqe_num = arg->vk_args.sqe_num; vsm->timeout = arg->vk_args.timeout; memcpy(vsm->sqe, arg->vk_args.sqe, XCU_SQE_SIZE_MAX); @@ -610,8 +597,6 @@ struct xsched_cu *xcu_find(uint32_t *type, /* Vstream metadata proccesing functions.*/ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); -void submit_kick(struct vstream_info *vs, struct xcu_op_handler_params *params, - struct vstream_metadata *vsm); /* Xsched group manage functions */ int xsched_group_inherit(struct task_struct *tsk, struct xsched_entity *xse); void xcu_cg_init_common(struct xsched_group *xcg); diff --git a/include/uapi/linux/xcu_vstream.h b/include/uapi/linux/xcu_vstream.h index 46d5a32db68e..f72d4720e993 100644 --- a/include/uapi/linux/xcu_vstream.h +++ b/include/uapi/linux/xcu_vstream.h @@ -6,6 +6,9 @@ #define PAYLOAD_SIZE_MAX 512 #define XCU_SQE_SIZE_MAX 64 +#define XCU_CQE_SIZE_MAX 32 +#define XCU_CQE_REPORT_NUM 4 +#define XCU_CQE_BUF_SIZE (XCU_CQE_REPORT_NUM * XCU_CQE_SIZE_MAX) /* * VSTREAM_ALLOC: alloc a vstream, buffer for tasks @@ -28,7 +31,6 @@ typedef struct vstream_free_args { } vstream_free_args_t; typedef struct vstream_kick_args { __u32 sqe_num; - __u32 exec_time; __s32 timeout; __s8 sqe[XCU_SQE_SIZE_MAX]; } vstream_kick_args_t; diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 78808f6ae561..744db05e36ec 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -54,15 +54,13 @@ static void put_prev_ctx(struct xsched_entity *xse) static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) { - int kick_count; + int kick_count, scheduled = 0, not_empty; struct vstream_info *vs; - unsigned int sum_exec_time = 0; - size_t kicks_submitted = 0; + struct xcu_op_handler_params params; struct vstream_metadata *vsm; - int not_empty; kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); - XSCHED_DEBUG("Before decrement XSE kick_count=%u @ %s\n", + XSCHED_DEBUG("Before decrement XSE kick_count=%d @ %s\n", kick_count, __func__); if (kick_count == 0) { @@ -74,31 +72,44 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) do { not_empty = 0; for_each_vstream_in_ctx(vs, xse->ctx) { + if (scheduled >= XSCHED_CFS_KICK_SLICE) + break; + spin_lock(&vs->stream_lock); vsm = xsched_vsm_fetch_first(vs); spin_unlock(&vs->stream_lock); - if (vsm) { - list_add_tail(&vsm->node, &xcu->vsm_list); - - sum_exec_time += vsm->exec_time; - kicks_submitted++; - xsched_dec_pending_kicks_xse(xse); - XSCHED_DEBUG( - "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", - vs->id, vsm->exec_time, vsm->sq_tail, - vsm->sqe_num, vsm->sq_id, __func__); - not_empty++; + + if (!vsm) + continue; + list_add_tail(&vsm->node, &xcu->vsm_list); + scheduled++; + xsched_dec_pending_kicks_xse(xse); + not_empty++; + } + } while ((scheduled < XSCHED_CFS_KICK_SLICE) && (not_empty)); + + /* + * Iterate over all vstreams in context: + * Set wr_cqe bit in last computing task in vsm_list + */ + for_each_vstream_in_ctx(vs, xse->ctx) { + list_for_each_entry_reverse(vsm, &xcu->vsm_list, node) { + if (vsm->parent == vs) { + params.group = vsm->parent->xcu->group; + params.param_1 = &(int){SQE_SET_NOTIFY}; + params.param_2 = &vsm->sqe; + xcu_sqe_op(¶ms); + break; } } - } while ((sum_exec_time < XSCHED_CFS_MIN_TIMESLICE) && (not_empty)); + } kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); XSCHED_DEBUG("After decrement XSE kick_count=%d @ %s\n", kick_count, __func__); - xse->total_scheduled += kicks_submitted; - - return kicks_submitted; + xse->total_scheduled += scheduled; + return scheduled; } static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) @@ -196,7 +207,6 @@ static int delete_ctx(struct xsched_context *ctx) atomic_read(&xse->kicks_pending_ctx_cnt), __func__); xsched_group_xse_detach(xse); - return 0; } @@ -368,7 +378,7 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) err = xsched_xse_set_class(xse); if (err) { - XSCHED_ERR("Failed to set xse class @ %s\n", __func__); + XSCHED_ERR("Fail to set xse class @ %s\n", __func__); return err; } @@ -401,70 +411,97 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) return err; } -/* - * A function for submitting stream's commands (sending commands to a XCU). - */ -static int xsched_proc(struct xsched_cu *xcu, struct vstream_info *vs, - struct vstream_metadata *vsm) +static void submit_kick(struct vstream_metadata *vsm) { + struct vstream_info *vs = vsm->parent; struct xcu_op_handler_params params; - struct xsched_entity *xse; + params.group = vs->xcu->group; + params.fd = vs->fd; + params.param_1 = &vs->id; + params.param_2 = &vs->channel_id; + params.param_3 = vsm->sqe; + params.param_4 = &vsm->sqe_num; + params.param_5 = &vsm->timeout; + params.param_6 = &vs->sqcq_type; + params.param_7 = vs->drv_ctx; + params.param_8 = &vs->logic_vcq_id; - XSCHED_CALL_STUB(); - - xse = &vs->ctx->xse; - - /* Init input parameters for xcu_run and xcu_wait callbacks. */ - params.group = xcu->group; - - /* Increase process time by abstract kick handling time. */ - xse->last_exec_runtime += vsm->exec_time; - - XSCHED_DEBUG("Process vsm sq_tail %d exec_time %u sqe_num %d sq_id %d@ %s\n", - vsm->sq_tail, vsm->exec_time, vsm->sqe_num, vsm->sq_id, __func__); - submit_kick(vs, ¶ms, vsm); + /* Send vstream on a device for processing. */ + if (xcu_run(¶ms)) { + XSCHED_ERR( + "Fail to send Vstream id %u tasks to a device for processing.\n", + vs->id); + } - xse->total_submitted++; + XSCHED_DEBUG("Vstream id %u submit vsm: sq_tail %u\n", vs->id, vsm->sq_tail); +} - XSCHED_DEBUG("xse %d total_submitted = %lu @ %s\n", - xse->tgid, xse->total_submitted, __func__); +static void submit_wait(struct vstream_metadata *vsm) +{ + struct vstream_info *vs = vsm->parent; + struct xcu_op_handler_params params; + /* Wait timeout in ms. */ + int32_t timeout = 500; + + params.group = vs->xcu->group; + params.param_1 = &vs->channel_id; + params.param_2 = &vs->logic_vcq_id; + params.param_3 = &vs->user_stream_id; + params.param_4 = &vsm->sqe; + params.param_5 = vsm->cqe; + params.param_6 = vs->drv_ctx; + params.param_7 = &timeout; + + /* Wait for a device to complete processing. */ + if (xcu_wait(¶ms)) { + XSCHED_ERR("Fail to wait Vstream id %u tasks, logic_cq_id %u.\n", + vs->id, vs->logic_vcq_id); + } - XSCHED_EXIT_STUB(); - return 0; + XSCHED_DEBUG("Vstream id %u wait finish, logic_cq_id %u\n", + vs->id, vs->logic_vcq_id); } static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) { struct vstream_metadata *vsm, *tmp; - unsigned int submit_exec_time = 0; - size_t kicks_submitted = 0; - unsigned long wait_us; + int submitted = 0; + long submit_exec_time = 0; + ktime_t t_start = 0; + struct xcu_op_handler_params params; XSCHED_DEBUG("%s called for xse %d on xcu %u\n", __func__, xse->tgid, xcu->id); list_for_each_entry_safe(vsm, tmp, &xcu->vsm_list, node) { - xsched_proc(xcu, vsm->parent, vsm); - submit_exec_time += vsm->exec_time; - kicks_submitted++; + submit_kick(vsm); + XSCHED_DEBUG("Xse %d vsm %u sched_delay: %lld ns\n", + xse->tgid, vsm->sq_id, ktime_to_ns(ktime_sub(ktime_get(), vsm->add_time))); + + params.group = vsm->parent->xcu->group; + params.param_1 = &(int){SQE_IS_NOTIFY}; + params.param_2 = &vsm->sqe; + if (xcu_sqe_op(¶ms)) { + mutex_unlock(&xcu->xcu_lock); + t_start = ktime_get(); + submit_wait(vsm); + submit_exec_time += ktime_to_ns(ktime_sub(ktime_get(), t_start)); + mutex_lock(&xcu->xcu_lock); + } + submitted++; + list_del(&vsm->node); + kfree(vsm); } + xse->last_exec_runtime += submit_exec_time; + xse->total_submitted += submitted; + atomic_add(submitted, &xse->submitted_one_kick); INIT_LIST_HEAD(&xcu->vsm_list); + XSCHED_DEBUG("Xse %d submitted=%d total=%zu, exec_time=%ld @ %s\n", + xse->tgid, submitted, xse->total_submitted, + submit_exec_time, __func__); - mutex_unlock(&xcu->xcu_lock); - - wait_us = div_u64(submit_exec_time, NSEC_PER_USEC); - XSCHED_DEBUG("XCU kicks_submitted=%lu wait_us=%lu @ %s\n", - kicks_submitted, wait_us, __func__); - - if (wait_us > 0) { - /* Sleep shift not larger than 12.5% */ - usleep_range(wait_us, wait_us + (wait_us >> 3)); - } - - mutex_lock(&xcu->xcu_lock); - - return kicks_submitted; + return submitted; } static inline bool should_preempt(struct xsched_entity *xse) @@ -521,35 +558,6 @@ static int xsched_schedule(void *input_xcu) return err; } -void submit_kick(struct vstream_info *vs, - struct xcu_op_handler_params *params, - struct vstream_metadata *vsm) -{ - int ret; - - params->fd = vs->fd; - params->param_1 = &vs->id; - params->param_2 = &vs->channel_id; - params->param_3 = vsm->sqe; - params->param_4 = &vsm->sqe_num; - params->param_5 = &vsm->timeout; - params->param_6 = &vs->sqcq_type; - params->param_7 = vs->drv_ctx; - /* Send vstream on a device for processing. */ - ret = xcu_run(params); - if (ret) { - XSCHED_ERR( - "Failed to send vstream tasks vstreamId=%d to a device for processing.\n", - vs->id); - } - - XSCHED_DEBUG("Vstream_id %d submit vsm: sq_tail %d\n", vs->id, vsm->sq_tail); - - kfree(vsm); - - return; -} - /* Initialize xsched rt runqueue during kernel init. * Should only be called from xsched_rq_init function. */ @@ -643,7 +651,7 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) new_vsm = kmalloc(sizeof(struct vstream_metadata), GFP_KERNEL); if (!new_vsm) { - XSCHED_ERR("Failed to alloc kick metadata for vs %u @ %s\n", + XSCHED_ERR("Fail to alloc kick metadata for vs %u @ %s\n", vs->id, __func__); return -ENOMEM; } diff --git a/kernel/xsched/rt.c b/kernel/xsched/rt.c index 3629fba3a48d..1a0a7f542060 100644 --- a/kernel/xsched/rt.c +++ b/kernel/xsched/rt.c @@ -186,6 +186,7 @@ static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) int kick_count, scheduled = 0; struct vstream_info *vs; struct vstream_metadata *vsm; + struct xcu_op_handler_params params; kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); XSCHED_DEBUG("Before decrement XSE kick_count=%d @ %s\n", @@ -207,6 +208,22 @@ static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) spin_unlock(&vs->stream_lock); } + /* + * Iterate over all vstreams in context: + * Set wr_cqe bit in last computing task in vsm_list + */ + for_each_vstream_in_ctx(vs, xse->ctx) { + list_for_each_entry_reverse(vsm, &xcu->vsm_list, node) { + if (vsm->parent == vs) { + params.group = vsm->parent->xcu->group; + params.param_1 = &(int){SQE_SET_NOTIFY}; + params.param_2 = &vsm->sqe; + xcu_sqe_op(¶ms); + break; + } + } + } + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); XSCHED_DEBUG("After decrement XSE kick_count=%d @ %s\n", kick_count, __func__); -- 2.34.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Adapt 910b npu driver for xsched Signed-off-by: Hui Tang <tanghui20@huawei.com> Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> --- .../0001-Adapt-910b-npu-driver-for-xsched.txt | 918 ++++++++++++++++++ 1 file changed, 918 insertions(+) create mode 100644 drivers/xcu/0001-Adapt-910b-npu-driver-for-xsched.txt diff --git a/drivers/xcu/0001-Adapt-910b-npu-driver-for-xsched.txt b/drivers/xcu/0001-Adapt-910b-npu-driver-for-xsched.txt new file mode 100644 index 000000000000..83fada81dbb9 --- /dev/null +++ b/drivers/xcu/0001-Adapt-910b-npu-driver-for-xsched.txt @@ -0,0 +1,918 @@ +From fe53ea5d5abcc587972079bcae5a706e54f52749 Mon Sep 17 00:00:00 2001 +From: Hui Tang <tanghui20@huawei.com> +Date: Tue, 25 Feb 2025 10:18:24 +0000 +Subject: [PATCH openEuler-25.03] Adapt 910b npu driver for xsched + +hulk inclusion +category: feature +bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB + +----------------------------------------- + +Adapt 910b npu driver for xsched + +Signed-off-by: Hui Tang <tanghui20@huawei.com> +Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> +Signed-off-by: Liu Kai <liukai284@huawei.com> +Signed-off-by: Xia Fukun <xiafukun@huawei.com> +--- + .../depends/inc_driver/ascend_hal_define.h | 2 +- + rms/trs_drv/chan/chan_rxtx.c | 2 +- + .../lba/near/comm/adapt/trs_near_adapt_init.h | 2 + + rms/trs_drv/lba/near/sia/adapt/Makefile | 7 +- + .../lba/near/sia/adapt/trs_host_init.c | 3 + + .../lba/near/sia/adapt/xsched_xpu_interface.c | 263 ++++++++++++++++++ + rms/trs_drv/trs_core/Makefile | 1 + + rms/trs_drv/trs_core/trs_fops.c | 143 +++++++--- + rms/trs_drv/trs_core/trs_hw_sqcq.c | 3 +- + rms/trs_drv/trs_core/trs_hw_sqcq.h | 3 +- + rms/trs_drv/trs_core/trs_logic_cq.c | 100 ++++--- + rms/trs_drv/trs_core/trs_logic_cq.h | 3 +- + rms/trs_drv/trs_core/trs_sqcq_map.c | 4 + + ts_agent/src/ts_agent_update_sqe.c | 6 + + 14 files changed, 466 insertions(+), 76 deletions(-) + create mode 100755 rms/trs_drv/lba/near/sia/adapt/xsched_xpu_interface.c + +diff --git a/dev_inc_open/inc/depends/inc_driver/ascend_hal_define.h b/dev_inc_open/inc/depends/inc_driver/ascend_hal_define.h +index a76efda..cc51c4d 100644 +--- a/dev_inc_open/inc/depends/inc_driver/ascend_hal_define.h ++++ b/dev_inc_open/inc/depends/inc_driver/ascend_hal_define.h +@@ -893,7 +893,7 @@ typedef enum tagDrvSqCqType { + } drvSqCqType_t; + + struct halSqCqInputInfo { +- drvSqCqType_t type; // normal : 0, callback : 1 ++ drvSqCqType_t type; // normal : 0, callback : 1, logic : 2 + uint32_t tsId; + /* The size and depth of each cqsq can be configured in normal mode, but this function is not yet supported */ + uint32_t sqeSize; // normal : 64Byte +diff --git a/rms/trs_drv/chan/chan_rxtx.c b/rms/trs_drv/chan/chan_rxtx.c +index 1fc72da..1e4ef38 100755 +--- a/rms/trs_drv/chan/chan_rxtx.c ++++ b/rms/trs_drv/chan/chan_rxtx.c +@@ -156,7 +156,7 @@ static int trs_chan_fill_sqe(struct trs_chan *chan, u8 *sqe, int timeout, int ad + /* if using bar to r/w sqe, it should use stack value to store sqe to avoid waster time */ + sqe_addr = trs_chan_mem_is_local_mem(&sq->mem_attr) ? dst_addr : sqe_tmp; + +- if (addr_domain == CHAN_ADDR_DOMAIN_KERNEL) { ++ if (addr_domain == CHAN_ADDR_DOMAIN_KERNEL || !access_ok(sqe, sq->para.sqe_size)) { + memcpy_s(sqe_addr, sq->para.sqe_size, sqe, sq->para.sqe_size); + } else { + ret_cpy = copy_from_user(sqe_addr, sqe, sq->para.sqe_size); +diff --git a/rms/trs_drv/lba/near/comm/adapt/trs_near_adapt_init.h b/rms/trs_drv/lba/near/comm/adapt/trs_near_adapt_init.h +index 3a60d1d..6b4598f 100755 +--- a/rms/trs_drv/lba/near/comm/adapt/trs_near_adapt_init.h ++++ b/rms/trs_drv/lba/near/comm/adapt/trs_near_adapt_init.h +@@ -21,4 +21,6 @@ + void trs_ts_adapt_init(struct trs_id_inst *inst); + void trs_ts_adapt_uninit(struct trs_id_inst *inst); + ++int xsched_xcu_group_init(u32 dev_id, u32 ts_num, u32 version); ++ + #endif /* TRS_NEAR_ADAPT_INIT_H */ +diff --git a/rms/trs_drv/lba/near/sia/adapt/Makefile b/rms/trs_drv/lba/near/sia/adapt/Makefile +index 16a3f05..2cbdd43 100755 +--- a/rms/trs_drv/lba/near/sia/adapt/Makefile ++++ b/rms/trs_drv/lba/near/sia/adapt/Makefile +@@ -59,8 +59,13 @@ EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/rms/trs_drv/lba/near/sia/adapt/comm/tsc + EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/dms/include/ + EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/dms/config/ + EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/dbl/dev_urd/ ++EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/rms/trs_drv/trs_core ++EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/dev/inc/ ++EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/drv_davinci_intf_host ++EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/tsch/ ++ + obj-m += ascend_trs_pm_adapt.o +-ascend_trs_pm_adapt-objs := trs_host_init.o near_comm/trs_host_comm.o near_comm/trs_host_db.o near_comm/trs_host_id.o near_comm/trs_host_mbox.o near_comm/trs_host_msg.o near_comm/trs_near_adapt_init.o ++ascend_trs_pm_adapt-objs := trs_host_init.o near_comm/trs_host_comm.o near_comm/trs_host_db.o near_comm/trs_host_id.o near_comm/trs_host_mbox.o near_comm/trs_host_msg.o near_comm/trs_near_adapt_init.o xsched_xpu_interface.o + + ascend_trs_pm_adapt-objs += near_comm/soc_adapt/soc_adapt.o trs_host_init/trs_host.o trs_host_chan/trs_host_chan.o trs_host_chan/trs_sqe_update.o trs_host_core/trs_host_core.o + ascend_trs_pm_adapt-objs += near_comm/trs_host_chan/stars_v1/trs_chan_stars_v1_ops.o near_comm/trs_host_chan/stars_v1/trs_chan_stars_v1_ops_stars.o +diff --git a/rms/trs_drv/lba/near/sia/adapt/trs_host_init.c b/rms/trs_drv/lba/near/sia/adapt/trs_host_init.c +index abdabc6..9de8549 100755 +--- a/rms/trs_drv/lba/near/sia/adapt/trs_host_init.c ++++ b/rms/trs_drv/lba/near/sia/adapt/trs_host_init.c +@@ -17,6 +17,7 @@ + #include <linux/types.h> + #include <linux/init.h> + #include <linux/module.h> ++#include <linux/xcu_group.h> + + #include "soc_res.h" + #include "trs_pub_def.h" +@@ -162,6 +163,8 @@ int trs_host_init(u32 phy_devid) + } + } + ++ xsched_xcu_group_init(phy_devid, ts_num, XCU_HW_V2); ++ + return 0; + } + EXPORT_SYMBOL(trs_host_init); +diff --git a/rms/trs_drv/lba/near/sia/adapt/xsched_xpu_interface.c b/rms/trs_drv/lba/near/sia/adapt/xsched_xpu_interface.c +new file mode 100755 +index 0000000..a7c01ba +--- /dev/null ++++ b/rms/trs_drv/lba/near/sia/adapt/xsched_xpu_interface.c +@@ -0,0 +1,263 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2019-2020. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 and ++ * only version 2 as published by the Free Software Foundation. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * Description: ++ * Author: Huawei ++ * Create: 2024-06-17 ++ */ ++ ++#ifndef TSDRV_KERNEL_UT ++#include <linux/cdev.h> ++#include <linux/slab.h> ++#include <linux/spinlock.h> ++#include <linux/errno.h> ++#include <linux/list.h> ++#include <linux/platform_device.h> ++#include <linux/vmalloc.h> ++#include <linux/xcu_group.h> ++#include <linux/file.h> ++ ++#include "securec.h" ++#include "devdrv_manager_comm.h" ++#include "ascend_hal_define.h" ++#include "trs_pub_def.h" ++#include "trs_res_id_def.h" ++#include "trs_proc.h" ++#include "trs_cmd.h" ++#include "davinci_api.h" ++#include "davinci_interface.h" ++#include "davinci_intf_init.h" ++#include "task_struct.h" ++ ++int ioctl_trs_sqcq_send(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg); ++int ioctl_trs_sqcq_alloc(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg); ++int ioctl_trs_sqcq_free(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg); ++int ioctl_trs_sqcq_recv(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg); ++ ++extern struct xcu_group *xcu_group_root; ++extern int xsched_xcu_register(struct xcu_group *group, uint32_t phys_id); ++ ++/* Gets device driver TS context from a file descriptor of opened device. */ ++static void *get_tsdrv_ctx(int fd) ++{ ++ struct davinci_intf_private_stru *file_private_data; ++ void *ctx = NULL; ++ struct fd f; ++ ++ f = fdget(fd); ++ if (!f.file) ++ goto out; ++ ++ file_private_data = f.file->private_data; ++ if (!file_private_data) ++ goto out; ++ ++ ctx = file_private_data->priv_filep.private_data; ++ ++out: ++ fdput(f); ++ return ctx; ++} ++ ++int trs_xsched_ctx_run(struct xcu_op_handler_params *params) ++{ ++ uint32_t sq_id = *(uint32_t *)params->param_1; ++ uint32_t tsId = *(uint32_t *)params->param_2; ++ uint8_t *sqe_addr = params->param_3; ++ uint32_t sqe_num = *(uint32_t *)params->param_4; ++ int32_t timeout = *(int32_t *)params->param_5; ++ int32_t type = *(int32_t *)params->param_6; ++ struct halTaskSendInfo input = {0}; ++ struct trs_proc_ctx *ctx = params->param_7; ++ uint32_t logic_cqId = *(uint32_t *)params->param_8; ++ ++ input.tsId = tsId; ++ input.sqId = sq_id; ++ input.timeout = timeout; ++ input.sqe_addr = sqe_addr; ++ input.sqe_num = sqe_num; ++ input.type = type; ++ ++ trs_debug("%s %d: tsId %u sqId %u timeout %d num %u\n", ++ __FUNCTION__, __LINE__, tsId, sq_id, timeout, sqe_num); ++ ++ /* Send SQ tail to a doorbel. */ ++ return ioctl_trs_sqcq_send(ctx, logic_cqId, (unsigned long)&input);; ++} ++ ++int trs_xsched_ctx_free(struct xcu_op_handler_params *params) ++{ ++ struct trs_proc_ctx *ctx; ++ ++ ctx = get_tsdrv_ctx(params->fd); ++ if (!ctx) ++ return -ENOENT; ++ ++ return ioctl_trs_sqcq_free(ctx, 0, (unsigned long)params->payload); ++} ++ ++int trs_xsched_ctx_wait(struct xcu_op_handler_params *params) ++{ ++ uint32_t tsId = *(uint32_t *)params->param_1; ++ uint32_t cqId = *(uint32_t *)params->param_2; ++ uint32_t streamId = *(uint32_t *)params->param_3; ++ struct ts_stars_sqe_header *sqe = params->param_4; ++ uint8_t *cqe_addr = params->param_5; ++ struct trs_proc_ctx *ctx = params->param_6; ++ int32_t timeout = *(uint32_t *)params->param_7; ++ int32_t cqe_num = 1; ++ struct halReportRecvInfo input = {0}; ++ uint32_t task_id = sqe->task_id; ++ ++ input.type = DRV_LOGIC_TYPE; ++ input.tsId = tsId; ++ input.cqId = cqId; ++ input.timeout = timeout; ++ input.cqe_num = cqe_num; ++ input.cqe_addr = cqe_addr; ++ input.stream_id = streamId; ++ input.task_id = task_id; ++ input.res[0] = 1; /* version 1 for new runtime. */ ++ ++ trs_debug("%s %d: tdId %u logic_cqId %u streamid %u task_id %d timeout %d \n", ++ __FUNCTION__, __LINE__, tsId, cqId, streamId, task_id, timeout); ++ ++ /* Wait for cq irq and read result. */ ++ return ioctl_trs_sqcq_recv(ctx, 0, (unsigned long)&input); ++} ++ ++int trs_xsched_ctx_complete(struct xcu_op_handler_params *params) ++{ ++ return 0; ++} ++ ++int trs_xsched_ctx_alloc(struct xcu_op_handler_params *params) ++{ ++ struct halSqCqInputInfo *input_info = params->payload; ++ uint32_t *tgid = (uint32_t *)params->param_1; ++ uint32_t *sq_id = (uint32_t *)params->param_2; ++ uint32_t *cq_id = (uint32_t *)params->param_3; ++ uint32_t *user_stream_id = (uint32_t *)params->param_4; ++ struct trs_proc_ctx *ctx; ++ int ret = 0; ++ ++ trs_debug("%s %d, input_info %lx, type: %d\n", ++ __FUNCTION__, __LINE__, (unsigned long)input_info, input_info->type); ++ ++ ctx = get_tsdrv_ctx(params->fd); ++ if (!ctx) ++ return -ENOENT; ++ ++ trs_debug("%s %d, pid %d, task_id %d, size %ld\n", ++ __FUNCTION__, __LINE__, ctx->pid, ctx->task_id, sizeof(*ctx)); ++ ret = ioctl_trs_sqcq_alloc(ctx, 0, (unsigned long)input_info); ++ if (ret != 0) ++ return ret; ++ ++ *tgid = ctx->pid; ++ *sq_id = input_info->sqId; ++ *cq_id = input_info->cqId; ++ *user_stream_id = input_info->info[0]; ++ params->param_5 = ctx; ++ return 0; ++} ++ ++int trs_xsched_ctx_logic_alloc(struct xcu_op_handler_params *params) ++{ ++ struct halSqCqInputInfo *input_info = params->payload; ++ uint32_t *logic_cq_id = (uint32_t *)params->param_1; ++ struct trs_proc_ctx *ctx; ++ int ret = 0; ++ ++ trs_debug("%s %d, type: %d\n", __FUNCTION__, __LINE__, input_info->type); ++ ++ ctx = get_tsdrv_ctx(params->fd); ++ if (!ctx) ++ return -ENOENT; ++ ++ trs_debug("%s %d, pid %d, task_id %d, size %ld\n", ++ __FUNCTION__, __LINE__, ctx->pid, ctx->task_id, sizeof(*ctx)); ++ ++ ret = ioctl_trs_sqcq_alloc(ctx, 0, (unsigned long)input_info); ++ if (ret != 0) ++ return ret; ++ ++ *logic_cq_id = input_info->cqId; ++ trs_debug("%s %d, type: %d, cq_id: %u\n", ++ __FUNCTION__, __LINE__, input_info->type, *logic_cq_id); ++ return 0; ++} ++ ++int trs_xsched_ctx_sqe_op(struct xcu_op_handler_params *params) ++{ ++ struct ts_stars_sqe_header *sqe = params->param_2; ++ int op_type = *(int *)(params->param_1); ++ ++ switch (op_type) { ++ case SQE_IS_NOTIFY: ++ return (sqe->type == 0) && (sqe->wr_cqe == 1); ++ case SQE_SET_NOTIFY: ++ if (sqe->type == 0) ++ sqe->wr_cqe = 1; ++ break; ++ default: ++ break; ++ } ++ ++ return 0; ++} ++ ++static struct xcu_operation trs_xsched_ctx_xcu_ops = { ++ .run = trs_xsched_ctx_run, ++ .finish = trs_xsched_ctx_free, ++ .wait = trs_xsched_ctx_wait, ++ .complete = trs_xsched_ctx_complete, ++ .alloc = trs_xsched_ctx_alloc, ++ .logic_alloc = trs_xsched_ctx_logic_alloc, ++ .sqe_op = trs_xsched_ctx_sqe_op, ++}; ++ ++int xsched_xcu_group_init(u32 dev_id, u32 ts_num, u32 version) ++{ ++ struct xcu_group *type_group; ++ struct xcu_group *dev_group; ++ struct xcu_group *ts_group; ++ int tsid; ++ ++ trs_debug("dev_id %u ts_num %u\n", dev_id, ts_num); ++ type_group = xcu_group_find(xcu_group_root, XCU_TYPE_XPU); ++ ++ if (!type_group) { ++ type_group = xcu_group_init(XCU_TYPE_XPU); ++ xcu_group_attach(type_group, xcu_group_root); ++ } ++ ++ dev_group = xcu_group_init(dev_id); ++ ++ trs_debug("%s %d deviceid is %d\n", __FUNCTION__, __LINE__, dev_id); ++ dev_group->id = dev_id; ++ xcu_group_attach(dev_group, type_group); ++ ++ for (tsid = 0; tsid < ts_num; tsid++) { ++ ts_group = xcu_group_init(tsid); ++ ts_group->ver = version; ++ ts_group->opt = &trs_xsched_ctx_xcu_ops; ++ ++ xcu_group_attach(ts_group, dev_group); ++ xsched_xcu_register(ts_group, dev_id); ++ ++ cond_resched(); ++ } ++ ++ return 0; ++} ++#endif +diff --git a/rms/trs_drv/trs_core/Makefile b/rms/trs_drv/trs_core/Makefile +index e0a6a55..8d27ad9 100755 +--- a/rms/trs_drv/trs_core/Makefile ++++ b/rms/trs_drv/trs_core/Makefile +@@ -41,6 +41,7 @@ endif + + EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/rms/trs_drv/inc + EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/rms/trs_drv/trs_core ++EXTRA_CFLAGS += -I$(DRIVER_SRC_BASE_DIR)/tsch/ + + obj-m += ascend_trs_core.o + ascend_trs_core-objs := trs_fops.o trs_ts_inst.o trs_proc.o trs_res_mng.o trs_sqcq_map.o trs_hw_sqcq.o trs_sw_sqcq.o trs_logic_cq.o trs_cb_sqcq.o trs_shm_sqcq.o trs_proc_fs.o +diff --git a/rms/trs_drv/trs_core/trs_fops.c b/rms/trs_drv/trs_core/trs_fops.c +index e5702d2..1a9b3c7 100755 +--- a/rms/trs_drv/trs_core/trs_fops.c ++++ b/rms/trs_drv/trs_core/trs_fops.c +@@ -21,6 +21,7 @@ + #include <linux/mm.h> + #include <linux/sched/mm.h> + #include <linux/version.h> ++#include <linux/vstream.h> + + #include "ascend_hal_define.h" + +@@ -33,6 +34,8 @@ + #include "trs_ts_inst.h" + #include "trs_cmd.h" + #include "trs_fops.h" ++#include "trs_logic_cq.h" ++#include "task_struct.h" + + static int (*const trs_res_id_handles[TRS_MAX_CMD])(struct trs_proc_ctx *proc_ctx, + struct trs_core_ts_inst *ts_inst, struct trs_res_id_para *para) = { +@@ -195,8 +198,17 @@ static int (*const trs_sqcq_alloc_handles[DRV_INVALID_TYPE])(struct trs_proc_ctx + [DRV_CTRL_TYPE] = trs_sw_sqcq_alloc + }; + +-static int ioctl_trs_sqcq_alloc(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) ++static bool is_xsched_used(void __user *ptr, int size) + { ++ if (access_ok(ptr, size)) ++ return false; ++ ++ return true; ++} ++ ++int ioctl_trs_sqcq_alloc(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) ++{ ++ bool xsched_used = is_xsched_used((void __user *)arg, sizeof(struct halSqCqInputInfo)); + struct trs_core_ts_inst *ts_inst = NULL; + struct halSqCqInputInfo para; + struct trs_alloc_para *alloc_para = NULL; +@@ -204,10 +216,14 @@ static int ioctl_trs_sqcq_alloc(struct trs_proc_ctx *proc_ctx, unsigned int cmd, + struct trs_uio_info uio_info; + int ret; + +- ret = copy_from_user(¶, (struct halSqCqInputInfo __user *)arg, sizeof(para)); +- if (ret != 0) { +- trs_err("Copy from user failed. (ret=%d)\n", ret); +- return ret; ++ if (xsched_used) { ++ memcpy(¶, (struct halSqCqInputInfo *)arg, sizeof(para)); ++ } else { ++ ret = copy_from_user(¶, (struct halSqCqInputInfo __user *)arg, sizeof(para)); ++ if (ret != 0) { ++ trs_err("Copy from user failed. (ret=%d)\n", ret); ++ return ret; ++ } + } + + alloc_para = get_alloc_para_addr(¶); +@@ -238,15 +254,24 @@ static int ioctl_trs_sqcq_alloc(struct trs_proc_ctx *proc_ctx, unsigned int cmd, + trs_core_inst_put(ts_inst); + + if (ret == 0) { +- ret = copy_to_user((struct halSqCqInputInfo __user *)arg, ¶, sizeof(para)); +- ret |= copy_to_user((struct trs_uio_info __user *)user_uio_info, &uio_info, sizeof(uio_info)); +- if (ret != 0) { +- trs_err("Copy to user failed. (ret=%d)\n", ret); ++ if (xsched_used) { ++ memcpy((struct halSqCqInputInfo *)arg, ¶, sizeof(para)); ++ ret = copy_to_user((struct trs_uio_info __user *)user_uio_info, &uio_info, sizeof(uio_info)); ++ if (ret != 0) { ++ trs_err("Copy to user failed. (ret=%d)\n", ret); ++ } ++ } else { ++ ret = copy_to_user((struct halSqCqInputInfo __user *)arg, ¶, sizeof(para)); ++ ret |= copy_to_user((struct trs_uio_info __user *)user_uio_info, &uio_info, sizeof(uio_info)); ++ if (ret != 0) { ++ trs_err("Copy to user failed. (ret=%d)\n", ret); ++ } + } + } + + return ret; + } ++EXPORT_SYMBOL(ioctl_trs_sqcq_alloc); + + static int (*const trs_sqcq_free_handles[DRV_INVALID_TYPE])(struct trs_proc_ctx *proc_ctx, + struct trs_core_ts_inst *ts_inst, struct halSqCqFreeInfo *para) = { +@@ -257,16 +282,20 @@ static int (*const trs_sqcq_free_handles[DRV_INVALID_TYPE])(struct trs_proc_ctx + [DRV_CTRL_TYPE] = trs_sw_sqcq_free + }; + +-static int ioctl_trs_sqcq_free(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) ++int ioctl_trs_sqcq_free(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) + { + struct trs_core_ts_inst *ts_inst = NULL; + struct halSqCqFreeInfo para; + int ret; + +- ret = copy_from_user(¶, (struct halSqCqFreeInfo __user *)arg, sizeof(para)); +- if (ret != 0) { +- trs_err("Copy from user failed. (ret=%d)\n", ret); +- return ret; ++ if (is_xsched_used((void *)arg, sizeof(struct halSqCqFreeInfo))) { ++ memcpy(¶, (struct halSqCqFreeInfo *)arg, sizeof(para)); ++ } else { ++ ret = copy_from_user(¶, (struct halSqCqFreeInfo __user *)arg, sizeof(para)); ++ if (ret != 0) { ++ trs_err("Copy from user failed. (ret=%d)\n", ret); ++ return ret; ++ } + } + + if ((para.type < 0) || (para.type >= DRV_INVALID_TYPE)) { +@@ -287,6 +316,7 @@ static int ioctl_trs_sqcq_free(struct trs_proc_ctx *proc_ctx, unsigned int cmd, + trs_core_inst_put(ts_inst); + return ret; + } ++EXPORT_SYMBOL(ioctl_trs_sqcq_free); + + static int ioctl_trs_sqcq_config(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) + { +@@ -362,17 +392,26 @@ static int (*const trs_sqcq_send_handles[DRV_INVALID_TYPE])(struct trs_proc_ctx + [DRV_CALLBACK_TYPE] = trs_cb_sqcq_send, + }; + +-static int ioctl_trs_sqcq_send(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) ++int ioctl_trs_sqcq_send(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) + { + struct trs_core_ts_inst *ts_inst = NULL; ++ struct halTaskSendInfo *kern_para = (struct halTaskSendInfo *)arg; + struct halTaskSendInfo __user *usr_para = (struct halTaskSendInfo __user *)arg; + struct halTaskSendInfo para; ++ struct trs_logic_cq *logic_cq = NULL; ++ struct ts_stars_sqe_header *sqe_header = NULL; ++ uint32_t logic_cqId = cmd; ++ bool xsched_used = is_xsched_used((void *)arg, sizeof(struct halTaskSendInfo)); + int ret; + +- ret = copy_from_user(¶, usr_para, sizeof(para)); +- if (ret != 0) { +- trs_err("Copy from user failed. (ret=%d)\n", ret); +- return ret; ++ if (xsched_used) { ++ memcpy(¶, (struct halTaskSendInfo *)arg, sizeof(para)); ++ } else { ++ ret = copy_from_user(¶, usr_para, sizeof(para)); ++ if (ret != 0) { ++ trs_err("Copy from user failed. (ret=%d)\n", ret); ++ return ret; ++ } + } + + if ((para.type < 0) || (para.type >= DRV_INVALID_TYPE) || (trs_sqcq_send_handles[para.type] == NULL) || +@@ -387,37 +426,69 @@ static int ioctl_trs_sqcq_send(struct trs_proc_ctx *proc_ctx, unsigned int cmd, + return -EINVAL; + } + ++ if (xsched_used) { ++ logic_cq = &ts_inst->logic_cq_ctx.cq[logic_cqId]; ++ if (logic_cq == NULL) { ++ trs_err("Invalid para. (logic_cqId=%u)\n", logic_cqId); ++ return -EINVAL; ++ } ++ ++ sqe_header = (struct ts_stars_sqe_header *)para.sqe_addr; ++ trs_debug("sqe_header->type=%u logic_cqId=%u stream_id=%u task_id=%u\n", ++ sqe_header->type, logic_cqId, sqe_header->rt_stream_id, sqe_header->task_id); ++ ++ if ((sqe_header->type == 0) && (sqe_header->wr_cqe == 1)) { ++ trs_debug("logic_cq->wakeup_num=%u\n", atomic_read(&logic_cq->wakeup_num)); ++ ++ if (atomic_read(&logic_cq->wakeup_num) > 0) { ++ atomic_dec(&logic_cq->wakeup_num); ++ trs_debug("logic_cq->wakeup_num=%u\n", atomic_read(&logic_cq->wakeup_num)); ++ } ++ } ++ } ++ + ret = trs_sqcq_send_handles[para.type](proc_ctx, ts_inst, ¶); + + trs_core_inst_put(ts_inst); + + if ((ret == 0) && (para.type == DRV_NORMAL_TYPE)) { +- ret = put_user(para.pos, &usr_para->pos); +- if (ret != 0) { +- trs_err("Put to user fail. (devid=%u; tsid=%u; sqId=%u)\n", proc_ctx->devid, para.tsId, para.sqId); ++ if (xsched_used) { ++ kern_para->pos = para.pos; ++ } else { ++ ret = put_user(para.pos, &usr_para->pos); ++ if (ret != 0) { ++ trs_err("Put to user fail. (devid=%u; tsid=%u; sqId=%u)\n", proc_ctx->devid, para.tsId, para.sqId); ++ } + } + } + + return ret; + } ++EXPORT_SYMBOL(ioctl_trs_sqcq_send); + + static int (*const trs_sqcq_recv_handles[DRV_INVALID_TYPE])(struct trs_proc_ctx *proc_ctx, +- struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para) = { ++ struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para, bool is_xsched) = { + [DRV_NORMAL_TYPE] = trs_hw_sqcq_recv, + [DRV_LOGIC_TYPE] = trs_logic_cq_recv, + }; + +-static int ioctl_trs_sqcq_recv(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) ++int ioctl_trs_sqcq_recv(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) + { + struct trs_core_ts_inst *ts_inst = NULL; +- struct halReportRecvInfo *usr_para = (struct halReportRecvInfo __user *)arg; ++ struct halReportRecvInfo *kern_para = (struct halReportRecvInfo *)arg; ++ struct halReportRecvInfo __user *usr_para = (struct halReportRecvInfo __user *)arg; + struct halReportRecvInfo para; + int ret; ++ bool xsched_used = is_xsched_used((void *)arg, sizeof(struct halReportRecvInfo)); + +- ret = copy_from_user(¶, usr_para, sizeof(para)); +- if (ret != 0) { +- trs_err("Copy from user failed. (ret=%d)\n", ret); +- return ret; ++ if (xsched_used) { ++ memcpy(¶, (struct halReportRecvInfo *)arg, sizeof(para)); ++ } else { ++ ret = copy_from_user(¶, usr_para, sizeof(para)); ++ if (ret != 0) { ++ trs_err("Copy from user failed. (ret=%d)\n", ret); ++ return ret; ++ } + } + + if ((para.type < 0) || (para.type >= DRV_INVALID_TYPE) || (trs_sqcq_recv_handles[para.type] == NULL) || +@@ -432,11 +503,16 @@ static int ioctl_trs_sqcq_recv(struct trs_proc_ctx *proc_ctx, unsigned int cmd, + return -EINVAL; + } + +- ret = trs_sqcq_recv_handles[para.type](proc_ctx, ts_inst, ¶); ++ ret = trs_sqcq_recv_handles[para.type](proc_ctx, ts_inst, ¶, xsched_used); ++ + if (ret == 0) { +- ret = put_user(para.report_cqe_num, &usr_para->report_cqe_num); +- if (ret != 0) { +- trs_err("Put to user fail. (devid=%u; tsid=%u; cqId=%u)\n", proc_ctx->devid, para.tsId, para.cqId); ++ if (xsched_used) { ++ kern_para->report_cqe_num = para.report_cqe_num; ++ } else { ++ ret = put_user(para.report_cqe_num, &usr_para->report_cqe_num); ++ if (ret != 0) { ++ trs_err("Put to user fail. (devid=%u; tsid=%u; cqId=%u)\n", proc_ctx->devid, para.tsId, para.cqId); ++ } + } + } else { + u32 ts_status; +@@ -449,6 +525,7 @@ static int ioctl_trs_sqcq_recv(struct trs_proc_ctx *proc_ctx, unsigned int cmd, + + return ret; + } ++EXPORT_SYMBOL(ioctl_trs_sqcq_recv); + + int ioctl_trs_stl_bind(struct trs_proc_ctx *proc_ctx, unsigned int cmd, unsigned long arg) + { +diff --git a/rms/trs_drv/trs_core/trs_hw_sqcq.c b/rms/trs_drv/trs_core/trs_hw_sqcq.c +index 825d603..10f3903 100755 +--- a/rms/trs_drv/trs_core/trs_hw_sqcq.c ++++ b/rms/trs_drv/trs_core/trs_hw_sqcq.c +@@ -1160,7 +1160,8 @@ int trs_hw_sqcq_send(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_ + return ret; + } + +-int trs_hw_sqcq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para) ++int trs_hw_sqcq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para, ++ bool is_xsched) + { + struct trs_id_inst *inst = &ts_inst->inst; + struct trs_chan_recv_para recv_para; +diff --git a/rms/trs_drv/trs_core/trs_hw_sqcq.h b/rms/trs_drv/trs_core/trs_hw_sqcq.h +index b32cd64..b6affdf 100755 +--- a/rms/trs_drv/trs_core/trs_hw_sqcq.h ++++ b/rms/trs_drv/trs_core/trs_hw_sqcq.h +@@ -32,7 +32,8 @@ int trs_sqcq_config(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_i + int trs_sqcq_query(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halSqCqQueryInfo *para); + + int trs_hw_sqcq_send(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halTaskSendInfo *para); +-int trs_hw_sqcq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para); ++int trs_hw_sqcq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para, ++ bool is_xsched); + void trs_proc_diable_sq_status(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, + int res_type, u32 res_id); + void trs_hw_sqcq_recycle(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, int res_type, u32 res_id); +diff --git a/rms/trs_drv/trs_core/trs_logic_cq.c b/rms/trs_drv/trs_core/trs_logic_cq.c +index d35b8d5..72cf64a 100755 +--- a/rms/trs_drv/trs_core/trs_logic_cq.c ++++ b/rms/trs_drv/trs_core/trs_logic_cq.c +@@ -265,13 +265,15 @@ static bool trs_logic_is_cqe_match(struct trs_logic_cq *logic_cq, void *cqe, u32 + } + + static int trs_logic_cq_recv_para_check(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, +- struct halReportRecvInfo *para) ++ struct halReportRecvInfo *para, bool is_xsched) + { + struct trs_id_inst *inst = &ts_inst->inst; + +- if (!trs_proc_has_res(proc_ctx, ts_inst, TRS_LOGIC_CQ, para->cqId)) { +- trs_err("Not proc owner cq. (devid=%u; tsid=%u; logic_cqid=%u)\n", inst->devid, inst->tsid, para->cqId); +- return -EINVAL; ++ if (!is_xsched) { ++ if (!trs_proc_has_res(proc_ctx, ts_inst, TRS_LOGIC_CQ, para->cqId)) { ++ trs_err("Not proc owner cq. (devid=%u; tsid=%u; logic_cqid=%u)\n", inst->devid, inst->tsid, para->cqId); ++ return -EINVAL; ++ } + } + + if (((para->timeout < 0) && (para->timeout != -1)) || (para->cqe_num == 0) || (para->cqe_addr == NULL)) { +@@ -441,7 +443,7 @@ static void trs_logic_cq_eliminate_holes(struct trs_logic_cq *logic_cq, u32 star + } + + static int trs_logic_cq_match_copy(struct trs_core_ts_inst *ts_inst, struct trs_logic_cq *logic_cq, +- struct halReportRecvInfo *para) ++ struct halReportRecvInfo *para, bool is_xsched) + { + u32 start, report_cnt, tail; + u32 rollback = 0; +@@ -463,11 +465,17 @@ static int trs_logic_cq_match_copy(struct trs_core_ts_inst *ts_inst, struct trs_ + } + + trs_logic_cq_copy_trace("Logic Cq Recv Match", ts_inst, logic_cq, start, report_cnt); +- ret = copy_to_user((void __user *)para->cqe_addr, logic_cq->addr + ((unsigned long)start * logic_cq->cqe_size), +- (unsigned long)report_cnt * logic_cq->cqe_size); +- if (ret != 0) { +- trs_err("copy to user fail, cqid=%u report_cnt=%u\n", logic_cq->cqid, report_cnt); +- return ret; ++ ++ if (is_xsched) { ++ memcpy((void *)para->cqe_addr, logic_cq->addr + ((unsigned long)start * logic_cq->cqe_size), ++ (unsigned long)report_cnt * logic_cq->cqe_size); ++ } else { ++ ret = copy_to_user((void __user *)para->cqe_addr, logic_cq->addr + ((unsigned long)start * logic_cq->cqe_size), ++ (unsigned long)report_cnt * logic_cq->cqe_size); ++ if (ret != 0) { ++ trs_err("copy to user fail, cqid=%u report_cnt=%u\n", logic_cq->cqid, report_cnt); ++ return ret; ++ } + } + + para->report_cqe_num = report_cnt; +@@ -480,7 +488,7 @@ static int trs_logic_cq_match_copy(struct trs_core_ts_inst *ts_inst, struct trs_ + } + + static int trs_logic_cq_non_match_copy(struct trs_core_ts_inst *ts_inst, struct trs_logic_cq *logic_cq, +- struct halReportRecvInfo *para) ++ struct halReportRecvInfo *para, bool is_xsched) + { + u32 start, report_cnt, tail; + int ret; +@@ -490,11 +498,17 @@ static int trs_logic_cq_non_match_copy(struct trs_core_ts_inst *ts_inst, struct + report_cnt = (tail > start) ? tail - start : logic_cq->cq_depth - start; + + trs_logic_cq_copy_trace("Logic Cq Recv NoMatch", ts_inst, logic_cq, start, report_cnt); +- ret = copy_to_user((void __user *)para->cqe_addr, logic_cq->addr + ((unsigned long)start * logic_cq->cqe_size), +- (unsigned long)report_cnt * logic_cq->cqe_size); +- if (ret != 0) { +- trs_err("copy to user fail, cqid=%u report_cnt=%u\n", logic_cq->cqid, report_cnt); +- return ret; ++ ++ if (is_xsched) { ++ memcpy((void *)para->cqe_addr, logic_cq->addr + ((unsigned long)start * logic_cq->cqe_size), ++ (unsigned long)report_cnt * logic_cq->cqe_size); ++ } else { ++ ret = copy_to_user((void __user *)para->cqe_addr, logic_cq->addr + ((unsigned long)start * logic_cq->cqe_size), ++ (unsigned long)report_cnt * logic_cq->cqe_size); ++ if (ret != 0) { ++ trs_err("copy to user fail, cqid=%u report_cnt=%u\n", logic_cq->cqid, report_cnt); ++ return ret; ++ } + } + + para->report_cqe_num = report_cnt; +@@ -503,7 +517,7 @@ static int trs_logic_cq_non_match_copy(struct trs_core_ts_inst *ts_inst, struct + } + + static int trs_logic_cq_copy_report(struct trs_core_ts_inst *ts_inst, +- struct trs_logic_cq *logic_cq, struct halReportRecvInfo *para) ++ struct trs_logic_cq *logic_cq, struct halReportRecvInfo *para, bool is_xsched) + { + u32 version = para->res[0]; + int full_flag = 0; +@@ -522,9 +536,9 @@ static int trs_logic_cq_copy_report(struct trs_core_ts_inst *ts_inst, + } + + if (version == 1) { +- ret = trs_logic_cq_match_copy(ts_inst, logic_cq, para); // runtime new version ++ ret = trs_logic_cq_match_copy(ts_inst, logic_cq, para, is_xsched); // runtime new version + } else { +- ret = trs_logic_cq_non_match_copy(ts_inst, logic_cq, para); ++ ret = trs_logic_cq_non_match_copy(ts_inst, logic_cq, para, is_xsched); + } + if (ret != 0) { + return ret; +@@ -553,8 +567,8 @@ static int trs_logic_cq_wait_event(struct trs_logic_cq *logic_cq, int timeout) + long ret, tm; + + atomic_inc(&logic_cq->wait_thread_num); +- trs_debug("Wake wait start. (logic_cqid=%u; timeout=%d; wait_thread_num=%d)\n", +- logic_cq->cqid, timeout, atomic_read(&logic_cq->wait_thread_num)); ++ trs_debug("Wake wait start. (logic_cqid=%u; timeout=%d; wait_thread_num=%d, wakeup_num=%d)\n", ++ logic_cq->cqid, timeout, atomic_read(&logic_cq->wait_thread_num), atomic_read(&logic_cq->wakeup_num)); + + tm = (timeout == -1) ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies((u32)timeout); + (void)prepare_to_wait_exclusive(&logic_cq->wait_queue, &wq_entry, TASK_INTERRUPTIBLE); +@@ -592,12 +606,13 @@ static int trs_logic_cq_wait_event(struct trs_logic_cq *logic_cq, int timeout) + return ret; + } + +-int trs_logic_cq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para) ++int trs_logic_cq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para, ++ bool is_xsched) + { + struct trs_logic_cq *logic_cq = NULL; + int ret; + +- ret = trs_logic_cq_recv_para_check(proc_ctx, ts_inst, para); ++ ret = trs_logic_cq_recv_para_check(proc_ctx, ts_inst, para, is_xsched); + if (ret != 0) { + return ret; + } +@@ -609,24 +624,35 @@ int trs_logic_cq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts + return trs_thread_bind_irq_wait(logic_cq, para->timeout); + } + trs_logic_cq_recv_trace("Recv start", ts_inst, para); +- do { +- mutex_lock(&logic_cq->mutex); +- ret = trs_logic_cq_copy_report(ts_inst, logic_cq, para); +- mutex_unlock(&logic_cq->mutex); +- if (ret == 0) { +- logic_cq->stat.recv++; +- trs_logic_cq_recv_trace("Recv finish", ts_inst, para); +- return ret; +- } + +- if (ret == -EAGAIN) { ++ if (is_xsched) { + if (para->timeout == 0) { +- para->report_cqe_num = 0; +- return 0; ++ para->report_cqe_num = 0; ++ return 0; + } + ret = trs_logic_cq_wait_event(logic_cq, para->timeout); +- } +- } while (ret >= 0); ++ trs_debug("Skip reading report for xsched, waiting for cq irq: logic_cqid=%u, timeout=%u, ret=%u)\n", ++ para->cqId, para->timeout, ret); ++ } else { ++ do { ++ mutex_lock(&logic_cq->mutex); ++ ret = trs_logic_cq_copy_report(ts_inst, logic_cq, para, is_xsched); ++ mutex_unlock(&logic_cq->mutex); ++ if (ret == 0) { ++ logic_cq->stat.recv++; ++ trs_logic_cq_recv_trace("Recv finish", ts_inst, para); ++ return ret; ++ } ++ ++ if (ret == -EAGAIN) { ++ if (para->timeout == 0) { ++ para->report_cqe_num = 0; ++ return 0; ++ } ++ ret = trs_logic_cq_wait_event(logic_cq, para->timeout); ++ } ++ } while (ret >= 0); ++ } + + return ret; + } +diff --git a/rms/trs_drv/trs_core/trs_logic_cq.h b/rms/trs_drv/trs_core/trs_logic_cq.h +index a45b110..b776b7f 100755 +--- a/rms/trs_drv/trs_core/trs_logic_cq.h ++++ b/rms/trs_drv/trs_core/trs_logic_cq.h +@@ -90,7 +90,8 @@ struct trs_core_ts_inst; + + int trs_logic_cq_alloc(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halSqCqInputInfo *para); + int trs_logic_cq_free(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halSqCqFreeInfo *para); +-int trs_logic_cq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para); ++int trs_logic_cq_recv(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst, struct halReportRecvInfo *para, ++ bool is_xsched); + + void trs_logic_set_cqe_version(struct trs_core_ts_inst *ts_inst, u32 logic_cqid, u32 cqe_verion); + int trs_logic_cq_enque(struct trs_core_ts_inst *ts_inst, u32 logic_cq_id, u32 stream_id, u32 task_id, void *cqe); +diff --git a/rms/trs_drv/trs_core/trs_sqcq_map.c b/rms/trs_drv/trs_core/trs_sqcq_map.c +index 8103d65..998ecfb 100755 +--- a/rms/trs_drv/trs_core/trs_sqcq_map.c ++++ b/rms/trs_drv/trs_core/trs_sqcq_map.c +@@ -305,6 +305,10 @@ int trs_sq_remap(struct trs_proc_ctx *proc_ctx, struct trs_core_ts_inst *ts_inst + int sq_reg_type = TRS_MAP_TYPE_REG; + int ret; + ++ ret = 0; ++ goto out; ++ ++ + if ((sq_info->sq_phy_addr == 0) || (sq_info->db_addr == 0) || (uio_info->sq_que_addr == 0)) { + ret = 0; + goto out; +diff --git a/ts_agent/src/ts_agent_update_sqe.c b/ts_agent/src/ts_agent_update_sqe.c +index bb4e3b2..01fe60c 100755 +--- a/ts_agent/src/ts_agent_update_sqe.c ++++ b/ts_agent/src/ts_agent_update_sqe.c +@@ -1146,6 +1146,12 @@ static void cqe_set_drop_flag(ts_stars_cqe_t *cqe) + if (cqe->warn || (cqe->sqe_type == TS_STARS_SQE_TYPE_PCIE_DMA)) { + /* cqe has been processed in ts_agent, no need to send to runtime */ + cqe->drop_flag = 1U; ++ ts_agent_debug("cqe has been processed in ts_agent, no need to send to runtime, drop_flag=%u\n", cqe->drop_flag); ++ ++ /* no drop, xsched needs to proc cqe */ ++ cqe->drop_flag = 0U; ++ ts_agent_debug("send cqe to runtime/xsched anyway, drop_flag=%u\n", cqe->drop_flag); ++ + return; + } + cqe->drop_flag = 0U; +-- +2.34.1 -- 2.34.1