From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB -------------------------------- Add sys_vstream_manage() syscall. Add the basic function framework. Add basic header files. Add new Kconfig.xsched with XCU_SCHEDULER and XCU_VSTREAM configurations. Create new dir kernel/xsched with vstream.c file with base xsched syscalls stubs. Add Makefile in kernel/xsched. Update main kernel Makefile to include kernel/xsched in build. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- arch/x86/entry/syscalls/syscall_64.tbl | 2 +- include/linux/syscalls.h | 3 ++ include/linux/vstream.h | 9 ++++ include/uapi/asm-generic/unistd.h | 4 +- include/uapi/linux/xcu_vstream.h | 54 +++++++++++++++++++ init/Kconfig | 1 + kernel/Makefile | 1 + kernel/xsched/Kconfig | 27 ++++++++++ kernel/xsched/Makefile | 2 + kernel/xsched/vstream.c | 73 ++++++++++++++++++++++++++ 10 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 include/linux/vstream.h create mode 100644 include/uapi/linux/xcu_vstream.h create mode 100644 kernel/xsched/Kconfig create mode 100644 kernel/xsched/Makefile create mode 100644 kernel/xsched/vstream.c diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index f88268a37ec2..504d1a1701d4 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -377,7 +377,7 @@ 453 64 map_shadow_stack sys_map_shadow_stack 454 common kabi_reserved454 sys_ni_syscall 455 common kabi_reserved455 sys_ni_syscall -456 common kabi_reserved456 sys_ni_syscall +456 common vstream_manage sys_vstream_manage 457 common kabi_reserved457 sys_ni_syscall 458 common kabi_reserved458 sys_ni_syscall 459 common kabi_reserved459 sys_ni_syscall diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 36c592e43d65..0324115b304a 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -948,6 +948,7 @@ asmlinkage long sys_cachestat(unsigned int fd, struct cachestat __user *cstat, unsigned int flags); asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags); +asmlinkage long sys_vstream_manage(struct vstream_args __user *arg, int cmd); /* * Architecture-specific system calls */ @@ -1060,6 +1061,8 @@ asmlinkage long sys_sysfs(int option, unsigned long arg1, unsigned long arg2); asmlinkage long sys_fork(void); +asmlinkage long sys_vstream_manage(struct vstream_args *arg, int cmd); + /* obsolete */ asmlinkage long sys_stime(__kernel_old_time_t __user *tptr); asmlinkage long sys_stime32(old_time32_t __user *tptr); diff --git a/include/linux/vstream.h b/include/linux/vstream.h new file mode 100644 index 000000000000..627f754f83c5 --- /dev/null +++ b/include/linux/vstream.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_VSTREAM_H +#define _LINUX_VSTREAM_H + +#include <uapi/linux/xcu_vstream.h> + +typedef int vstream_manage_t(struct vstream_args *arg); + +#endif /* _LINUX_VSTREAM_H */ diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index bf2b30463784..52151361036e 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -830,8 +830,8 @@ __SYSCALL(__NR_map_shadow_stack, sys_map_shadow_stack) __SYSCALL(__NR_kabi_reserved454, sys_ni_syscall) #define __NR_kabi_reserved455 455 __SYSCALL(__NR_kabi_reserved455, sys_ni_syscall) -#define __NR_kabi_reserved456 456 -__SYSCALL(__NR_kabi_reserved456, sys_ni_syscall) +#define __NR_vstream_manage 456 +__SYSCALL(__NR_vstream_manage, sys_vstream_manage) #define __NR_kabi_reserved457 457 __SYSCALL(__NR_kabi_reserved457, sys_ni_syscall) #define __NR_kabi_reserved458 458 diff --git a/include/uapi/linux/xcu_vstream.h b/include/uapi/linux/xcu_vstream.h new file mode 100644 index 000000000000..095d203ad422 --- /dev/null +++ b/include/uapi/linux/xcu_vstream.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_XCU_VSTREAM_H +#define _UAPI_XCU_VSTREAM_H + +#include <linux/types.h> + +#define PAYLOAD_SIZE_MAX 512 +#define XCU_SQE_SIZE_MAX 64 + +/* + * VSTREAM_ALLOC: alloc a vstream, buffer for tasks + * VSTREAM_FREE: free a vstream + * VSTREAM_KICK: there are tasks to be executed in the vstream + */ +typedef enum VSTREAM_COMMAND { + VSTREAM_ALLOC = 0, + VSTREAM_FREE, + VSTREAM_KICK, + MAX_COMMAND +} vstream_command_t; + +typedef struct vstream_alloc_args { + __s32 type; + __u32 user_stream_id; +} vstream_alloc_args_t; + +typedef struct vstream_free_args { } vstream_free_args_t; + +typedef struct vstream_kick_args { + __u32 sqe_num; + __s32 timeout; + __s8 sqe[XCU_SQE_SIZE_MAX]; +} vstream_kick_args_t; + +typedef struct vstream_args { + __u32 channel_id; + __u32 fd; + __u32 devid; + __u32 task_type; + __u32 sq_id; + __u32 cq_id; + + /* Device related structures. */ + union { + vstream_alloc_args_t va_args; + vstream_free_args_t vf_args; + vstream_kick_args_t vk_args; + }; + + __u32 payload_size; + __s8 payload[PAYLOAD_SIZE_MAX]; +} vstream_args_t; + +#endif /* _UAPI_LINUX_SCHED_H */ diff --git a/init/Kconfig b/init/Kconfig index 5af21834fbff..52290ec7c8db 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -478,6 +478,7 @@ source "kernel/irq/Kconfig" source "kernel/time/Kconfig" source "kernel/bpf/Kconfig" source "kernel/Kconfig.preempt" +source "kernel/xsched/Kconfig" menu "CPU/Task time and stats accounting" diff --git a/kernel/Makefile b/kernel/Makefile index 1fe46db40806..0baddecc349f 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -50,6 +50,7 @@ obj-y += rcu/ obj-y += livepatch/ obj-y += dma/ obj-y += entry/ +obj-y += xsched/ obj-$(CONFIG_MODULES) += module/ obj-$(CONFIG_KCMP) += kcmp.o diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig new file mode 100644 index 000000000000..9ec1011e85bf --- /dev/null +++ b/kernel/xsched/Kconfig @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0 + +config XCU_SCHEDULER + bool "Enable XSched functionality" + select CGROUP_XCU + default n + help + This option enables the XSched scheduler, a custom scheduling mechanism + designed for heterogeneous compute units (e.g., XPUs). It provides: + - Priority-based task scheduling with latency-sensitive optimizations. + - Integration with cgroups (via CGROUP_XCU) for resource isolation. + + Enable this only if your system requires advanced scheduling for XPU workloads. + If unsure, say N. + +config XCU_VSTREAM + bool "Enable vstream SQ/CQ buffers maintaining for XPU" + default n + depends on XCU_SCHEDULER + help + This option enables virtual stream (vstream) support for XPUs, managing + submission queues (SQ) and completion queues (CQ) in kernel space. Key features: + - Zero-copy buffer management between user and kernel space. + - Batch processing of XPU commands to reduce MMIO overhead. + + Requires XCU_SCHEDULER to be enabled. May increase kernel memory usage. + Recommended for high-throughput XPU workloads. If unsure, say N. diff --git a/kernel/xsched/Makefile b/kernel/xsched/Makefile new file mode 100644 index 000000000000..e972cd93b607 --- /dev/null +++ b/kernel/xsched/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += vstream.o diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c new file mode 100644 index 000000000000..5723c359e0f2 --- /dev/null +++ b/kernel/xsched/vstream.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Vstream manage for XPU device + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/syscalls.h> +#include <linux/vstream.h> + +#ifdef CONFIG_XCU_VSTREAM + +int vstream_alloc(struct vstream_args *arg) +{ + return 0; +} + +int vstream_free(struct vstream_args *arg) +{ + return 0; +} + +int vstream_kick(struct vstream_args *arg) +{ + return 0; +} + +/* + * vstream_manage_cmd table + */ +static vstream_manage_t(*vstream_command_table[MAX_COMMAND + 1]) = { + vstream_alloc, // VSTREAM_ALLOC + vstream_free, // VSTREAM_FREE + vstream_kick, // VSTREAM_KICK + NULL // MAX_COMMAND +}; + +SYSCALL_DEFINE2(vstream_manage, struct vstream_args __user *, arg, int, cmd) +{ + int res = 0; + struct vstream_args vstream_arg; + + if (copy_from_user(&vstream_arg, arg, sizeof(struct vstream_args))) { + pr_err("copy_from_user failed\n"); + return -EFAULT; + } + + res = vstream_command_table[cmd](&vstream_arg); + if (copy_to_user(arg, &vstream_arg, sizeof(struct vstream_args))) { + pr_err("copy_to_user failed\n"); + return -EFAULT; + } + + pr_debug("vstream_manage: cmd %d\n", cmd); + return res; +} +#else +SYSCALL_DEFINE2(vstream_manage, struct vstream_args __user *, arg, int, cmd) +{ + return 0; +} +#endif -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add xsched.h with xsched related prints. Add XSCHED_DEBUG_PRINTS configuration to switch_on/off xsched prints. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- include/linux/xsched.h | 51 ++++++++++++++++++++++++++++++++++++++++++ kernel/xsched/Kconfig | 11 +++++++++ 2 files changed, 62 insertions(+) create mode 100644 include/linux/xsched.h diff --git a/include/linux/xsched.h b/include/linux/xsched.h new file mode 100644 index 000000000000..c2f56678dda4 --- /dev/null +++ b/include/linux/xsched.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_XSCHED_H__ +#define __LINUX_XSCHED_H__ + +#ifndef pr_fmt +#define pr_fmt(fmt) fmt +#endif + +#define XSCHED_ERR_PREFIX "XSched [ERROR]: " +#define XSCHED_ERR(fmt, ...) \ + pr_err(XSCHED_ERR_PREFIX fmt, ##__VA_ARGS__) + +#define XSCHED_WARN_PREFIX "XSched [WARNING]: " +#define XSCHED_WARN(fmt, ...) \ + pr_warn(XSCHED_WARN_PREFIX fmt, ##__VA_ARGS__) + +/* Debug specific prints that are enabled + * if CONFIG_XSCHED_DEBUG_PRINTS = y + */ +#ifdef CONFIG_XSCHED_DEBUG_PRINTS + +#define XSCHED_INFO_PREFIX "XSched [INFO]: " +#define XSCHED_INFO(fmt, ...) \ + pr_info(XSCHED_INFO_PREFIX fmt, ##__VA_ARGS__) + +#define XSCHED_DEBUG_PREFIX "XSched [DEBUG]: " +#define XSCHED_DEBUG(fmt, ...) \ + pr_debug(XSCHED_DEBUG_PREFIX fmt, ##__VA_ARGS__) + +#define XSCHED_CALL_STUB() \ + XSCHED_INFO("-----* %s @ %s called *-----\n", __func__, __FILE__) + +#define XSCHED_EXIT_STUB() \ + XSCHED_INFO("-----* %s @ %s exited *-----\n", __func__, __FILE__) + +#define XSCHED_TRACE_PREFIX "XSCHED [TRACE]: " + +#define __XSCHED_TRACE(fmt, ...) \ + pr_info(XSCHED_TRACE_PREFIX fmt, ##__VA_ARGS__) +#define XSCHED_TRACE(event, xcu, xse, vs, type) \ + __XSCHED_TRACE("event_type=%u %u %s %s; xcu=%u; xse=%u; vs=%u; type=%s;", \ + xse, vs, event, type, xcu, xse, vs, type) +#else +#define XSCHED_INFO(fmt, ...) +#define XSCHED_DEBUG(fmt, ...) +#define XSCHED_EXIT_STUB() +#define XSCHED_CALL_STUB() +#define __XSCHED_TRACE(fmt, ...) +#endif + +#endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig index 9ec1011e85bf..cbe937a90d80 100644 --- a/kernel/xsched/Kconfig +++ b/kernel/xsched/Kconfig @@ -25,3 +25,14 @@ config XCU_VSTREAM Requires XCU_SCHEDULER to be enabled. May increase kernel memory usage. Recommended for high-throughput XPU workloads. If unsure, say N. + +config XSCHED_DEBUG_PRINTS + bool "Enable info and debug prints in XSched mechanism for analysis and debugging purposes" + default n + depends on XCU_SCHEDULER + help + This option enables verbose logging (info and debug levels) for the XSched + scheduling mechanism. It is primarily used for debugging performance issues, + analyzing task scheduling behavior, and tracking internal state changes. + Enabling this may impact performance due to increased log output. + If unsure, say N. -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add vstream related data structures: - vstream_info. Add vstream related context and entity data structures: - xsched_entity - xsched_context Add xsched_init() implementation. Add vstream_alloc() and xcu_alloc() implementation. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- drivers/xcu/xcu_group.c | 47 ++++- include/linux/vstream.h | 45 +++++ include/linux/xcu_group.h | 16 ++ include/linux/xsched.h | 72 ++++++++ kernel/xsched/core.c | 77 ++++++++ kernel/xsched/vstream.c | 359 +++++++++++++++++++++++++++++++++++++- 6 files changed, 614 insertions(+), 2 deletions(-) diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index a2419db384ac..b70e9e7392e3 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -193,7 +193,52 @@ int xcu_finish(struct xcu_op_handler_params *params) */ int xcu_alloc(struct xcu_op_handler_params *params) { - return 0; + int ret = 0; + + if (params->group->opt && params->group->opt->alloc) + ret = params->group->opt->alloc(params); + else + XSCHED_DEBUG("No function [alloc] called.\n"); + + return ret; +} + +/* This function runs a "logic_alloc" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement allocation + * and registering memory of logic CQ buffer. + */ +int xcu_logic_alloc(struct xcu_op_handler_params *params) +{ + int ret = 0; + + if (params->group->opt && params->group->opt->logic_alloc) + ret = params->group->opt->logic_alloc(params); + else + XSCHED_DEBUG("No function [logic_alloc] called.\n"); + + return ret; +} + +/* This function runs a "logic_free" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement deallocation + * and unregistering memory of a logic CQ buffer. + */ +int xcu_logic_free(struct xcu_op_handler_params *params) +{ + int ret = 0; + + if (params->group->opt && params->group->opt->logic_free) + ret = params->group->opt->logic_free(params); + else + XSCHED_DEBUG("No function [logic_free] called.\n"); + + return ret; } static struct xcu_group __xcu_group_root = { diff --git a/include/linux/vstream.h b/include/linux/vstream.h index 627f754f83c5..39cc6a26db50 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -4,6 +4,51 @@ #include <uapi/linux/xcu_vstream.h> +typedef struct vstream_info { + uint32_t user_streamId; + uint32_t id; + uint32_t vcqId; + uint32_t logic_vcqId; + uint32_t devId; + uint32_t channel_id; + uint32_t fd; + uint32_t task_type; + int tgid; + int sqcq_type; + + void *drv_ctx; + + /* Pointer to corresponding context. */ + struct xsched_context *ctx; + + /* List node in context's vstream list. */ + struct list_head ctx_node; + + /* Pointer to an CU object on which this + * vstream is currently being processed. + * NULL if vstream is not being processed. + */ + struct xsched_cu *xcu; + + /* List node in an CU list of vstreams that + * are currently being processed by this specific CU. + */ + struct list_head xcu_node; + + /* Private vstream data. */ + void *data; + + spinlock_t stream_lock; + + uint32_t kicks_count; + + /* List of metadata a.k.a. all recorded unprocesed + * kicks for this exact vstream. + */ + struct list_head metadata_list; + +} vstream_info_t; + typedef int vstream_manage_t(struct vstream_args *arg); #endif /* _LINUX_VSTREAM_H */ diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h index 55facb0e5760..fc4f23fce726 100644 --- a/include/linux/xcu_group.h +++ b/include/linux/xcu_group.h @@ -23,6 +23,20 @@ enum xcu_version { }; struct xcu_op_handler_params { + int fd; + struct xcu_group *group; + void *payload; + union { + struct { + void *param_1; + void *param_2; + void *param_3; + void *param_4; + void *param_5; + void *param_6; + void *param_7; + }; + }; }; typedef int (*xcu_op_handler_fn_t)(struct xcu_op_handler_params *params); @@ -33,6 +47,8 @@ struct xcu_operation { xcu_op_handler_fn_t wait; xcu_op_handler_fn_t complete; xcu_op_handler_fn_t alloc; + xcu_op_handler_fn_t logic_alloc; + xcu_op_handler_fn_t logic_free; }; struct xcu_group { diff --git a/include/linux/xsched.h b/include/linux/xsched.h index bdac1653ae36..dc930d7692ba 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -3,6 +3,8 @@ #define __LINUX_XSCHED_H__ #include <linux/xcu_group.h> +#include <linux/kref.h> +#include <linux/vstream.h> #ifndef pr_fmt #define pr_fmt(fmt) fmt #endif @@ -88,5 +90,75 @@ struct xsched_cu { wait_queue_head_t wq_xcore_running; }; +struct xsched_entity { + uint32_t task_type; + + bool on_rq; + + pid_t owner_pid; + pid_t tgid; + + /* File descriptor coming from an associated context + * used for identifying a given xsched entity in + * info and error prints. + */ + uint32_t fd; + + /* Xsched class for this xse. */ + const struct xsched_class *class; + + /* Pointer to context object. */ + struct xsched_context *ctx; + + /* Pointer to an XCU object that represents an XCU + * on which this xse is to be processed or is being + * processed currently. + */ + struct xsched_cu *xcu; + + /* General purpose xse lock. */ + spinlock_t xse_lock; +}; + +struct xsched_context { + uint32_t fd; + uint32_t devId; + pid_t tgid; + + struct list_head vstream_list; + struct list_head ctx_node; + + struct xsched_entity xse; + + spinlock_t ctx_lock; + struct mutex ctx_mutex; + struct kref kref; +}; + +extern struct list_head xsched_ctx_list; +extern struct mutex xsched_ctx_list_mutex; + +/* Returns a pointer to xsched_context object corresponding to a given + * device file descriptor provided by fd argument. + */ +static inline struct xsched_context *find_ctx_by_tgid(pid_t tgid) +{ + struct xsched_context *ctx; + struct xsched_context *ret = NULL; + + list_for_each_entry(ctx, &xsched_ctx_list, ctx_node) { + if (ctx->tgid == tgid) { + ret = ctx; + break; + } + } + + return ret; +} + int xsched_register_xcu(struct xcu_group *group); +int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs); +int bind_ctx_to_xcu(vstream_info_t *vstream_info, struct xsched_context *ctx); +int bind_vstream_to_xcu(vstream_info_t *vstream_info); +struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 96a814275963..52a600b945f4 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -32,6 +32,73 @@ spinlock_t xcu_mgr_lock; DECLARE_BITMAP(xcu_online_mask, XSCHED_NR_CUS); struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; +/* Storage list for contexts. */ +struct list_head xsched_ctx_list; +DEFINE_MUTEX(xsched_ctx_list_mutex); + +int bind_vstream_to_xcu(vstream_info_t *vstream_info) +{ + struct xsched_cu *xcu_found = NULL; + __u32 type = XCU_TYPE_NPU; + + xcu_found = xcu_find(&type, vstream_info->devId, vstream_info->channel_id); + + if (!xcu_found) + return -1; + + /* Bind vstream to a xcu. */ + vstream_info->xcu = xcu_found; + + XSCHED_INFO("XCU bound to a vstream: type=%u, dev_id=%u, chan_id=%u.\n", + type, vstream_info->devId, vstream_info->channel_id); + + return 0; +} + +struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id) +{ + struct xcu_group *group = NULL; + __u32 local_type = *type; + + /* Find xcu by type. */ + group = xcu_group_find_noalloc(xcu_group_root, local_type); + if (group == NULL) { + XSCHED_INFO("Find XCU group with real device is failed.\n"); + + local_type = XCU_TYPE_NPU; + group = xcu_group_find_noalloc(xcu_group_root, local_type); + if (group == NULL) { + XSCHED_ERR("Find XCU with qemu device is failed.\n"); + return NULL; + } + } + + /* Find by device id group. */ + group = xcu_group_find_noalloc(group, devId); + if (group == NULL) { + XSCHED_ERR("Find device group is failed.\n"); + return NULL; + } + /* Find channel id group. */ + group = xcu_group_find_noalloc(group, channel_id); + if (group == NULL) { + XSCHED_ERR("Find channel group is failed.\n"); + return NULL; + } + + *type = local_type; + + XSCHED_INFO("XCU found: type=%u, dev_id=%u, chan_id=%u.\n", local_type, + devId, channel_id); + + return group->xcu; +} + +int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) +{ + return 0; +} + static int xsched_schedule(void *input_xcu) { return 0; @@ -109,3 +176,13 @@ int xsched_register_xcu(struct xcu_group *group) return 0; } EXPORT_SYMBOL(xsched_register_xcu); + +int __init xsched_init(void) +{ + /* Initializing global Xsched context list. */ + INIT_LIST_HEAD(&xsched_ctx_list); + + return 0; +} + +late_initcall(xsched_init); diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 5723c359e0f2..73017da55ff2 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -18,12 +18,369 @@ */ #include <linux/syscalls.h> #include <linux/vstream.h> +#include <linux/xsched.h> #ifdef CONFIG_XCU_VSTREAM +#define MAX_VSTREAM_NUM (512 - 1) -int vstream_alloc(struct vstream_args *arg) +static DEFINE_MUTEX(vs_mutex); +static vstream_info_t *vstream_array[MAX_VSTREAM_NUM]; +static void init_xsched_ctx(struct xsched_context *ctx, + const struct vstream_info *vs) +{ + ctx->tgid = vs->tgid; + ctx->fd = vs->fd; + ctx->devId = vs->devId; + kref_init(&ctx->kref); + + INIT_LIST_HEAD(&ctx->vstream_list); + INIT_LIST_HEAD(&ctx->ctx_node); + + spin_lock_init(&ctx->ctx_lock); + mutex_init(&ctx->ctx_mutex); +} + +/* Allocates a new xsched_context if a new vstream_info is bound + * to a device that no other vstream that is currently present + * is bound to. + */ +static int alloc_ctx_from_vstream(struct vstream_info *vstream_info, + struct xsched_context **ctx) +{ + int err = 0; + + XSCHED_CALL_STUB(); + + *ctx = find_ctx_by_tgid(vstream_info->tgid); + if (*ctx) { + XSCHED_INFO("Ctx %d found @ %s\n", + vstream_info->tgid, __func__); + goto out_err; + } + + *ctx = kzalloc(sizeof(struct xsched_context), GFP_KERNEL); + + if (!*ctx) { + XSCHED_ERR("Could not allocate xsched context (tgid=%d) @ %s\n", + vstream_info->tgid, __func__); + err = -ENOMEM; + goto out_err; + } + + init_xsched_ctx(*ctx, vstream_info); + + err = xsched_ctx_init_xse(*ctx, vstream_info); + + if (err) { + XSCHED_ERR("Failed to initialize XSE for context @ %s\n", + __func__); + kfree(*ctx); + err = -EINVAL; + goto out_err; + } + + list_add(&(*ctx)->ctx_node, &xsched_ctx_list); + +out_err: + XSCHED_EXIT_STUB(); + + return err; +} + +/* Bounds a new vstream_info object to a corresponding xsched context. */ +static int vstream_bind_to_ctx(struct vstream_info *vs) +{ + struct xsched_context *ctx = NULL; + int alloc_err = 0; + + XSCHED_CALL_STUB(); + + XSCHED_INFO("Ctx list mutext taken @ %s\n", __func__); + mutex_lock(&xsched_ctx_list_mutex); + + ctx = find_ctx_by_tgid(vs->tgid); + if (ctx) { + XSCHED_INFO("Ctx %d found @ %s\n", vs->tgid, __func__); + kref_get(&ctx->kref); + } else { + alloc_err = alloc_ctx_from_vstream(vs, &ctx); + if (alloc_err) + goto out_err; + } + + XSCHED_INFO("Ctx = %p @ %s\n", ctx, __func__); + + vs->ctx = ctx; + vs->xcu = ctx->xse.xcu; + ctx->devId = vs->devId; + list_add(&vs->ctx_node, &vs->ctx->vstream_list); + +out_err: + mutex_unlock(&xsched_ctx_list_mutex); + XSCHED_INFO("Ctx list mutex released @ %s\n", __func__); + + XSCHED_EXIT_STUB(); + + return alloc_err; +} + +static vstream_info_t *vstream_create_info(struct vstream_args *arg) +{ + struct vstream_info *vstream; + + XSCHED_CALL_STUB(); + + vstream = kzalloc(sizeof(vstream_info_t), GFP_KERNEL); + if (!vstream) { + XSCHED_ERR("Failed to allocate vstream.\n"); + vstream = NULL; + goto out_err; + } + + vstream->devId = arg->devid; + vstream->channel_id = arg->channel_id; + + INIT_LIST_HEAD(&vstream->ctx_node); + INIT_LIST_HEAD(&vstream->xcu_node); + INIT_LIST_HEAD(&vstream->metadata_list); + + spin_lock_init(&vstream->stream_lock); + + vstream->kicks_count = 0; + + vstream->xcu = NULL; + +out_err: + XSCHED_EXIT_STUB(); + + return vstream; +} + +static int vstream_add(vstream_info_t *vstream, uint32_t id) +{ + XSCHED_CALL_STUB(); + XSCHED_INFO("Adding vstream %u @ %s\n", id, __func__); + + if (id >= MAX_VSTREAM_NUM) { + XSCHED_ERR("vstreamId out of range.\n"); + return -EINVAL; + } + + mutex_lock(&vs_mutex); + if (vstream_array[id] != NULL) { + mutex_unlock(&vs_mutex); + XSCHED_ERR("VstreamId=%u cell is busy.\n", id); + return -EINVAL; + } + vstream_array[id] = vstream; + mutex_unlock(&vs_mutex); + + return 0; +} + +static vstream_info_t * +vstream_get_by_user_stream_id(uint32_t user_streamId) +{ + int id; + + for (id = 0; id < MAX_VSTREAM_NUM; id++) { + if (vstream_array[id] != NULL) + if (vstream_array[id]->user_streamId == + user_streamId) + return vstream_array[id]; + } + return NULL; +} + +static int sqcq_alloc(struct vstream_args *arg) { + vstream_alloc_args_t *va_args = &arg->va_args; + struct xsched_context *ctx = NULL; + struct xcu_op_handler_params params; + uint32_t logic_cq_id = 0; + vstream_info_t *vstream; + int err = -EINVAL; + uint32_t tgid = 0; + uint32_t cq_id = 0; + uint32_t sq_id = 0; + + XSCHED_CALL_STUB(); + + vstream = vstream_create_info(arg); + if (!vstream) { + XSCHED_ERR("vstream create failed.\n"); + err = -ENOSPC; + goto out_err; + } + vstream->fd = arg->fd; + vstream->task_type = arg->task_type; + + err = bind_vstream_to_xcu(vstream); + if (err) { + XSCHED_ERR( + "Couldn't find valid xcu for vstream dev_id=%u chan_id=%u @ %s\n", + vstream->devId, vstream->channel_id, __func__); + err = -EINVAL; + goto out_err_vstream_free; + } + + /* Allocates vstream's SQ and CQ memory on a XCU for processing. */ + params.group = vstream->xcu->group; + params.fd = arg->fd; + params.payload = arg->payload; + params.param_1 = &tgid; + params.param_2 = &sq_id; + params.param_3 = &cq_id; + params.param_4 = &logic_cq_id; + err = xcu_alloc(¶ms); + if (err) { + XSCHED_ERR( + "Failed to allocate SQ and CQ memory to an app stream.\n"); + goto out_err_vstream_free; + } + vstream->drv_ctx = params.param_5; + vstream->id = sq_id; + vstream->vcqId = cq_id; + vstream->logic_vcqId = logic_cq_id; + XSCHED_INFO("New vstream id %u, cq_id=%u, logic_cqid=%u @ %s\n", + vstream->id, vstream->vcqId, vstream->logic_vcqId, __func__); + + vstream->user_streamId = va_args->user_stream_id; + vstream->tgid = tgid; + vstream->sqcq_type = va_args->type; + + + err = vstream_bind_to_ctx(vstream); + if (err) { + XSCHED_ERR( + "Failed to bind vstream %u to an app context.\n", + vstream->id); + goto out_err_vstream_free; + } + + ctx = vstream->ctx; + + err = vstream_add(vstream, vstream->id); + if (err) { + XSCHED_ERR( + "Failed to add vstream id=%u to vstream_array.\n", + vstream->id); + goto out_err_vstream_free; + } + + XSCHED_INFO( + "vstream allocation success: user_streamId=%u, sqid=%u, cqid=%u, ctx=%p.\n", + vstream->user_streamId, vstream->id, vstream->vcqId, ctx); + + XSCHED_EXIT_STUB(); + return 0; + +out_err_vstream_free: + kfree(vstream); + +out_err: + XSCHED_INFO( + "Exit %s with error, current_pid=%d, err=%d.\n", + __func__, current->pid, err); + + return err; +} + +static int logic_cq_alloc(struct vstream_args *arg) +{ + int err = 0; + struct xcu_op_handler_params params; + vstream_info_t *vstream = NULL; + vstream_alloc_args_t *logic_cq_alloc_para = &arg->va_args; + uint32_t logic_cq_id = 0; + + XSCHED_CALL_STUB(); + + XSCHED_INFO( + "Enter %s, current_pid=%d, fd=%u.\n", + __func__, current->pid, arg->fd); + + vstream = vstream_get_by_user_stream_id( + logic_cq_alloc_para->user_stream_id); + if (!vstream) { + struct xsched_cu *xcu_found = NULL; + __u32 type = XCU_TYPE_NPU; + + xcu_found = xcu_find(&type, arg->devid, arg->channel_id); + if (xcu_found == NULL) { + XSCHED_ERR( + "Couldn't find valid xcu for control vstream dev_id=%u chan_id=%u @ %s\n", + arg->devid, arg->channel_id, __func__); + err = -EINVAL; + goto out_err; + } + + /* Allocates control vstream's SQ and CQ memory on a XCU for processing. */ + params.group = xcu_found->group; + params.fd = arg->fd; + params.payload = arg->payload; + params.param_1 = &logic_cq_id; + + err = xcu_logic_alloc(¶ms); + if (err) { + XSCHED_ERR( + "Failed to allocate logic CQ memory to a vstream.\n"); + goto out_err; + } + + XSCHED_INFO( + "vstream logic CQ memory allocation success: logic_cq_id=%u.\n", + logic_cq_id); + + return 0; + } + + params.group = vstream->xcu->group; + params.fd = arg->fd; + params.payload = arg->payload; + params.param_1 = &logic_cq_id; + + err = xcu_logic_alloc(¶ms); + if (err) { + XSCHED_ERR( + "Failed to allocate logic CQ memory to an app stream.\n"); + goto out_err; + } + + vstream->logic_vcqId = logic_cq_id; + + XSCHED_INFO( + "Vstream logic CQ memory allocation success: user_streamId=%u, logic_cqid=%u.\n", + vstream->user_streamId, vstream->logic_vcqId); + + return 0; + +out_err: + XSCHED_INFO( + "Exit %s with error, current_pid=%d, err=%d.\n", + __func__, current->pid, err); + + return err; +} + +int vstream_alloc(struct vstream_args *arg) +{ + vstream_alloc_args_t *va_args = &arg->va_args; + int ret; + + XSCHED_CALL_STUB(); + XSCHED_INFO( + "Enter %s, current_pid=%d, fd=%u, type=%d.\n", + __func__, current->pid, arg->fd, va_args->type); + + if (!va_args->type) + ret = sqcq_alloc(arg); + else + ret = logic_cq_alloc(arg); + + XSCHED_EXIT_STUB(); + return ret; } int vstream_free(struct vstream_args *arg) -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add vstream_metadata data structures. Add vstream_kick() and xcu_run() implementation. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> --- include/linux/vstream.h | 24 ++++++++++++++++++ include/linux/xsched.h | 16 ++++++++++++ kernel/xsched/core.c | 32 +++++++++++++++++++++++ kernel/xsched/vstream.c | 56 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/include/linux/vstream.h b/include/linux/vstream.h index 39cc6a26db50..8e7bf5992898 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -4,6 +4,30 @@ #include <uapi/linux/xcu_vstream.h> +#define MAX_VSTREAM_SIZE 2048 + +/* Vstream metadata describes each incoming kick + * that gets stored into a list of pending kicks + * inside a vstream to keep track of what is left + * to be processed by a driver. + */ +typedef struct vstream_metadata { + uint32_t exec_time; + /* A value of SQ tail that has been passed with the + * kick that is described by this exact metadata object. + */ + uint32_t sq_tail; + uint32_t sqe_num; + uint32_t sq_id; + int32_t timeout; + int8_t sqe[XCU_SQE_SIZE_MAX]; + + /* A node for metadata list */ + struct list_head node; + + struct vstream_info *parent; +} vstream_metadata_t; + typedef struct vstream_info { uint32_t user_streamId; uint32_t id; diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 15c67bb1ac95..902f4a4fba34 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -156,10 +156,26 @@ static inline struct xsched_context *find_ctx_by_tgid(pid_t tgid) return ret; } + +static inline void xsched_init_vsm(struct vstream_metadata *vsm, + struct vstream_info *vs, + vstream_args_t *arg) +{ + vsm->sq_id = arg->sq_id; + vsm->sqe_num = arg->vk_args.sqe_num; + vsm->timeout = arg->vk_args.timeout; + memcpy(vsm->sqe, arg->vk_args.sqe, XCU_SQE_SIZE_MAX); + vsm->parent = vs; + INIT_LIST_HEAD(&vsm->node); +} + int xsched_register_xcu(struct xcu_group *group); void xsched_free_task(struct kref *kref); int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs); int bind_ctx_to_xcu(vstream_info_t *vstream_info, struct xsched_context *ctx); int bind_vstream_to_xcu(vstream_info_t *vstream_info); struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id); + +/* Vstream metadata proccesing functions.*/ +int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 98f412c1ca7b..8e531297d4c5 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -176,6 +176,38 @@ static int alloc_xcu_id(void) return xcu_id; } +/* Adds vstream_metadata object to a specified vstream. */ +int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) +{ + int err = 0; + struct vstream_metadata *new_vsm; + + new_vsm = kmalloc(sizeof(struct vstream_metadata), GFP_KERNEL); + if (!new_vsm) { + XSCHED_ERR("Failed to allocate kick metadata for vs %u @ %s\n", + vs->id, __func__); + err = -ENOMEM; + goto out_err; + } + + xsched_init_vsm(new_vsm, vs, arg); + + if (vs->kicks_count > MAX_VSTREAM_SIZE) { + err = -EBUSY; + kfree(new_vsm); + goto out_err; + } + + list_add_tail(&new_vsm->node, &vs->metadata_list); + vs->kicks_count += 1; + + XSCHED_INFO("Vstream_id %u Add vsm: sq_tail %u, sqe_num %u, kicks_count %u\n", + vs->id, new_vsm->sq_tail, new_vsm->sqe_num, vs->kicks_count); + +out_err: + return err; +} + /* * Initialize and register xcu in xcu_manager array. */ diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 30f8073593ab..1d1b4e436f69 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -468,7 +468,61 @@ int vstream_free(struct vstream_args *arg) int vstream_kick(struct vstream_args *arg) { - return 0; + vstream_info_t *vstream; + int vstreamId = arg->sq_id; + struct xsched_entity *xse; + int err = 0; + + struct xsched_cu *xcu = NULL; + + XSCHED_CALL_STUB(); + + /* Get vstream. */ + vstream = vstream_get(vstreamId); + if (!vstream || !vstream->ctx) { + XSCHED_ERR( + "Vstream NULL or doesn't have a context.\n"); + err = -EINVAL; + goto out_err; + } + + xse = &vstream->ctx->xse; + xcu = vstream->xcu; + XSCHED_INFO("New kick on xse %d @ %s\n", xse->tgid, __func__); + +repeat_kick: + mutex_lock(&xcu->xcu_lock); + XSCHED_INFO("xcu lock taken @ %s\n", __func__); + spin_lock(&vstream->stream_lock); + XSCHED_INFO("vstream lock taken @ %s\n", __func__); + + /* Adding kick metadata. */ + err = xsched_vsm_add_tail(vstream, arg); + if (err) { + if (err == -EBUSY) { + spin_unlock(&vstream->stream_lock); + XSCHED_INFO("no space: vstream lock released @ %s\n", __func__); + mutex_unlock(&xcu->xcu_lock); + goto repeat_kick; + } + XSCHED_ERR("Failed to add kick metadata to vs %u @ %s\n", + vstream->id, __func__); + + spin_unlock(&vstream->stream_lock); + XSCHED_INFO("vstream lock released @ %s\n", __func__); + mutex_unlock(&xcu->xcu_lock); + XSCHED_INFO("Xcu lock released @ %s\n", __func__); + goto out_err; + } + + spin_unlock(&vstream->stream_lock); + XSCHED_INFO("vstream lock released @ %s\n", __func__); + mutex_unlock(&xcu->xcu_lock); + +out_err: + XSCHED_EXIT_STUB(); + + return err; } /* -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add XCU xsched_schedule() implementation. Add xsched_rq data structures and related process. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- include/linux/xsched.h | 79 ++++++++++++++++++ kernel/xsched/core.c | 180 +++++++++++++++++++++++++++++++++++++++- kernel/xsched/vstream.c | 8 ++ 3 files changed, 266 insertions(+), 1 deletion(-) diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 465b61b07fd6..1652541e39e3 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -59,6 +59,28 @@ #define GET_VS_TASK_TYPE(vs_ptr) __GET_VS_TASK_TYPE((vs_ptr)->task_type) +enum xsched_rq_state { + XRQ_STATE_INACTIVE = 0x00, + XRQ_STATE_IDLE = 0x01, + XRQ_STATE_BUSY = 0x02, + XRQ_STATE_SUBMIT = 0x04, + XRQ_STATE_WAIT_RUNNING = 0x08 +}; + +#define for_each_vstream_in_ctx(vs, ctx) \ + list_for_each_entry((vs), &((ctx)->vstream_list), ctx_node) + + +/* Base Xsched runqueue object structure that contains both mutual and + * individual parameters for different scheduling classes. + */ +struct xsched_rq { + struct xsched_entity *curr_xse; + + int state; + int nr_running; +}; + enum xcu_state { XCU_INACTIVE, XCU_IDLE, @@ -89,10 +111,15 @@ struct xsched_cu { struct task_struct *worker; + struct xsched_rq xrq; + struct list_head vsm_list; + struct xcu_group *group; struct mutex xcu_lock; + atomic_t has_active; + wait_queue_head_t wq_xcu_idle; wait_queue_head_t wq_xcu_running; wait_queue_head_t wq_xcore_running; @@ -134,6 +161,56 @@ struct xsched_entity { spinlock_t xse_lock; }; +/* Increments pending kicks counter for an XCU that the given + * xsched entity is attached to and for xsched entity's xsched + * class. + */ +static inline int xsched_inc_pending_kicks_xse(struct xsched_entity *xse) +{ + /* Icrement pending kicks for current XSE. */ + atomic_inc(&xse->kicks_pending_ctx_cnt); + + return 0; +} + +/* Decrements pending kicks counter for an XCU that the given + * xsched entity is attached to and for XSched entity's sched + * class. + */ +static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) +{ + /* Decrementing pending kicks for current XSE. */ + atomic_dec(&xse->kicks_pending_ctx_cnt); + + return 0; +} + +/* Checks if there are pending kicks left on a given XCU for all + * xsched classes. + */ +static inline bool xsched_check_pending_kicks_xcu(struct xsched_cu *xcu) +{ + return 0; +} + +static inline int xse_integrity_check(const struct xsched_entity *xse) +{ + int err = !xse || !xse->class ? 0 : 1; + + if (!xse) { + XSCHED_ERR("xse is null @ %s\n", __func__); + goto out_err; + } + + if (!xse->class) { + XSCHED_ERR("xse->class is null @ %s\n", __func__); + goto out_err; + } + +out_err: + return err; +} + struct xsched_context { uint32_t fd; uint32_t devId; @@ -191,4 +268,6 @@ struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id); /* Vstream metadata proccesing functions.*/ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); +struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); +void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 3ae3c78d8222..8a85fc0db45d 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -16,6 +16,7 @@ * more details. * */ +#include <linux/delay.h> #include <linux/kthread.h> #include <linux/slab.h> #include <linux/spinlock_types.h> @@ -39,6 +40,70 @@ DEFINE_MUTEX(xsched_ctx_list_mutex); static DEFINE_MUTEX(revmap_mutex); static DEFINE_HASHTABLE(ctx_revmap, XCU_HASH_ORDER); +static void put_prev_ctx(struct xsched_entity *xse) +{ +} + +static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) +{ + return NULL; +} + +void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) +{ +} + +void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) +{ +} + +static int delete_ctx(struct xsched_context *ctx) +{ + struct xsched_cu *xcu = ctx->xse.xcu; + struct xsched_entity *curr_xse = xcu->xrq.curr_xse; + struct xsched_entity *xse = &ctx->xse; + + XSCHED_CALL_STUB(); + + if (!xse_integrity_check(xse)) { + XSCHED_ERR("Failed xse integrity check! %s\n", __func__); + return -EINVAL; + } + + if (!xse->xcu) { + XSCHED_ERR("Trying to delete ctx that is not attached to an XCU @ %s\n", __func__); + return -EINVAL; + } + + /* Wait till context has been submitted. */ + while (atomic_read(&xse->kicks_pending_ctx_cnt)) { + XSCHED_INFO( + "Deleting ctx %d, xse->kicks_pending_ctx_cnt=%d @ %s\n", + xse->tgid, atomic_read(&xse->kicks_pending_ctx_cnt), + __func__); + } + + if (atomic_read(&xse->kicks_pending_ctx_cnt)) { + XSCHED_ERR("Deleting ctx %d that has pending kicks left @ %s\n", + xse->tgid, __func__); + return -EINVAL; + } + + mutex_lock(&xcu->xcu_lock); + if (curr_xse == xse) + xcu->xrq.curr_xse = NULL; + + dequeue_ctx(xse, xcu); + mutex_unlock(&xcu->xcu_lock); + + XSCHED_INFO("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, + atomic_read(&xse->kicks_pending_ctx_cnt), __func__); + + XSCHED_EXIT_STUB(); + + return 0; +} + /* Frees a given vstream and also frees and dequeues it's context * if a given vstream is the last and only vstream attached to it's * corresponding context object. @@ -52,6 +117,9 @@ void xsched_free_task(struct kref *kref) ctx = container_of(kref, struct xsched_context, kref); + while (READ_ONCE(ctx->xse.on_rq)) + usleep_range(100, 200); + mutex_lock(&xsched_ctx_list_mutex); list_for_each_entry_safe(vs, tmp, &ctx->vstream_list, ctx_node) { list_del(&vs->ctx_node); @@ -59,6 +127,7 @@ void xsched_free_task(struct kref *kref) kfree(vs); } + delete_ctx(ctx); list_del(&ctx->ctx_node); mutex_unlock(&xsched_ctx_list_mutex); @@ -264,11 +333,82 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) return err; } -static int xsched_schedule(void *input_xcu) +static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) { return 0; } +static int xsched_schedule(void *input_xcu) +{ + struct xsched_cu *xcu = input_xcu; + int err = 0; + struct xsched_entity *curr_xse = NULL; + struct xsched_entity *next_xse = NULL; + + XSCHED_CALL_STUB(); + + while (!kthread_should_stop()) { + XSCHED_INFO("%s: Xcu lock released\n", __func__); + mutex_unlock(&xcu->xcu_lock); + + wait_event_interruptible(xcu->wq_xcu_idle, + atomic_read(&xcu->has_active) || xcu->xrq.nr_running); + + XSCHED_INFO("%s: rt_nr_running = %d, has_active = %d\n", + __func__, xcu->xrq.nr_running, atomic_read(&xcu->has_active)); + + mutex_lock(&xcu->xcu_lock); + XSCHED_INFO("%s: Xcu lock taken\n", __func__); + + if (!xsched_check_pending_kicks_xcu(xcu)) { + XSCHED_ERR("%s: No pending kicks for xcu %u\n", __func__, xcu->id); + continue; + } + + next_xse = __raw_pick_next_ctx(xcu); + + if (!next_xse) { + XSCHED_ERR("%s: Couldn't find next xse for xcu %u\n", __func__, xcu->id); + continue; + } + + xcu->xrq.curr_xse = next_xse; + __XSCHED_TRACE("%s: Pick next ctx returned xse %d\n", __func__, next_xse->tgid); + + if (__xsched_submit(xcu, next_xse)) { + XSCHED_ERR("%s: Xse %d on XCU %u tried to submit with zero kicks\n", + __func__, next_xse->tgid, xcu->id); + continue; + } + + curr_xse = xcu->xrq.curr_xse; + if (curr_xse) { /* if not deleted yet */ + put_prev_ctx(curr_xse); + if (!atomic_read(&curr_xse->kicks_pending_ctx_cnt)) { + dequeue_ctx(curr_xse, xcu); + XSCHED_INFO( + "%s: Dequeue xse %d due to zero kicks on xcu %u\n", + __func__, curr_xse->tgid, xcu->id); + curr_xse = xcu->xrq.curr_xse = NULL; + } + } + } + + XSCHED_INFO("Xsched_schedule finished for xcu %u\n", xcu->id); + + XSCHED_EXIT_STUB(); + + return err; +} + +/* Initialize xsched classes' runqueues. */ +static inline void xsched_rq_init(struct xsched_cu *xcu) +{ + xcu->xrq.nr_running = 0; + xcu->xrq.curr_xse = NULL; + xcu->xrq.state = XRQ_STATE_IDLE; +} + /* Initializes all xsched XCU objects. * Should only be called from xsched_register_xcu function. */ @@ -281,6 +421,12 @@ static void xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, xcu->state = XSCHED_XCU_NONE; xcu->group = group; + atomic_set(&xcu->has_active, 0); + + INIT_LIST_HEAD(&xcu->vsm_list); + + init_waitqueue_head(&xcu->wq_xcu_idle); + mutex_init(&xcu->xcu_lock); /* Mark current XCU in a mask inside XCU root group. */ @@ -344,6 +490,38 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) return err; } +/* Fetch the first vstream metadata from vstream metadata list + * and removes it from that list. Returned vstream metadata pointer + * to be freed after. + */ +struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs) +{ + struct vstream_metadata *vsm; + + if (list_empty(&vs->metadata_list)) { + XSCHED_INFO("No metadata to fetch from vs %u @ %s\n", vs->id, + __func__); + goto out_null; + } + + vsm = list_first_entry(&vs->metadata_list, struct vstream_metadata, node); + + if (!vsm) { + XSCHED_ERR( + "Tried to delete metadata from empty list in vs %u @ %s\n", + vs->id, __func__); + goto out_null; + } + + list_del(&vsm->node); + vs->kicks_count -= 1; + + return vsm; + +out_null: + return NULL; +} + /* * Initialize and register xcu in xcu_manager array. */ diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 1d1b4e436f69..c7e3772009bf 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -515,9 +515,17 @@ int vstream_kick(struct vstream_args *arg) goto out_err; } + enqueue_ctx(xse, xcu); + + /* Increasing a total amount of kicks on an CU to which this + * context is attached to based on sched_class. + */ + xsched_inc_pending_kicks_xse(&vstream->ctx->xse); + spin_unlock(&vstream->stream_lock); XSCHED_INFO("vstream lock released @ %s\n", __func__); mutex_unlock(&xcu->xcu_lock); + wake_up_interruptible(&xcu->wq_xcu_idle); out_err: XSCHED_EXIT_STUB(); -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add rt class callbacks implementation: - dequeue_ctx - enqueue_ctx - pick_next_ctx - put_prev_ctx - submit_prepare_ctx - select_work - check_preempt Add RT class callbacks support in core.c. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- include/linux/xsched.h | 3 + kernel/xsched/core.c | 150 ++++++++++++++++++++++++++++++- kernel/xsched/rt.c | 198 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 342 insertions(+), 9 deletions(-) diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 64436db90647..3f52cabbb99b 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -217,6 +217,9 @@ struct xsched_entity { /* Amount of submitted kicks context, used for resched decision. */ atomic_t kicks_submited; + size_t kicks_submitted; + size_t kicks_processed; + /* File descriptor coming from an associated context * used for identifying a given xsched entity in * info and error prints. diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 28ebf1791930..3fd8fe9cc7e1 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -42,19 +42,134 @@ static DEFINE_HASHTABLE(ctx_revmap, XCU_HASH_ORDER); static void put_prev_ctx(struct xsched_entity *xse) { + struct xsched_cu *xcu = xse->xcu; + + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + xse->class->put_prev_ctx(xse); + + xse->last_process_time = 0; + + XSCHED_EXIT_STUB(); +} + +static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) +{ + uint32_t kick_count; + struct vstream_info *vs; + unsigned int sum_exec_time = 0; + size_t kicks_submitted = 0; + struct vstream_metadata *vsm; + size_t not_empty; + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("Before decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + if (kick_count == 0) { + XSCHED_ERR("Tried to submit xse that has 0 kicks @ %s\n", + __func__); + goto out_err; + } + + do { + not_empty = 0; + for_each_vstream_in_ctx(vs, xse->ctx) { + spin_lock(&vs->stream_lock); + vsm = xsched_vsm_fetch_first(vs); + spin_unlock(&vs->stream_lock); + if (vsm) { + list_add_tail(&vsm->node, &xcu->vsm_list); + + sum_exec_time += vsm->exec_time; + kicks_submitted++; + xsched_dec_pending_kicks_xse(xse); + XSCHED_INFO( + "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", + vs->id, vsm->exec_time, vsm->sq_tail, + vsm->sqe_num, vsm->sq_id, __func__); + not_empty++; + } + } + } while (not_empty); + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + xse->kicks_submitted += kicks_submitted; + + XSCHED_INFO("xse %d kicks_submitted = %lu @ %s\n", + xse->tgid, xse->kicks_submitted, __func__); + +out_err: + return kicks_submitted; } static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) { - return NULL; + const struct xsched_class *class; + struct xsched_entity *next = NULL; + + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + for_each_xsched_class(class) { + next = class->pick_next_ctx(xcu); + if (next) { + if (class->select_work) + class->select_work(xcu, next); + else + select_work_def(xcu, next); + break; + } + } + + XSCHED_EXIT_STUB(); + return next; } void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) { + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + if (!xse_integrity_check(xse)) { + XSCHED_ERR("Failed xse integrity check @ %s\n", __func__); + return; + } + + if (!xse->on_rq) { + xse->on_rq = true; + xse->class->enqueue_ctx(xse, xcu); + __XSCHED_TRACE("Enqueue xse %d @ %s\n", xse->tgid, __func__); + } + + XSCHED_EXIT_STUB(); } void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu) { + XSCHED_CALL_STUB(); + + lockdep_assert_held(&xcu->xcu_lock); + + if (!xse_integrity_check(xse)) { + XSCHED_ERR("Failed xse integrity check @ %s\n", __func__); + return; + } + + if (xse->on_rq) { + xse->class->dequeue_ctx(xse); + xse->on_rq = false; + __XSCHED_TRACE("Dequeue xse %d @ %s\n", xse->tgid, __func__); + } + + XSCHED_EXIT_STUB(); } static int delete_ctx(struct xsched_context *ctx) @@ -288,6 +403,15 @@ struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id) int xsched_xse_set_class(struct xsched_entity *xse) { + switch (xse->task_type) { + case XSCHED_TYPE_RT: + xse->class = &rt_xsched_class; + XSCHED_INFO("Context is in RT class %s\n", __func__); + break; + default: + XSCHED_ERR("Xse has incorrect class @ %s\n", __func__); + return -EINVAL; + } return 0; } @@ -326,6 +450,25 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) goto out_err; } + if (xse_is_rt(xse)) { + xse->rt.state = XSE_PREPARE; + xse->rt.flag = XSE_TIF_NONE; + xse->rt.prio = GET_VS_TASK_PRIO_RT(vs); + xse->rt.kick_slice = XSCHED_RT_KICK_SLICE; + + /* XSE priority is being decreased by 1 here because + * in libucc priority counter starts from 1 while in the + * kernel counter starts with 0. + * + * This inconsistency has to be solve in libucc in the + * future rather that having this confusing decrement to + * priority inside the kernel. + */ + if (xse->rt.prio > 0) + xse->rt.prio -= 1; + + INIT_LIST_HEAD(&xse->rt.list_node); + } WRITE_ONCE(xse->on_rq, false); spin_lock_init(&xse->xse_lock); @@ -340,6 +483,11 @@ static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) return 0; } +static inline bool should_preempt(struct xsched_entity *xse) +{ + return xse->class->check_preempt(xse); +} + static int xsched_schedule(void *input_xcu) { struct xsched_cu *xcu = input_xcu; diff --git a/kernel/xsched/rt.c b/kernel/xsched/rt.c index 12b188dce567..da9959778366 100644 --- a/kernel/xsched/rt.c +++ b/kernel/xsched/rt.c @@ -23,25 +23,162 @@ #include <linux/xsched.h> #include <linux/vstream.h> -static void dequeue_ctx_rt(struct xsched_entity *xse) {} +/* Add xsched entitiy to a run list based on priority, set on_cu flag + * and set a corresponding curr_prios bit if necessary. + */ +static inline void +xse_rt_add(struct xsched_entity *xse, struct xsched_cu *xcu) +{ + list_add_tail(&xse->rt.list_node, &xcu->xrq.rt.rq[xse->rt.prio]); + __set_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +/* Delete xsched entitiy from a run list, unset on_cu flag and + * unset corresponding curr_prios bit if necessary. + */ +static inline void xse_rt_del(struct xsched_entity *xse) +{ + struct xsched_cu *xcu = xse->xcu; + + list_del_init(&xse->rt.list_node); + + if (list_empty(&xcu->xrq.rt.rq[xse->rt.prio])) + __clear_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +static inline void xse_rt_move_tail(struct xsched_entity *xse) +{ + struct xsched_cu *xcu = xse->xcu; + + list_move_tail(&xse->rt.list_node, &xcu->xrq.rt.rq[xse->rt.prio]); +} + +/* Increase RT runqueue total and per prio nr_running stat. */ +static inline void xrq_inc_nr_running(struct xsched_entity *xse, + struct xsched_cu *xcu) +{ + xcu->xrq.nr_running++; + xcu->xrq.rt.prio_nr_running[xse->rt.prio]++; + set_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +/* Decrease RT runqueue total and per prio nr_running stat + * and raise a bug if nr_running decrease beyond zero. + */ +static inline void xrq_dec_nr_running(struct xsched_entity *xse) +{ + struct xsched_cu *xcu = xse->xcu; + + xcu->xrq.nr_running--; + xcu->xrq.rt.prio_nr_running[xse->rt.prio]--; + + if (!xcu->xrq.rt.prio_nr_running[xse->rt.prio]) + clear_bit(xse->rt.prio, xcu->xrq.rt.curr_prios); +} + +static void dequeue_ctx_rt(struct xsched_entity *xse) +{ + xse_rt_del(xse); + xrq_dec_nr_running(xse); +} + +static void enqueue_ctx_rt(struct xsched_entity *xse, struct xsched_cu *xcu) +{ + xse_rt_add(xse, xcu); + xrq_inc_nr_running(xse, xcu); +} + +static inline struct xsched_entity *xrq_next_xse(struct xsched_cu *xcu, + int prio) +{ + return list_first_entry(&xcu->xrq.rt.rq[prio], struct xsched_entity, + rt.list_node); +} + +/* Returns the next priority for pick_next_ctx taking into + * account if there are pending kicks on certain priority. + */ +static inline uint32_t get_next_prio_rt(struct xsched_rq *xrq) +{ + int32_t curr_prio; + bool bit_val; + unsigned long *prios = xrq->rt.curr_prios; + atomic_t *prio_nr_kicks = xrq->rt.prio_nr_kicks; -static void enqueue_ctx_rt(struct xsched_entity *xse, struct xsched_cu *xcu) {} + /* Using generic for loop instead of for_each_set_bit + * because it will be faster than for_each_set_bit. + */ + for (curr_prio = NR_XSE_PRIO - 1; curr_prio >= 0; curr_prio--) { + bit_val = test_bit(curr_prio, prios); + if (!bit_val && atomic_read(&prio_nr_kicks[curr_prio])) { + XSCHED_ERR( + "kicks > 0 on RT priority with the priority bit unset\n"); + WARN_ON_ONCE(1); + return NR_XSE_PRIO; + } + + if (bit_val && atomic_read(&prio_nr_kicks[curr_prio])) + return curr_prio; + } + return NR_XSE_PRIO; +} static struct xsched_entity *pick_next_ctx_rt(struct xsched_cu *xcu) { - return NULL; + struct xsched_entity *result; + int next_prio; + + next_prio = get_next_prio_rt(&xcu->xrq); + if (next_prio >= NR_XSE_PRIO) { + XSCHED_INFO("No pending kicks in RT class @ %s\n", __func__); + return NULL; + } + + if (!xcu->xrq.rt.prio_nr_running[next_prio]) { + XSCHED_ERR( + "In RT runqueue nr_running is zero while there are pending kicks for %u prio\n", + next_prio); + return NULL; + } + + result = xrq_next_xse(xcu, next_prio); + if (!result) + XSCHED_ERR("Next XSE not found @ %s\n", __func__); + + return result; } -static void put_prev_ctx_rt(struct xsched_entity *xse) {} +static void put_prev_ctx_rt(struct xsched_entity *xse) +{ + xse->rt.kick_slice -= atomic_read(&xse->kicks_submited); + XSCHED_INFO( + "Update XSE=%d kick_slice=%lld, XSE kicks_submited=%d in RT class @ %s\n", + xse->tgid, xse->rt.kick_slice, + atomic_read(&xse->kicks_submited), __func__); + + if (xse->rt.kick_slice <= 0) { + xse->rt.kick_slice = XSCHED_RT_KICK_SLICE; + XSCHED_INFO("Refill XSE=%d kick_slice=%lld in RT class @ %s\n", + xse->tgid, xse->rt.kick_slice, __func__); + xse_rt_move_tail(xse); + } + + atomic_set(&xse->kicks_submited, 0); +} static int submit_prepare_ctx_rt(struct xsched_entity *xse, struct xsched_cu *xcu) { - return 0; -} + if (!atomic_read(&xse->kicks_pending_ctx_cnt)) { + XSCHED_INFO("xse %d doesn't have pending kicks @ %s\n", + xse->tgid, __func__); + xse->rt.state = XSE_READY; + xse->rt.kick_slice = 0; + return -EAGAIN; + } + + xse->rt.state = XSE_RUNNING; -static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) -{ return 0; } @@ -50,6 +187,51 @@ static bool check_preempt_ctx_rt(struct xsched_entity *xse) return true; } +static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) +{ + uint32_t kick_count; + struct vstream_info *vs; + size_t kicks_submitted = 0; + struct vstream_metadata *vsm; + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("Before decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + if (kick_count == 0) { + XSCHED_ERR("Tried to submit xse that has 0 kicks @ %s\n", + __func__); + goto out_err; + } + + for_each_vstream_in_ctx(vs, xse->ctx) { + spin_lock(&vs->stream_lock); + while ((vsm = xsched_vsm_fetch_first(vs))) { + list_add_tail(&vsm->node, &xcu->vsm_list); + vsm->exec_time = 0; + kicks_submitted++; + xsched_dec_pending_kicks_xse(xse); + XSCHED_INFO( + "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", + vs->id, vsm->exec_time, vsm->sq_tail, + vsm->sqe_num, vsm->sq_id, __func__); + } + spin_unlock(&vs->stream_lock); + } + + kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); + XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", + kick_count, __func__); + + xse->kicks_submitted += kicks_submitted; + + XSCHED_INFO("xse %d kicks_submitted = %lu @ %s\n", + xse->tgid, xse->kicks_submitted, __func__); + +out_err: + return kicks_submitted; +} + const struct xsched_class rt_xsched_class = { .next = NULL, .dequeue_ctx = dequeue_ctx_rt, -- 2.34.1

From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add xsched CFS class base template with function stubs: - dequeue_ctx_fair. - enqueue_ctx_fair. - pick_next_ctx_fair. - check_preempt_fair. - put_prev_ctx_fair. - submit_prepare_ctx_fair. Add xsched_cfs.c in /kernel/xsched Makefile. Add cfs class related data structure. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- include/linux/xsched.h | 45 +++++++++++++++++++++++-- kernel/xsched/Makefile | 2 +- kernel/xsched/cfs.c | 76 ++++++++++++++++++++++++++++++++++++++++++ kernel/xsched/core.c | 16 ++++++++- kernel/xsched/rt.c | 2 +- 5 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 kernel/xsched/cfs.c diff --git a/include/linux/xsched.h b/include/linux/xsched.h index f093d82dfb95..0f3c0f7f21ad 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -55,6 +55,8 @@ #define XCU_HASH_ORDER 6 +#define XSCHED_CFS_MIN_TIMESLICE (10 * NSEC_PER_MSEC) + #define __GET_VS_TASK_TYPE(t) ((t)&0xFF) #define __GET_VS_TASK_PRIO_RT(t) (((t) >> 8) & 0xFF) @@ -72,6 +74,7 @@ enum xcu_sched_type { XSCHED_TYPE_RT, XSCHED_TYPE_DFLT = XSCHED_TYPE_RT, + XSCHED_TYPE_CFS, XSCHED_TYPE_NUM, }; @@ -105,6 +108,7 @@ enum xse_flag { extern const struct xsched_class rt_xsched_class; +extern const struct xsched_class fair_xsched_class; #define xsched_first_class (&rt_xsched_class) @@ -117,6 +121,12 @@ extern const struct xsched_class rt_xsched_class; #define for_each_vstream_in_ctx(vs, ctx) \ list_for_each_entry((vs), &((ctx)->vstream_list), ctx_node) +/* Manages xsched CFS-like class rbtree based runqueue. */ +struct xsched_rq_cfs { + unsigned int load; + u64 min_xruntime; + struct rb_root_cached ctx_timeline; +}; /* Manages xsched RT-like class linked list based runqueue. * @@ -144,6 +154,8 @@ struct xsched_rq { /* RT class run queue.*/ struct xsched_rq_rt rt; + /* CFS class run queue.*/ + struct xsched_rq_cfs cfs; }; enum xcu_state { @@ -176,6 +188,8 @@ struct xsched_cu { /* RT class kick counter. */ atomic_t pending_kicks_rt; + /* CFS class kick counter. */ + atomic_t pending_kicks_cfs; struct task_struct *worker; @@ -203,6 +217,22 @@ struct xsched_entity_rt { s64 kick_slice; }; +struct xsched_entity_cfs { + struct rb_node run_node; + + /* Rq on which this entity is (to be) queued. */ + struct xsched_rq_cfs *cfs_rq; + + /* Value of "virtual" runtime to sort entities in rbtree */ + u64 xruntime; + u32 weight; + + /* Clean execution time of scheduling entity */ + u64 exec_start; + u64 last_exec_runtime; + u64 sum_exec_runtime; +}; + struct xsched_entity { uint32_t task_type; @@ -231,6 +261,8 @@ struct xsched_entity { /* RT class entity. */ struct xsched_entity_rt rt; + /* CFS class entity. */ + struct xsched_entity_cfs cfs; /* Pointer to context object. */ struct xsched_context *ctx; @@ -252,6 +284,11 @@ static inline bool xse_is_rt(const struct xsched_entity *xse) return xse && xse->class == &rt_xsched_class; } +static inline bool xse_is_cfs(const struct xsched_entity *xse) +{ + return xse && xse->class == &fair_xsched_class; +} + /* Returns a pointer to an atomic_t variable representing a counter * of currently pending vstream kicks on a given XCU and for a * given xsched class. @@ -273,6 +310,8 @@ xsched_get_pending_kicks_class(const struct xsched_class *class, if (class == &rt_xsched_class) return &xcu->pending_kicks_rt; + if (class == &fair_xsched_class) + return &xcu->pending_kicks_cfs; XSCHED_ERR("Xsched entity has an invalid class @ %s\n", __func__); return NULL; @@ -384,13 +423,15 @@ static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) static inline bool xsched_check_pending_kicks_xcu(struct xsched_cu *xcu) { atomic_t *kicks_rt; + atomic_t *kicks_cfs; kicks_rt = xsched_get_pending_kicks_class(&rt_xsched_class, xcu); + kicks_cfs = xsched_get_pending_kicks_class(&fair_xsched_class, xcu); - if (!kicks_rt) + if (!kicks_rt || !kicks_cfs) return 0; - return !!atomic_read(kicks_rt); + return (!!atomic_read(kicks_rt) || !!atomic_read(kicks_cfs)); } static inline int xse_integrity_check(const struct xsched_entity *xse) diff --git a/kernel/xsched/Makefile b/kernel/xsched/Makefile index f882518d54ab..fe212f228cf6 100644 --- a/kernel/xsched/Makefile +++ b/kernel/xsched/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += vstream.o -obj-$(CONFIG_XCU_SCHEDULER) += core.o rt.o +obj-$(CONFIG_XCU_SCHEDULER) += core.o rt.o cfs.o diff --git a/kernel/xsched/cfs.c b/kernel/xsched/cfs.c new file mode 100644 index 000000000000..36922d91e85b --- /dev/null +++ b/kernel/xsched/cfs.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Completely Fair Scheduling (CFS) Class for XPU device + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/xsched.h> + +#define CFS_INNER_RQ_EMPTY(cfs_xse) \ + ((cfs_xse)->xruntime == XSCHED_TIME_INF) + +/* For test xsched_cfs_grp_test.c */ +atomic64_t virtual_sched_clock = ATOMIC_INIT(0); + +/* + * Xsched Fair class methods + * For rq manipulation we rely on root runqueue lock already acquired in core. + * Access xsched_group_xcu_priv requires no locks because one thread per XCU. + */ +static void dequeue_ctx_fair(struct xsched_entity *xse) +{ +} + +/** + * enqueue_ctx_fair() - Add context to the runqueue + * @xse: xsched entity of context + * @xcu: executor + * + * In contrary to enqueue_task it is called once on context init. + * Although groups reside in tree, their nodes not counted in nr_running. + * The xruntime of a group xsched entitry represented by min xruntime inside. + */ +static void enqueue_ctx_fair(struct xsched_entity *xse, struct xsched_cu *xcu) +{ +} + +static struct xsched_entity *pick_next_ctx_fair(struct xsched_cu *xcu) +{ + return NULL; +} + +static inline bool xs_should_preempt_fair(struct xsched_entity *xse) +{ + return 0; +} + +static void put_prev_ctx_fair(struct xsched_entity *xse) +{ +} + +int submit_prepare_ctx_fair(struct xsched_entity *xse, struct xsched_cu *xcu) +{ + return 0; +} + +const struct xsched_class fair_xsched_class = { + .next = NULL, + .dequeue_ctx = dequeue_ctx_fair, + .enqueue_ctx = enqueue_ctx_fair, + .pick_next_ctx = pick_next_ctx_fair, + .put_prev_ctx = put_prev_ctx_fair, + .submit_prepare_ctx = submit_prepare_ctx_fair, + .check_preempt = xs_should_preempt_fair, +}; diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 81bdb1bfc9a2..5d96a22337fa 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -93,7 +93,7 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) not_empty++; } } - } while (not_empty); + } while ((sum_exec_time < XSCHED_CFS_MIN_TIMESLICE) && (not_empty)); kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", @@ -408,6 +408,10 @@ int xsched_xse_set_class(struct xsched_entity *xse) xse->class = &rt_xsched_class; XSCHED_INFO("Context is in RT class %s\n", __func__); break; + case XSCHED_TYPE_CFS: + xse->class = &fair_xsched_class; + XSCHED_INFO("Context is in CFS class %s\n", __func__); + break; default: XSCHED_ERR("Xse has incorrect class @ %s\n", __func__); return -EINVAL; @@ -658,6 +662,14 @@ static inline void xsched_rt_rq_init(struct xsched_cu *xcu) } } +/* Initialize xsched cfs runqueue during kernel init. + * Should only be called from xsched_init function. + */ +static inline void xsched_cfs_rq_init(struct xsched_cu *xcu) +{ + xcu->xrq.cfs.ctx_timeline = RB_ROOT_CACHED; +} + /* Initialize xsched classes' runqueues. */ static inline void xsched_rq_init(struct xsched_cu *xcu) { @@ -666,6 +678,7 @@ static inline void xsched_rq_init(struct xsched_cu *xcu) xcu->xrq.class = &rt_xsched_class; xcu->xrq.state = XRQ_STATE_IDLE; xsched_rt_rq_init(xcu); + xsched_cfs_rq_init(xcu); } /* Initializes all xsched XCU objects. @@ -682,6 +695,7 @@ static void xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, xcu->group = group; atomic_set(&xcu->pending_kicks_rt, 0); + atomic_set(&xcu->pending_kicks_cfs, 0); atomic_set(&xcu->has_active, 0); INIT_LIST_HEAD(&xcu->vsm_list); diff --git a/kernel/xsched/rt.c b/kernel/xsched/rt.c index da9959778366..8bd3f7c7f403 100644 --- a/kernel/xsched/rt.c +++ b/kernel/xsched/rt.c @@ -233,7 +233,7 @@ static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) } const struct xsched_class rt_xsched_class = { - .next = NULL, + .next = &fair_xsched_class, .dequeue_ctx = dequeue_ctx_rt, .enqueue_ctx = enqueue_ctx_rt, .pick_next_ctx = pick_next_ctx_rt, -- 2.34.1

From: Alekseev Dmitry <alekseev.dmitry@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add cgroup initialization inculing root cgroup. Add xcu cgroup callbacks: alloc, free, attach, detach, etc. Add xsched_group cgroup management files and methods for: - sched type - shares Add xcu cgroup subsys and option CONFIG_CGROUP_XCU Add cgroup.c in /kernel/xsched Makefile. Signed-off-by: Alekseev Dmitry <alekseev.dmitry@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> --- include/linux/cgroup_subsys.h | 4 + include/linux/xsched.h | 105 +++++- kernel/cgroup/cgroup.c | 2 +- kernel/xsched/Kconfig | 19 ++ kernel/xsched/Makefile | 1 + kernel/xsched/cfs.c | 50 ++- kernel/xsched/cgroup.c | 604 ++++++++++++++++++++++++++++++++++ kernel/xsched/core.c | 13 +- 8 files changed, 790 insertions(+), 8 deletions(-) create mode 100644 kernel/xsched/cgroup.c diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h index 85fa78049bd0..e65ae90946c2 100644 --- a/include/linux/cgroup_subsys.h +++ b/include/linux/cgroup_subsys.h @@ -61,6 +61,10 @@ SUBSYS(pids) SUBSYS(rdma) #endif +#if IS_ENABLED(CONFIG_CGROUP_XCU) +SUBSYS(xcu) +#endif + #if IS_ENABLED(CONFIG_CGROUP_MISC) SUBSYS(misc) #endif diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 3186bae9d2c2..51be002b1970 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -5,7 +5,7 @@ #include <linux/hash.h> #include <linux/hashtable.h> #include <linux/xcu_group.h> -#include <linux/kref.h> +#include <linux/cgroup.h> #include <linux/vstream.h> #ifndef pr_fmt #define pr_fmt(fmt) fmt @@ -59,6 +59,7 @@ #define XSCHED_TIME_INF RUNTIME_INF #define XSCHED_CFS_ENTITY_WEIGHT_DFLT 1 #define XSCHED_CFS_MIN_TIMESLICE (10 * NSEC_PER_MSEC) +#define XSCHED_CFG_SHARE_DFLT 1024 #define __GET_VS_TASK_TYPE(t) ((t)&0xFF) @@ -68,6 +69,8 @@ #define GET_VS_TASK_PRIO_RT(vs_ptr) __GET_VS_TASK_PRIO_RT((vs_ptr)->task_type) +extern struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; + #define XSCHED_RT_TIMESLICE_MS (10 * NSEC_PER_MSEC) /* * A default kick slice for RT class XSEs. @@ -112,6 +115,7 @@ enum xse_flag { extern const struct xsched_class rt_xsched_class; extern const struct xsched_class fair_xsched_class; +extern struct xsched_group *root_xcg; #define xsched_first_class (&rt_xsched_class) @@ -210,6 +214,11 @@ struct xsched_cu { wait_queue_head_t wq_xcore_running; }; +extern int num_active_xcu; +#define for_each_active_xcu(xcu, id) \ + for ((id) = 0, xcu = xsched_cu_mgr[(id)]; \ + (id) < num_active_xcu && (xcu = xsched_cu_mgr[(id)]); (id)++) + struct xsched_entity_rt { struct list_head list_node; enum xse_state state; @@ -279,6 +288,11 @@ struct xsched_entity { */ struct xsched_cu *xcu; + /* Link to list of xsched_group items */ + struct list_head group_node; + struct xsched_group *parent_grp; + bool is_group; + /* General purpose xse lock. */ spinlock_t xse_lock; }; @@ -292,6 +306,90 @@ static inline bool xse_is_cfs(const struct xsched_entity *xse) return xse && xse->class == &fair_xsched_class; } +/* xsched_group's xcu related stuff */ +struct xsched_group_xcu_priv { + /* Owner of this group */ + struct xsched_group *self; + /* xcu id */ + int32_t xcu_id; + /* Link to scheduler */ + struct xsched_entity xse; /* xse of this group on runqueue */ + struct xsched_rq_cfs *rq; /* runqueue "owned" by this group */ + /* Statistics */ + int nr_throttled; + u64 throttled_time; + u64 overrun_time; +}; + +/* Xsched scheduling control group */ +struct xsched_group { + /* Cgroups controller structure */ + struct cgroup_subsys_state css; + + /* Control group settings: */ + int sched_type; + int prio; + + /* Bandwidth setting: shares value set by user */ + u64 shares_cfg; + u64 shares_cfg_red; + u32 weight; + u64 children_shares_sum; + + /* Bandwidth setting: maximal quota in period */ + s64 quota; + s64 rt_exec; + s64 period; + struct hrtimer quota_timeout; + struct work_struct refill_work; + u64 qoslevel; + + struct xsched_group_xcu_priv perxcu_priv[XSCHED_NR_CUS]; + + /* Groups hierarchcy */ + struct xsched_group *parent; + struct list_head children_groups; + struct list_head group_node; + + spinlock_t lock; + + /* for XSE to move in perxcu ? */ + struct list_head members; +}; + +#define XSCHED_RQ_OF(xse) \ + (container_of(((xse)->cfs.cfs_rq), struct xsched_rq, cfs)) + +#define XSCHED_RQ_OF_CFS_XSE(cfs_xse) \ + (container_of(((cfs_xse)->cfs_rq), struct xsched_rq, cfs)) + +#define XSCHED_SE_OF(cfs_xse) \ + (container_of((cfs_xse), struct xsched_entity, cfs)) + +#define xcg_parent_grp_xcu(xcg) \ + ((xcg)->self->parent->perxcu_priv[(xcg)->xcu_id]) + +#define xse_parent_grp_xcu(xse_cfs) \ + (&((XSCHED_SE_OF(xse_cfs) \ + ->parent_grp \ + ->perxcu_priv[(XSCHED_SE_OF(xse_cfs))->xcu->id]))) + +static inline struct xsched_group_xcu_priv * +xse_this_grp_xcu(struct xsched_entity_cfs *xse_cfs) +{ + struct xsched_entity *xse; + + xse = xse_cfs ? container_of(xse_cfs, struct xsched_entity, cfs) : NULL; + return xse ? container_of(xse, struct xsched_group_xcu_priv, xse) : + NULL; +} + +static inline struct xsched_group * +xse_this_grp(struct xsched_entity_cfs *xse_cfs) +{ + return xse_cfs ? xse_this_grp_xcu(xse_cfs)->self : NULL; +} + /* Returns a pointer to an atomic_t variable representing a counter * of currently pending vstream kicks on a given XCU and for a * given xsched class. @@ -542,5 +640,10 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); void submit_kick(struct vstream_info *vs, struct xcu_op_handler_params *params, struct vstream_metadata *vsm); +/* Xsched group manage functions */ +int xsched_group_inherit(struct task_struct *tsk, struct xsched_entity *xse); +void xcu_cg_init_common(struct xsched_group *xcg); +void xcu_grp_shares_update(struct xsched_group *xg); +void xsched_group_xse_detach(struct xsched_entity *xse); void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index c26a9b3a3576..b632590eae0f 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -6190,7 +6190,7 @@ int __init cgroup_init(void) struct cgroup_subsys *ss; int ssid; - BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16); + BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 17); BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files)); BUG_ON(cgroup_init_cftypes(NULL, cgroup_psi_files)); BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files)); diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig index cbd2eec8bfad..d09c77aa4cdd 100644 --- a/kernel/xsched/Kconfig +++ b/kernel/xsched/Kconfig @@ -41,3 +41,22 @@ config XSCHED_NR_CUS int "Number of CUs (a.k.a. XCUs) available to XSched mechanism" default 8 depends on XCU_SCHEDULER + help + This option defines the maximum number of Compute Units (CUs) that can be + managed by the XSched scheduler, consider changing this value proportionally + to the number of available XCU cores. + +config CGROUP_XCU + bool "XCU bandwidth control and group scheduling for xsched_cfs" + default n + depends on XCU_SCHEDULER + help + This option enables the eXtended Compute Unit (XCU) resource controller for + CFS task groups, providing hierarchical scheduling and fine-grained bandwidth + allocation capabilities. Key features include: + - Proportional XCU time distribution across cgroups based on shares/quotas + - Nested group scheduling with latency isolation + - Integration with xsched_cfs for fair CPU resource management + + Required for systems requiring fine-grained resource control in cgroups. + If unsure, say N. diff --git a/kernel/xsched/Makefile b/kernel/xsched/Makefile index fe212f228cf6..c4c06b6038ff 100644 --- a/kernel/xsched/Makefile +++ b/kernel/xsched/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += vstream.o obj-$(CONFIG_XCU_SCHEDULER) += core.o rt.o cfs.o +obj-$(CONFIG_CGROUP_XCU) += cgroup.o diff --git a/kernel/xsched/cfs.c b/kernel/xsched/cfs.c index d20ba5cf3165..ea0f49488fb7 100644 --- a/kernel/xsched/cfs.c +++ b/kernel/xsched/cfs.c @@ -84,10 +84,45 @@ xs_pick_first(struct xsched_rq_cfs *cfs_rq) */ static void xs_update(struct xsched_entity_cfs *xse_cfs, u64 delta) { - u64 new_xrt = xse_cfs->xruntime + delta * xse_cfs->weight; + struct xsched_group_xcu_priv *xg = xse_parent_grp_xcu(xse_cfs); - xs_cfs_rq_update(xse_cfs, new_xrt); - xse_cfs->sum_exec_runtime += delta; + for (; xg; xse_cfs = &xg->xse.cfs, xg = &xcg_parent_grp_xcu(xg)) { + u64 new_xrt = xse_cfs->xruntime + delta * xse_cfs->weight; + + xs_cfs_rq_update(xse_cfs, new_xrt); + xse_cfs->sum_exec_runtime += delta; + + if (xg->self->parent == NULL) + break; + } +} + +/** + * xg_update() - Update container group's xruntime + * @gxcu: Descendant xsched group's private xcu control structure + * + * No locks required to access xsched_group_xcu_priv members, + * because only one worker thread works for one XCU. + */ +static void xg_update(struct xsched_group_xcu_priv *xg) +{ + u64 new_xrt; + struct xsched_entity_cfs *entry; + + for (; xg; xg = &xcg_parent_grp_xcu(xg)) { + entry = xs_pick_first(xg->rq); + if (entry) + new_xrt = entry->xruntime * xg->xse.cfs.weight; + else + new_xrt = XSCHED_TIME_INF; + + xg->rq->min_xruntime = new_xrt; + + if (xg->self->parent) + xs_cfs_rq_update(&xg->xse.cfs, new_xrt); + else + break; + } } /* @@ -102,6 +137,7 @@ static void dequeue_ctx_fair(struct xsched_entity *xse) struct xsched_entity_cfs *xse_cfs = &xse->cfs; xs_rq_remove(xse_cfs); + xg_update(xse_parent_grp_xcu(xse_cfs)); first = xs_pick_first(&xcu->xrq.cfs); xcu->xrq.cfs.min_xruntime = (first) ? first->xruntime : XSCHED_TIME_INF; @@ -128,7 +164,7 @@ static void enqueue_ctx_fair(struct xsched_entity *xse, struct xsched_cu *xcu) struct xsched_entity_cfs *xse_cfs = &xse->cfs; xse_cfs->weight = XSCHED_CFS_ENTITY_WEIGHT_DFLT; - rq = xse_cfs->cfs_rq = &xcu->xrq.cfs; + rq = xse_cfs->cfs_rq = xse_parent_grp_xcu(xse_cfs)->rq; /* If no XSE of only empty groups */ if (xs_pick_first(rq) == NULL || rq->min_xruntime == XSCHED_TIME_INF) @@ -137,6 +173,7 @@ static void enqueue_ctx_fair(struct xsched_entity *xse, struct xsched_cu *xcu) xse_cfs->xruntime = max(xse_cfs->xruntime, rq->min_xruntime); xs_rq_add(xse_cfs); + xg_update(xse_parent_grp_xcu(xse_cfs)); first = xs_pick_first(&xcu->xrq.cfs); xcu->xrq.cfs.min_xruntime = (first) ? first->xruntime : XSCHED_TIME_INF; @@ -155,6 +192,11 @@ static struct xsched_entity *pick_next_ctx_fair(struct xsched_cu *xcu) xse = xs_pick_first(rq); if (!xse) return NULL; + for (; XSCHED_SE_OF(xse)->is_group; xse = xs_pick_first(rq)) { + if (!xse || CFS_INNER_RQ_EMPTY(xse)) + return NULL; + rq = xse_this_grp_xcu(xse)->rq; + } return container_of(xse, struct xsched_entity, cfs); } diff --git a/kernel/xsched/cgroup.c b/kernel/xsched/cgroup.c new file mode 100644 index 000000000000..758b8b9c183d --- /dev/null +++ b/kernel/xsched/cgroup.c @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Support cgroup for xpu device + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/err.h> +#include <linux/cgroup.h> +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/xsched.h> + +enum xcu_file_type { + XCU_FILE_PERIOD_MS, + XCU_FILE_QUOTA_MS, + XCU_FILE_SHARES +}; + +static struct xsched_group root_xsched_group; +struct xsched_group *root_xcg = &root_xsched_group; +static bool root_cg_inited; + +static struct xsched_group *old_xcg; +static DECLARE_WAIT_QUEUE_HEAD(xcg_attach_wq); +static bool attach_in_progress; +static DEFINE_MUTEX(xcg_mutex); + +static const char xcu_sched_name[XSCHED_TYPE_NUM][4] = { + [XSCHED_TYPE_RT] = "rt", + [XSCHED_TYPE_CFS] = "cfs" +}; + +void xcu_cg_init_common(struct xsched_group *xcg) +{ + spin_lock_init(&xcg->lock); + INIT_LIST_HEAD(&xcg->members); + INIT_LIST_HEAD(&xcg->children_groups); +} + +static void xcu_cfs_root_cg_init(void) +{ + uint32_t id; + struct xsched_cu *xcu; + + for_each_active_xcu(xcu, id) { + root_xcg->perxcu_priv[id].xcu_id = id; + root_xcg->perxcu_priv[id].self = root_xcg; + root_xcg->perxcu_priv[id].rq = &xcu->xrq.cfs; + root_xcg->perxcu_priv[id].xse.cfs.weight = 1; + } + + root_xcg->sched_type = XSCHED_TYPE_DFLT; +} + +/** + * xcu_cfs_cg_init() - Initialize xsched_group cfs runqueues and bw control. + * @xcg: new xsched_cgroup + * @parent_xg: parent's group + * + * One xsched_group can host many processes with contexts on different devices. + * Function creates xsched_entity for every XCU, and places it in runqueue + * of parent group. Create new cfs rq for xse inside group. + */ +static void xcu_cfs_cg_init(struct xsched_group *xcg, + struct xsched_group *parent_xg) +{ + uint32_t id; + struct xsched_cu *xcu; + struct xsched_rq_cfs *sub_cfs_rq; + + if (unlikely(!root_cg_inited)) { + xcu_cfs_root_cg_init(); + root_cg_inited = true; + } + + for_each_active_xcu(xcu, id) { + xcg->perxcu_priv[id].xcu_id = id; + xcg->perxcu_priv[id].self = xcg; + + sub_cfs_rq = kzalloc(sizeof(struct xsched_rq_cfs), GFP_KERNEL); + xcg->perxcu_priv[id].rq = sub_cfs_rq; + xcg->perxcu_priv[id].rq->ctx_timeline = RB_ROOT_CACHED; + + xcg->perxcu_priv[id].xse.is_group = true; + xcg->perxcu_priv[id].xse.xcu = xcu; + xcg->perxcu_priv[id].xse.class = &fair_xsched_class; + + /* Put new empty groups to the right in parent's rbtree: */ + xcg->perxcu_priv[id].xse.cfs.xruntime = XSCHED_TIME_INF; + xcg->perxcu_priv[id].xse.cfs.weight = + XSCHED_CFS_ENTITY_WEIGHT_DFLT; + xcg->perxcu_priv[id].xse.parent_grp = parent_xg; + + mutex_lock(&xcu->xcu_lock); + enqueue_ctx(&xcg->perxcu_priv[id].xse, xcu); + mutex_unlock(&xcu->xcu_lock); + } + + xcg->shares_cfg = XSCHED_CFG_SHARE_DFLT; + xcu_grp_shares_update(parent_xg); +} + +static void xcu_cfs_cg_deinit(struct xsched_group *xcg) +{ + uint32_t id; + struct xsched_cu *xcu; + + for_each_active_xcu(xcu, id) { + mutex_lock(&xcu->xcu_lock); + fair_xsched_class.dequeue_ctx(&xcg->perxcu_priv[id].xse); + mutex_unlock(&xcu->xcu_lock); + kfree(xcg->perxcu_priv[id].rq); + } + xcu_grp_shares_update(xcg->parent); +} + +/** + * xcu_cg_init() - Initialize non-root xsched_group structure. + * @xcg: new xsched_cgroup + * @parent_xg: parent's group + */ +static void xcu_cg_init(struct xsched_group *xcg, + struct xsched_group *parent_xg) +{ + xcu_cg_init_common(xcg); + xcg->parent = parent_xg; + list_add_tail(&xcg->group_node, &parent_xg->children_groups); + xcg->sched_type = parent_xg->sched_type; + + switch (xcg->sched_type) { + case XSCHED_TYPE_CFS: + xcu_cfs_cg_init(xcg, parent_xg); + break; + default: + pr_debug("xcu_cgroup: init RT group css=0x%lx\n", (uintptr_t)&xcg->css); + break; + } +} + +inline struct xsched_group *xcu_cg_from_css(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct xsched_group, css) : NULL; +} + +/** + * xcu_css_alloc() - Allocate and init xcu cgroup. + * @parent_css: css of parent xcu cgroup + * + * Called from kernel/cgroup.c with cgroup_lock() held. + * First called in subsys initialization to create root xcu cgroup, when + * XCUs haven't been initialized yet. Func used on every new cgroup creation, + * on second call to set root xsched_group runqueue. + * + * Return: pointer of new xcu cgroup css on success, -ENOMEM otherwise. + */ +static struct cgroup_subsys_state * +xcu_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct xsched_group *parent_xg; + struct xsched_group *xg; + + if (!parent_css) + return &root_xsched_group.css; + + xg = kzalloc(sizeof(*xg), GFP_KERNEL); + if (!xg) + return ERR_PTR(-ENOMEM); + + mutex_lock(&xcg_mutex); + + parent_xg = xcu_cg_from_css(parent_css); + + xcu_cg_init(xg, parent_xg); + + mutex_unlock(&xcg_mutex); + return &xg->css; +} + +static void xcu_css_free(struct cgroup_subsys_state *css) +{ + struct xsched_group *xcg; + + mutex_lock(&xcg_mutex); + + xcg = xcu_cg_from_css(css); + + if (xcg->parent != NULL) { + switch (xcg->sched_type) { + case XSCHED_TYPE_CFS: + xcu_cfs_cg_deinit(xcg); + break; + default: + pr_debug("xcu_cgroup: deinit RT group css=0x%lx\n", (uintptr_t)&xcg->css); + break; + } + } + list_del(&xcg->group_node); + + mutex_unlock(&xcg_mutex); + + kfree(xcg); +} + +int xcu_css_online(struct cgroup_subsys_state *css) +{ + return 0; +} + +static void xcu_css_offline(struct cgroup_subsys_state *css) +{ + ; +} + +static void xsched_group_xse_attach(struct xsched_group *xg, + struct xsched_entity *xse) +{ + spin_lock(&xg->lock); + list_add_tail(&xse->group_node, &xg->members); + spin_unlock(&xg->lock); + xse->parent_grp = xg; +} + +void xsched_group_xse_detach(struct xsched_entity *xse) +{ + struct xsched_group *xcg = xse->parent_grp; + + spin_lock(&xcg->lock); + list_del(&xse->group_node); + spin_unlock(&xcg->lock); +} + +static int xcu_task_can_attach(struct task_struct *task, + struct xsched_group *old, + struct xsched_group *dst) +{ + struct xsched_entity *xse; + bool has_xse = false; + + spin_lock(&old->lock); + list_for_each_entry(xse, &old->members, group_node) { + if (xse->owner_pid == task_pid_nr(task)) { + has_xse = true; + break; + } + } + spin_unlock(&old->lock); + + return has_xse ? -EINVAL : 0; +} + +static int xcu_can_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *dst_css; + int ret = 0; + + mutex_lock(&xcg_mutex); + cgroup_taskset_for_each(task, dst_css, tset) { + struct cgroup_subsys_state *old_css = + task_css(task, xcu_cgrp_id); + struct xsched_group *dst_xcg = xcu_cg_from_css(dst_css); + + old_xcg = xcu_cg_from_css(old_css); + ret = xcu_task_can_attach(task, old_xcg, dst_xcg); + if (ret) + break; + } + if (!ret) + attach_in_progress = true; + mutex_unlock(&xcg_mutex); + return ret; +} + +static void xcu_cancel_attach(struct cgroup_taskset *tset) +{ + mutex_lock(&xcg_mutex); + attach_in_progress = false; + wake_up(&xcg_attach_wq); + mutex_unlock(&xcg_mutex); +} + +void xcu_move_task(struct task_struct *task, struct xsched_group *old_xcg, + struct xsched_group *new_xcg) +{ + struct xsched_entity *xse, *tmp; + struct xsched_cu *xcu; + + spin_lock(&old_xcg->lock); + list_for_each_entry_safe(xse, tmp, &old_xcg->members, group_node) { + if (xse->owner_pid == task_pid_nr(task)) { + xcu = xse->xcu; + + WARN_ON_ONCE(old_xcg != xse->parent_grp); + + /* delete from the old_xcg */ + list_del(&xse->group_node); + + mutex_lock(&xcu->xcu_lock); + + /* dequeue from the current runqueue */ + xse->class->dequeue_ctx(xse); + /* attach to the new_xcg */ + xsched_group_xse_attach(new_xcg, xse); + /* enqueue to the runqueue in new_xcg */ + enqueue_ctx(xse, xcu); + mutex_unlock(&xcu->xcu_lock); + } + } + spin_unlock(&old_xcg->lock); +} + +static void xcu_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + + mutex_lock(&xcg_mutex); + cgroup_taskset_for_each(task, css, tset) { + xcu_move_task(task, old_xcg, xcu_cg_from_css(css)); + } + attach_in_progress = false; + wake_up(&xcg_attach_wq); + mutex_unlock(&xcg_mutex); +} + +/** + * xsched_group_inherit() - Attach new entity to task's xsched_group. + * @task: task_struct + * @xse: xsched entity + * + * Called in xsched context initialization to attach xse to task's group + * and inherit its xse scheduling class and bandwidth control policy. + * + * Return: Zero on success. + */ +int xsched_group_inherit(struct task_struct *task, struct xsched_entity *xse) +{ + struct cgroup_subsys_state *css; + struct xsched_group *xg; + +retry: + wait_event(xcg_attach_wq, !attach_in_progress); + mutex_lock(&xcg_mutex); + if (attach_in_progress) { + mutex_unlock(&xcg_mutex); + goto retry; + } + xse->owner_pid = task_pid_nr(task); + css = task_get_css(task, xcu_cgrp_id); + xg = xcu_cg_from_css(css); + xsched_group_xse_attach(xg, xse); + + css_put(css); + mutex_unlock(&xcg_mutex); + return 0; +} + +static int xcu_sched_show(struct seq_file *sf, void *v) +{ + struct cgroup_subsys_state *css = seq_css(sf); + struct xsched_group *xg = xcu_cg_from_css(css); + + seq_printf(sf, "%s\n", xcu_sched_name[xg->sched_type]); + return 0; +} + +/** + * xcu_cg_set_sched() - Set scheduling type for group. + * @xg: xsched group + * @type: scheduler type + * + * Scheduler type can be changed if task is child of root group + * and haven't got scheduling entities. + * + * Return: Zero on success or -EINVAL + */ +int xcu_cg_set_sched(struct xsched_group *xg, int type) +{ + if (type == xg->sched_type) + return 0; + + if (xg->parent != root_xcg || !list_empty(&xg->members)) + return -EINVAL; + + if (xg->sched_type == XSCHED_TYPE_CFS) + xcu_cfs_cg_deinit(xg); + + if (type == XSCHED_TYPE_CFS) { + xg->sched_type = XSCHED_TYPE_CFS; + xcu_cfs_cg_init(xg, xg->parent); + } + + xg->sched_type = type; + + return 0; +} + +static ssize_t xcu_sched_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct cgroup_subsys_state *css = of_css(of); + struct xsched_group *xg = xcu_cg_from_css(css); + char type_name[4]; + int type = -1; + + ssize_t ret = sscanf(buf, "%3s", type_name); + + if (ret < 1) + return -EINVAL; + + for (type = 0; type < XSCHED_TYPE_NUM; type++) { + if (!strcmp(type_name, xcu_sched_name[type])) + break; + } + + if (type == XSCHED_TYPE_NUM) + return -EINVAL; + + if (!list_empty(&css->children)) + return -EINVAL; + + mutex_lock(&xcg_mutex); + ret = xcu_cg_set_sched(xg, type); + mutex_unlock(&xcg_mutex); + + return (ret) ? ret : nbytes; +} + +static s64 xcu_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) +{ + s64 ret = 0; + struct xsched_group *xcucg = xcu_cg_from_css(css); + + spin_lock(&xcucg->lock); + switch (cft->private) { + case XCU_FILE_SHARES: + ret = xcucg->shares_cfg; + break; + default: + break; + } + spin_unlock(&xcucg->lock); + return ret; +} + +static inline u64 gcd(u64 a, u64 b) +{ + while (a != 0 && b != 0) { + if (a > b) + a %= b; + else + b %= a; + } + return (a) ? a : b; +} + +void xcu_grp_shares_update(struct xsched_group *xg) +{ + int id; + struct xsched_cu *xcu; + struct xsched_group *xgi, *parent = xg; + u64 sh_sum = 0, sh_gcd = 0, w_gcd = 0, sh_prod_red = 1; + + spin_lock(&parent->lock); + + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) + sh_gcd = gcd(sh_gcd, xgi->shares_cfg); + } + + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) { + sh_sum += xgi->shares_cfg; + xgi->shares_cfg_red = div_u64(xgi->shares_cfg, sh_gcd); + + if ((sh_prod_red % xgi->shares_cfg_red) != 0) + sh_prod_red *= xgi->shares_cfg_red; + } + } + + parent->children_shares_sum = sh_sum; + + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) { + xgi->weight = div_u64(sh_prod_red, xgi->shares_cfg_red); + w_gcd = gcd(w_gcd, xgi->weight); + } + } + + list_for_each_entry((xgi), &(parent)->children_groups, group_node) { + if ((xgi)->sched_type == XSCHED_TYPE_CFS) { + xgi->weight = div_u64(xgi->weight, w_gcd); + for_each_active_xcu(xcu, id) { + mutex_lock(&xcu->xcu_lock); + xgi->perxcu_priv[id].xse.cfs.weight = xgi->weight; + mutex_unlock(&xcu->xcu_lock); + } + } + } + + spin_unlock(&parent->lock); +} + +static int xcu_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, + s64 val) +{ + int ret = 0; + struct xsched_group *xcucg = xcu_cg_from_css(css); + + spin_lock(&xcucg->lock); + switch (cft->private) { + case XCU_FILE_SHARES: + if (val <= 0) { + ret = -EINVAL; + break; + } + xcucg->shares_cfg = val; + xcu_grp_shares_update(xcucg->parent); + break; + default: + ret = -EINVAL; + break; + } + spin_unlock(&xcucg->lock); + + return ret; +} + +static int xcu_stat(struct seq_file *sf, void *v) +{ + struct cgroup_subsys_state *css = seq_css(sf); + struct xsched_group *xcucg = xcu_cg_from_css(css); + + u64 nr_throttled = 0; + u64 throttled_time = 0; + u64 exec_runtime = 0; + + int xcu_id; + struct xsched_cu *xcu; + + if (xcucg->sched_type == XSCHED_TYPE_RT) { + seq_puts(sf, "RT group stat is not supported\n"); + return 0; + } + + for_each_active_xcu(xcu, xcu_id) { + nr_throttled += xcucg->perxcu_priv[xcu_id].nr_throttled; + throttled_time += xcucg->perxcu_priv[xcu_id].throttled_time; + exec_runtime += + xcucg->perxcu_priv[xcu_id].xse.cfs.sum_exec_runtime; + } + + seq_printf(sf, "exec_runtime: %llu\n", exec_runtime); + seq_printf(sf, "shares cfg: %llu/%llu x%u\n", xcucg->shares_cfg, + xcucg->parent->children_shares_sum, xcucg->weight); + + return 0; +} + +static struct cftype xcu_cg_files[] = { + { + .name = "shares", + .flags = CFTYPE_NOT_ON_ROOT, + .read_s64 = xcu_read_s64, + .write_s64 = xcu_write_s64, + .private = XCU_FILE_SHARES, + }, + { + .name = "stat", + .seq_show = xcu_stat, + }, + { + .name = "sched", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = xcu_sched_show, + .write = xcu_sched_write, + }, + {} /* terminate */ +}; + +struct cgroup_subsys xcu_cgrp_subsys = { + .css_alloc = xcu_css_alloc, + .css_online = xcu_css_online, + .css_offline = xcu_css_offline, + .css_free = xcu_css_free, + .can_attach = xcu_can_attach, + .cancel_attach = xcu_cancel_attach, + .attach = xcu_attach, + .dfl_cftypes = xcu_cg_files, + .legacy_cftypes = xcu_cg_files, + .early_init = false, +}; diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 5d96a22337fa..b28a11909fd6 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -49,6 +49,8 @@ static void put_prev_ctx(struct xsched_entity *xse) lockdep_assert_held(&xcu->xcu_lock); xse->class->put_prev_ctx(xse); + XSCHED_INFO("Put current xse %d sum_exec_runtime %llu @ %s\n", + xse->tgid, xse->cfs.sum_exec_runtime, __func__); xse->last_process_time = 0; @@ -214,6 +216,8 @@ static int delete_ctx(struct xsched_context *ctx) XSCHED_INFO("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, atomic_read(&xse->kicks_pending_ctx_cnt), __func__); + xsched_group_xse_detach(xse); + XSCHED_EXIT_STUB(); return 0; @@ -403,7 +407,10 @@ struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id) int xsched_xse_set_class(struct xsched_entity *xse) { - switch (xse->task_type) { +#ifdef CONFIG_CGROUP_XCU + xsched_group_inherit(current, xse); +#endif + switch (xse->parent_grp->sched_type) { case XSCHED_TYPE_RT: xse->class = &rt_xsched_class; XSCHED_INFO("Context is in RT class %s\n", __func__); @@ -428,7 +435,8 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) atomic_set(&xse->kicks_pending_ctx_cnt, 0); atomic_set(&xse->kicks_submited, 0); - xse->task_type = XSCHED_TYPE_RT; + + xse->task_type = GET_VS_TASK_TYPE(vs); xse->last_process_time = 0; xse->fd = ctx->fd; @@ -836,6 +844,7 @@ int __init xsched_init(void) /* Initializing global Xsched context list. */ INIT_LIST_HEAD(&xsched_ctx_list); + xcu_cg_init_common(root_xcg); return 0; } -- 2.34.1

From: Dmitriy Alekseev <alekseev.dmitry@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- The xpu device should be exclusive in sched slice. So add xcu_wait() implementation which return after xpu kernels executed completed. It is called after submit_kick(). Signed-off-by: Dmitriy Alekseev <alekseev.dmitry@huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> --- drivers/xcu/xcu_group.c | 24 +++- include/linux/vstream.h | 7 +- include/linux/xcu_group.h | 8 ++ include/linux/xsched.h | 15 +-- include/uapi/linux/xcu_vstream.h | 4 +- kernel/xsched/core.c | 181 +++++++++++++++++-------------- kernel/xsched/rt.c | 6 +- 7 files changed, 145 insertions(+), 100 deletions(-) diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index 0d4b0de482fc..45725758683e 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -167,7 +167,11 @@ int xcu_run(struct xcu_op_handler_params *params) */ int xcu_wait(struct xcu_op_handler_params *params) { - return 0; + if (!params->group->opt || !params->group->opt->wait) { + XSCHED_DEBUG("No function [wait] called.\n"); + return 0; + } + return params->group->opt->wait(params); } /* This function runs "complete" callback for a given xcu_group @@ -252,6 +256,24 @@ int xcu_logic_free(struct xcu_op_handler_params *params) return ret; } +/* This function runs a "sqe_op" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to set or get sqe info. + */ +int xcu_sqe_op(struct xcu_op_handler_params *params) +{ + int ret = 0; + + if (params->group->opt && params->group->opt->sqe_op) + ret = params->group->opt->sqe_op(params); + else + XSCHED_DEBUG("No function [sqe_op] called.\n"); + + return ret; +} + static struct xcu_group __xcu_group_root = { .id = 0, .type = XCU_TYPE_ROOT, diff --git a/include/linux/vstream.h b/include/linux/vstream.h index 8e7bf5992898..3c35b9466832 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -12,15 +12,18 @@ * to be processed by a driver. */ typedef struct vstream_metadata { - uint32_t exec_time; /* A value of SQ tail that has been passed with the * kick that is described by this exact metadata object. */ uint32_t sq_tail; uint32_t sqe_num; uint32_t sq_id; + uint8_t sqe[XCU_SQE_SIZE_MAX]; + + /* Report buffer for fake read. */ + int8_t cqe[XCU_CQE_BUF_SIZE]; + uint32_t cqe_num; int32_t timeout; - int8_t sqe[XCU_SQE_SIZE_MAX]; /* A node for metadata list */ struct list_head node; diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h index a2ecfbe87dc8..c74387c760c3 100644 --- a/include/linux/xcu_group.h +++ b/include/linux/xcu_group.h @@ -22,6 +22,11 @@ enum xcu_version { XCU_HW_V2 }; +enum xcu_sqe_op_type { + SQE_SET_NOTIFY, + SQE_IS_NOTIFY +}; + /** * @group: value for this entry. * @hash_node: hash node list. @@ -46,6 +51,7 @@ struct xcu_op_handler_params { void *param_5; void *param_6; void *param_7; + void *param_8; }; }; }; @@ -60,6 +66,7 @@ struct xcu_operation { xcu_op_handler_fn_t alloc; xcu_op_handler_fn_t logic_alloc; xcu_op_handler_fn_t logic_free; + xcu_op_handler_fn_t sqe_op; }; struct xcu_group { @@ -110,6 +117,7 @@ extern int xcu_finish(struct xcu_op_handler_params *params); extern int xcu_alloc(struct xcu_op_handler_params *params); extern int xcu_logic_alloc(struct xcu_op_handler_params *params); extern int xcu_logic_free(struct xcu_op_handler_params *params); +extern int xcu_sqe_op(struct xcu_op_handler_params *params); #endif /* CONFIG_XCU_SCHEDULER */ #endif /* !__XSCHED_XCU_GROUP_H__ */ diff --git a/include/linux/xsched.h b/include/linux/xsched.h index 5de53d9c231b..7c8a02c862d1 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -463,12 +463,10 @@ static inline int xsched_inc_pending_kicks_xse(struct xsched_entity *xse) /* Incrementing prio based pending kicks counter for RT class */ if (xse_is_rt(xse)) { atomic_inc(&xse->xcu->xrq.rt.prio_nr_kicks[xse->rt.prio]); - XSCHED_INFO("xcu increased pending kicks @ %s\n", __func__); - } else { - XSCHED_INFO("xse %u isn't rt class @ %s\n", xse->tgid, - __func__); } + XSCHED_INFO("xcu increased pending kicks @ %s\n", __func__); + return 0; } @@ -511,12 +509,10 @@ static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) } atomic_dec(kicks_prio_rt); - XSCHED_INFO("xcu decreased pending kicks @ %s\n", __func__); - } else { - XSCHED_INFO("xse %u isn't rt class @ %s\n", xse->tgid, - __func__); } + XSCHED_INFO("xcu decreased pending kicks @ %s\n", __func__); + return 0; } @@ -623,7 +619,6 @@ static inline void xsched_init_vsm(struct vstream_metadata *vsm, vstream_args_t *arg) { vsm->sq_id = arg->sq_id; - vsm->exec_time = arg->vk_args.exec_time; vsm->sqe_num = arg->vk_args.sqe_num; vsm->timeout = arg->vk_args.timeout; memcpy(vsm->sqe, arg->vk_args.sqe, XCU_SQE_SIZE_MAX); @@ -641,8 +636,6 @@ struct xsched_cu *xcu_find(__u32 *type, __u32 devId, __u32 channel_id); /* Vstream metadata proccesing functions.*/ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg); struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs); -void submit_kick(struct vstream_info *vs, struct xcu_op_handler_params *params, - struct vstream_metadata *vsm); /* Xsched group manage functions */ int xsched_group_inherit(struct task_struct *tsk, struct xsched_entity *xse); void xcu_cg_init_common(struct xsched_group *xcg); diff --git a/include/uapi/linux/xcu_vstream.h b/include/uapi/linux/xcu_vstream.h index 1024f5af5c2d..0c19421e1650 100644 --- a/include/uapi/linux/xcu_vstream.h +++ b/include/uapi/linux/xcu_vstream.h @@ -6,6 +6,9 @@ #define PAYLOAD_SIZE_MAX 512 #define XCU_SQE_SIZE_MAX 64 +#define XCU_CQE_SIZE_MAX 32 +#define XCU_CQE_REPORT_NUM 4 +#define XCU_CQE_BUF_SIZE (XCU_CQE_REPORT_NUM * XCU_CQE_SIZE_MAX) /* * VSTREAM_ALLOC: alloc a vstream, buffer for tasks @@ -28,7 +31,6 @@ typedef struct vstream_free_args { } vstream_free_args_t; typedef struct vstream_kick_args { __u32 sqe_num; - __u32 exec_time; __s32 timeout; __s8 sqe[XCU_SQE_SIZE_MAX]; } vstream_kick_args_t; diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index 98f4965d97e5..288cad6a5652 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -62,8 +62,7 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) { uint32_t kick_count; struct vstream_info *vs; - unsigned int sum_exec_time = 0; - size_t kicks_submitted = 0; + size_t kicks_scheduled = 0; struct vstream_metadata *vsm; size_t not_empty; @@ -85,30 +84,46 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) spin_unlock(&vs->stream_lock); if (vsm) { list_add_tail(&vsm->node, &xcu->vsm_list); - - sum_exec_time += vsm->exec_time; - kicks_submitted++; + kicks_scheduled++; xsched_dec_pending_kicks_xse(xse); XSCHED_INFO( - "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", - vs->id, vsm->exec_time, vsm->sq_tail, - vsm->sqe_num, vsm->sq_id, __func__); + "vs id = %u sq_tail %u @ %s\n", + vs->id, vsm->sq_tail, __func__); not_empty++; } } - } while ((sum_exec_time < XSCHED_CFS_MIN_TIMESLICE) && (not_empty)); + } while ((kicks_scheduled < 10) && (not_empty)); + + /* + * Iterate over all vstreams in context: + * Set wr_cqe bit in last computing task in vsm_list + */ + for_each_vstream_in_ctx(vs, xse->ctx) { + list_for_each_entry_reverse(vsm, &xcu->vsm_list, node) { + if (vsm->parent == vs) { + struct xcu_op_handler_params params; + int op_type = SQE_SET_NOTIFY; + + params.group = vsm->parent->xcu->group; + params.param_1 = &op_type; + params.param_2 = &vsm->sqe; + xcu_sqe_op(¶ms); + break; + } + } + } kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); XSCHED_INFO("After decrement XSE kick_count=%u @ %s\n", kick_count, __func__); - xse->kicks_submitted += kicks_submitted; + xse->kicks_submitted += kicks_scheduled; - XSCHED_INFO("xse %d kicks_submitted = %lu @ %s\n", + XSCHED_INFO("xse %d kicks_scheduled = %lu @ %s\n", xse->tgid, xse->kicks_submitted, __func__); out_err: - return kicks_submitted; + return kicks_scheduled; } static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) @@ -208,10 +223,13 @@ static int delete_ctx(struct xsched_context *ctx) } mutex_lock(&xcu->xcu_lock); + XSCHED_INFO("%s: Xcu lock taken\n", __func__); if (curr_xse == xse) xcu->xrq.curr_xse = NULL; dequeue_ctx(xse, xcu); + + XSCHED_INFO("%s: Xcu lock released\n", __func__); mutex_unlock(&xcu->xcu_lock); XSCHED_INFO("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, @@ -491,71 +509,99 @@ int xsched_ctx_init_xse(struct xsched_context *ctx, struct vstream_info *vs) return err; } -/* - * A function for submitting stream's commands (sending commands to a XCU). - */ -static int xsched_proc(struct xsched_cu *xcu, struct vstream_info *vs, - struct vstream_metadata *vsm) +static void submit_kick(struct vstream_metadata *vsm) { + struct vstream_info *vs = vsm->parent; struct xcu_op_handler_params params; - struct xsched_entity *xse; - - XSCHED_CALL_STUB(); + params.group = vs->xcu->group; + params.fd = vs->fd; + params.param_1 = &vs->id; + params.param_2 = &vs->channel_id; + params.param_3 = vsm->sqe; + params.param_4 = &vsm->sqe_num; + params.param_5 = &vsm->timeout; + params.param_6 = &vs->sqcq_type; + params.param_7 = vs->drv_ctx; + params.param_8 = &vs->logic_vcqId; - xse = &vs->ctx->xse; - - /* Init input parameters for xcu_run and xcu_wait callbacks. */ - params.group = xcu->group; - - /* Increase process time by abstract kick handling time. */ - xse->last_process_time += vsm->exec_time; - - XSCHED_INFO("Process vsm sq_tail %d exec_time %u sqe_num %d sq_id %d@ %s\n", - vsm->sq_tail, vsm->exec_time, vsm->sqe_num, vsm->sq_id, __func__); - submit_kick(vs, ¶ms, vsm); + /* Send vstream on a device for processing. */ + if (xcu_run(¶ms) != 0) { + XSCHED_ERR( + "Failed to send vstream tasks vstreamId=%u to a device for processing.\n", + vs->id); + } - xse->kicks_processed++; + XSCHED_INFO("Vstream_id %u submit vsm: sq_tail %u\n", vs->id, vsm->sq_tail); +} - XSCHED_INFO("xse %d kicks_processed = %lu @ %s\n", - xse->tgid, xse->kicks_processed, __func__); +static void submit_wait(struct vstream_metadata *vsm) +{ + struct vstream_info *vs = vsm->parent; + struct xcu_op_handler_params params; + /* Wait timeout in ms. */ + int32_t timeout = 500; + + params.group = vs->xcu->group; + params.param_1 = &vs->channel_id; + params.param_2 = &vs->logic_vcqId; + params.param_3 = &vs->user_streamId; + params.param_4 = &vsm->sqe; + params.param_5 = vsm->cqe; + params.param_6 = vs->drv_ctx; + params.param_7 = &timeout; + + /* Wait for a device to complete processing. */ + if (xcu_wait(¶ms) != 0) { + XSCHED_ERR( + "Failed to wait vstream tasks vstreamId=%u, cq_logic=%u.\n", + vs->id, vs->logic_vcqId); + } - XSCHED_EXIT_STUB(); - return 0; + XSCHED_INFO("Vstream_id %u wait finish, cq_logic %u\n", vs->id, vs->logic_vcqId); } static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) { int err = 0; struct vstream_metadata *vsm, *tmp; - unsigned int submit_exec_time = 0; size_t kicks_submitted = 0; - unsigned long wait_us; + ktime_t submit_exec_time = 0; + ktime_t t_start = 0; XSCHED_CALL_STUB(); - XSCHED_INFO("%s called for xse %d on xcu %u\n", __func__, xse->tgid, xcu->id); list_for_each_entry_safe(vsm, tmp, &xcu->vsm_list, node) { - xsched_proc(xcu, vsm->parent, vsm); - submit_exec_time += vsm->exec_time; + struct xcu_op_handler_params params; + int op_type = SQE_IS_NOTIFY; + + submit_kick(vsm); + + params.group = vsm->parent->xcu->group; + params.param_1 = &op_type; + params.param_2 = &vsm->sqe; + if (xcu_sqe_op(¶ms)) { + XSCHED_INFO("%s: Xcu lock released\n", __func__); + mutex_unlock(&xcu->xcu_lock); + t_start = ktime_get(); + submit_wait(vsm); + submit_exec_time += ktime_get() - t_start; + mutex_lock(&xcu->xcu_lock); + XSCHED_INFO("%s: Xcu lock taken\n", __func__); + } kicks_submitted++; + list_del(&vsm->node); + kfree(vsm); } - INIT_LIST_HEAD(&xcu->vsm_list); - - mutex_unlock(&xcu->xcu_lock); - - wait_us = div_u64(submit_exec_time, NSEC_PER_USEC); - XSCHED_INFO("XCU kicks_submitted=%lu wait_us=%lu @ %s\n", - kicks_submitted, wait_us, __func__); - - if (wait_us > 0) { - /* Sleep shift not larger than 12.5% */ - usleep_range(wait_us, wait_us + (wait_us >> 3)); - } + xse->last_process_time += submit_exec_time; + xse->kicks_processed += kicks_submitted; + XSCHED_INFO("XCU kicks submitted=%lu total=%lu exec_time = %lld @ %s\n", + kicks_submitted, xse->kicks_processed, submit_exec_time, + __func__); - mutex_lock(&xcu->xcu_lock); + INIT_LIST_HEAD(&xcu->vsm_list); XSCHED_EXIT_STUB(); @@ -640,33 +686,6 @@ static int xsched_schedule(void *input_xcu) return err; } -void submit_kick(struct vstream_info *vs, - struct xcu_op_handler_params *params, - struct vstream_metadata *vsm) -{ - int ret; - - params->fd = vs->fd; - params->param_1 = &vs->id; - params->param_2 = &vs->channel_id; - params->param_3 = vsm->sqe; - params->param_4 = &vsm->sqe_num; - params->param_5 = &vsm->timeout; - params->param_6 = &vs->sqcq_type; - params->param_7 = vs->drv_ctx; - /* Send vstream on a device for processing. */ - ret = xcu_run(params); - if (ret) { - XSCHED_ERR( - "Failed to send vstream tasks vstreamId=%d to a device for processing.\n", - vs->id); - } - - XSCHED_INFO("Vstream_id %d submit vsm: sq_tail %d\n", vs->id, vsm->sq_tail); - - kfree(vsm); -} - /* Initialize xsched rt runqueue during kernel init. * Should only be called from xsched_init function. */ diff --git a/kernel/xsched/rt.c b/kernel/xsched/rt.c index 8bd3f7c7f403..60845981114c 100644 --- a/kernel/xsched/rt.c +++ b/kernel/xsched/rt.c @@ -208,13 +208,11 @@ static size_t select_work_rt(struct xsched_cu *xcu, struct xsched_entity *xse) spin_lock(&vs->stream_lock); while ((vsm = xsched_vsm_fetch_first(vs))) { list_add_tail(&vsm->node, &xcu->vsm_list); - vsm->exec_time = 0; kicks_submitted++; xsched_dec_pending_kicks_xse(xse); XSCHED_INFO( - "vs id = %d Kick submit exec_time %u sq_tail %u sqe_num %u sq_id %u @ %s\n", - vs->id, vsm->exec_time, vsm->sq_tail, - vsm->sqe_num, vsm->sq_id, __func__); + "vs id = %u Kick submit sq_tail %u sqe_num %u sq_id %u @ %s\n", + vs->id, vsm->sq_tail, vsm->sqe_num, vsm->sq_id, __func__); } spin_unlock(&vs->stream_lock); } -- 2.34.1