From: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC5EHB ----------------------------------------- Add base xcu_group structure, xcu_type enum, xcu_operation struct Add build support in Makefiles. Signed-off-by: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> Signed-off-by: Hui Tang <tanghui20@.huawei.com> Signed-off-by: Liu Kai <liukai284@huawei.com> Signed-off-by: Xia Fukun <xiafukun@huawei.com> Signed-off-by: Zicheng Qu <quzicheng@huawei.com> --- drivers/Makefile | 1 + drivers/xcu/Makefile | 2 + drivers/xcu/xcu_group.c | 362 ++++++++++++++++++++++++++++++++++++++ include/linux/xcu_group.h | 89 ++++++++++ include/linux/xsched.h | 28 +++ kernel/xsched/Kconfig | 9 + kernel/xsched/Makefile | 1 + kernel/xsched/core.c | 58 ++++++ 8 files changed, 550 insertions(+) create mode 100644 drivers/xcu/Makefile create mode 100644 drivers/xcu/xcu_group.c create mode 100644 include/linux/xcu_group.h create mode 100644 kernel/xsched/core.c diff --git a/drivers/Makefile b/drivers/Makefile index f8e58f0ca2d1..57826d4b5cd7 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -197,6 +197,7 @@ obj-$(CONFIG_GNSS) += gnss/ obj-$(CONFIG_INTERCONNECT) += interconnect/ obj-$(CONFIG_COUNTER) += counter/ obj-$(CONFIG_MOST) += most/ +obj-$(CONFIG_XCU_SCHEDULER) += xcu/ obj-$(CONFIG_PECI) += peci/ obj-$(CONFIG_HTE) += hte/ obj-$(CONFIG_DRM_ACCEL) += accel/ diff --git a/drivers/xcu/Makefile b/drivers/xcu/Makefile new file mode 100644 index 000000000000..575115b148ec --- /dev/null +++ b/drivers/xcu/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_XCU_SCHEDULER) += xcu_group.o diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c new file mode 100644 index 000000000000..3215f37e4ece --- /dev/null +++ b/drivers/xcu/xcu_group.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Code for NPU driver support + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/kthread.h> +#include <linux/rwsem.h> +#include <linux/slab.h> +#include <linux/xcu_group.h> +#include <linux/xsched.h> + +static int num_active_xcu; +static DEFINE_SPINLOCK(xcu_mgr_lock); +struct xsched_cu *xsched_cu_mgr[XSCHED_NR_CUS]; +static DECLARE_RWSEM(xcu_group_rwsem); +struct xcu_group *xcu_group_init(int id) +{ + struct xcu_group *node = kzalloc(sizeof(*node), GFP_KERNEL); + + if (!node) + return NULL; + + node->id = id; + node->type = XCU_TYPE_XPU; + idr_init(&node->next_layer); + return node; +} +EXPORT_SYMBOL(xcu_group_init); + +int __xcu_group_attach(struct xcu_group *new_group, + struct xcu_group *previous_group) +{ + int id = new_group->id; + + if (id == -1) + id = idr_alloc(&previous_group->next_layer, new_group, 0, + INT_MAX, GFP_KERNEL); + else + id = idr_alloc(&previous_group->next_layer, new_group, id, + id + 1, GFP_KERNEL); + + if (id < 0) { + XSCHED_ERR("Fail to attach xcu_group: id conflict @ %s\n", + __func__); + return -EEXIST; + } + new_group->id = id; + new_group->previous_layer = previous_group; + + return 0; +} + +int xcu_group_attach(struct xcu_group *new_group, + struct xcu_group *previous_group) +{ + int ret; + + down_write(&xcu_group_rwsem); + ret = __xcu_group_attach(new_group, previous_group); + up_write(&xcu_group_rwsem); + + return ret; +} +EXPORT_SYMBOL(xcu_group_attach); + +static inline void __xcu_group_detach(struct xcu_group *group) +{ + if (!group || !group->previous_layer) + return; + + idr_remove(&group->previous_layer->next_layer, group->id); + group->previous_layer = NULL; +} + +void xcu_group_detach(struct xcu_group *group) +{ + down_write(&xcu_group_rwsem); + __xcu_group_detach(group); + up_write(&xcu_group_rwsem); +} +EXPORT_SYMBOL(xcu_group_detach); + +void xcu_group_free(struct xcu_group *group) +{ + idr_destroy(&group->next_layer); + if (group != xcu_group_root) + kfree(group); +} +EXPORT_SYMBOL(xcu_group_free); + +static struct xcu_group *__xcu_group_find_nolock(struct xcu_group *group, int id) +{ + return idr_find(&group->next_layer, id); +} + +struct xcu_group *xcu_group_find(struct xcu_group *group, int id) +{ + struct xcu_group *result; + + down_read(&xcu_group_rwsem); + result = __xcu_group_find_nolock(group, id); + up_read(&xcu_group_rwsem); + + return result; +} +EXPORT_SYMBOL(xcu_group_find); + +/* This function runs "run" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object + */ +int xcu_run(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->run) { + XSCHED_ERR("No function [run] called.\n"); + return -EINVAL; + } + + return params->group->opt->run(params); +} + +/* This function runs "wait" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object + */ +int xcu_wait(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->wait) { + XSCHED_ERR("No function [wait] called.\n"); + return -EINVAL; + } + + return params->group->opt->wait(params); +} + +/* This function runs "complete" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + */ +int xcu_complete(struct xcu_op_handler_params *params) +{ + return 0; +} + +/* This function runs "finish" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement deallocation + * and freeing memory for SQ and CQ buffers. + */ +int xcu_finish(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->finish) { + XSCHED_ERR("No function [finish] called.\n"); + return -EINVAL; + } + + return params->group->opt->finish(params); +} + +/* This function runs a "alloc" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement allocation + * and registering memory for SQ and CQ buffers. + */ +int xcu_alloc(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->alloc) { + XSCHED_ERR("No function [alloc] called.\n"); + return -EINVAL; + } + + return params->group->opt->alloc(params); +} + +/* This function runs a "logic_alloc" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement allocation + * and registering memory of logic CQ buffer. + */ +int xcu_logic_alloc(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->logic_alloc) { + XSCHED_ERR("No function [logic_alloc] called.\n"); + return -EINVAL; + } + + return params->group->opt->logic_alloc(params); +} + +/* This function runs a "logic_free" callback for a given xcu_group + * and a given vstream that are passed within + * xcu_op_handler_params object. + * + * This handler provides an interface to implement deallocation + * and unregistering memory of a logic CQ buffer. + */ +int xcu_logic_free(struct xcu_op_handler_params *params) +{ + if (!params->group->opt || !params->group->opt->logic_free) { + XSCHED_ERR("No function [logic_free] called.\n"); + return -EINVAL; + } + + return params->group->opt->logic_free(params); +} + +static struct xcu_group __xcu_group_root = { + .id = 0, + .type = XCU_TYPE_ROOT, + .next_layer = IDR_INIT(next_layer), +}; + +struct xcu_group *xcu_group_root = &__xcu_group_root; +EXPORT_SYMBOL(xcu_group_root); + +static int nr_active_cu_inc(void) +{ + int cur_num = -1; + + spin_lock(&xcu_mgr_lock); + if (num_active_xcu >= XSCHED_NR_CUS) + goto out_unlock; + + cur_num = num_active_xcu; + num_active_xcu++; + +out_unlock: + spin_unlock(&xcu_mgr_lock); + return cur_num; +} + +static int nr_active_cu_dec(void) +{ + int cur_num = -1; + + spin_lock(&xcu_mgr_lock); + if (num_active_xcu <= 0) + goto out_unlock; + + cur_num = num_active_xcu; + num_active_xcu--; + +out_unlock: + spin_unlock(&xcu_mgr_lock); + return cur_num; +} + +/* + * Initialize and register xcu in xcu_manager array. + */ +int xsched_xcu_register(struct xcu_group *group, uint32_t phys_id) +{ + int xcu_cur_num, ret = 0; + struct xsched_cu *xcu; + + if (phys_id >= XSCHED_NR_CUS) { + XSCHED_ERR("phys_id [%u] is out of valid range [0, %d).\n", + phys_id, XSCHED_NR_CUS); + return -EINVAL; + } + + if (!group) { + XSCHED_ERR("group cannot be NULL.\n"); + return -EINVAL; + } + + xcu_cur_num = nr_active_cu_inc(); + if (xcu_cur_num < 0) { + XSCHED_ERR("Number of present XCU's exceeds %d: %d.\n", + XSCHED_NR_CUS, num_active_xcu); + return -ENOSPC; + } + + xcu = kzalloc(sizeof(struct xsched_cu), GFP_KERNEL); + if (!xcu) { + if (nr_active_cu_dec() < 0) { + XSCHED_ERR("num_active_xcu [%d] must be > 0.\n", + num_active_xcu); + } + XSCHED_ERR("Fail to alloc xcu for phys_id [%u].\n", phys_id); + return -ENOMEM; + } + + group->xcu = xcu; + xsched_cu_mgr[phys_id] = xcu; + + /* Init xcu's internals. */ + ret = xsched_xcu_init(xcu, group, phys_id); + if (ret != 0) { + group->xcu = NULL; + xsched_cu_mgr[phys_id] = NULL; + kfree(xcu); + XSCHED_ERR("Fail to init xcu[%u].", xcu->id); + return ret; + } + + return 0; +} +EXPORT_SYMBOL(xsched_xcu_register); + +int xsched_xcu_unregister(struct xcu_group *group, uint32_t phys_id) +{ + struct xsched_cu *xcu; + + if (phys_id >= XSCHED_NR_CUS) { + XSCHED_ERR("phys_id [%u] is out of valid range [0, %d).\n", + phys_id, XSCHED_NR_CUS); + return -EINVAL; + } + + if (!group || !group->xcu || group->xcu != xsched_cu_mgr[phys_id]) { + XSCHED_ERR("group is invalid or cannot mapping to phys_id [%u].\n", phys_id); + return -EINVAL; + } + + xcu = group->xcu; + if (!xcu) { + XSCHED_ERR("xcu for phys_id [%u] not found.\n", phys_id); + return -EINVAL; + } + + if (nr_active_cu_dec() < 0) { + XSCHED_ERR("No active XCU\n"); + return -EPERM; + }; + + if (xcu->worker) { + mutex_lock(&xcu->xcu_lock); + wake_up_interruptible(&xcu->wq_xcu_idle); + mutex_unlock(&xcu->xcu_lock); + + kthread_stop(xcu->worker); + xcu->worker = NULL; + } else { + XSCHED_ERR("The worker for xcu [%u] must not be NULL.\n", xcu->id); + } + + group->xcu = NULL; + xsched_cu_mgr[phys_id] = NULL; + kfree(xcu); + + return 0; +} +EXPORT_SYMBOL(xsched_xcu_unregister); + diff --git a/include/linux/xcu_group.h b/include/linux/xcu_group.h new file mode 100644 index 000000000000..b24641b98e6a --- /dev/null +++ b/include/linux/xcu_group.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __XSCHED_XCU_GROUP_H__ +#define __XSCHED_XCU_GROUP_H__ + +#include <linux/idr.h> +#include <uapi/linux/xcu_vstream.h> + +#ifndef CONFIG_XSCHED_NR_CUS +#define CONFIG_XSCHED_NR_CUS 128 +#endif /* !CONFIG_XSCHED_NR_CUS */ +#define XSCHED_NR_CUS CONFIG_XSCHED_NR_CUS + +extern struct xcu_group *xcu_group_root; + +enum xcu_type { + XCU_TYPE_ROOT, + XCU_TYPE_XPU, +}; + +struct xcu_op_handler_params { + int fd; + struct xcu_group *group; + void *payload; + union { + struct { + void *param_1; + void *param_2; + void *param_3; + void *param_4; + void *param_5; + void *param_6; + void *param_7; + void *param_8; + }; + }; +}; + +typedef int (*xcu_op_handler_fn_t)(struct xcu_op_handler_params *params); + +struct xcu_operation { + xcu_op_handler_fn_t run; + xcu_op_handler_fn_t finish; + xcu_op_handler_fn_t wait; + xcu_op_handler_fn_t complete; + xcu_op_handler_fn_t alloc; + xcu_op_handler_fn_t logic_alloc; + xcu_op_handler_fn_t logic_free; +}; + +struct xcu_group { + /* sq id. */ + uint32_t id; + + /* Type of XCU group. */ + enum xcu_type type; + + /* IDR for the next layer of XCU group tree. */ + struct idr next_layer; + + /* Pointer to the previous XCU group in the XCU group tree. */ + struct xcu_group *previous_layer; + + /* Pointer to operation fn pointers object describing + * this XCU group's callbacks. + */ + struct xcu_operation *opt; + + /* Pointer to the XCU related to this XCU group. */ + struct xsched_cu *xcu; +}; + +int xcu_group_attach(struct xcu_group *new_group, + struct xcu_group *previous_group); +void xcu_group_detach(struct xcu_group *group); +struct xcu_group *xcu_group_find(struct xcu_group *group, int id); +struct xcu_group *xcu_group_init(int id); +void xcu_group_free(struct xcu_group *group); + +extern int xcu_run(struct xcu_op_handler_params *params); +extern int xcu_wait(struct xcu_op_handler_params *params); +extern int xcu_complete(struct xcu_op_handler_params *params); +extern int xcu_finish(struct xcu_op_handler_params *params); +extern int xcu_alloc(struct xcu_op_handler_params *params); +extern int xcu_logic_alloc(struct xcu_op_handler_params *params); +extern int xcu_logic_free(struct xcu_op_handler_params *params); + +int xsched_xcu_register(struct xcu_group *group, uint32_t phys_id); +int xsched_xcu_unregister(struct xcu_group *group, uint32_t phys_id); +#endif /* __XSCHED_XCU_GROUP_H__ */ diff --git a/include/linux/xsched.h b/include/linux/xsched.h index dc840136a35f..ed3314ec6572 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -32,4 +32,32 @@ #define XSCHED_EXIT_STUB() \ XSCHED_DEBUG(" -----* %s @ %s exited *-----\n", __func__, __FILE__) +enum xsched_cu_status { + /* Worker not initialized. */ + XSCHED_XCU_NONE, + + /* Worker is sleeping in idle state. */ + XSCHED_XCU_WAIT_IDLE, + + /* Worker is sleeping in running state. */ + XSCHED_XCU_WAIT_RUNNING, + + /* Worker is active but not processing anything. */ + XSCHED_XCU_ACTIVE, + + NR_XSCHED_XCU_STATUS, +}; + +/* This is the abstraction object of the xcu computing unit. */ +struct xsched_cu { + uint32_t id; + uint32_t state; + struct task_struct *worker; + struct xcu_group *group; + struct mutex xcu_lock; + wait_queue_head_t wq_xcu_idle; +}; + +int xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, int xcu_id); +int xsched_schedule(void *input_xcu); #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig index c2d587f6d57a..8d12b8db5f6d 100644 --- a/kernel/xsched/Kconfig +++ b/kernel/xsched/Kconfig @@ -25,3 +25,12 @@ config XCU_VSTREAM Requires XCU_SCHEDULER to be enabled. May increase kernel memory usage. Recommended for high-throughput XPU workloads. If unsure, say N. + +config XSCHED_NR_CUS + int "Number of CUs (a.k.a. XCUs) available to XSched mechanism" + default 128 + depends on XCU_SCHEDULER + help + This option defines the maximum number of Compute Units (CUs) that can be + managed by the XSched scheduler, consider changing this value proportionally + to the number of available XCU cores. diff --git a/kernel/xsched/Makefile b/kernel/xsched/Makefile index 8e6ef8718d5e..e0dd73bfb5f5 100644 --- a/kernel/xsched/Makefile +++ b/kernel/xsched/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_XCU_VSTREAM) += vstream.o +obj-$(CONFIG_XCU_SCHEDULER) += core.o diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c new file mode 100644 index 000000000000..d85379d914f5 --- /dev/null +++ b/kernel/xsched/core.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Core kernel scheduler code for XPU device + * + * Copyright (C) 2025-2026 Huawei Technologies Co., Ltd + * + * Author: Konstantin Meskhidze <konstantin.meskhidze@huawei.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ +#include <linux/delay.h> +#include <linux/kthread.h> +#include <linux/slab.h> +#include <linux/spinlock_types.h> +#include <linux/types.h> +#include <linux/xsched.h> + +int xsched_schedule(void *input_xcu) +{ + return 0; +} + +/* Initializes all xsched XCU objects. + * Should only be called from xsched_xcu_register function. + */ +int xsched_xcu_init(struct xsched_cu *xcu, struct xcu_group *group, int xcu_id) +{ + int err; + + xcu->id = xcu_id; + xcu->state = XSCHED_XCU_NONE; + xcu->group = group; + + mutex_init(&xcu->xcu_lock); + + /* This worker should set XCU to XSCHED_XCU_WAIT_IDLE. + * If after initialization XCU still has XSCHED_XCU_NONE + * status then we can assume that there was a problem + * with XCU kthread job. + */ + xcu->worker = kthread_run(xsched_schedule, xcu, "xcu_%u", xcu->id); + + if (IS_ERR(xcu->worker)) { + err = PTR_ERR(xcu->worker); + xcu->worker = NULL; + XSCHED_DEBUG("Fail to run the worker to schedule for xcu[%u].", xcu->id); + return err; + } + return 0; +} -- 2.34.1