From: Junchong Pan <panjunchong@h-partners.com> In the updated UADK framework, the management functions for hardware queue allocation and release have been decoupled from the framework layer to the user-space driver layer. However, for existing uacce hardware queues and pure software computation queues, we have created a set of common allocation and release interfaces, thereby reducing redundancy in the driver layer code and improving the reusability of the framework code. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- Makefile.am | 9 +- drv/wd_drv.c | 340 +++++++++++ drv/wd_drv.h | 62 ++ wd_util.c | 1623 ++++++++++++++++++++++++++++++++------------------ 4 files changed, 1457 insertions(+), 577 deletions(-) create mode 100644 drv/wd_drv.c create mode 100644 drv/wd_drv.h diff --git a/Makefile.am b/Makefile.am index 5cc8c39..c08c825 100644 --- a/Makefile.am +++ b/Makefile.am @@ -84,6 +84,7 @@ libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_aead.c wd_aead.h wd_aead_drv.h \ + wd.c wd.h wd_alg.h \ wd_rsa.c wd_rsa.h wd_rsa_drv.h \ wd_dh.c wd_dh.h wd_dh_drv.h \ wd_ecc.c wd_ecc.h wd_ecc_drv.h \ @@ -93,24 +94,24 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h \n drv/wd_drv.h drv/wd_drv.c libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h if ARCH_ARM64 libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ - drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h + drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h \n drv/wd_drv.h drv/wd_drv.c libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ drv/hash_mb/sm3_sve_common.S drv/hash_mb/sm3_mb_asimd_x1.S \ drv/hash_mb/sm3_mb_asimd_x4.S drv/hash_mb/sm3_mb_sve.S \ drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ - drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S + drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S \n drv/wd_drv.h drv/wd_drv.c endif libhisi_dae_la_SOURCES=drv/hisi_dae.c hisi_dae.h drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h drv/hisi_dae_join_gather.c drv/hisi_dae_common.c + hisi_qm_udrv.h drv/hisi_dae_join_gather.c drv/hisi_dae_common.c \n drv/wd_drv.h drv/wd_drv.c libhisi_udma_la_SOURCES=drv/hisi_udma.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h diff --git a/drv/wd_drv.c b/drv/wd_drv.c new file mode 100644 index 0000000..d011d3f --- /dev/null +++ b/drv/wd_drv.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. */ +#include <stdlib.h> +#include <sched.h> + +#include "wd_internal.h" +#include "wd_alg.h" +#include "wd_util.h" +#include "wd_drv.h" + +int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx) +{ + struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; + struct wd_soft_ctx *sfctx; + + if (!params || !ctx) { + WD_ERR("invalid: params, or ctx is NULL!\n"); + return -WD_EINVAL; + } + + /* Allocate ONE software context structure */ + sfctx = calloc(1, sizeof(struct wd_soft_ctx)); + if (!sfctx) { + WD_ERR("failed to alloc ctx!\n"); + return -WD_ENOMEM; + } + + /* Initialize as software context */ + sfctx->fd = -1; + sfctx->ctx_type = UADK_ALG_SOFT; + pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_PRIVATE); + + /* Return context handle */ + *ctx = (handle_t)sfctx; + + return WD_SUCCESS; +} + +void wd_soft_free_ctx(handle_t ctx) +{ + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + + if (!sfctx) { + WD_ERR("invalid: ctx is NULL!\n"); + return; + } + + /* Simply free the allocated wd_ctx_h structure */ + pthread_spin_destroy(&sfctx->slock); + pthread_spin_destroy(&sfctx->rlock); + free(sfctx); +} + +static int wd_compare_dev_distance(const void *a, const void *b) +{ + struct uacce_dev_list *node_a = *(struct uacce_dev_list **)a; + struct uacce_dev_list *node_b = *(struct uacce_dev_list **)b; + unsigned int curr_node; + int dist_a, dist_b; + + if (getcpu(NULL, &curr_node) || curr_node == (unsigned int)NUMA_NO_NODE) + return 0; + + dist_a = numa_distance((int)curr_node, node_a->dev->numa_id); + dist_b = numa_distance((int)curr_node, node_b->dev->numa_id); + + return dist_a - dist_b; +} + +static struct uacce_dev_list *wd_sort_dev_list(struct uacce_dev_list *list, int list_count) +{ + struct uacce_dev_list *p, **nodes = NULL; + struct uacce_dev_list *result = NULL; + int i; + + if (!list || !list_count) + return NULL; + + /* Convert to array */ + nodes = calloc(list_count, sizeof(struct uacce_dev_list *)); + if (!nodes) + return list; /* Return original list on allocation failure */ + + p = list; + for (i = 0; i < list_count; i++) { + if (!p) + break; + nodes[i] = p; + p = p->next; + } + + /* Sort by NUMA distance */ + qsort(nodes, list_count, sizeof(struct uacce_dev_list *), wd_compare_dev_distance); + + /* Rebuild sorted list */ + for (i = 0; i < list_count; i++) { + nodes[i]->next = NULL; + if (!result) + result = nodes[i]; + else + wd_add_dev_to_list(result, nodes[i]); + } + free(nodes); + + return result; +} + +struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, int target_numa) +{ + struct uacce_dev_list *p, *node, *result = NULL; + struct uacce_dev_list *ret, *head = NULL; + struct uacce_dev *dev; + int count = 0; + int numa_id; + + p = list; + while (p) { + dev = p->dev; + numa_id = dev->numa_id; + if (numa_id != target_numa) { + p = p->next; + continue; + } + + node = calloc(1, sizeof(*node)); + if (!node) { + ret = WD_ERR_PTR(-WD_ENOMEM); + goto out_free_list; + } + + node->dev = wd_clone_dev(dev); + if (!node->dev) { + ret = WD_ERR_PTR(-WD_ENOMEM); + goto out_free_node; + } + + if (!head) + head = node; + else + wd_add_dev_to_list(head, node); + + count++; + p = p->next; + } + + if (!count) + return NULL; + + /* Sort by NUMA distance */ + result = wd_sort_dev_list(head, count); + if (!result) { + ret = WD_ERR_PTR(-WD_ENODEV); + goto out_free_list; + } + + return result; + +out_free_node: + free(node); +out_free_list: + wd_free_list_accels(head); + return ret; +} + +/** + * wd_hw_alloc_ctx() - HW driver's alloc_ctx callback. + * + * Allocates ONE hardware context from UACCE device. + * Device selection strategy: + * 1. Filter devices by bmp (NUMA bitmask) + * 2. Sort by NUMA distance (nearest first) + * 3. Prefer devices on target_numa; fall back to others in distance order + * + * @alg_name: The algorithm name + * @params: Minimal allocation parameters (ctx_mode, op_type, numa_id, bmp) + * @ctx: (output) Allocated context handle + * + * Return: 0 on success, negative on failure + */ +int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) +{ + struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; + struct uacce_dev_list *dev_list, *used_list = NULL; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct uacce_dev_list *curr; + struct wd_ctx_h *ctx_h; + int target_numa; + handle_t hctx; + int ret; + + if (!params || !ctx) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + target_numa = ctx_params->numa_id; + + /* Get algorithm type and device list */ + ret = wd_get_alg_type(alg_name, alg_type); + if (ret) { + WD_ERR("invalid: alg_name is NULL!\n"); + return -WD_EINVAL; + } + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) { + WD_ERR("failed to get device list for alg %s\n", alg_name); + return -WD_ENODEV; + } + + /* Filter by bmp and sort by NUMA distance */ + used_list = wd_get_usable_list(dev_list, target_numa); + if (WD_IS_ERR(used_list) || !used_list) { + WD_INFO("Info: No usable device detected on numa<%d>\n", target_numa); + used_list = NULL; + ret = -WD_ENODEV; + goto out; + } + + curr = used_list; + while (curr) { + if (curr->dev) { + hctx = wd_request_ctx(curr->dev); + if (hctx) + goto success; + } + curr = curr->next; + } + + WD_ERR("failed to request ctx on NUMA node %d for %s\n", + target_numa, alg_name); + ret = -WD_EBUSY; + goto out; + +success: + ctx_h = (struct wd_ctx_h *)hctx; + ctx_h->priv = NULL; + ctx_h->ctx_type = UADK_ALG_HW; + *ctx = hctx; + ret = 0; +out: + if (dev_list) + wd_free_list_accels(dev_list); + if (used_list && !WD_IS_ERR(used_list)) + wd_free_list_accels(used_list); + + return ret; +} + +/** + * wd_hw_free_ctx() - HW driver's free_ctx callback. + * + * Releases ONE hardware context back to UACCE device. + * + * @ctx: The context handle to release + */ +void wd_hw_free_ctx(handle_t ctx) +{ + struct wd_ctx_h *ctx_h = (struct wd_ctx_h *)ctx; + + if (!ctx_h) { + WD_ERR("invalid: ctx is NULL!\n"); + return; + } + + /* Release hardware context back to device */ + wd_release_ctx(ctx); +} + +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id) +{ + struct wd_soft_sqe *sqe = NULL; + + if (!sctx) { + WD_ERR("invalid: sctx is NULL!\n"); + return -WD_EINVAL; + } + + pthread_spin_lock(&sctx->slock); + sqe = &sctx->qfifo[sctx->head]; + if (!sqe->used && !sqe->complete) { // find the next not used sqe + sctx->head++; + if (unlikely(sctx->head == MAX_SOFT_QUEUE_LENGTH)) + sctx->head = 0; + + sqe->used = 1; + sqe->complete = 1; + sqe->id = tag_id; + sqe->result = 0; + __atomic_fetch_add(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->slock); + } else { + pthread_spin_unlock(&sctx->slock); + return -WD_EBUSY; + } + + return WD_SUCCESS; +} + +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result) +{ + struct wd_soft_sqe *sqe = NULL; + + /* The queue is not used */ + if (!sctx || !tag_id || !result || sctx->run_num < 1) + return -WD_EAGAIN; + + if (pthread_spin_trylock(&sctx->rlock)) + return -WD_EAGAIN; + sqe = &sctx->qfifo[sctx->tail]; + if (sqe->used && sqe->complete) { // find a used sqe + sctx->tail++; + if (unlikely(sctx->tail == MAX_SOFT_QUEUE_LENGTH)) + sctx->tail = 0; + + *tag_id = sqe->id; + *result = sqe->result; + sqe->used = 0x0; + sqe->complete = 0x0; + __atomic_fetch_sub(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->rlock); + } else { + pthread_spin_unlock(&sctx->rlock); + return -WD_EAGAIN; + } + + return WD_SUCCESS; +} + +int wd_queue_is_busy(struct wd_soft_ctx *sctx) +{ + /* The queue is not used */ + if (!sctx) + return -WD_EINVAL; + + if (__atomic_load_n(&sctx->run_num, __ATOMIC_ACQUIRE) >= MAX_SOFT_QUEUE_LENGTH - 1) + return -WD_EBUSY; + + return WD_SUCCESS; +} diff --git a/drv/wd_drv.h b/drv/wd_drv.h new file mode 100644 index 0000000..17c7cc1 --- /dev/null +++ b/drv/wd_drv.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. */ +#ifndef __WD_DRV_H +#define __WD_DRV_H + +#include <numa.h> +#include <stdlib.h> +#include <stdio.h> +#include <sys/ipc.h> +#include <sys/shm.h> +#include <pthread.h> + +#include "wd.h" +#include "wd_alg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SOFT_QUEUE_LENGTH 1024U + +/** + * default queue length set to 1024 + */ +struct wd_soft_sqe { + __u8 used; + __u8 result; + __u8 complete; + __u8 rsv; + __u32 id; +}; + +struct wd_soft_ctx { + __u8 ctx_type; + __u8 rsv[3]; + pthread_spinlock_t slock; + pthread_spinlock_t rlock; + __u32 head; + __u32 tail; + __u32 run_num; + int fd; + void *priv; + struct wd_soft_sqe qfifo[MAX_SOFT_QUEUE_LENGTH]; +}; + +/* Public function declarations */ +int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx); +void wd_hw_free_ctx(handle_t ctx); + +int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx); +void wd_soft_free_ctx(handle_t ctx); + +int wd_queue_is_busy(struct wd_soft_ctx *sctx); +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id); +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result); +struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, int target_numa); + +#ifdef __cplusplus +} +#endif + +#endif /* __WD_DRV_H */ diff --git a/wd_util.c b/wd_util.c index 707cdb5..1b4c74e 100644 --- a/wd_util.c +++ b/wd_util.c @@ -8,15 +8,15 @@ #include <dirent.h> #include <dlfcn.h> #include <pthread.h> +#include <stdlib.h> #include <string.h> #include <ctype.h> #include "wd_sched.h" #include "wd_util.h" #include "wd_alg.h" #include "wd_bmm.h" +#include "wd_internal.h" -#define WD_ASYNC_DEF_POLL_NUM 1 -#define WD_ASYNC_DEF_QUEUE_DEPTH 1024 #define WD_BALANCE_THRHD 1280 #define WD_RECV_MAX_CNT_SLEEP 60000000 #define WD_RECV_MAX_CNT_NOSLEEP 200000000 @@ -25,18 +25,14 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define WD_INIT_SLEEP_UTIME 1000 -#define WD_INIT_RETRY_TIMES 10000 #define US2S(us) ((us) >> 20) #define WD_INIT_RETRY_TIMEOUT 3 -#define WD_SOFT_CTX_NUM 2 -#define WD_SOFT_SYNC_CTX 0 -#define WD_SOFT_ASYNC_CTX 1 - #define WD_DRV_LIB_DIR "uadk" #define WD_DRV_CONF_FILE "uadk.cnf" #define WD_PATH_DIR_NUM 2 +#define UADK_MAX_NUMA_NODES 64 struct msg_pool { /* message array allocated dynamically */ @@ -56,7 +52,7 @@ static const char *comp_ctx_type[2][2] = { }; /* define two ctx mode here for cipher and other alg */ -static const char *ctx_type[2][1] = { {"sync:"}, {"async:"} }; +static const char *ctx_mode_type[2][1] = { {"sync:"}, {"async:"} }; static const char *wd_env_name[WD_TYPE_MAX] = { "WD_COMP_CTX_NUM", @@ -78,7 +74,7 @@ struct drv_lib_list { static void *wd_internal_alloc(void *usr, size_t size) { - if (size != 0) + if (size) return malloc(size); else return NULL; @@ -86,7 +82,7 @@ static void *wd_internal_alloc(void *usr, size_t size) static void wd_internal_free(void *usr, void *va) { - if (va != NULL) + if (va) free(va); } @@ -163,33 +159,8 @@ int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type) return 0; } -static int wd_parse_dev_id(handle_t h_ctx) -{ - struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; - char *dev_path = ctx->dev_path; - char *last_str = NULL; - char *endptr; - int dev_id; - - if (!dev_path) - return -WD_EINVAL; - - /* Find the last '-' in the string. */ - last_str = strrchr(dev_path, '-'); - if (!last_str || *(last_str + 1) == '\0') - return -WD_EINVAL; - - /* Parse the following number */ - dev_id = strtol(last_str + 1, &endptr, DECIMAL_NUMBER); - /* Check whether it is truly all digits */ - if (*endptr != '\0' || dev_id < 0) - return -WD_EINVAL; - - return dev_id; -} - static void clone_ctx_to_internal(struct wd_ctx *ctx, - struct wd_ctx_internal *ctx_in) + struct wd_ctx_internal *ctx_in) { ctx_in->ctx = ctx->ctx; ctx_in->op_type = ctx->op_type; @@ -242,7 +213,7 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, struct wd_ctx_config *cfg) { struct wd_ctx_internal *ctxs; - char *alg_name; + const char *alg_name; __u32 i, j; int ret; @@ -264,9 +235,8 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, for (i = 0; i < cfg->ctx_num; i++) { if (!cfg->ctxs[i].ctx) { - WD_ERR("invalid: ctx is NULL!\n"); - ret = -WD_EINVAL; - goto err_out; + WD_ERR("invalid: ctx<%u> is NULL!\n", i); + break; } clone_ctx_to_internal(cfg->ctxs + i, ctxs + i); ret = pthread_spin_init(&ctxs[i].lock, PTHREAD_PROCESS_SHARED); @@ -305,6 +275,12 @@ err_shm_del: return ret; } +static void wd_sched_set_param_default(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + +} + int wd_init_sched(struct wd_sched *in, struct wd_sched *from) { if (!from->name || !from->sched_init || @@ -319,8 +295,15 @@ int wd_init_sched(struct wd_sched *in, struct wd_sched *from) return -WD_ENOMEM; in->sched_init = from->sched_init; + in->sched_uninit = from->sched_uninit; in->pick_next_ctx = from->pick_next_ctx; in->poll_policy = from->poll_policy; + in->set_param = from->set_param; + + if (!from->set_param) { + WD_ERR("set param is NULL, use default!\n"); + in->set_param = wd_sched_set_param_default; + } return 0; } @@ -334,15 +317,17 @@ void wd_clear_sched(struct wd_sched *in) in->h_sched_ctx = 0; in->name = NULL; in->sched_init = NULL; + in->sched_uninit = NULL; in->pick_next_ctx = NULL; in->poll_policy = NULL; + in->set_param = NULL; } void wd_clear_ctx_config(struct wd_ctx_config_internal *in) { __u32 i; - for (i = 0; i < in->ctx_num; i++) + for (i = 0; in->ctxs && i < in->ctx_num; i++) pthread_spin_destroy(&in->ctxs[i].lock); in->priv = NULL; @@ -352,7 +337,6 @@ void wd_clear_ctx_config(struct wd_ctx_config_internal *in) in->ctxs = NULL; } - wd_remove_ctx_list(); wd_shm_delete(in); } @@ -470,7 +454,7 @@ void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, struct msg_pool *p; __u32 msg_num; - if ((__u32)ctx_idx > pool->pool_num) { + if ((__u32)ctx_idx >= pool->pool_num) { WD_ERR("invalid: message ctx id index is %d!\n", ctx_idx); return NULL; } @@ -478,7 +462,7 @@ void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, msg_num = p->msg_num; /* tag value start from 1 */ - if (tag == 0 || tag > msg_num) { + if (!tag || tag > msg_num) { WD_ERR("invalid: message cache tag is %u!\n", tag); return NULL; } @@ -707,7 +691,7 @@ static int wd_alloc_numa(struct wd_env_config *config, /* get numa num and device num of each numa from uacce_dev list */ config->numa_num = wd_get_dev_numa(head, numa_dev_num, max_node); - if (config->numa_num == 0 || config->numa_num > max_node) { + if (!config->numa_num || config->numa_num > max_node) { WD_ERR("invalid: numa number is %u!\n", config->numa_num); ret = -WD_ENODEV; goto free_list; @@ -743,7 +727,7 @@ static int is_number(const char *str) return 0; len = strlen(str); - if (len == 0) + if (!len) return 0; if (len != 1 && str[0] == '0') @@ -875,7 +859,7 @@ static int get_and_fill_ctx_num(struct wd_env_config_per_numa *config_numa, for (i = 0; i < CTX_MODE_MAX; i++) for (j = 0; j < config_numa->op_type_num; j++) { if (config_numa->op_type_num == 1) - type = ctx_type[i][j]; + type = ctx_mode_type[i][j]; else type = comp_ctx_type[i][j]; @@ -1265,6 +1249,7 @@ static int wd_sched_fill_table(struct wd_env_config_per_numa *config_numa, param.type = i; param.begin = ctx_table[mode][i].begin; param.end = ctx_table[mode][i].end; + param.ctx_prop = UADK_ALG_HW; ret = wd_sched_rr_instance(sched, ¶m); if (ret) return ret; @@ -1439,6 +1424,9 @@ int wd_check_ctx(struct wd_ctx_config_internal *config, __u8 mode, __u32 idx) { struct wd_ctx_internal *ctx; + if (unlikely(idx == QUEUE_FULL_POS)) + return -WD_EBUSY; + if (unlikely(idx >= config->ctx_num)) { WD_ERR("failed to pick a proper ctx: idx %u!\n", idx); return -WD_EINVAL; @@ -1476,8 +1464,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en) return 0; } -int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en) +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en) { __u64 timeout = WD_RECV_MAX_CNT_NOSLEEP; __u64 rx_cnt = 0; @@ -1486,7 +1474,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand if (balance) timeout = WD_RECV_MAX_CNT_SLEEP; - ret = msg_handle->send(drv, ctx, msg); + ret = msg_handle->send(ctx, msg); if (unlikely(ret < 0)) { WD_ERR("failed to send msg to hw, ret = %d!\n", ret); return ret; @@ -1499,7 +1487,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand WD_ERR("wd ctx wait timeout(%d)!\n", ret); } - ret = msg_handle->recv(drv, ctx, msg); + ret = msg_handle->recv(ctx, msg); if (ret != -WD_EAGAIN) { if (unlikely(ret < 0)) { WD_ERR("failed to recv msg: error = %d!\n", ret); @@ -1527,18 +1515,50 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) { if (!config || !config->ctxs || !config->ctxs[0].ctx) { - WD_ERR("invalid: config or config->ctxs is NULL!\n"); + WD_ERR("invalid: wd_ctx_config is NULL!\n"); return -WD_EINVAL; } if (!sched) { - WD_ERR("invalid: sched is NULL!\n"); + WD_ERR("invalid: wd_sched is NULL!\n"); return -WD_EINVAL; } return 0; } +int wd_alg_try_init(enum wd_status *status) +{ + enum wd_status expected; + __u32 count = 0; + bool ret; + + /* + * Here is aimed to protect the security of the initialization interface + * in the multi-thread scenario. Only one thread can get the WD_INITING + * status to initialize algorithm. Other thread will wait for the result. + * And the algorithm initialization interfaces is a liner process. + * So the initing thread will return a result to notify other thread go on. + */ + do { + expected = WD_UNINIT; + ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if (expected == WD_INIT) { + WD_ERR("The algorithm has been initialized!\n"); + return -WD_EEXIST; + } + usleep(WD_INIT_SLEEP_UTIME); + + if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { + WD_ERR("The algorithm initialize wait timeout!\n"); + return -WD_ETIMEDOUT; + } + } while (!ret); + + return 0; +} + static int wd_alg_init_fallback(struct wd_alg_driver *fb_driver) { if (!fb_driver->init) { @@ -1561,47 +1581,103 @@ static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) fb_driver->exit(NULL); } -int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) +static int wd_ctx_init_driver(struct wd_ctx_config_internal *config, + struct wd_alg_driver *driver) { + void *priv; int ret; - if (!driver->init) { - driver->fallback = 0; - WD_ERR("driver have no init interface.\n"); - ret = -WD_EINVAL; - goto err_alloc; + if (!driver || !driver->priv_size) + return -WD_EINVAL; + + if (driver->ops_size) { + driver->extend_ops = calloc(1, driver->ops_size); + if (!driver->extend_ops) + return -WD_ENOMEM; + } else { + driver->extend_ops = NULL; + } + + priv = calloc(1, driver->priv_size); + if (!priv) { + if (driver->extend_ops) + free(driver->extend_ops); + + return -WD_ENOMEM; } - ret = driver->init(driver, config); + ret = driver->init(config, priv); if (ret < 0) { - WD_ERR("driver init failed.\n"); - goto err_alloc; + if (driver->extend_ops) + free(driver->extend_ops); + + free(priv); + return ret; } + driver->drv_data = priv; if (driver->fallback) { ret = wd_alg_init_fallback((struct wd_alg_driver *)driver->fallback); - if (ret) { + if (ret) driver->fallback = 0; - WD_ERR("soft alg driver init failed.\n"); - } } return 0; +} -err_alloc: - return ret; +static void wd_ctx_uninit_driver(struct wd_alg_driver *driver) +{ + void *priv; + + if (!driver) + return; + priv = driver->drv_data; + if (!priv) + return; + driver->exit(priv); + free(priv); + driver->drv_data = NULL; + if (driver->extend_ops) { + free(driver->extend_ops); + driver->extend_ops = NULL; + } + + if (driver->fallback) + wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); } -void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) +int wd_alg_init_driver(struct wd_ctx_config_internal *config) { - driver->exit(driver); + __u32 i, j; + int ret; + + /* Only initialize the drivers that have been filtered and selected for use. */ + for (i = 0; i < config->drv_count; i++) { + ret = wd_ctx_init_driver(config, config->drv_array[i]); + if (ret) + goto init_err; + } + + return 0; + +init_err: + for (j = 0; j < i; j++) + wd_ctx_uninit_driver(config->drv_array[j]); /* Ctx config just need clear once */ wd_clear_ctx_config(config); - if (driver->fallback) - wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); + return ret; +} + +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config) +{ + __u32 i; + + for (i = 0; i < config->drv_count; i++) + wd_ctx_uninit_driver(config->drv_array[i]); + + /* Ctx config just need clear once */ + wd_clear_ctx_config(config); } void wd_dlclose_drv(void *dlh_list) @@ -1633,12 +1709,11 @@ static void add_lib_to_list(struct drv_lib_list *head, tmp->next = node; } -static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, struct uacce_dev_list *list, +static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, const char *section, __u32 op_type_num, int is_comp) { struct wd_ctx_nums *ctxs = ctx_params->ctx_set_num; int ret, ctx_num, node; - struct uacce_dev *dev; char *ctx_section; const char *type; __u32 i, j; @@ -1657,13 +1732,16 @@ static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, struct uacce_dev_li if (!ctx_num) return 0; - dev = wd_find_dev_by_numa(list, node); - if (WD_IS_ERR(dev)) - return -WD_ENODEV; + /* Validate node is within the system's NUMA node range */ + if (node < 0 || node > numa_max_node()) { + WD_ERR("invalid: numa node %d exceeds system max node %d!\n", + node, numa_max_node()); + return -WD_EINVAL; + } for (i = 0; i < CTX_MODE_MAX; i++) { for (j = 0; j < op_type_num; j++) { - type = is_comp ? comp_ctx_type[i][j] : ctx_type[i][0]; + type = is_comp ? comp_ctx_type[i][j] : ctx_mode_type[i][0]; if (strncmp(section, type, strlen(type))) continue; @@ -1685,14 +1763,12 @@ static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, struct uacce_dev_li static int wd_env_set_ctx_nums(const char *alg_name, const char *name, const char *var_s, struct wd_ctx_params *ctx_params, __u32 op_type_num) { - char alg_type[CRYPTO_MAX_ALG_NAME]; char *left, *section, *start; - struct uacce_dev_list *list; int is_comp; int ret; /* COMP environment variable's format is different, mark it */ - is_comp = strncmp(name, "WD_COMP_CTX_NUM", strlen(name)) ? 0 : 1; + is_comp = strncmp(name, "WD_COMP_CTX_NUM", sizeof("WD_COMP_CTX_NUM") - 1) ? 0 : 1; if (is_comp && op_type_num > ARRAY_SIZE(comp_ctx_type)) return -WD_EINVAL; @@ -1700,26 +1776,13 @@ static int wd_env_set_ctx_nums(const char *alg_name, const char *name, const cha if (!start) return -WD_ENOMEM; - ret = wd_get_alg_type(alg_name, alg_type); - if (ret) - goto free_start; - - list = wd_get_accel_list(alg_type); - if (!list) { - WD_ERR("failed to get devices!\n"); - ret = -WD_ENODEV; - goto free_start; - } - left = start; while ((section = strsep(&left, ","))) { - ret = wd_set_ctx_nums(ctx_params, list, section, op_type_num, is_comp); + ret = wd_set_ctx_nums(ctx_params, section, op_type_num, is_comp); if (ret < 0) break; } - wd_free_list_accels(list); -free_start: free(start); return ret; } @@ -1731,8 +1794,8 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params) int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type) + char *alg, enum wd_type type, + int max_op_type) { const char *env_name = wd_env_name[type]; const char *var_s; @@ -1746,9 +1809,9 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, /* Only hw driver support environment variable */ var_s = secure_getenv(env_name); - if (var_s && strlen(var_s) && driver->calc_type == UADK_ALG_HW) { + if (var_s && strlen(var_s)) { /* environment variable has the highest priority */ - ret = wd_env_set_ctx_nums(driver->alg_name, env_name, var_s, + ret = wd_env_set_ctx_nums(alg, env_name, var_s, ctx_params, max_op_type); if (ret) { WD_ERR("fail to init ctx nums from %s!\n", env_name); @@ -1758,38 +1821,26 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, } else { /* environment variable is not set, try to use user_ctx_params first */ if (user_ctx_params) { - if (user_ctx_params->bmp) { - copy_bitmask_to_bitmask(user_ctx_params->bmp, ctx_params->bmp); - } else { - /* default value */ - numa_bitmask_setall(ctx_params->bmp); - } - ctx_params->cap = user_ctx_params->cap; - ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; - ctx_params->op_type_num = user_ctx_params->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { + copy_bitmask_to_bitmask(user_ctx_params->bmp, ctx_params->bmp); + if (user_ctx_params->op_type_num > (__u32)max_op_type) { WD_ERR("fail to check user op type numbers.\n"); numa_free_nodemask(ctx_params->bmp); return -WD_EINVAL; } + ctx_params->cap = user_ctx_params->cap; + ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; + ctx_params->op_type_num = user_ctx_params->op_type_num; return 0; } - /* user_ctx_params is also not set, use driver's defalut queue_num */ numa_bitmask_setall(ctx_params->bmp); - for (i = 0; i < driver->op_type_num; i++) { - ctx_params->ctx_set_num[i].sync_ctx_num = driver->queue_num; - ctx_params->ctx_set_num[i].async_ctx_num = driver->queue_num; + for (i = 0; i < max_op_type; i++) { + ctx_params->ctx_set_num[i].sync_ctx_num = 1; + ctx_params->ctx_set_num[i].async_ctx_num = 1; } } - - ctx_params->op_type_num = driver->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { - WD_ERR("fail to check driver op type numbers.\n"); - numa_free_nodemask(ctx_params->bmp); - return -WD_EAGAIN; - } + ctx_params->op_type_num = max_op_type; return 0; } @@ -1949,7 +2000,7 @@ static void create_lib_to_list(const char *lib_path, struct drv_lib_list **head) node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); if (!node->dlhandle) { - WD_ERR("failed to open lib file: %s, skipped\n", lib_path); + WD_ERR("failed to open lib file: %s, err: %s\n", lib_path, dlerror()); free(node); return; } @@ -2117,604 +2168,1030 @@ free_path: return (void *)head; } -struct wd_alg_driver *wd_alg_drv_bind(int task_type, const char *alg_name) +/** + * wd_ctx_unbind_drivers() - Unbind drivers from internal contexts. + * + * Decrements driver refcounts and clears all drv pointers. + * + * @config: Internal ctx config + */ +void wd_ctx_unbind_drivers(struct wd_ctx_config_internal *config) +{ + __u32 i; + + if (!config || !config->drv_array) + return; + + wd_alg_drv_ref_dec(config->drv_array, config->drv_count); + + for (i = 0; i < config->ctx_num; i++) + config->ctxs[i].drv = NULL; +} + +/** + * wd_ctx_bind_drivers() - Bind drivers to internal contexts via round-robin. + * + * This is the single write point for ctxs[i].drv in the entire lifecycle. + * Uses RR rule: ctxs[i].drv = drv_array[i % drv_count] + * + * Also: + * - Sets up soft fallback for HW drivers (once per unique HW driver) + * - Caches drv_array in config for session queries + * - Increments driver refcounts (deduplicated: each unique driver +1) + * + * @config: Internal ctx config (ctxs[] already copied by wd_init_ctx_config) + * @drv_array: Discovered unique drivers + * @drv_count: Number of unique drivers + * Return: 0 on success, negative on failure + */ +int wd_ctx_bind_drivers(struct wd_ctx_config_internal *config_api, + struct wd_ctx_config_internal *config_in, int init_type) { - struct wd_alg_driver *set_driver = NULL; struct wd_alg_driver *drv; + __u32 i; + + if (!config_api || init_type > WD_TYPE_V2) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } - /* Get alg driver and dev name */ - switch (task_type) { - case TASK_INSTR: - drv = wd_request_drv(alg_name, true); - if (!drv) { - WD_ERR("no soft %s driver support\n", alg_name); - return NULL; + if (init_type == WD_TYPE_V1) { + if (!config_api->drv_array || config_api->drv_count != 1) { + WD_ERR("invalid: config driver number is error!\n"); + return -WD_EINVAL; } - set_driver = drv; - set_driver->fallback = 0; - break; - case TASK_HW: - case TASK_MIX: - drv = wd_request_drv(alg_name, false); - if (!drv) { - WD_ERR("no HW %s driver support\n", alg_name); - return NULL; + for (i = 0; i < config_api->ctx_num; i++) { + config_api->ctxs[i].drv = config_api->drv_array[0]; + config_api->ctxs[i].ctx_type = config_api->drv_array[0]->calc_type; } - set_driver = drv; - set_driver->fallback = 0; - if (task_type == TASK_MIX) { - drv = wd_request_drv(alg_name, true); - if (!drv) { - set_driver->fallback = 0; - WD_ERR("no soft %s driver support\n", alg_name); - } else { - set_driver->fallback = (handle_t)drv; - WD_ERR("successful to get soft driver\n"); - } + + drv = config_api->drv_array[0]; + if (!drv->fallback) { + drv->fallback = (handle_t)wd_request_drv( + config_api->alg_name, ALG_DRV_FB); } - break; - default: - WD_ERR("task type error.\n"); - return NULL; + return WD_SUCCESS; } - return set_driver; + if (!config_in || !config_in->drv_array || !config_in->drv_count) { + WD_ERR("invalid: V2 parameters, config_in=%p, drv_array=%p, ctx_num=%u!\n", + config_in, config_in ? config_in->drv_array : NULL, + config_in ? config_in->ctx_num : 0); + return -WD_EINVAL; + } + + WD_DEBUG("discovered %u drivers for ctx binding\n", config_in->drv_count); + for (i = 0; i < config_in->ctx_num; i++) { + if (!config_in->ctxs[i].drv) { + WD_ERR("failed to check ctx<%u> driver bound in internal config!\n", i); + continue; + } + /* + * The internally allocated queues have already been bound to the drivers, + * so only direct assignment processing is required here. + */ + config_api->ctxs[i].drv = config_in->ctxs[i].drv; + config_api->ctxs[i].ctx_type = config_in->ctxs[i].ctx_type; + } + + /* HW driver needs soft fallback — set once per unique driver */ + for (i = 0; i < config_in->drv_count; i++) { + drv = config_in->drv_array[i]; + if (!drv) + continue; + + if (drv->calc_type == UADK_ALG_HW && !drv->fallback) { + drv->fallback = (handle_t)wd_request_drv( + config_api->alg_name, ALG_DRV_FB); + WD_DEBUG("Set fallback for HW driver %s\n", drv->drv_name); + } + } + + /* Cache driver array for session queries */ + config_api->drv_array = config_in->drv_array; + config_api->drv_count = config_in->drv_count; + + /* Deduplicated refcount increment */ + wd_alg_drv_ref_inc(config_in->drv_array, config_in->drv_count); + + return WD_SUCCESS; } -void wd_alg_drv_unbind(struct wd_alg_driver *drv) +/** + * wd_alg_config_uninit() - Free driver discovery result. + * + * Releases the drv_array allocated by wd_alg_drv_discover(). + * Does NOT touch the drivers themselves (refcount managed separately). + * + * @attrs: Initialization attributes + */ +void wd_alg_config_uninit(struct wd_init_attrs *attrs) { - struct wd_alg_driver *fb_drv = NULL; + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; - if (!drv) + if (!internal_config || !internal_config->drv_array) return; - fb_drv = (struct wd_alg_driver *)drv->fallback; - if (fb_drv) - wd_release_drv(fb_drv); - wd_release_drv(drv); + /* Release wd_get_drv_array alloc memory */ + wd_put_drv_array(internal_config->drv_array, internal_config->drv_count); + internal_config->drv_array = NULL; + internal_config->drv_count = 0; + + /* Release ctx_config_internal */ + if (internal_config->ctxs) + free(internal_config->ctxs); + free(internal_config); + attrs->ctx_config_internal = NULL; } -int wd_alg_try_init(enum wd_status *status) +static __u32 wd_ctx_num_sum(struct wd_ctx_params *ctx_params, + struct wd_alg_driver **drv_array, + __u32 drv_count) { - enum wd_status expected; - __u32 count = 0; - bool ret; + __u32 total_ctx_num = 0; + __u32 async_num = 0; + __u32 sync_num = 0; + __u32 numa_count = 0; + __u32 per_driver_ctx; + int max_node, n; + __u32 i; - do { - expected = WD_UNINIT; - ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); - if (expected == WD_INIT) { - WD_ERR("The algorithm has been initialized!\n"); - return -WD_EEXIST; + for (i = 0; i < ctx_params->op_type_num; i++) { + sync_num += ctx_params->ctx_set_num[i].sync_ctx_num; + async_num += ctx_params->ctx_set_num[i].async_ctx_num; + } + per_driver_ctx = sync_num + async_num; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > NUMA_NUM_NODES) + max_node = NUMA_NUM_NODES; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(ctx_params->bmp, n)) + numa_count++; + } + if (!numa_count) + numa_count = 1; + + for (i = 0; i < drv_count; i++) { + if (drv_array[i]->calc_type == UADK_ALG_HW) + total_ctx_num += per_driver_ctx * numa_count; + else + total_ctx_num += per_driver_ctx; + } + + WD_DEBUG("total ctxs: %u (per_driver=%u, hw_numa=%u, drv_count=%u)\n", + total_ctx_num, per_driver_ctx, numa_count, drv_count); + + return total_ctx_num; +} + +/** + * wd_alg_config_init() - Discover matching drivers. + * + * Normalizes attrs->alg to alg_type ("cipher", "digest", etc.), + * then calls wd_get_drv_array() to find all unique drivers. + * Results stored in attrs->drv_array and attrs->drv_count. + * Filter drivers by sched_policy: NONE→CE, SINGLE→SVE. + * In-place removes non-matching entries, keeps at most 1 (highest priority). + * Returns filtered count via drv_count, or negative on no match. + * + * Pure query — no resource allocation, no refcount changes. + * + * @attrs: Initialization attributes (input: alg, task_type; output: drv_array, drv_count) + * Return: 0 on success, negative on failure + */ +static int wd_filter_drv_by_sched(__u32 sched_type, + struct wd_alg_driver **drv_array, + __u32 *drv_count) +{ + int primary_type, fallback_type = -1; + __u32 drv_array_cnt = *drv_count; + __u32 i, kept = 0; + + if (sched_type == SCHED_POLICY_NONE) { + primary_type = UADK_ALG_CE_INSTR; + } else if (sched_type == SCHED_POLICY_SINGLE) { + /* SINGLE: prefer SVE, fall back to CE when SVE unavailable */ + primary_type = UADK_ALG_SVE_INSTR; + fallback_type = UADK_ALG_CE_INSTR; + } else { + return 0; + } + + for (i = 0; i < drv_array_cnt; i++) { + if (drv_array[i]->calc_type == primary_type) { + if (kept != i) + drv_array[kept] = drv_array[i]; + kept++; } - usleep(WD_INIT_SLEEP_UTIME); + } - if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { - WD_ERR("The algorithm initialize wait timeout!\n"); - return -WD_ETIMEDOUT; + /* Fallback to secondary type only if primary yielded nothing */ + if (!kept && fallback_type >= 0) { + for (i = 0; i < drv_array_cnt; i++) { + if (drv_array[i]->calc_type == fallback_type) { + if (kept != i) + drv_array[kept] = drv_array[i]; + kept++; + } } - } while (!ret); + } + + *drv_count = kept; + + if (!kept) { + WD_ERR("invalid: no %s driver found for %s scheduler\n", + sched_type == SCHED_POLICY_NONE ? "CE" : "SVE/CE", + sched_type == SCHED_POLICY_NONE ? "NONE" : "SINGLE"); + return -WD_EINVAL; + } + /* Keep only the first (highest priority) */ + if (kept > 1) + *drv_count = 1; return 0; } -static __u32 wd_get_ctx_numbers(struct wd_ctx_params ctx_params, int end) +int wd_alg_config_init(struct wd_init_attrs *attrs) { - __u32 count = 0; - int i; + struct wd_ctx_config_internal *internal_config = NULL; + struct wd_alg_driver **temp_drv_array = NULL; + char alg_type[CRYPTO_MAX_ALG_NAME] = {0}; + __u32 tmp_drv_count; + __u32 tmp_ctx_num; + int ret; - for (i = 0; i < end; i++) { - count += ctx_params.ctx_set_num[i].sync_ctx_num; - count += ctx_params.ctx_set_num[i].async_ctx_num; + if (!attrs || !attrs->alg[0] || !attrs->ctx_params) + return -WD_EINVAL; + + /* Normalize alg to alg_type (e.g. "cipher", "digest") */ + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret || !alg_type[0]) { + WD_ERR("failed to get alg type for %s!\n", attrs->alg); + return -WD_EINVAL; } - return count; + /* Driver discovery */ + ret = wd_get_drv_array(alg_type, attrs->task_type, NULL, + &temp_drv_array, &tmp_drv_count); + if (ret || !tmp_drv_count) { + WD_ERR("failed to get %s's driver array!\n", attrs->alg); + goto driver_error; + } + + /* Filter drivers by sched_policy semantic constraints */ + ret = wd_filter_drv_by_sched(attrs->sched_type, temp_drv_array, &tmp_drv_count); + if (ret) + goto driver_error; + + /* Calculate total sync/async context counts */ + tmp_ctx_num = wd_ctx_num_sum(attrs->ctx_params, temp_drv_array, tmp_drv_count); + if (!tmp_ctx_num) { + WD_ERR("invalid: total_ctx_num is zero!\n"); + ret = -WD_EINVAL; + goto driver_error; + } + + /* Allocate internal ctx_config structure */ + internal_config = calloc(1, sizeof(*internal_config)); + if (!internal_config) { + ret = -WD_ENOMEM; + WD_ERR("failed to allocate ctx_config_internal!\n"); + goto driver_error; + } + + /* Allocate internal ctx array */ + internal_config->ctxs = calloc(tmp_ctx_num, sizeof(struct wd_ctx_internal)); + if (!internal_config->ctxs) { + WD_ERR("failed to allocate internal ctxs array!\n"); + ret = -WD_ENOMEM; + goto clean_config; + } + + /* Initialize configuration */ + internal_config->ctx_num = tmp_ctx_num; + internal_config->drv_array = temp_drv_array; + internal_config->drv_count = tmp_drv_count; + attrs->ctx_config_internal = internal_config; + + WD_DEBUG("Algorithm initialization started: alg=%s, task_type=%u\n", + attrs->alg, attrs->task_type); + + return WD_SUCCESS; + +clean_config: + free(internal_config); +driver_error: + wd_put_drv_array(temp_drv_array, tmp_drv_count); + attrs->ctx_config_internal = NULL; + return ret; } -static struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, struct bitmask *bmp) +static int wd_parse_dev_id(handle_t h_ctx) { - struct uacce_dev_list *p, *node, *result = NULL; - struct uacce_dev *dev; - int numa_id, ret; + struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + char *dev_path = ctx->dev_path; + char *last_str = NULL; + char *endptr; + int dev_id; - if (!bmp) { - WD_ERR("invalid: bmp is NULL!\n"); - return WD_ERR_PTR(-WD_EINVAL); - } + if (!dev_path) + return -WD_EINVAL; - p = list; - while (p) { - dev = p->dev; - numa_id = dev->numa_id; - ret = numa_bitmask_isbitset(bmp, numa_id); - if (!ret) { - p = p->next; - continue; - } + last_str = strrchr(dev_path, '-'); + if (!last_str || *(last_str + 1) == '\0') + return -WD_EINVAL; - node = calloc(1, sizeof(*node)); - if (!node) { - result = WD_ERR_PTR(-WD_ENOMEM); - goto out_free_list; - } + dev_id = strtol(last_str + 1, &endptr, DECIMAL_NUMBER); + if (*endptr != '\0' || dev_id < 0) + return -WD_EINVAL; - node->dev = wd_clone_dev(dev); - if (!node->dev) { - result = WD_ERR_PTR(-WD_ENOMEM); - goto out_free_node; - } + return dev_id; +} - if (!result) - result = node; - else - wd_add_dev_to_list(result, node); +static int wd_sched_ctx_region_key(handle_t ctx, __u8 ctx_type, + bool is_dev_policy, int *numa_id) +{ + struct wd_ctx_h *hctx; + int dev_id; + + if (is_dev_policy) { + dev_id = wd_parse_dev_id(ctx); + if (dev_id < 0) + return dev_id; - p = p->next; + *numa_id = dev_id; + return dev_id; } - return result ? result : WD_ERR_PTR(-WD_ENODEV); + *numa_id = 0; + if (ctx_type != UADK_ALG_HW) + return 0; + + hctx = (struct wd_ctx_h *)ctx; + if (hctx->dev) + *numa_id = hctx->dev->numa_id; -out_free_node: - free(node); -out_free_list: - wd_free_list_accels(result); - return result; + return 0; } -static int wd_init_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *list, - __u32 idx, int numa_id, __u32 op_type) +static int wd_alg_sched_instance(struct wd_sched *sched, + struct wd_ctx_config_internal *internal_config) { - struct wd_ctx_nums ctx_nums = attrs->ctx_params->ctx_set_num[op_type]; - __u32 ctx_set_num = ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num; - struct wd_ctx_config *ctx_config = attrs->ctx_config; - __u32 count = idx + ctx_set_num; - struct uacce_dev *dev; - __u32 i, cnt = 0; - - /* If the ctx set number is 0, the initialization is skipped. */ - if (!ctx_set_num) - return -WD_ENOPROC; - - dev = wd_find_dev_by_numa(list, numa_id); - if (WD_IS_ERR(dev)) - return WD_PTR_ERR(dev); - - for (i = idx; i < count; i++) { - ctx_config->ctxs[i].ctx = wd_request_ctx(dev); - if (errno == WD_EBUSY) { - dev = wd_find_dev_by_numa(list, numa_id); - if (WD_IS_ERR(dev)) - return WD_PTR_ERR(dev); - - if (cnt++ > WD_INIT_RETRY_TIMES) { - WD_ERR("failed to request enough ctx due to timeout!\n"); - return -WD_ETIMEDOUT; + struct wd_ctx_internal *cur, *nxt = NULL; + struct sched_params sparams; + int cur_rgn_key, cur_numa_id; + int nxt_rgn_key, nxt_numa_id; + __u32 seg_begin, seg_end; + __u8 mode, inctx_type; + __u32 op_type, i = 0; + bool is_dev; + int ret; + + if (!sched || !internal_config) { + WD_ERR("invalid: sched, ctx_config, or ctx_params is NULL!\n"); + return -WD_EINVAL; + } + + if (!sched || !internal_config || !internal_config->ctxs) { + WD_ERR("invalid: internal_config->ctxs is NULL!\n"); + return -WD_EINVAL; + } + + is_dev = (sched->sched_policy == SCHED_POLICY_DEV); + for (i = 0; i < internal_config->ctx_num;) { + cur = &internal_config->ctxs[i]; + mode = internal_config->ctxs[i].ctx_mode; + op_type = internal_config->ctxs[i].op_type; + inctx_type = internal_config->ctxs[i].ctx_type; + + cur_rgn_key = wd_sched_ctx_region_key(cur->ctx, inctx_type, is_dev, &cur_numa_id); + if (cur_rgn_key < 0) { + WD_ERR("failed to parse region key for ctx %u!\n", i); + return -WD_EINVAL; + } + + /* Scan forward for contiguous ctxs with identical segment key */ + seg_begin = i; + for (seg_end = seg_begin; seg_end + 1 < internal_config->ctx_num; seg_end++) { + nxt = &internal_config->ctxs[seg_end + 1]; + if (nxt->ctx_mode != mode || nxt->op_type != op_type || + nxt->ctx_type != inctx_type) + break; + + nxt_rgn_key = wd_sched_ctx_region_key(nxt->ctx, nxt->ctx_type, + is_dev, &nxt_numa_id); + if (nxt_rgn_key < 0) { + WD_ERR("failed to parse region key for ctx %u!\n", seg_end + 1); + return -WD_EINVAL; } + if (nxt_rgn_key != cur_rgn_key || nxt_numa_id != cur_numa_id) + break; + } - /* self-decrease i to eliminate self-increase on next loop */ - i--; - continue; - } else if (!ctx_config->ctxs[i].ctx) { - /* - * wd_release_ctx_set will release ctx in - * caller wd_init_ctx_and_sched. - */ - return -WD_ENOMEM; + /* Register segment to scheduler */ + memset(&sparams, 0, sizeof(sparams)); + sparams.numa_id = cur_numa_id; + sparams.dev_id = cur_rgn_key; + sparams.type = op_type; + sparams.mode = mode; + sparams.begin = seg_begin; + sparams.end = seg_end; + sparams.ctx_prop = inctx_type; + + ret = wd_sched_rr_instance(sched, &sparams); + if (ret) { + WD_ERR("failed to register ctx[%u, %u] (op_type=%u, mode=%u, prop=%d)!\n", + seg_begin, seg_end, op_type, mode, inctx_type); + return ret; } - ctx_config->ctxs[i].op_type = op_type; - ctx_config->ctxs[i].ctx_mode = - ((i - idx) < ctx_nums.sync_ctx_num) ? - CTX_MODE_SYNC : CTX_MODE_ASYNC; + + i = seg_end + 1; } - return 0; + return WD_SUCCESS; } -static void wd_release_ctx_set(struct wd_ctx_config *ctx_config) +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) { + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_alg_driver *drv; __u32 i; - for (i = 0; i < ctx_config->ctx_num; i++) - if (ctx_config->ctxs[i].ctx) { - wd_release_ctx(ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; - } + if (!internal_config || !internal_config->ctxs || !allocated_count) + return; + + for (i = 0; i < allocated_count; i++) { + if (!internal_config->ctxs[i].ctx) + continue; + + drv = internal_config->ctxs[i].drv; + if (drv && drv->free_ctx) + drv->free_ctx(internal_config->ctxs[i].ctx); + + internal_config->ctxs[i].ctx = 0; + } } -static int wd_instance_sched_set(struct wd_init_attrs *attrs, struct wd_ctx_nums ctx_nums, - int idx, int numa_id, int op_type) +static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + __u8 ctx_mode, __u8 op_type, + __u32 *ctx_idx) { - struct wd_sched *sched = attrs->sched; - struct sched_params sparams; - int i, end, dev_id, ret = 0; - - dev_id = wd_parse_dev_id(attrs->ctx_config->ctxs[idx].ctx); - if (dev_id < 0) - return -WD_EINVAL; + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + struct wd_drv_ctx_params dparams; + int numa_nodes[UADK_MAX_NUMA_NODES]; + __u32 mode_ctx_num, numa_count = 0; + __u32 numa_idx, j; + int max_node, n; + handle_t ctx; + int ret; - for (i = 0; i < CTX_MODE_MAX; i++) { - sparams.numa_id = numa_id; - sparams.type = op_type; - sparams.dev_id = dev_id; - sparams.mode = i; - sparams.begin = idx + ctx_nums.sync_ctx_num * i; - end = idx - 1 + ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num * i; - if (end < 0 || sparams.begin > (__u32)end) - continue; + if (ctx_mode == CTX_MODE_SYNC) + mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; + else + mode_ctx_num = ctx_params->ctx_set_num[op_type].async_ctx_num; + if (!mode_ctx_num) + return WD_SUCCESS; + + if ((attrs->sched_type == SCHED_POLICY_NONE || + attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) + mode_ctx_num = 1; + + if (drv->calc_type == UADK_ALG_HW) { + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(ctx_params->bmp, n)) + numa_nodes[numa_count++] = n; + } + } else { + numa_nodes[0] = 0; + numa_count = 1; + } + if (!numa_count) { + numa_nodes[0] = 0; + numa_count = 1; + } + + for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + for (j = 0; j < mode_ctx_num; j++) { + memset(&dparams, 0, sizeof(dparams)); + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.numa_id = numa_nodes[numa_idx]; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); + if (!ctx || ret < 0) { + if (ret == -WD_ENODEV) + break; + WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", + *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); + return ret; + } - sparams.end = end; - ret = wd_sched_rr_instance(sched, &sparams); - if (ret) - goto out; + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + } } -out: - return ret; + return WD_SUCCESS; } -static int wd_init_ctx_and_sched(struct wd_init_attrs *attrs, struct bitmask *bmp, - struct uacce_dev_list *list) +static int wd_alloc_ctxs_batch(struct wd_init_attrs *attrs, + __u8 ctx_mode, __u32 *start_idx) { + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - __u32 op_type_num = ctx_params->op_type_num; - int i, ret, max_node = numa_max_node() + 1; - struct wd_ctx_nums ctx_nums; - __u32 j, idx = 0; + struct wd_alg_driver *drv; + __u8 op_type, op_type_num; + __u32 ctx_idx, drv_idx; + int ret; - for (i = 0; i < max_node; i++) { - if (!numa_bitmask_isbitset(bmp, i)) - continue; - for (j = 0; j < op_type_num; j++) { - ctx_nums = ctx_params->ctx_set_num[j]; - ret = wd_init_ctx_set(attrs, list, idx, i, j); - if (ret == -WD_ENOPROC) - continue; - else if (ret) - goto free_ctxs; - ret = wd_instance_sched_set(attrs, ctx_nums, idx, i, j); + op_type_num = ctx_params->op_type_num; + if (attrs->sched_type == SCHED_POLICY_NONE || + attrs->sched_type == SCHED_POLICY_SINGLE) + op_type_num = 1; + + ctx_idx = *start_idx; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + + for (op_type = 0; op_type < op_type_num; op_type++) { + ret = wd_alloc_single_drv_ctxs(attrs, drv, + ctx_mode, op_type, + &ctx_idx); if (ret) - goto free_ctxs; - idx += (ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num); + goto err_ctxs; } } + *start_idx = ctx_idx; - return 0; - -free_ctxs: - wd_release_ctx_set(attrs->ctx_config); + return WD_SUCCESS; +err_ctxs: + wd_free_ctxs_batch(attrs, ctx_idx); return ret; } -static void wd_init_device_nodemask(struct uacce_dev_list *list, struct bitmask *bmp) +/** + * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. + * + * Releases resources in reverse allocation order: + * 1. Release scheduler + * 2. Release ctxs via RR rule (drv->free_ctx) + * 3. Free ctx_config and ctxs array + * + * @attrs: Initialization attributes + */ +void wd_alg_ctx_uninit(struct wd_init_attrs *attrs) { - struct uacce_dev_list *p = list; + struct wd_ctx_config_internal *internal_config; - numa_bitmask_clearall(bmp); - while (p) { - numa_bitmask_setbit(bmp, p->dev->numa_id); - p = p->next; - } -} + if (!attrs) + return; -static int wd_alg_ctx_init(struct wd_init_attrs *attrs) -{ - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct bitmask *used_bmp = ctx_params->bmp; - struct uacce_dev_list *list, *used_list = NULL; - __u32 ctx_set_num, op_type_num; - int numa_cnt, ret; + internal_config = attrs->ctx_config_internal; - list = wd_get_accel_list(attrs->alg); - if (!list) { - WD_ERR("failed to get devices for alg: %s\n", attrs->alg); - return -WD_ENODEV; + WD_DEBUG("releasing ctxs, scheduler, and ctx_config\n"); + /* Release scheduler */ + if (attrs->sched) { + wd_sched_rr_release(attrs->sched); + attrs->sched = NULL; } - op_type_num = ctx_params->op_type_num; - ctx_set_num = wd_get_ctx_numbers(*ctx_params, op_type_num); - if (!ctx_set_num || !op_type_num) { - WD_ERR("invalid: ctx_set_num is %u, op_type_num is %u!\n", - ctx_set_num, op_type_num); - ret = -WD_EINVAL; - goto out_freelist; - } + /* Release ctxs via RR rule */ + if (internal_config) + wd_free_ctxs_batch(attrs, internal_config->ctx_num); - /* - * Not every numa has a device. Therefore, the first thing is to - * filter the devices in the selected numa node, and the second - * thing is to obtain the distribution of devices. - */ - used_list = wd_get_usable_list(list, used_bmp); - if (WD_IS_ERR(used_list)) { - ret = WD_PTR_ERR(used_list); - WD_ERR("failed to get usable devices(%d)!\n", ret); - goto out_freelist; + /* Release user-visible ctx_config */ + if (attrs->ctx_config) { + if (attrs->ctx_config->ctxs) { + free(attrs->ctx_config->ctxs); + attrs->ctx_config->ctxs = NULL; + } + free(attrs->ctx_config); + attrs->ctx_config = NULL; } - wd_init_device_nodemask(used_list, used_bmp); + WD_DEBUG("ctx uninit complete\n"); +} - numa_cnt = numa_bitmask_weight(used_bmp); - if (!numa_cnt) { - ret = numa_cnt; - WD_ERR("invalid: bmp is clear!\n"); - goto out_freeusedlist; +static int wd_init_ctx_config_sched(struct wd_init_attrs *attrs) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + __u32 total_ctx_num = internal_config->ctx_num; + __u32 i; + int ret; + + /* Allocate user-visible wd_ctx_config structure */ + attrs->ctx_config = calloc(1, sizeof(struct wd_ctx_config)); + if (!attrs->ctx_config) { + WD_ERR("failed to allocate ctx_config!\n"); + return -WD_ENOMEM; } - ctx_config->ctx_num = ctx_set_num * numa_cnt; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { + /* Allocate user-visible wd_ctx array */ + attrs->ctx_config->ctxs = calloc(total_ctx_num, sizeof(struct wd_ctx)); + if (!attrs->ctx_config->ctxs) { + WD_ERR("failed to allocate ctxs array!\n"); ret = -WD_ENOMEM; - WD_ERR("failed to alloc ctxs!\n"); - goto out_freeusedlist; + goto cleanup_config; } - ret = wd_init_ctx_and_sched(attrs, used_bmp, used_list); - if (ret) - free(ctx_config->ctxs); + attrs->ctx_config->ctx_num = total_ctx_num; + /* Copy queue information from internal to user-visible config */ + for (i = 0; i < total_ctx_num; i++) { + attrs->ctx_config->ctxs[i].ctx = internal_config->ctxs[i].ctx; + attrs->ctx_config->ctxs[i].op_type = internal_config->ctxs[i].op_type; + attrs->ctx_config->ctxs[i].ctx_mode = internal_config->ctxs[i].ctx_mode; + } + + /* ── Call algorithm-specific init ── */ + attrs->ctx_config->cap = ctx_params->cap; + ret = attrs->alg_init(attrs->ctx_config, attrs->sched); + if (ret) { + WD_ERR("failed to initialize algorithm!\n"); + goto cleanup_ctxs; + } -out_freeusedlist: - wd_free_list_accels(used_list); -out_freelist: - wd_free_list_accels(list); + return WD_SUCCESS; +cleanup_ctxs: + free(attrs->ctx_config->ctxs); + attrs->ctx_config->ctxs = NULL; +cleanup_config: + free(attrs->ctx_config); + attrs->ctx_config = NULL; return ret; } -static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs) +/** + * wd_alg_ctx_init() - Allocate contexts, scheduler, and initialize algorithm. + * + * Uses drivers discovered by wd_alg_config_init(). + * Allocates contexts via RR: ctx[i] -> drv_array[i % drv_count]->alloc_ctx() + * Then allocates scheduler, registers context ranges, and calls alg_init + * which performs wd_init_ctx_config() (wd_ctx[] -> wd_ctx_internal[] copy). + * + * On return: + * - attrs->ctx_config: user-visible context array (populated) + * - attrs->sched: scheduler (allocated and populated) + * - attrs->ctx_config_internal: MUST be set by alg_init callback + * + * NOTE: ctxs[i].drv is still NULL after this function — set later by + * wd_ctx_bind_drivers(). + * + * @attrs: Initialization attributes (input: drv_array, ctx_params, alg_init, etc.) + * Return: 0 on success, negative on failure + */ +int wd_alg_ctx_init(struct wd_init_attrs *attrs) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ce_ctx *ctx; + struct wd_ctx_config_internal *internal_config; + int numa_num_int = numa_max_node(); + __u16 region_num, numa_num; + __u32 ctx_idx = 0; + int ret; - ctx_config->ctx_num = 1; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { - WD_ERR("failed to alloc ctxs!\n"); - return -WD_ENOMEM; - } + if (numa_num_int < 0) + numa_num = 0; + else + numa_num = (__u16)numa_num_int; - ctx = calloc(1, sizeof(struct wd_ce_ctx)); - if (!ctx) { - free(ctx_config->ctxs); - return -WD_ENOMEM; + if (!attrs || !attrs->ctx_params || !attrs->ctx_config_internal || + !attrs->ctx_config_internal->drv_array) { + WD_ERR("invalid: attrs, ctx_params, or drv_array is NULL/empty!\n"); + return -WD_EINVAL; } - ctx->fd = -1; - ctx_config->ctxs[0].ctx = (handle_t)ctx; - return WD_SUCCESS; -} + /* + * Ensure that contexts (ctx) with the same attributes are allocated first, + * thereby maintaining queue continuity within the contexts. + */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; -static void wd_alg_ce_ctx_uninit(struct wd_ctx_config *ctx_config) -{ - __u32 i; + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; - for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - free((struct wd_ce_ctx *)ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; - } + /* Backfill actual allocated count */ + internal_config = attrs->ctx_config_internal; + internal_config->ctx_num = ctx_idx; + if (!ctx_idx) { + WD_ERR("no contexts allocated on any NUMA node!\n"); + ret = -WD_EINVAL; + goto cleanup_ctxs; } - free(ctx_config->ctxs); -} + /* ── Allocate scheduler ── */ + if (attrs->sched_type == SCHED_POLICY_DEV) + region_num = DEVICE_REGION_MAX; + else if (numa_num == 0) + region_num = 1; + else + region_num = numa_num + 1; + + attrs->sched = wd_sched_rr_alloc(attrs->sched_type, + attrs->ctx_params->op_type_num, + region_num, + attrs->alg_poll_ctx); + if (!attrs->sched) { + WD_ERR("failed to allocate scheduler!\n"); + ret = -WD_ENOMEM; + goto cleanup_ctxs; + } -static void wd_alg_ctx_uninit(struct wd_ctx_config *ctx_config) -{ - __u32 i; + /* ── Register contexts to scheduler ── */ + internal_config = attrs->ctx_config_internal; + ret = wd_alg_sched_instance(attrs->sched, internal_config); + if (ret) { + WD_ERR("failed to register contexts to scheduler!\n"); + goto cleanup_sched; + } - for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - wd_release_ctx(ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; - } + /* ── Allocate user ctx_config and initialize algorithm ── */ + ret = wd_init_ctx_config_sched(attrs); + if (ret) { + WD_ERR("failed to allocate user ctx_config and initialize algorithm!\n"); + goto cleanup_user_config; } - free(ctx_config->ctxs); + WD_DEBUG("ctx init complete: %u ctxs from %u drivers\n", + internal_config->ctx_num, internal_config->drv_count); + + return WD_SUCCESS; + + /* ── Error cleanup (LIFO) ── */ +cleanup_user_config: + if (attrs->ctx_config) { + if (attrs->ctx_config->ctxs) + free(attrs->ctx_config->ctxs); + free(attrs->ctx_config); + attrs->ctx_config = NULL; + } +cleanup_sched: + wd_sched_rr_release(attrs->sched); + attrs->sched = NULL; +cleanup_ctxs: + /* Free ctxs allocated so far using RR rule */ + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; } -static int wd_alg_init_sve_ctx(struct wd_ctx_config *ctx_config) +/** + * wd_alg_attrs_uninit() - Release all algorithm resources. + * + * Releases resources in reverse order of allocation: + * 1. wd_alg_ctx_uninit() — free contexts, scheduler, ctx_config + * 2. wd_alg_config_uninit() — free driver array and internal config + * + * @attrs: Initialization attributes + */ +void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) { - struct wd_soft_ctx *ctx_sync, *ctx_async; + if (!attrs) + return; - ctx_config->ctx_num = WD_SOFT_CTX_NUM; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) - return -WD_ENOMEM; + WD_DEBUG("Algorithm cleanup started: alg=%s\n", attrs->alg); - ctx_sync = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_sync) - goto free_ctxs; + /* Release ctxs, scheduler, ctx_config */ + wd_alg_ctx_uninit(attrs); - ctx_sync->fd = -1; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx_mode = CTX_MODE_SYNC; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx = (handle_t)ctx_sync; + /* Free driver array and internal config */ + wd_alg_config_uninit(attrs); - ctx_async = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_async) - goto free_ctx_sync; + WD_DEBUG("Algorithm cleanup complete\n"); +} - ctx_async->fd = -1; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx_mode = CTX_MODE_ASYNC; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx = (handle_t)ctx_async; +static bool wd_check_sva_mode(const char *alg_type) +{ + struct uacce_dev_list *dev_list; + bool is_sva = true; - return 0; + dev_list = wd_get_accel_list(alg_type); + if (!dev_list || !dev_list->dev) + return true; -free_ctx_sync: - free(ctx_sync); -free_ctxs: - free(ctx_config->ctxs); - return -WD_ENOMEM; -} + if (!(dev_list->dev->flags & UACCE_DEV_SVA)) + is_sva = false; -static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) -{ - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); - free(ctx_config->ctxs); + wd_free_list_accels(dev_list); + return is_sva; } -int wd_alg_attrs_init(struct wd_init_attrs *attrs) +static int wd_check_nosva_type(struct wd_init_attrs *attrs) { - wd_alg_poll_ctx alg_poll_func = attrs->alg_poll_ctx; - wd_alg_init alg_init_func = attrs->alg_init; - __u32 sched_type = attrs->sched_type; - struct wd_ctx_config *ctx_config = NULL; - struct wd_sched *alg_sched = NULL; - char alg_type[CRYPTO_MAX_ALG_NAME]; - int driver_type = UADK_ALG_HW; - const char *alg = attrs->alg; - int ret = -WD_EINVAL; + char alg_type_buf[CRYPTO_MAX_ALG_NAME] = {0}; + bool is_sva; + int ret; - if (!attrs->ctx_params) + ret = wd_get_alg_type(attrs->alg, alg_type_buf); + if (ret) { + WD_ERR("failed to get alg type for No-SVA check!\n"); return -WD_EINVAL; + } - if (attrs->driver) - driver_type = attrs->driver->calc_type; + /* These two special types require the use of the original algorithm names. */ + if (strcmp(alg_type_buf, "comp") == 0 || + strcmp(alg_type_buf, "ecc") == 0) + (void)strcpy(alg_type_buf, attrs->alg); - switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; - - /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - goto out_ctx_config; + is_sva = wd_check_sva_mode(alg_type_buf); + if (is_sva) { + if (attrs->sched_type == SCHED_POLICY_DEV) { + WD_ERR("invalid: SVA mode does not support SCHED_POLICY_DEV!\n"); + return -WD_EINVAL; } + return WD_SUCCESS; + } - attrs->sched = alg_sched; + /* + * No-SVA with TASK_INSTR: CE/SVE drivers execute on CPU and + * do not perform DMA, no dependency on hardware SVA mode. + */ + if (attrs->task_type == TASK_INSTR) + return WD_SUCCESS; - ret = wd_alg_ce_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init ce ctx\n"); - goto out_freesched; - } + /* No-SVA only allows TASK_HW */ + if (attrs->task_type != TASK_HW) { + WD_ERR("invalid: No-SVA mode only supports TASK_HW, got %u!\n", + attrs->task_type); + return -WD_EINVAL; + } - ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; + /* TASK_HW + DEV: correct combination */ + if (attrs->sched_type == SCHED_POLICY_DEV) + return WD_SUCCESS; - break; - case UADK_ALG_SVE_INSTR: - /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_SINGLE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - return -WD_EINVAL; - } - attrs->sched = alg_sched; + /* TASK_HW + RR/LOOP/HUNGRY: auto-switch to DEV */ + if (attrs->sched_type == SCHED_POLICY_RR || + attrs->sched_type == SCHED_POLICY_LOOP || + attrs->sched_type == SCHED_POLICY_HUNGRY) { + WD_INFO("info: No-SVA mode auto-switching sched %u to SCHED_POLICY_DEV!\n", + attrs->sched_type); + attrs->sched_type = SCHED_POLICY_DEV; + return WD_SUCCESS; + } - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - goto out_freesched; - } - attrs->ctx_config = ctx_config; + /* TASK_HW + NONE/SINGLE/INSTR: reject */ + WD_ERR("invalid: No-SVA mode requires SCHED_POLICY_DEV, got %u!\n", + attrs->sched_type); + return -WD_EINVAL; +} - ret = wd_alg_init_sve_ctx(ctx_config); - if (ret) { - WD_ERR("fail to init sve ctx!\n"); - goto out_freesched; - } +/** + * wd_task_sched_check() - Validate task_type + sched_type combination. + * + * After driver discovery, check that the requested scheduling + * policy is compatible with the discovered drivers and task type. + * + * Invalid combinations: + * TASK_HW + SCHED_POLICY_INSTR - instr poll only polls ctx[0], + * losing HW async completions. + * TASK_INSTR + SCHED_POLICY_DEV - CE/SVE/SOFT drivers have no + * dev_id for device-level domains. + * SCHED_POLICY_NONE + >1 driver - NONE always picks ctx[0]; + * different sessions may route to + * a driver that doesn\'t support + * their algorithm. + * + * @attrs: Initialization attributes (drv_count must be populated) + * Return: 0 on success, -WD_EINVAL on invalid combination + */ +static int wd_task_sched_check(struct wd_init_attrs *attrs) +{ + struct wd_alg_driver **drv_arr; - ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) { - wd_alg_uninit_sve_ctx(ctx_config); - goto out_freesched; - } - break; - case UADK_ALG_HW: - if (wd_get_alg_type(alg, alg_type)) - return -WD_EINVAL; - (void)strcpy(attrs->alg, alg_type); + if (attrs->task_type == TASK_HW && + attrs->sched_type == SCHED_POLICY_INSTR) { + WD_ERR("invalid: HW tasks must not use INSTR scheduler\n"); + return -WD_EINVAL; + } - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; + if (attrs->task_type == TASK_MIX && + attrs->sched_type == SCHED_POLICY_INSTR) { + WD_ERR("invalid: MIX tasks must not use INSTR scheduler\n"); + return -WD_EINVAL; + } - if (sched_type == SCHED_POLICY_DEV) - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - DEVICE_REGION_MAX, alg_poll_func); - else - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - numa_max_node() + 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to instance scheduler\n"); - goto out_ctx_config; - } - attrs->sched = alg_sched; + if (attrs->sched_type == SCHED_POLICY_NONE && + attrs->ctx_config_internal->drv_count > 1) { + WD_ERR("invalid: NONE scheduler requires single driver\n"); + return -WD_EINVAL; + } - ret = wd_alg_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init ctx\n"); - goto out_freesched; - } + /* NONE scheduler: only TASK_INSTR (CE driver) is valid */ + if (attrs->sched_type == SCHED_POLICY_NONE && + (attrs->task_type == TASK_HW || attrs->task_type == TASK_MIX)) { + WD_ERR("invalid: NONE scheduler only supports TASK_INSTR!\n"); + return -WD_EINVAL; + } - ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; - break; - default: - WD_ERR("driver type error: %d\n", driver_type); + /* SINGLE scheduler: only TASK_INSTR (SVE driver) is valid */ + if (attrs->sched_type == SCHED_POLICY_SINGLE && + (attrs->task_type == TASK_HW || attrs->task_type == TASK_MIX)) { + WD_ERR("invalid: SINGLE scheduler only supports TASK_INSTR!\n"); return -WD_EINVAL; } - return 0; + /* NONE with TASK_INSTR: must be CE driver */ + if (attrs->sched_type == SCHED_POLICY_NONE && + attrs->task_type == TASK_INSTR) { + drv_arr = attrs->ctx_config_internal->drv_array; + if (drv_arr && drv_arr[0] && + drv_arr[0]->calc_type != UADK_ALG_CE_INSTR) { + WD_ERR("invalid: NONE scheduler requires CE driver, got type %d!\n", + drv_arr[0]->calc_type); + return -WD_EINVAL; + } + } -out_pre_init: - if (driver_type == UADK_ALG_CE_INSTR || driver_type == UADK_ALG_SOFT) - wd_alg_ce_ctx_uninit(ctx_config); - else - wd_alg_ctx_uninit(ctx_config); -out_freesched: - wd_sched_rr_release(alg_sched); -out_ctx_config: - if (ctx_config) - free(ctx_config); - return ret; + /* SINGLE with TASK_INSTR: must be SVE driver */ + if (attrs->sched_type == SCHED_POLICY_SINGLE && + attrs->task_type == TASK_INSTR) { + drv_arr = attrs->ctx_config_internal->drv_array; + if (drv_arr && drv_arr[0] && + (drv_arr[0]->calc_type != UADK_ALG_SVE_INSTR && + drv_arr[0]->calc_type != UADK_ALG_CE_INSTR)) { + WD_ERR("invalid: SINGLE scheduler requires CE/SVE driver, got type %d!\n", + drv_arr[0]->calc_type); + return -WD_EINVAL; + } + } + + /* No-SVA mode: only TASK_HW + SCHED_POLICY_DEV is allowed */ + return wd_check_nosva_type(attrs); } -void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) +/** + * wd_alg_attrs_init() - Initialize algorithm with auto-discovered drivers. + * + * Initialization sequence: + * 1. wd_alg_config_init() — discover matching drivers, allocate internal config + * 2. wd_task_sched_check() — validate task_type + sched_type compatibility + * 3. wd_alg_ctx_init() — allocate contexts, scheduler, and initialize algorithm + * + * After this, driver init is done by the caller via wd_alg_init_driver(). + * + * @attrs: Initialization attributes (input/output) + * Return: 0 on success, negative on failure + */ +int wd_alg_attrs_init(struct wd_init_attrs *attrs) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_sched *alg_sched = attrs->sched; - int driver_type = attrs->driver->calc_type; + int ret; - if (!ctx_config) { - wd_sched_rr_release(alg_sched); - return; + if (!attrs) { + WD_ERR("invalid: attrs is NULL!\n"); + return -WD_EINVAL; } - switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - wd_alg_ce_ctx_uninit(ctx_config); - break; - case UADK_ALG_SVE_INSTR: - wd_alg_uninit_sve_ctx(ctx_config); - break; - case UADK_ALG_HW: - wd_alg_ctx_uninit(ctx_config); - break; - default: - break; + /* Driver discovery and configuration setup */ + ret = wd_alg_config_init(attrs); + if (ret) { + WD_ERR("failed to discover drivers!\n"); + goto out_undiscover; } + WD_DEBUG("discovered %u unique drivers\n", attrs->ctx_config_internal->drv_count); - free(ctx_config); - wd_sched_rr_release(alg_sched); + /* Scheduler compatibility check */ + ret = wd_task_sched_check(attrs); + if (ret) { + WD_ERR("failed to match task type with sched type!\n"); + goto out_undiscover; + } + + /* Allocate ctxs, init scheduler and algorithm */ + ret = wd_alg_ctx_init(attrs); + if (ret) { + WD_ERR("failed to init ctx!\n"); + goto out_undiscover; + } + + WD_DEBUG("Algorithm initialization complete: %u contexts from %u drivers\n", + attrs->ctx_config_internal->ctx_num, attrs->ctx_config_internal->drv_count); + + return WD_SUCCESS; + +out_undiscover: + wd_alg_config_uninit(attrs); + return ret; } -- 2.43.0