[Acc] [PATCH 23/24] uadk: uadk: Unify the init and init2 for the UADK framework.

From: Longfang Liu <liulongfang@huawei.com> On the new UADK framework, we have unified the queue abstraction layer and logically merged the init and init2 paths. This has ultimately achieved framework decoupling and solution updates, laying the foundation for future maintenance and expansion. Signed-off-by: Longfang Liu <liulongfang@huawei.com> --- Makefile.am | 13 +- drv/wd_drv.c | 284 +++++++++++ drv/wd_drv.h | 61 +++ include/wd_alg.h | 89 +++- include/wd_alg_common.h | 39 +- include/wd_internal.h | 34 +- include/wd_util.h | 32 +- libwd.map | 6 +- wd.c | 52 +- wd_alg.c | 666 +++++++++++++++++------- wd_cipher.c | 98 +++- wd_sched.c | 34 +- wd_util.c | 1070 ++++++++++++++++----------------------- 13 files changed, 1500 insertions(+), 978 deletions(-) create mode 100644 drv/wd_drv.c create mode 100644 drv/wd_drv.h diff --git a/Makefile.am b/Makefile.am index fc14529..9292f9f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -58,23 +58,26 @@ libwd_la_SOURCES=wd.c wd_mempool.c wd_bmm.c wd_bmm.h wd.h wd_alg.c wd_alg.h \ # Crypto library with cipher and aead support only libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_aead.c wd_aead.h wd_aead_drv.h \ + wd.c wd.h wd_alg.h \ wd_util.c wd_util.h \ - wd_sched.c wd_sched.h \ - wd.c wd.h + wd_sched.c wd_sched.h # Cipher driver for hisi_sec hardware libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h \ + drv/wd_drv.h drv/wd_drv.c libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ - drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h + drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h \ + drv/wd_drv.h drv/wd_drv.c libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ drv/hash_mb/sm3_sve_common.S drv/hash_mb/sm3_mb_asimd_x1.S \ drv/hash_mb/sm3_mb_asimd_x4.S drv/hash_mb/sm3_mb_sve.S \ drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ - drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S + drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S \ + drv/wd_drv.h drv/wd_drv.c # Static driver build configuration if WD_STATIC_DRV diff --git a/drv/wd_drv.c b/drv/wd_drv.c new file mode 100644 index 0000000..fcb03b1 --- /dev/null +++ b/drv/wd_drv.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 +/* + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +#include <stdlib.h> + +#include "wd_internal.h" +#include "wd_alg.h" +#include "wd_util.h" +#include "wd_drv.h" + +int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx) +{ + struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; + struct wd_soft_ctx *sfctx; + + if (!params || !ctx) { + WD_ERR("invalid: params, or ctx is NULL!\n"); + return -WD_EINVAL; + } + + /* Allocate ONE software context structure */ + sfctx = calloc(1, sizeof(struct wd_soft_ctx)); + if (!sfctx) { + WD_ERR("failed to alloc ctx!\n"); + return -WD_ENOMEM; + } + + /* Initialize as software context */ + sfctx->fd = -1; + pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_SHARED); + pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_SHARED); + + /* Return context handle */ + *ctx = (handle_t)sfctx; + + WD_INFO("SW context allocated: alg=%s, type=%d, mode=%d\n", + alg_name, ctx_params->op_type, ctx_params->ctx_mode); + + return 0; +} + +void wd_soft_free_ctx(handle_t ctx) +{ + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + + if (!sfctx) { + WD_ERR("invalid: ctx is NULL!\n"); + return; + } + + /* Simply free the allocated wd_ctx_h structure */ + pthread_spin_destroy(&sfctx->slock); + pthread_spin_destroy(&sfctx->rlock); + free(sfctx); + + WD_INFO("SW context released\n"); +} + +struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, struct bitmask *bmp) +{ + struct uacce_dev_list *p, *node, *result = NULL; + struct uacce_dev *dev; + int numa_id, ret; + + if (!bmp) { + WD_ERR("invalid: bmp is NULL!\n"); + return WD_ERR_PTR(-WD_EINVAL); + } + + p = list; + while (p) { + dev = p->dev; + numa_id = dev->numa_id; + ret = numa_bitmask_isbitset(bmp, numa_id); + if (!ret) { + p = p->next; + continue; + } + + node = calloc(1, sizeof(*node)); + if (!node) { + result = WD_ERR_PTR(-WD_ENOMEM); + goto out_free_list; + } + + node->dev = wd_clone_dev(dev); + if (!node->dev) { + result = WD_ERR_PTR(-WD_ENOMEM); + goto out_free_node; + } + + if (!result) + result = node; + else + wd_add_dev_to_list(result, node); + + p = p->next; + } + + return result ? result : WD_ERR_PTR(-WD_ENODEV); + +out_free_node: + free(node); +out_free_list: + wd_free_list_accels(result); + return result; +} + +/** + * wd_hw_alloc_ctx() - HW driver's alloc_ctx callback. + * + * Allocates ONE hardware context from UACCE device. + * This is a driver callback, called by framework for each context. + * + * @alg: The alg name + * @params: Minimal allocation parameters (ctx_mode, op_type, bmp) + * @result: (output) Allocated context information + * + * Return: 0 on success, negative on failure + */ +int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) +{ + struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; + struct uacce_dev_list *dev_list, *used_list = NULL; + struct bitmask *used_bmp = ctx_params->bmp; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct uacce_dev *dev = NULL; + struct uacce_dev_list *curr; + struct wd_ctx_h *ctx_h; + int ret = -WD_EINVAL; + + if (!params || !ctx) { + WD_ERR("invalid parameters!\n"); + return -WD_EINVAL; + } + + /* Get algorithm type and device list */ + wd_get_alg_type(alg_name, alg_type); + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) { + WD_ERR("failed to get device list for alg %s\n", alg_name); + return -WD_ENODEV; + } + + /* Get usable device list based on NUMA mask */ + used_list = wd_get_usable_list(dev_list, used_bmp); + if (!used_list) { + WD_ERR("failed to get usable device list\n"); + ret = -WD_ENODEV; + goto out; + } + + /* + * After allocating all queues on the current device, proceed to + * request queues from the next device to ensure NUMA affinity handling. + * + * Try each device in the usable list until success + */ + curr = used_list; + while (curr) { + dev = curr->dev; + if (WD_IS_ERR(dev) || !dev) { + WD_ERR("invalid device in list, skip\n"); + curr = curr->next; + continue; + } + + /* Request hardware context from current device */ + ctx_h = wd_request_ctx(dev); + if (ctx_h) { + /* Success: return context handle */ + ctx_h->priv = NULL; + *ctx = (handle_t)ctx_h; + ret = 0; + WD_INFO("successful to alloc ctx from device %s, ctx: %p\n", + dev->dev_root, ctx_h); + goto out; + } + + WD_DEBUG("failed to request ctx from device %s, try next\n", + dev->dev_root); + curr = curr->next; + } + + /* All devices failed */ + WD_ERR("failed to request ctx from all available devices for driver %s\n", + alg_name); + ret = -WD_EBUSY; + +out: + if (dev_list) + wd_free_list_accels(dev_list); + + return ret; +} + +/** + * wd_hw_free_ctx() - HW driver's free_ctx callback. + * + * Releases ONE hardware context back to UACCE device. + * + * @ctx: The context handle to release + */ +void wd_hw_free_ctx(handle_t ctx) +{ + struct wd_ctx_h *ctx_h = (struct wd_ctx_h *)ctx; + + if (!ctx_h) { + WD_ERR("invalid: ctx is NULL!\n"); + return; + } + + /* Release hardware context back to device */ + wd_release_ctx(ctx); + + WD_INFO("HW context released\n"); +} + +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id) +{ + struct wd_soft_sqe *sqe = NULL; + + pthread_spin_lock(&sctx->slock); + sqe = &sctx->qfifo[sctx->head]; + if (!sqe->used && !sqe->complete) { // find the next not used sqe + sctx->head++; + if (unlikely(sctx->head == MAX_SOFT_QUEUE_LENGTH)) + sctx->head = 0; + + sqe->used = 1; + sqe->complete = 1; + sqe->id = tag_id; + sqe->result = 0; + __atomic_fetch_add(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->slock); + } else { + pthread_spin_unlock(&sctx->slock); + return -WD_EBUSY; + } + + return 0; +} + +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result) +{ + struct wd_soft_sqe *sqe = NULL; + + /* The queue is not used */ + if (sctx->run_num < 1) + return -WD_EAGAIN; + + if (pthread_spin_trylock(&sctx->rlock)) + return -WD_EAGAIN; + sqe = &sctx->qfifo[sctx->tail]; + if (sqe->used && sqe->complete) { // find a used sqe + sctx->tail++; + if (unlikely(sctx->tail == MAX_SOFT_QUEUE_LENGTH)) + sctx->tail = 0; + + *tag_id = sqe->id; + *result = sqe->result; + sqe->used = 0x0; + sqe->complete = 0x0; + __atomic_fetch_sub(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->rlock); + } else { + pthread_spin_unlock(&sctx->rlock); + return -WD_EAGAIN; + } + + return 0; +} + +int wd_queue_is_busy(struct wd_soft_ctx *sctx) +{ + /* The queue is not used */ + if (sctx->run_num >= MAX_SOFT_QUEUE_LENGTH - 1) + return -WD_EBUSY; + + return 0; +} + diff --git a/drv/wd_drv.h b/drv/wd_drv.h new file mode 100644 index 0000000..ddf3728 --- /dev/null +++ b/drv/wd_drv.h @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +/* + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +#ifndef __WD_DRV_H +#define __WD_DRV_H + +#include <numa.h> +#include <stdlib.h> +#include <stdio.h> +#include <sys/ipc.h> +#include <sys/shm.h> +#include <pthread.h> + +#include "wd.h" +#include "wd_alg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SOFT_QUEUE_LENGTH 1024U + +/** + * default queue length set to 1024 + */ +struct wd_soft_sqe { + __u8 used; + __u8 result; + __u8 complete; + __u32 id; +}; + +struct wd_soft_ctx { + int fd; + pthread_spinlock_t slock; + __u32 head; + struct wd_soft_sqe qfifo[MAX_SOFT_QUEUE_LENGTH]; + pthread_spinlock_t rlock; + __u32 tail; + __u32 run_num; + void *priv; +}; + +/* Public function declarations */ +int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx); +void wd_hw_free_ctx(handle_t ctx); + +int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx); +void wd_soft_free_ctx(handle_t ctx); + +int wd_queue_is_busy(struct wd_soft_ctx *sctx); +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id); +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result); + +#ifdef __cplusplus +} +#endif + +#endif /* __WD_DRV_H */ diff --git a/include/wd_alg.h b/include/wd_alg.h index 05e022c..237d15a 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,11 +64,14 @@ typedef unsigned char __u8; # define HWCAP2_RNG (1 << 16) #endif -enum alg_priority { - UADK_ALG_HW = 0x0, - UADK_ALG_CE_INSTR = 0x1, +enum alg_dev_type { + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_SOFT = 0x3 + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; enum alg_drv_type { @@ -80,6 +83,25 @@ enum alg_drv_type { ALG_DRV_FB, }; +/** + * struct wd_ctx_alloc_params - Minimal parameters for single context allocation. + * + * Used to pass only necessary information to driver's alloc_ctx callback. + * Keeps driver layer simple and focused. + * + * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC + * @op_type: Operation type + * @bmp: NUMA node bitmask (optional, NULL if not needed) + */ +struct wd_drv_ctx_params { + __u8 ctx_mode; + __u8 op_type; + int numa_id; + __u32 idx; + bool epoll_en; + struct bitmask *bmp; +}; + /** * @drv_name: name of the current device driver * @alg_name: name of the algorithm supported by the driver @@ -108,7 +130,13 @@ enum alg_drv_type { * @get_usage: callback interface used to obtain the * utilization rate of devices. * @get_extend_ops: callback interface to get private operation of drivers. + * @alloc_ctx: Allocate contexts for this driver. + * HW drivers use wd_hw_alloc_ctx. + * Non-HW drivers use wd_drv_alloc_ctx_array. + * @free_ctx: Release all resources allocated by alloc_ctx. + * @sched_policy: Preferred scheduling policy. */ + struct wd_alg_driver { const char *drv_name; const char *alg_name; @@ -127,6 +155,9 @@ struct wd_alg_driver { int (*recv)(handle_t ctx, void *drv_msg); int (*get_usage)(void *param); int (*get_extend_ops)(void *ops); + + int (*alloc_ctx)(char *alg_name, void *params, handle_t *ctx); + void (*free_ctx)(handle_t ctx); }; struct hisi_dev_usage { @@ -144,30 +175,49 @@ struct hisi_dev_usage { int wd_alg_driver_register(struct wd_alg_driver *drv); void wd_alg_driver_unregister(struct wd_alg_driver *drv); +#define MAX_DRV_ALG_NUM 64 /** - * @alg_name: name of the algorithm supported by the driver - * @drv_name: name of the current device driver + * Secondary structure: Algorithm entry (only algorithm-specific attributes) + * @alg_name: Specific algorithm name, e.g., "cbc(aes)" + * @avaiblable: Availability depends on specific CE/SVE instructions + */ +struct wd_alg_entry { + char alg_name[ALG_NAME_SIZE]; + bool available; +}; +/** + * Primary structure: Driver node (List backbone, contains driver-level shared attributes) + * @drv_name: name of the current device driver e.g., "hisi_sec" + * @alg_type: Algorithm class, e.g., "cipher" (Promoted to driver level) * @available: Indicates whether the current driver still has resources available * @priority: priority of the type of algorithm supported by the driver - * @calc_type: the calculation method of algorithm supported by the driver - * @refcnt: the number of times the algorithm driver is being cited by the task + * @calc_type: Driver calc type (HW, CE, SVE, SOFT) + * @refcnt: Driver-level global reference count * - * @drv: device Drivers Supporting Algorithms + * @drv: Pointer to driver implementation + * @algs: Static array for supported algorithms + * @alg_count: Current number of registered algorithms * @next: pointer to the next node of the algorithm linked list */ -struct wd_alg_list { - char alg_name[ALG_NAME_SIZE]; +struct wd_drv_node { char drv_name[DEV_NAME_LEN]; - bool available; - int priority; - int calc_type; - int refcnt; - - struct wd_alg_driver *drv; - struct wd_alg_list *next; char alg_type[ALG_NAME_SIZE]; + int priority; + int calc_type; + int refcnt; + struct wd_alg_driver *drv; + struct wd_alg_entry algs[MAX_DRV_ALG_NUM]; + int alg_count; + struct wd_drv_node *next; }; +int wd_get_drv_array(const char *alg_type, int task_type, char *drv_name, + struct wd_alg_driver ***drv_array, __u32 *drv_count); +void wd_put_drv_array(struct wd_alg_driver **drv_array, __u32 drv_count); + +void wd_alg_drv_ref_inc(struct wd_alg_driver **drv_array, __u32 drv_count); +void wd_alg_drv_ref_dec(struct wd_alg_driver **drv_array, __u32 drv_count); + /** * wd_request_drv() - Apply for an algorithm driver. * @alg_name: task algorithm name. @@ -176,7 +226,6 @@ struct wd_alg_list { * Returns the applied algorithm driver, non means error. */ struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type); -void wd_release_drv(struct wd_alg_driver *drv); /** * wd_drv_alg_support() - Check the algorithms supported by the driver. @@ -197,7 +246,7 @@ void wd_disable_drv(struct wd_alg_driver *drv); int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 op_type); int wd_get_alg_type(const char *alg_name, char *alg_type); -struct wd_alg_list *wd_get_alg_head(void); +struct wd_drv_node *wd_get_alg_head(void); #ifdef WD_STATIC_DRV /** diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index ac50cb2..00d4430 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -92,7 +92,6 @@ struct wd_ctx { handle_t ctx; __u8 op_type; __u8 ctx_mode; - __u8 ctx_type; }; /** @@ -124,31 +123,16 @@ struct wd_ctx_config { struct wd_cap_config *cap; }; -/* 0x0 mean calloc init value */ -enum wd_ctx_property { - UADK_CTX_HW = 0x0, - UADK_CTX_CE_INS = 0x1, - UADK_CTX_SVE_INS = 0x2, - UADK_CTX_SOFT = 0x3, - UADK_CTX_MAX -}; - /** * struct wd_ctx_nums - Define the ctx sets numbers. * @sync_ctx_num: The ctx numbers which are used for sync mode for each * ctx sets. * @async_ctx_num: The ctx numbers which are used for async mode for each * ctx sets. - * @ctx_prop: Indicates the properties of the current queue - * @ctx_begin: The encoding starting position of the current device ctx - * @other_ctx: Other types of queues configured */ struct wd_ctx_nums { __u32 sync_ctx_num; __u32 async_ctx_num; - __u8 ctx_prop; - __u16 ctx_begin; - struct wd_ctx_nums *other_ctx; }; /** @@ -194,9 +178,30 @@ struct wd_sched { handle_t h_sched_ctx; }; -typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); +typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched, void *attrs); typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); +/** + * struct wd_init_attrs - Algorithm initialization attributes. + * + * Updated: No longer contains driver field. + * Initialization path determined solely by task_type. + */ +struct wd_init_attrs { + __u32 sched_type; + __u32 task_type; + char alg[CRYPTO_MAX_ALG_NAME]; + struct wd_sched *sched; + struct wd_ctx_params *ctx_params; + struct wd_ctx_config *ctx_config; + wd_alg_init alg_init; + wd_alg_poll_ctx alg_poll_ctx; + + struct wd_ctx_config_internal *ctx_config_internal; + struct wd_alg_driver **drv_array; + __u32 drv_count; +}; + #ifdef __cplusplus } #endif diff --git a/include/wd_internal.h b/include/wd_internal.h index c12500b..455e283 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -9,6 +9,7 @@ #include <pthread.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -17,7 +18,6 @@ extern "C" { #define DEVICE_REGION_MAX 16 #define DECIMAL_NUMBER 10 #define MAX_FD_NUM 65535 -#define MAX_SOFT_QUEUE_LENGTH 1024U struct wd_ctx_h { int fd; @@ -30,33 +30,6 @@ struct wd_ctx_h { void *priv; }; -struct wd_soft_sqe { - __u8 used; - __u8 result; - __u8 complete; - __u32 id; -}; - -/** - * default queue length set to 1024 - */ -struct wd_soft_ctx { - int fd; - pthread_spinlock_t slock; - __u32 head; - struct wd_soft_sqe qfifo[MAX_SOFT_QUEUE_LENGTH]; - pthread_spinlock_t rlock; - __u32 tail; - __u32 run_num; - void *priv; -}; - -struct wd_ce_ctx { - int fd; - char *drv_name; - void *priv; -}; - struct wd_ctx_internal { __u8 op_type; __u8 ctx_mode; @@ -66,8 +39,8 @@ struct wd_ctx_internal { __u16 sqn; pthread_spinlock_t lock; struct wd_alg_driver *drv; - void *extend_ops; void *drv_priv; + void *extend_ops; }; struct wd_ctx_config_internal { @@ -78,6 +51,9 @@ struct wd_ctx_config_internal { bool epoll_en; unsigned long *msg_cnt; const char *alg_name; + + struct wd_alg_driver **drv_array; + __u32 drv_count; }; struct wd_datalist { diff --git a/include/wd_util.h b/include/wd_util.h index c24c554..1fc7d60 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -118,17 +118,6 @@ struct wd_msg_handle { int (*recv)(handle_t sess, void *msg); }; -struct wd_init_attrs { - __u32 sched_type; - __u32 task_type; - char alg[CRYPTO_MAX_ALG_NAME]; - struct wd_sched *sched; - struct wd_ctx_params *ctx_params; - struct wd_ctx_config *ctx_config; - wd_alg_init alg_init; - wd_alg_poll_ctx alg_poll_ctx; -}; - /* * wd_init_ctx_config() - Init internal ctx configuration. * @in: ctx configuration in global setting. @@ -435,17 +424,6 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); int wd_alg_attrs_init(struct wd_init_attrs *attrs); void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); -/** - * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain - * with the given devices list, ctxs number and numa mask. - * @ctx_type: the type of ctx specified by the current algorithm. - * @alg_name: the name of the algorithm specified by the task. - * - * Return device driver if succeed and other NULL if fail. - */ -struct wd_alg_driver *wd_alg_drv_bind(__u8 ctx_prop, char *alg_name); -void wd_alg_drv_unbind(struct wd_alg_driver *drv); - /** * wd_alg_init_driver() - Initialize the current device driver according * to the obtained queue resource and the applied driver. @@ -519,9 +497,13 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type) int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type); -int wd_queue_is_busy(struct wd_soft_ctx *sctx); -int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id); -int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result); +int wd_alg_drv_discover(struct wd_init_attrs *attrs); +void wd_alg_drv_undiscover(struct wd_init_attrs *attrs); +int wd_alg_ctx_init(struct wd_init_attrs *attrs); +void wd_alg_ctx_uninit(struct wd_init_attrs *attrs); +int wd_ctx_bind_drivers(struct wd_ctx_config_internal *config, + struct wd_alg_driver **drv_array, __u32 drv_count); +void wd_ctx_unbind_drivers(struct wd_ctx_config_internal *config); #ifdef __cplusplus } diff --git a/libwd.map b/libwd.map index 0635198..ce8c3ce 100644 --- a/libwd.map +++ b/libwd.map @@ -45,11 +45,15 @@ global: wd_alg_driver_register; wd_alg_driver_unregister; wd_request_drv; - wd_release_drv; wd_drv_alg_support; wd_enable_drv; wd_disable_drv; wd_get_alg_head; + wd_get_drv_array; + wd_put_drv_array; + wd_alg_drv_ref_inc; + wd_alg_drv_ref_dec; + wd_alg_driver_init; wd_alg_driver_exit; wd_alg_driver_send; diff --git a/wd.c b/wd.c index 01334e1..dd75ca2 100644 --- a/wd.c +++ b/wd.c @@ -996,33 +996,39 @@ void wd_release_alg_cap(struct wd_capability *head) struct wd_capability *wd_get_alg_cap(void) { - struct wd_alg_list *head = wd_get_alg_head(); - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *head = wd_get_alg_head(); + struct wd_drv_node *drv_node = head->next; struct wd_capability *cap_head = NULL; struct wd_capability *cap_pnext = NULL; struct wd_capability *cap_node; - - while (pnext) { - cap_node = calloc(1, sizeof(struct wd_capability)); - if (!cap_node) { - WD_ERR("fail to alloc wd capability head\n"); - goto alloc_err; - } - - (void)strcpy(cap_node->alg_name, pnext->alg_name); - (void)strcpy(cap_node->drv_name, pnext->drv_name); - cap_node->available = pnext->available; - cap_node->priority = pnext->priority; - cap_node->calc_type = pnext->calc_type; - cap_node->next = NULL; - - pnext = pnext->next; - if (!cap_pnext) { - cap_head = cap_node; - cap_pnext = cap_node; + int i; + + while (drv_node) { + /* Traverse the static algorithm array inside each driver node */ + for (i = 0; i < drv_node->alg_count; i++) { + cap_node = calloc(1, sizeof(struct wd_capability)); + if (!cap_node) { + WD_ERR("fail to alloc wd capability head\n"); + goto alloc_err; + } + /* Flatten the secondary structure into the original binary-tuple format */ + (void)strcpy(cap_node->alg_name, drv_node->algs[i].alg_name); + (void)strcpy(cap_node->drv_name, drv_node->drv_name); + cap_node->available = drv_node->algs[i].available; + cap_node->priority = drv_node->priority; + cap_node->calc_type = drv_node->calc_type; + cap_node->next = NULL; + + /* Append to the capability linked list */ + if (!cap_head) { + cap_head = cap_node; + cap_pnext = cap_node; + } else { + cap_pnext->next = cap_node; + cap_pnext = cap_node; + } } - cap_pnext->next = cap_node; - cap_pnext = cap_node; + drv_node = drv_node->next; } return cap_head; diff --git a/wd_alg.c b/wd_alg.c index 787dcad..3de8539 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -6,6 +6,7 @@ #define _GNU_SOURCE #include <dirent.h> #include <errno.h> +#include <stdio.h> #include <stdbool.h> #include <stdlib.h> #include <pthread.h> @@ -19,10 +20,21 @@ #define DEV_SVA_SIZE 32 #define STR_DECIMAL 0xA -static struct wd_alg_list alg_list_head; -static struct wd_alg_list *alg_list_tail = &alg_list_head; +/* Registry structure (List manager) */ +struct wd_alg_registry { + struct wd_drv_node *head; + struct wd_drv_node *tail; + pthread_mutex_t mutex; + int drv_type_num; /* Number of unique driver nodes in the list */ +}; -static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static struct wd_drv_node drv_list_head; +static struct wd_alg_registry alg_registry = { + .head = &drv_list_head, + .tail = &drv_list_head, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .drv_type_num = 0, +}; struct acc_alg_item { const char *name; @@ -208,47 +220,33 @@ static bool wd_alg_check_available(int calc_type, return ret; } -static bool wd_alg_driver_match(struct wd_alg_driver *drv, - struct wd_alg_list *node) +/** + * Mapping from task_type to calc_type filter: + * + * TASK_HW → calc_type == UADK_ALG_HW + * TASK_INSTR → calc_type != UADK_ALG_HW (CE_INSTR | SVE_INSTR | SOFT) + * TASK_MIX → all calc_type values + */ +static inline bool wd_alg_drv_type_match(int task_type, int drv_calc_type) { - if (strcmp(drv->alg_name, node->alg_name)) - return false; - - if (strcmp(drv->drv_name, node->drv_name)) - return false; - - if (drv->priority != node->priority) - return false; - - if (drv->calc_type != node->calc_type) + switch (task_type) { + case TASK_HW: + return drv_calc_type == UADK_ALG_HW; + case TASK_INSTR: + return drv_calc_type != UADK_ALG_HW; + case TASK_MIX: + return true; + default: return false; - - return true; -} - -static bool wd_alg_repeat_check(struct wd_alg_driver *drv) -{ - struct wd_alg_list *npre = &alg_list_head; - struct wd_alg_list *pnext = NULL; - - pthread_mutex_lock(&mutex); - pnext = npre->next; - while (pnext) { - if (wd_alg_driver_match(drv, pnext)) { - pthread_mutex_unlock(&mutex); - return true; - } - npre = pnext; - pnext = pnext->next; } - pthread_mutex_unlock(&mutex); - - return false; } int wd_alg_driver_register(struct wd_alg_driver *drv) { - struct wd_alg_list *new_alg; + struct wd_drv_node *node = alg_registry.head->next; + struct wd_drv_node *target_node = NULL; + char alg_type[ALG_NAME_SIZE]; + int i, ret; if (!drv) { WD_ERR("invalid: register drv is NULL!\n"); @@ -260,155 +258,285 @@ int wd_alg_driver_register(struct wd_alg_driver *drv) return -WD_EINVAL; } - if (wd_alg_repeat_check(drv)) - return 0; + ret = wd_get_alg_type(drv->alg_name, alg_type); + if (ret) { + WD_ERR("failed to get alg_type for %s!\n", drv->alg_name); + return -WD_EINVAL; + } - new_alg = calloc(1, sizeof(struct wd_alg_list)); - if (!new_alg) { - WD_ERR("failed to alloc alg driver memory!\n"); - return -WD_ENOMEM; + /* Search for an existing node with the same drv_name */ + pthread_mutex_lock(&alg_registry.mutex); + while (node) { + if (strcmp(node->drv_name, drv->drv_name) == 0) { + target_node = node; + break; + } + node = node->next; } - (void)wd_get_alg_type(drv->alg_name, new_alg->alg_type); - strncpy(new_alg->alg_name, drv->alg_name, ALG_NAME_SIZE - 1); - strncpy(new_alg->drv_name, drv->drv_name, DEV_NAME_LEN - 1); - new_alg->priority = drv->priority; - new_alg->calc_type = drv->calc_type; - new_alg->drv = drv; - new_alg->refcnt = 0; - new_alg->next = NULL; + if (target_node) { + /* Consistency check: a driver must strictly have uniform properties */ + if (strcmp(target_node->alg_type, alg_type) != 0 || + target_node->priority != drv->priority || + target_node->calc_type != drv->calc_type) { + WD_ERR("invalid: driver %s attributes mismatch on re-register!\n", drv->drv_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_EINVAL; + } - new_alg->available = wd_alg_check_available(drv->calc_type, - drv->alg_name, drv->drv_name); - if (!new_alg->available) { - free(new_alg); - return -WD_ENODEV; - } + /* Check if alg_name already exists in this driver's array */ + for (i = 0; i < target_node->alg_count; i++) { + if (strcmp(target_node->algs[i].alg_name, drv->alg_name) == 0) { + /* Algorithm already registered, skip duplicate */ + pthread_mutex_unlock(&alg_registry.mutex); + return 0; + } + } + + /* Check array capacity */ + if (target_node->alg_count >= MAX_DRV_ALG_NUM) { + WD_ERR("driver %s alg array overflow (max %d)!\n", drv->drv_name, MAX_DRV_ALG_NUM); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENOMEM; + } + + /* Add new algorithm to existing driver node */ + strncpy(target_node->algs[target_node->alg_count].alg_name, drv->alg_name, ALG_NAME_SIZE - 1); + target_node->algs[target_node->alg_count].available = + wd_alg_check_available(drv->calc_type, drv->alg_name, drv->drv_name); + if (!target_node->algs[target_node->alg_count].available) { + WD_ERR("driver %s alg %s not available on current system!\n", drv->drv_name, drv->alg_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENODEV; + } + target_node->alg_count++; + } else { + /* Create a new driver node */ + target_node = calloc(1, sizeof(struct wd_drv_node)); + if (!target_node) { + WD_ERR("failed to alloc drv node memory!\n"); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENOMEM; + } + + strncpy(target_node->drv_name, drv->drv_name, DEV_NAME_LEN - 1); + strncpy(target_node->alg_type, alg_type, ALG_NAME_SIZE - 1); + target_node->priority = drv->priority; + target_node->calc_type = drv->calc_type; + target_node->drv = drv; + target_node->refcnt = 0; + target_node->alg_count = 0; + + /* Add the first algorithm to the new node's array */ + strncpy(target_node->algs[0].alg_name, drv->alg_name, ALG_NAME_SIZE - 1); + target_node->algs[0].available = + wd_alg_check_available(drv->calc_type, drv->alg_name, drv->drv_name); + if (!target_node->algs[0].available) { + free(target_node); + WD_ERR("driver %s alg %s not available on current system!\n", drv->drv_name, drv->alg_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENODEV; + } + target_node->alg_count = 1; + target_node->next = NULL; - pthread_mutex_lock(&mutex); - alg_list_tail->next = new_alg; - alg_list_tail = new_alg; - pthread_mutex_unlock(&mutex); + /* Append to list tail */ + alg_registry.tail->next = target_node; + alg_registry.tail = target_node; + alg_registry.drv_type_num++; + } + pthread_mutex_unlock(&alg_registry.mutex); return 0; } void wd_alg_driver_unregister(struct wd_alg_driver *drv) { - struct wd_alg_list *npre = &alg_list_head; - struct wd_alg_list *pnext = npre->next; + struct wd_drv_node *npre = alg_registry.head; + struct wd_drv_node *pnext = npre->next; + int i; - /* Alg driver list has no drivers */ if (!pnext || !drv) return; - pthread_mutex_lock(&mutex); + pthread_mutex_lock(&alg_registry.mutex); + /* Find the driver node matching drv_name */ while (pnext) { - if (wd_alg_driver_match(drv, pnext)) + if (strcmp(drv->drv_name, pnext->drv_name) == 0) break; npre = pnext; pnext = pnext->next; } - /* The current algorithm is not registered */ if (!pnext) { - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); return; } - /* Used to locate the problem and ensure symmetrical use driver */ - if (pnext->refcnt > 0) - WD_ERR("driver<%s> still in used: %d\n", pnext->drv_name, pnext->refcnt); + /* Find and remove the specific alg_name from the node's array */ + for (i = 0; i < pnext->alg_count; i++) { + if (strcmp(pnext->algs[i].alg_name, drv->alg_name) == 0) { + /* Compact the array: move the last element to the removed slot */ + if (i != pnext->alg_count - 1) + pnext->algs[i] = pnext->algs[pnext->alg_count - 1]; + pnext->alg_count--; + break; + } + } + + /* If the driver no longer supports any algorithms, remove the entire node */ + if (pnext->alg_count == 0) { + if (pnext->refcnt > 0) + WD_ERR("driver<%s> still in used: %d\n", pnext->drv_name, pnext->refcnt); + + if (pnext == alg_registry.tail) + alg_registry.tail = npre; - if (pnext == alg_list_tail) - alg_list_tail = npre; + npre->next = pnext->next; + free(pnext); + if (alg_registry.drv_type_num > 0) + alg_registry.drv_type_num--; + } - npre->next = pnext->next; - free(pnext); - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); } -struct wd_alg_list *wd_get_alg_head(void) +struct wd_drv_node *wd_get_alg_head(void) { - return &alg_list_head; + return alg_registry.head; +} + +/** + * wd_alg_match_drv() - Check if a given algorithm match a specific driver. + * @drv: Pointer to the driver instance + * @alg_name: Specific algorithm name to check (e.g., "cbc(aes)") + * + * Uses the new hierarchical structure: finds the driver node, then searches + * its internal static algorithm array. + * + * Return: true if supported and available, false otherwise. + */ +bool wd_alg_match_drv(struct wd_alg_driver *drv, const char *alg_name) +{ + struct wd_drv_node *node; + int i; + + if (!drv || !alg_name) + return false; + + pthread_mutex_lock(&alg_registry.mutex); + node = alg_registry.head->next; + while (node) { + if (node->drv == drv) { + /* Found the driver node, now search its algs array */ + for (i = 0; i < node->alg_count; i++) { + if (!strcmp(node->algs[i].alg_name, alg_name) && + node->algs[i].available) { + pthread_mutex_unlock(&alg_registry.mutex); + return true; + } + } + /* Driver found, but algorithm not in its array or not available */ + pthread_mutex_unlock(&alg_registry.mutex); + return false; + } + node = node->next; + } + pthread_mutex_unlock(&alg_registry.mutex); + + return false; } bool wd_drv_alg_support(const char *alg_name, void *param) { struct wd_ctx_config_internal *config = param; - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_driver *drv; - __u32 i; + struct wd_drv_node *head = alg_registry.head; + struct wd_drv_node *node; + __u32 i, j; if (!alg_name || !config) return false; + /* Check whether the currently allocated ctxs supports the specified algorithm. */ for (i = 0; i < config->ctx_num; i++) { - drv = config->ctxs[i].drv; - while (pnext) { - if (!strcmp(alg_name, pnext->alg_name) && - !strcmp(drv->drv_name, pnext->drv_name)) { - return true; + if (!config->ctxs[i].drv) + continue; + node = head->next; + while (node) { + /* Query the position of the driver matching the context in the list. */ + if (strcmp(config->ctxs[i].drv->drv_name, node->drv_name) == 0) { + for (j = 0; j < node->alg_count; j++) { + if (!strcmp(alg_name, node->algs[j].alg_name) && + node->algs[j].available) + return true; + } + break; } - pnext = pnext->next; + node = node->next; } } - return false; } void wd_enable_drv(struct wd_alg_driver *drv) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *node = alg_registry.head->next; + int i; - if (!pnext || !drv) + if (!node || !drv) return; - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext)) + pthread_mutex_lock(&alg_registry.mutex); + while (node) { + if (strcmp(drv->drv_name, node->drv_name) == 0) break; - pnext = pnext->next; + node = node->next; } - if (pnext) - pnext->available = wd_alg_check_available(drv->calc_type, - drv->alg_name, drv->drv_name); - pthread_mutex_unlock(&mutex); + if (node) { + /* Re-evaluate availability for each algorithm upon enabling */ + for (i = 0; i < node->alg_count; i++) { + node->algs[i].available = + wd_alg_check_available(node->calc_type, + node->algs[i].alg_name, + node->drv_name); + } + } + pthread_mutex_unlock(&alg_registry.mutex); } void wd_disable_drv(struct wd_alg_driver *drv) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *node = alg_registry.head->next; + int i; - if (!pnext || !drv) + if (!node || !drv) return; - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext) && pnext->available) + pthread_mutex_lock(&alg_registry.mutex); + while (node) { + if (strcmp(drv->drv_name, node->drv_name) == 0) break; - pnext = pnext->next; + node = node->next; } - if (pnext) - pnext->available = false; - pthread_mutex_unlock(&mutex); + if (node) { + /* Disable all algorithms for this driver */ + for (i = 0; i < node->alg_count; i++) + node->algs[i].available = false; + } + pthread_mutex_unlock(&alg_registry.mutex); } struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_list *select_node = NULL; + struct wd_drv_node *node = alg_registry.head->next; struct wd_alg_driver *drv = NULL; int tmp_priority = -1; + int i; - if (!pnext) { - WD_ERR("invalid: requset drv pnext is NULL!\n"); + if (!node) { + WD_ERR("invalid: request drv node is NULL!\n"); return NULL; } @@ -417,72 +545,59 @@ struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type) return NULL; } - /* Check the list to get an best driver */ - pthread_mutex_lock(&mutex); - while (pnext) { - if (!strcmp(alg_name, pnext->alg_name) && pnext->available) { - /* HW driver mean to used hardware dev */ - if (drv_type == ALG_DRV_HW && pnext->drv->calc_type == UADK_ALG_HW) - select_node = pnext; - /* CE driver mean to used CE dev */ - else if (drv_type == ALG_DRV_CE_INS && pnext->drv->calc_type == UADK_ALG_CE_INSTR) - select_node = pnext; - /* SVE driver mean to used SVE dev */ - else if (drv_type == ALG_DRV_SVE_INS && pnext->drv->calc_type == UADK_ALG_SVE_INSTR) - select_node = pnext; - /* INS driver mean to used CE and SVE dev */ - else if (drv_type == ALG_DRV_INS && (pnext->drv->calc_type == UADK_ALG_CE_INSTR || - pnext->drv->calc_type == UADK_ALG_SVE_INSTR)) - select_node = pnext; - /* Soft driver mean to used Soft, CE and SVE dev */ - else if (drv_type == ALG_DRV_SOFT && pnext->drv->calc_type != UADK_ALG_HW) - select_node = pnext; - /* Fallback driver mean to used Soft or CE dev */ - else if (drv_type == ALG_DRV_FB && (pnext->drv->calc_type == UADK_ALG_SOFT || - pnext->drv->calc_type == UADK_ALG_CE_INSTR)) - select_node = pnext; - - if (select_node && select_node->drv->priority > tmp_priority) { - drv = select_node->drv; - tmp_priority = select_node->drv->priority; + pthread_mutex_lock(&alg_registry.mutex); + while (node) { + bool type_match = false; + + /* Check calc_type against requested drv_type */ + if (drv_type == ALG_DRV_HW && node->calc_type == UADK_ALG_HW) + type_match = true; + else if (drv_type == ALG_DRV_CE_INS && node->calc_type == UADK_ALG_CE_INSTR) + type_match = true; + else if (drv_type == ALG_DRV_SVE_INS && node->calc_type == UADK_ALG_SVE_INSTR) + type_match = true; + else if (drv_type == ALG_DRV_INS && (node->calc_type == UADK_ALG_CE_INSTR || + node->calc_type == UADK_ALG_SVE_INSTR)) + type_match = true; + else if (drv_type == ALG_DRV_SOFT && node->calc_type != UADK_ALG_HW) + type_match = true; + else if (drv_type == ALG_DRV_FB && (node->calc_type == UADK_ALG_SOFT || + node->calc_type == UADK_ALG_CE_INSTR)) + type_match = true; + + if (type_match && node->drv->priority > tmp_priority) { + /* Check if this driver supports the requested alg_name and it's available */ + for (i = 0; i < node->alg_count; i++) { + if (!strcmp(alg_name, node->algs[i].alg_name) && + node->algs[i].available) { + drv = node->drv; + tmp_priority = node->drv->priority; + break; + } } } - pnext = pnext->next; + node = node->next; } - if (select_node) - select_node->refcnt++; - pthread_mutex_unlock(&mutex); - - return drv; -} - -void wd_release_drv(struct wd_alg_driver *drv) -{ - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_list *select_node = NULL; - - if (!pnext || !drv) - return; - - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext) && pnext->refcnt > 0) { - select_node = pnext; - break; + /* Increment refcnt on the selected driver node */ + if (drv) { + node = alg_registry.head->next; + while (node) { + if (node->drv == drv) { + node->refcnt++; + break; + } + node = node->next; } - pnext = pnext->next; } - if (select_node) - select_node->refcnt--; - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); + return drv; } int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op_type) { - struct wd_alg_list *pnext = alg_list_head.next; + struct wd_drv_node *node = alg_registry.head->next; struct hisi_dev_usage dev_usage; struct wd_alg_driver *drv; @@ -491,18 +606,19 @@ int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op return -WD_EINVAL; } - while (pnext) { - if (strstr(dev_name, pnext->drv_name) && - !strcmp(alg_type, pnext->alg_type)) + while (node) { + /* Match dev_name and alg_type at the driver node level */ + if (strstr(dev_name, node->drv_name) && + !strcmp(alg_type, node->alg_type)) break; - pnext = pnext->next; + node = node->next; } - if (!pnext) + if (!node) return -WD_EACCES; - drv = pnext->drv; + drv = node->drv; if (!drv->get_usage) return -WD_EINVAL; @@ -512,3 +628,201 @@ int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op return drv->get_usage(&dev_usage); } + +/** + * wd_put_drv_array() - Release driver array allocated by wd_get_drv_array(). + * + * Frees the driver pointer array. Does NOT touch the drivers themselves + * (refcount managed separately by wd_alg_drv_ref_inc/dec). + * + * @drv_array: Driver array from wd_get_drv_array() + * @drv_count: Number of entries (unused, for API symmetry) + */ +void wd_put_drv_array(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + __u32 i; + + for (i = 0; i < drv_count; i++) + drv_array[i] = NULL; + free(drv_array); +} + +/** + * wd_get_drv_array() - Discover all unique drivers matching alg_type and task_type. + * + * @alg_type: Algorithm class string ("cipher", "digest", "aead", "comp", etc.) + * @task_type: TASK_HW (hardware only), TASK_INSTR (instruction only), TASK_MIX (all) + * @drv_array: Output - newly allocated array of unique wd_alg_driver* pointers, + * caller must free with plain free() + * @drv_count: Output - number of unique drivers found + * + * Traverses wd_drv_node list once: + * 1. Matches by alg_type at node level (no need to traverse algs array for this). + * 2. Filters by task_type using wd_alg_drv_type_match(). + * 3. Deduplicates is inherently solved (each node is a unique driver). + * + * This is a PURE QUERY — no reference counting or resource allocation side effects. + * Reference counting is done separately by wd_alg_drv_ref_inc/dec(). + * + * Return: 0 on success, negative on failure. + */ +int wd_get_drv_array(const char *alg_type, int task_type, char *drv_name, + struct wd_alg_driver ***drv_array, __u32 *drv_count) +{ + struct wd_drv_node *head, *node; + struct wd_alg_driver **drivers; + __u32 max_driver_count, current_count = 0; + int i; + + if (!alg_type || !drv_array || !drv_count) { + WD_ERR("invalid: NULL parameter!\n"); + return -WD_EINVAL; + } + + *drv_array = NULL; + *drv_count = 0; + head = wd_get_alg_head(); + if (!head) { + WD_ERR("failed to get alg list head!\n"); + return -WD_EINVAL; + } + + max_driver_count = alg_registry.drv_type_num; + WD_INFO("drivers list drv_type_num: %d\n", alg_registry.drv_type_num); + + if (max_driver_count == 0) { + WD_ERR("no drivers registered for alg_type: %s\n", alg_type); + return -WD_EINVAL; + } + + drivers = calloc(max_driver_count, sizeof(struct wd_alg_driver *)); + if (!drivers) { + WD_ERR("failed to allocate drivers array!\n"); + return -WD_ENOMEM; + } + + /* + * Single traversal of wd_drv_node list: + * - Match by alg_type at node level + * - Filter by task_type + * - Deduplication inherently solved + */ + node = head->next; + while (node) { + if (strcmp(node->alg_type, alg_type) == 0 && + wd_alg_drv_type_match(task_type, node->calc_type)) { + + if (drv_name && strcmp(node->drv_name, drv_name) != 0) + continue; + + /* Check if at least one algorithm in this driver is available */ + bool has_available_alg = false; + for (i = 0; i < node->alg_count; i++) { + if (node->algs[i].available) { + has_available_alg = true; + break; + } + } + + if (!has_available_alg) { + node = node->next; + continue; + } + + if (current_count >= max_driver_count) { + WD_ERR("driver array overflow!\n"); + goto query_failed; + } + drivers[current_count] = node->drv; + current_count++; + } + node = node->next; + } + + if (current_count == 0) { + WD_ERR("no available drivers for alg_type: %s, task_type: %d\n", + alg_type, task_type); + goto query_failed; + } + + WD_INFO("Driver discovery: %u unique drivers for alg_type=%s\n", + current_count, alg_type); + *drv_array = drivers; + *drv_count = current_count; + return 0; + +query_failed: + free(drivers); + return -WD_EINVAL; +} + +/** + * wd_alg_drv_ref_inc() - Increment reference count for each unique driver. + * + * @drv_array: Array of unique driver pointers + * @drv_count: Number of drivers in the array + * + * For each unique driver, finds its node in wd_drv_node list and + * increments refcnt by exactly 1. This ensures refcnt reflects the + * number of configs using the driver, not the number of ctxs. + * + * Must be called after wd_get_drv_array() and after ctx binding. + */ +void wd_alg_drv_ref_inc(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + struct wd_drv_node *node; + __u32 i; + + if (!drv_array || drv_count == 0) + return; + + pthread_mutex_lock(&alg_registry.mutex); + for (i = 0; i < drv_count; i++) { + if (!drv_array[i]) + continue; + /* Directly find the unique driver node and increment refcnt */ + node = alg_registry.head->next; + while (node) { + if (node->drv == drv_array[i]) { + node->refcnt++; + break; + } + node = node->next; + } + } + pthread_mutex_unlock(&alg_registry.mutex); +} + +/** + * wd_alg_drv_ref_dec() - Decrement reference count for each unique driver. + * + * @drv_array: Array of unique driver pointers + * @drv_count: Number of drivers in the array + * + * Inverse of wd_alg_drv_ref_inc(). Decrements refcnt by 1 for each + * unique driver. Must be called during cleanup. + */ +void wd_alg_drv_ref_dec(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + struct wd_drv_node *node; + __u32 i; + + if (!drv_array || drv_count == 0) + return; + + pthread_mutex_lock(&alg_registry.mutex); + for (i = 0; i < drv_count; i++) { + if (!drv_array[i]) + continue; + /* Directly find the unique driver node and decrement refcnt */ + node = alg_registry.head->next; + while (node) { + if (node->drv == drv_array[i] && node->refcnt > 0) { + node->refcnt--; + break; + } + node = node->next; + } + } + pthread_mutex_unlock(&alg_registry.mutex); +} diff --git a/wd_cipher.c b/wd_cipher.c index a4d6c63..f312545 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -340,8 +340,9 @@ static void wd_cipher_clear_status(void) } static int wd_cipher_common_init(struct wd_ctx_config *config, - struct wd_sched *sched) + struct wd_sched *sched, void *attrs) { + struct wd_init_attrs *cipher_attrs = (struct wd_init_attrs *)attrs; int ret; ret = wd_set_epoll_en("WD_CIPHER_EPOLL_EN", @@ -358,6 +359,7 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, if (ret < 0) goto out_clear_ctx_config; + /* allocate async pool for every ctx */ ret = wd_init_async_request_pool(&wd_cipher_setting.pool, config, WD_POOL_MAX_ENTRIES, @@ -367,6 +369,10 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, wd_cipher_setting.priv = STATUS_ENABLE; + /* V2 path: let framework know where the internal config is */ + if (cipher_attrs) + cipher_attrs->ctx_config_internal = &wd_cipher_setting.config; + return 0; out_clear_sched: @@ -395,6 +401,7 @@ static int wd_cipher_common_uninit(void) int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; pthread_atfork(NULL, NULL, wd_cipher_clear_status); @@ -411,26 +418,46 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; - ret = wd_cipher_common_init(config, sched); + /* ═══ Phase 1: Internal copy (existing common_init) ═══ */ + ret = wd_cipher_common_init(config, sched, NULL); if (ret) goto out_close_driver; - ret = wd_ctx_drv_config("ecb(aes)", &wd_cipher_setting.config); - if (ret) - goto out_uninit_nolock; + /* ═══ Phase 2: Driver discovery ═══ */ + ret = wd_get_drv_array("cipher", TASK_HW, "hisi_sec2", + &wd_cipher_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_free_drv_array; + } + WD_INFO("discovered %u unique drivers\n", drv_count); + + /* ═══ Phase 2.5: RR bind drivers to internal ctxs ═══ */ + ret = wd_ctx_bind_drivers(&wd_cipher_setting.config, + wd_cipher_setting.config.drv_array, drv_count); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + /* ═══ Phase 3: Driver initialization ═══ */ ret = wd_alg_init_driver(&wd_cipher_setting.config); - if (ret) - goto out_drv_deconfig; + if (ret) { + WD_ERR("cipher driver init failed!\n"); + goto out_unbind_drivers; + } wd_alg_set_init(&wd_cipher_setting.status); return 0; -out_drv_deconfig: - wd_ctx_drv_deconfig(&wd_cipher_setting.config); -out_uninit_nolock: +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_cipher_setting.config); +out_common_uninit: wd_cipher_common_uninit(); +out_free_drv_array: + wd_put_drv_array(wd_cipher_setting.config.drv_array, drv_count); + wd_cipher_setting.config.drv_array = NULL; out_close_driver: wd_cipher_close_driver(WD_TYPE_V1); out_clear_init: @@ -443,17 +470,21 @@ void wd_cipher_uninit(void) int ret; wd_alg_uninit_driver(&wd_cipher_setting.config); - wd_ctx_drv_deconfig(&wd_cipher_setting.config); - + wd_ctx_unbind_drivers(&wd_cipher_setting.config); ret = wd_cipher_common_uninit(); if (ret) return; + wd_put_drv_array(wd_cipher_setting.config.drv_array, + wd_cipher_setting.config.drv_count); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_cipher_setting.status); } -int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) +int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, + struct wd_ctx_params *ctx_params) { struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; @@ -483,16 +514,17 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p goto out_uninit; while (ret != 0) { - memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + memset(&wd_cipher_setting.config, 0, + sizeof(struct wd_ctx_config_internal)); /* Init ctx param and prepare for ctx request */ cipher_ctx_params.ctx_set_num = cipher_ctx_num; ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, - alg, task_type, WD_CIPHER_TYPE, WD_CIPHER_DECRYPTION + 1); + alg, task_type, WD_CIPHER_TYPE, + WD_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { + if (ret == -WD_EAGAIN) continue; - } goto out_dlclose; } @@ -502,6 +534,8 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; + + /* ═══ Phase 1 + Phase 2 ═══ */ ret = wd_alg_attrs_init(&wd_cipher_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -513,23 +547,31 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p } } - WD_ERR("ctxs numbers: %u.\n", wd_cipher_setting.config.ctx_num); - ret = wd_ctx_drv_config(alg, &wd_cipher_setting.config); - if (ret) - goto out_uninit_nolock; + WD_INFO("ctxs numbers: %u.\n", wd_cipher_setting.config.ctx_num); + /* ═══ Phase 2.5: RR bind drivers ═══ */ + ret = wd_ctx_bind_drivers(&wd_cipher_setting.config, + wd_cipher_init_attrs.drv_array, + wd_cipher_init_attrs.drv_count); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + /* ═══ Phase 3: Driver initialization ═══ */ ret = wd_alg_init_driver(&wd_cipher_setting.config); - if (ret) - goto out_drv_deconfig; + if (ret) { + WD_ERR("driver init failed!\n"); + goto out_unbind_drivers; + } wd_alg_set_init(&wd_cipher_setting.status); wd_ctx_param_uninit(&cipher_ctx_params); return 0; -out_drv_deconfig: - wd_ctx_drv_deconfig(&wd_cipher_setting.config); -out_uninit_nolock: +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_cipher_setting.config); +out_common_uninit: wd_cipher_common_uninit(); wd_alg_attrs_uninit(&wd_cipher_init_attrs); out_params_uninit: @@ -545,12 +587,14 @@ void wd_cipher_uninit2(void) { int ret; - wd_ctx_drv_deconfig(&wd_cipher_setting.config); + wd_alg_uninit_driver(&wd_cipher_setting.config); + wd_ctx_unbind_drivers(&wd_cipher_setting.config); ret = wd_cipher_common_uninit(); if (ret) return; wd_alg_attrs_uninit(&wd_cipher_init_attrs); + wd_cipher_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_cipher_setting.status); } diff --git a/wd_sched.c b/wd_sched.c index edc893c..c4451a7 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1085,7 +1085,7 @@ static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct wd_sched_ctx_domain *domain = NULL; if (region_id >= sched_ctx->region_num || sched_mode >= SCHED_MODE_BUTT || - op_type >= sched_ctx->type_num || prop >= UADK_CTX_MAX) { + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", region_id, sched_mode, op_type, prop); return INVALID_POS; @@ -1147,7 +1147,7 @@ static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, skey->ctx_prop, SCHED_MODE_ASYNC); if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { - WD_ERR("failed to get valid sync_ctx or async_ctx!\n"); + WD_ERR("there is no valid sync_ctx or async_ctx domain!\n"); return -WD_EINVAL; } @@ -1209,7 +1209,7 @@ static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) } sched_skey_param_init(sched_ctx, skey); - WD_INFO("initialized RR scheduler with sync and async domains\n"); + WD_DEBUG("initialized RR scheduler with sync and async domains\n"); return hskey; } @@ -1399,14 +1399,14 @@ static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) skey = (struct wd_sched_key *)hskey; def_prop = skey->ctx_prop; /* Init and get ctx for every ctx mode */ - for (i = 0; i < UADK_CTX_MAX; i++) { + for (i = 0; i < UADK_ALG_TYPE_MAX; i++) { skey->ctx_prop = i; ret = session_sched_domain_init(sched_ctx, skey); - if (ret != 0) { - WD_ERR("Can't to request prop=%d type ctx!\n", i); + if (ret != 0) continue; - } + /* Request two Pre_fetch queues each time. */ + WD_INFO("Successful to request prop=%d type ctx!\n", i); req_ctx_num += 2; } if (!req_ctx_num) { @@ -1417,7 +1417,7 @@ static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) /* Restore the initialization prop settings. */ skey->ctx_prop = def_prop; sched_skey_param_init(sched_ctx, skey); - WD_INFO("initialized Hungry scheduler with sync and async domains\n"); + WD_DEBUG("initialized Hungry scheduler with sync and async domains\n"); return hskey; } @@ -1531,14 +1531,14 @@ static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) skey = (struct wd_sched_key *)hskey; def_prop = skey->ctx_prop; /* Init and get ctx for every ctx mode */ - for (i = 0; i < UADK_CTX_MAX; i++) { + for (i = 0; i < UADK_ALG_TYPE_MAX; i++) { skey->ctx_prop = i; ret = session_sched_domain_init(sched_ctx, skey); - if (ret != 0) { - WD_ERR("Can't to request prop=%d type ctx!\n", i); + if (ret != 0) continue; - } + /* Request two Pre_fetch queues each time. */ + WD_INFO("Successful to request prop=%d type ctx!\n", i); req_ctx_num += 2; } if (!req_ctx_num) { @@ -1549,7 +1549,7 @@ static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) /* Restore the initialization prop settings. */ skey->ctx_prop = def_prop; sched_skey_param_init(sched_ctx, skey); - WD_INFO("initialized Loop scheduler with sync and async domains\n"); + WD_DEBUG("initialized Loop scheduler with sync and async domains\n"); return (handle_t)skey; } @@ -1836,8 +1836,8 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - if (param->ctx_prop < 0 || param->ctx_prop > UADK_CTX_SOFT) - param->ctx_prop = UADK_CTX_HW; + if (param->ctx_prop < 0 || param->ctx_prop > UADK_ALG_SOFT) + param->ctx_prop = UADK_ALG_HW; /* Insert or get domain from hash table using four dimensions */ domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, @@ -1950,12 +1950,12 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, if (sched_type == SCHED_POLICY_DEV) { /* Device mode: region_num is actually device count */ - estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_CTX_MAX; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { /* NUMA mode: validate region_num */ if (numa_num_check(region_num)) goto err_out; - estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_CTX_MAX; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } /* Create single global hash table */ diff --git a/wd_util.c b/wd_util.c index c174d7c..a80d4af 100644 --- a/wd_util.c +++ b/wd_util.c @@ -32,6 +32,7 @@ #define WD_SOFT_CTX_NUM 2 #define WD_SOFT_SYNC_CTX 0 #define WD_SOFT_ASYNC_CTX 1 +#define WD_DRV_MAX_NUM 128 #define WD_DRV_LIB_DIR "uadk" #define WD_DRV_CONF_FILE "uadk.cnf" @@ -163,38 +164,12 @@ int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type) return 0; } -static int wd_parse_dev_id(handle_t h_ctx) -{ - struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; - char *dev_path = ctx->dev_path; - char *last_str = NULL; - char *endptr; - int dev_id; - - if (!dev_path) - return -WD_EINVAL; - - /* Find the last '-' in the string. */ - last_str = strrchr(dev_path, '-'); - if (!last_str || *(last_str + 1) == '\0') - return -WD_EINVAL; - - /* Parse the following number */ - dev_id = strtol(last_str + 1, &endptr, DECIMAL_NUMBER); - /* Check whether it is truly all digits */ - if (*endptr != '\0' || dev_id < 0) - return -WD_EINVAL; - - return dev_id; -} - static void clone_ctx_to_internal(struct wd_ctx *ctx, struct wd_ctx_internal *ctx_in) { ctx_in->ctx = ctx->ctx; ctx_in->op_type = ctx->op_type; ctx_in->ctx_mode = ctx->ctx_mode; - ctx_in->ctx_type = ctx->ctx_type; } static int wd_shm_create(struct wd_ctx_config_internal *in) @@ -1255,7 +1230,7 @@ static int wd_sched_fill_table(struct wd_env_config_per_numa *config_numa, param.type = i; param.begin = ctx_table[mode][i].begin; param.end = ctx_table[mode][i].end; - param.ctx_prop = UADK_CTX_HW; + param.ctx_prop = UADK_ALG_HW; ret = wd_sched_rr_instance(sched, ¶m); if (ret) return ret; @@ -1436,7 +1411,7 @@ int wd_check_ctx(struct wd_ctx_config_internal *config, __u8 mode, __u32 idx) } ctx = config->ctxs + idx; - if (ctx->ctx_type == UADK_CTX_HW && ctx->ctx_mode != mode) { + if (ctx->ctx_type == UADK_ALG_HW && ctx->ctx_mode != mode) { WD_ERR("invalid: ctx(%u) mode is %hhu!\n", idx, ctx->ctx_mode); return -WD_EINVAL; } @@ -1596,6 +1571,8 @@ static int wd_ctx_init_driver(struct wd_ctx_config_internal *config, if (!driver) return 0; + WD_INFO("driver init: drv name: %s, alg_name: %s \n", + driver->drv_name, driver->alg_name); /* Prevent repeated initialization */ if (driver->init_state) return 0; @@ -2203,739 +2180,556 @@ free_path: return (void *)head; } -struct wd_alg_driver *wd_alg_drv_bind(__u8 ctx_prop, char *alg_name) +int wd_ctx_drv_config(char *alg_name, struct wd_ctx_config_internal *ctx_config) +{ + return 0; +} +void wd_ctx_drv_deconfig(struct wd_ctx_config_internal *ctx_config) { - struct wd_alg_driver *fb_drv; - struct wd_alg_driver *drv; - - /* Get alg driver from ctx type and alg name */ - switch (ctx_prop) { - case UADK_CTX_HW: - drv = wd_request_drv(alg_name, ALG_DRV_HW); - if (!drv) { - WD_ERR("no HW %s driver support\n", alg_name); - return NULL; - } - - fb_drv = wd_request_drv(alg_name, ALG_DRV_SOFT); - if (!fb_drv) - drv->fallback = 0; - else - drv->fallback = (handle_t)fb_drv; - - break; - case UADK_CTX_CE_INS: - drv = wd_request_drv(alg_name, ALG_DRV_CE_INS); - if (!drv) { - WD_ERR("no CE instr soft %s driver support\n", alg_name); - return NULL; - } - drv->fallback = 0; - break; - case UADK_CTX_SVE_INS: - drv = wd_request_drv(alg_name, ALG_DRV_SVE_INS); - if (!drv) { - WD_ERR("no SVE instr soft %s driver support\n", alg_name); - return NULL; - } - drv->fallback = 0; - break; - case UADK_CTX_SOFT: - drv = wd_request_drv(alg_name, ALG_DRV_SOFT); - if (!drv) { - WD_ERR("no instr soft %s driver support\n", alg_name); - return NULL; - } - drv->fallback = 0; - break; - default: - WD_ERR("ctx type error: %d.\n", ctx_prop); - return WD_ERR_PTR(-WD_ENODEV); - } - - return drv; } -void wd_alg_drv_unbind(struct wd_alg_driver *drv) +/** + * wd_ctx_unbind_drivers() - Phase 2.5 reverse: Unbind drivers from internal ctxs. + * + * Decrements driver refcounts and clears all drv pointers. + * + * @config: Internal ctx config + */ +void wd_ctx_unbind_drivers(struct wd_ctx_config_internal *config) { - struct wd_alg_driver *fb_drv = NULL; + __u32 i; - if (!drv) + if (!config || !config->drv_array) return; - fb_drv = (struct wd_alg_driver *)drv->fallback; - if (fb_drv) - wd_release_drv(fb_drv); - wd_release_drv(drv); + wd_alg_drv_ref_dec(config->drv_array, config->drv_count); + + for (i = 0; i < config->ctx_num; i++) + config->ctxs[i].drv = NULL; } -static __u32 wd_ctxs_idx_init(struct wd_init_attrs *attrs, int numa_cnt, int task_type) +/** + * wd_ctx_bind_drivers() - Bind drivers to internal ctxs via RR. + * + * This is the SINGLE WRITE POINT for ctxs[i].drv in the entire lifecycle. + * Uses RR rule: ctxs[i].drv = drv_array[i % drv_count] + * + * Also: + * - Sets up soft fallback for HW drivers (once per unique HW driver) + * - Caches drv_array in config for session queries + * - Increments driver refcounts (deduplicated: each unique driver +1) + * + * and overwrote the RR mapping. + * + * @config: Internal ctx config (ctxs[] already copied by wd_init_ctx_config) + * @drv_array: Discovered unique drivers (from Phase 1) + * @drv_count: Number of unique drivers + * Return: 0 on success, negative on failure + */ +int wd_ctx_bind_drivers(struct wd_ctx_config_internal *config, + struct wd_alg_driver **drv_array, __u32 drv_count) { - struct wd_ctx_params *ctx_params = attrs->ctx_params; - int end = ctx_params->op_type_num; - struct wd_ctx_nums *ptr_ctx; - __u32 count = 0; - int i, uidx; + struct wd_alg_driver *drv; + __u32 i; - if (ctx_params->op_type_num > MAX_CTX_OP_TYPE) { - WD_ERR("invalid: max ctx op type<%u> is wrong!\n", ctx_params->op_type_num); - return 0; + if (!config || !drv_array || drv_count == 0) { + WD_ERR("invalid parameters!\n"); + return -WD_EINVAL; } - for (uidx = 0; uidx < UADK_CTX_MAX; uidx++) { - /* If it is a soft computing task, do not use HW queue */ - if (task_type == TASK_INSTR && uidx == UADK_CTX_HW) - continue; - for (i = 0; i < end; i++) { - ptr_ctx = &ctx_params->ctx_set_num[i]; - while (ptr_ctx) { - if (ptr_ctx->ctx_prop == uidx) { - ptr_ctx->ctx_begin = count; - if (uidx == UADK_CTX_HW) { - count += ptr_ctx->sync_ctx_num * numa_cnt; - count += ptr_ctx->async_ctx_num * numa_cnt; - } else { - count += ptr_ctx->sync_ctx_num; - count += ptr_ctx->async_ctx_num; - } - WD_ERR("optype<%d>, prop<%u>, begin<%u>, synx<%u>, async<%u>\n", - i, ptr_ctx->ctx_prop, ptr_ctx->ctx_begin, ptr_ctx->sync_ctx_num, ptr_ctx->async_ctx_num); - } - ptr_ctx = ptr_ctx->other_ctx; + WD_INFO("Phase 2: drivers array have <%u> drvers.\n", drv_count); + for (i = 0; i < config->ctx_num; i++) { + /* In the init process, only one hisi driver will be specified. */ + if (drv_count == 1) { + config->ctxs[i].drv = drv_array[0]; + config->ctxs[i].ctx_type = config->ctxs[0].drv->calc_type; + } else { + /* + * RR binding — the ONLY write to ctxs[i].drv in the + * entire lifecycle. After this, drv is read-only. + */ + config->ctxs[i].drv = drv_array[i % drv_count]; + config->ctxs[i].ctx_type = config->ctxs[i].drv->calc_type; + } + WD_INFO("driver bind: drv name: %s, alg_name: %s for ctx<%u>\n", + config->ctxs[i].drv->drv_name, config->ctxs[i].drv->alg_name, i); + + /* HW driver needs soft fallback — set once per unique driver */ + if (config->ctxs[i].ctx_type == UADK_ALG_HW) { + drv = config->ctxs[i].drv; + if (!drv->fallback) { + drv->fallback = (handle_t)wd_request_drv( + config->alg_name, ALG_DRV_SOFT); } } } - return count; -} + /* Cache driver array for session queries */ + config->drv_array = drv_array; + config->drv_count = drv_count; -static struct wd_ctx_nums *wd_get_ctx_ptr(struct wd_ctx_params *ctx_params, - __u32 op_type, int ctx_prop) -{ - struct wd_ctx_nums *ptr_ctx; + /* Deduplicated refcount increment */ + wd_alg_drv_ref_inc(drv_array, drv_count); - if (op_type > ctx_params->op_type_num) - return NULL; + WD_INFO("Phase 2.5: bound %u ctxs to %u drivers via RR\n", + config->ctx_num, drv_count); - ptr_ctx = &ctx_params->ctx_set_num[op_type]; - while (ptr_ctx) { - if (ptr_ctx->ctx_prop == ctx_prop) - return ptr_ctx; - ptr_ctx = ptr_ctx->other_ctx; - } - - return NULL; + return 0; } -void wd_ctx_drv_deconfig(struct wd_ctx_config_internal *ctx_config) +/** + * wd_alg_drv_undiscover() - Free driver discovery result. + * + * Releases the drv_array allocated by wd_alg_drv_discover(). + * Does NOT touch the drivers themselves (refcount managed separately). + * + * @attrs: Initialization attributes + */ +void wd_alg_drv_undiscover(struct wd_init_attrs *attrs) { - __u32 i; - - // wd_dlclose_drv after this - for (i = 0; i < ctx_config->ctx_num; i++) - wd_alg_drv_unbind(ctx_config->ctxs[i].drv); + if (!attrs || !attrs->drv_array) + return; + /* Release wd_get_drv_array alloc memory */ + wd_put_drv_array(attrs->drv_array, attrs->drv_count); + attrs->drv_array = NULL; + attrs->drv_count = 0; } -int wd_ctx_drv_config(char *alg_name, struct wd_ctx_config_internal *ctx_config) +/** + * wd_alg_drv_discover() - Discover matching drivers. + * + * Normalizes attrs->alg to alg_type ("cipher", "digest", etc.), + * then calls wd_get_drv_array() to find all unique drivers. + * Results stored in attrs->drv_array and attrs->drv_count. + * + * Pure query — no resource allocation, no refcount changes. + * + * @attrs: Initialization attributes (input: alg, task_type; output: drv_array, drv_count) + * Return: 0 on success, negative on failure + */ +int wd_alg_drv_discover(struct wd_init_attrs *attrs) { - __u32 i, j; + char alg_type[CRYPTO_MAX_ALG_NAME] = {0}; + int ret; - // wd_dlopen_drv before this - WD_ERR("debug: call function: %s!\n", __func__); - for (i = 0; i < ctx_config->ctx_num; i++) { - ctx_config->ctxs[i].drv = wd_alg_drv_bind(ctx_config->ctxs[i].ctx_type, alg_name); - if (WD_IS_ERR(ctx_config->ctxs[i].drv)) { - continue; - } else if (!ctx_config->ctxs[i].drv) { - WD_ERR("failed to bind %s driver.\n", alg_name); - goto bind_err; - } - } + if (!attrs || !attrs->alg[0]) + return -WD_EINVAL; - return 0; + /* Normalize alg to alg_type (e.g. "cipher", "digest") */ + wd_get_alg_type(attrs->alg, alg_type); + if (!alg_type[0]) { + WD_ERR("unknown alg type for %s\n", attrs->alg); + return -WD_EINVAL; + } -bind_err: - for (j = 0; j < i; j++) { - wd_alg_drv_unbind(ctx_config->ctxs[j].drv); + ret = wd_get_drv_array(alg_type, attrs->task_type, NULL, + &attrs->drv_array, &attrs->drv_count); + if (ret) { + WD_ERR("failed to get %s's driver array\n", attrs->alg); + return -WD_EINVAL; } - return -WD_EINVAL; + + return 0; } -struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, struct bitmask *bmp) +static int wd_alg_sched_instance(struct wd_sched *sched, + struct wd_ctx_config *ctx_config, + struct wd_ctx_params *ctx_params) { - struct uacce_dev_list *p, *node, *result = NULL; - struct uacce_dev *dev; - int numa_id, ret; + struct sched_params sparams; + __u32 sync_count, async_count, total_count; + __u32 i; + int ret; - if (!bmp) { - WD_ERR("invalid: bmp is NULL!\n"); - return WD_ERR_PTR(-WD_EINVAL); + if (!sched || !ctx_config || !ctx_params) { + WD_ERR("invalid: sched, ctx_config, or ctx_params is NULL!\n"); + return -WD_EINVAL; } - p = list; - while (p) { - dev = p->dev; - numa_id = dev->numa_id; - ret = numa_bitmask_isbitset(bmp, numa_id); - if (!ret) { - p = p->next; - continue; - } - - node = calloc(1, sizeof(*node)); - if (!node) { - result = WD_ERR_PTR(-WD_ENOMEM); - goto out_free_list; - } - - node->dev = wd_clone_dev(dev); - if (!node->dev) { - result = WD_ERR_PTR(-WD_ENOMEM); - goto out_free_node; - } - - if (!result) - result = node; - else - wd_add_dev_to_list(result, node); + /* Calculate total sync/async context counts */ + sync_count = 0; + async_count = 0; - p = p->next; + for (i = 0; i < ctx_params->op_type_num; i++) { + sync_count += ctx_params->ctx_set_num[i].sync_ctx_num; + async_count += ctx_params->ctx_set_num[i].async_ctx_num; } - return result ? result : WD_ERR_PTR(-WD_ENODEV); - -out_free_node: - free(node); -out_free_list: - wd_free_list_accels(result); - return result; -} - -static int wd_init_hw_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *list, - __u32 idx, int numa_id, __u32 op_type) -{ - struct wd_ctx_nums ctx_nums = attrs->ctx_params->ctx_set_num[op_type]; - __u32 ctx_set_num = ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num; - struct wd_ctx_config *ctx_config = attrs->ctx_config; - __u32 count = idx + ctx_set_num; - struct uacce_dev *dev; - __u32 i, cnt = 0; - - /* If the ctx set number is 0, the initialization is skipped. */ - if (!ctx_set_num) - return -WD_ENOPROC; - - dev = wd_find_dev_by_numa(list, numa_id); - if (WD_IS_ERR(dev)) - return WD_PTR_ERR(dev); - - for (i = idx; i < count; i++) { - ctx_config->ctxs[i].ctx = wd_request_ctx(dev); - if (errno == WD_EBUSY) { - dev = wd_find_dev_by_numa(list, numa_id); - if (WD_IS_ERR(dev)) - return WD_PTR_ERR(dev); - - if (cnt++ > WD_INIT_RETRY_TIMES) { - WD_ERR("failed to request enough ctx due to timeout!\n"); - return -WD_ETIMEDOUT; - } + total_count = sync_count + async_count; - /* self-decrease i to eliminate self-increase on next loop */ - i--; - continue; - } else if (!ctx_config->ctxs[i].ctx) { - /* - * wd_release_ctx_set will release ctx in - * caller wd_init_ctx_and_sched. - */ - return -WD_ENOMEM; - } - ctx_config->ctxs[i].op_type = op_type; - ctx_config->ctxs[i].ctx_mode = - ((i - idx) < ctx_nums.sync_ctx_num) ? - CTX_MODE_SYNC : CTX_MODE_ASYNC; - ctx_config->ctxs[i].ctx_type = UADK_CTX_HW; + if (total_count != ctx_config->ctx_num) { + WD_ERR("mismatch: expected %u contexts, got %u!\n", + total_count, ctx_config->ctx_num); + return -WD_EINVAL; } - return 0; -} + WD_INFO("Registering contexts: sync=%u, async=%u, total=%u\n", + sync_count, async_count, total_count); -static void wd_release_ctx_set(struct wd_ctx_config *ctx_config) -{ - __u32 i; + /* Register sync contexts range to scheduler */ + if (sync_count > 0) { + memset(&sparams, 0, sizeof(sparams)); + sparams.numa_id = 0; + sparams.type = 0; + sparams.mode = CTX_MODE_SYNC; + sparams.begin = 0; + sparams.end = sync_count - 1; + sparams.ctx_prop = 0; - for (i = 0; i < ctx_config->ctx_num; i++) - if (ctx_config->ctxs[i].ctx) { - wd_release_ctx(ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; + ret = wd_sched_rr_instance(sched, &sparams); + if (ret) { + WD_ERR("failed to register sync contexts to scheduler!\n"); + return ret; } -} - -static int wd_instance_sched_set(struct wd_init_attrs *attrs, struct wd_ctx_nums ctx_nums, - int idx, int numa_id, int op_type) -{ - struct wd_sched *sched = attrs->sched; - struct sched_params sparams; - int i, end, dev_id, ret = 0; - dev_id = wd_parse_dev_id(attrs->ctx_config->ctxs[idx].ctx); - if (dev_id < 0) - return -WD_EINVAL; - - for (i = 0; i < CTX_MODE_MAX; i++) { - sparams.numa_id = numa_id; - sparams.type = op_type; - sparams.dev_id = dev_id; - sparams.mode = i; - sparams.begin = idx + ctx_nums.sync_ctx_num * i; - sparams.ctx_prop = UADK_CTX_HW; - end = idx - 1 + ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num * i; - if (end < 0 || sparams.begin > (__u32)end) - continue; - - sparams.end = end; - ret = wd_sched_rr_instance(sched, &sparams); - if (ret) - goto out; + WD_INFO("Registered %u sync contexts to scheduler\n", sync_count); } -out: - return ret; -} - -static int wd_hw_ctx_and_sched(struct wd_init_attrs *attrs, struct bitmask *bmp, - struct uacce_dev_list *list) -{ - struct wd_ctx_params *ctx_params = attrs->ctx_params; - __u32 op_type_num = ctx_params->op_type_num; - int i, ret, max_node = numa_max_node() + 1; - struct wd_ctx_nums ctx_nums; - __u32 j, idx = 0; + /* Register async contexts range to scheduler */ + if (async_count > 0) { + memset(&sparams, 0, sizeof(sparams)); + sparams.numa_id = 0; + sparams.type = 0; + sparams.mode = CTX_MODE_ASYNC; + sparams.begin = sync_count; + sparams.end = total_count - 1; + sparams.ctx_prop = 0; - for (i = 0; i < max_node; i++) { - if (!numa_bitmask_isbitset(bmp, i)) - continue; - for (j = 0; j < op_type_num; j++) { - ctx_nums = ctx_params->ctx_set_num[j]; - ret = wd_init_hw_ctx_set(attrs, list, idx, i, j); - if (ret == -WD_ENOPROC) - continue; - else if (ret) - goto free_ctxs; - ret = wd_instance_sched_set(attrs, ctx_nums, idx, i, j); - if (ret) - goto free_ctxs; - idx += (ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num); + ret = wd_sched_rr_instance(sched, &sparams); + if (ret) { + WD_ERR("failed to register async contexts to scheduler!\n"); + return ret; } + + WD_INFO("Registered %u async contexts to scheduler\n", async_count); } return 0; - -free_ctxs: - wd_release_ctx_set(attrs->ctx_config); - - return ret; } -static void wd_init_device_nodemask(struct uacce_dev_list *list, struct bitmask *bmp) +/** + * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. + * + * Releases resources in reverse allocation order: + * 1. Release scheduler + * 2. Release ctxs via RR rule (drv->free_ctx) + * 3. Free ctx_config and ctxs array + * + * @attrs: Initialization attributes + */ +void wd_alg_ctx_uninit(struct wd_init_attrs *attrs) { - struct uacce_dev_list *p = list; + struct wd_ctx_config *ctx_config; + struct wd_alg_driver *drv; + __u32 i, drv_idx; - numa_bitmask_clearall(bmp); - while (p) { - numa_bitmask_setbit(bmp, p->dev->numa_id); - p = p->next; - } -} + if (!attrs) + return; -static int wd_alg_other_ctx_init(struct wd_init_attrs *attrs, int ctx_prop) -{ - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_ctx_nums *ptr_ctxs; - struct wd_soft_ctx *sfctx; - struct sched_params sparams; - __u32 begin, end, ctx_num; - int sync_type, ret; - __u32 i, j, k; + ctx_config = attrs->ctx_config; - WD_ERR("debug: call function: %s!\n", __func__); - for (i = 0; i < ctx_params->op_type_num; i++) { - ptr_ctxs = wd_get_ctx_ptr(ctx_params, i, ctx_prop); - if (!ptr_ctxs) - continue; + WD_INFO("Phase 1： wd_alg_ctx_uninit\n"); + /* Release scheduler */ + if (attrs->sched) { + wd_sched_rr_release(attrs->sched); + attrs->sched = NULL; + } - for (sync_type = CTX_MODE_SYNC; sync_type < CTX_MODE_MAX; sync_type++) { - if (sync_type == CTX_MODE_SYNC) { - ctx_num = ptr_ctxs->sync_ctx_num; - begin = ptr_ctxs->ctx_begin; - } else { - ctx_num = ptr_ctxs->async_ctx_num; - begin = ptr_ctxs->ctx_begin + ptr_ctxs->sync_ctx_num; - } - if (ctx_num == 0) + /* Release ctxs via RR rule */ + if (ctx_config && ctx_config->ctxs && attrs->drv_array && + attrs->drv_count > 0) { + for (i = 0; i < ctx_config->ctx_num; i++) { + if (!ctx_config->ctxs[i].ctx) continue; - end = begin + ctx_num; - for (j = begin; j < end; j++) { - ctx_config->ctxs[j].op_type = i; - ctx_config->ctxs[j].ctx_mode = sync_type; - ctx_config->ctxs[j].ctx_type = ctx_prop; - sfctx = calloc(1, sizeof(struct wd_soft_ctx)); - if (!sfctx) { - WD_ERR("failed to alloc ctx!\n"); - goto ctx_err; - } - ctx_config->ctxs[j].ctx = (handle_t)sfctx; - pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_SHARED); - pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_SHARED); - } + drv_idx = i % attrs->drv_count; + drv = attrs->drv_array[drv_idx]; - memset(&sparams, 0x0, sizeof(struct sched_params)); - sparams.begin = begin; - sparams.end = end - 1; - sparams.mode = sync_type; - sparams.numa_id = 0; - sparams.ctx_prop = ctx_prop; - ret = wd_sched_rr_instance(attrs->sched, &sparams); - if (ret) { - WD_ERR("fail to instance scheduler.\n"); - goto ctx_err; + if (drv && drv->free_ctx) { + drv->free_ctx(ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + WD_INFO("Phase 2： free drv--<%s>, ctx<%u>\n", drv->drv_name, i); } } } - return WD_SUCCESS; - -ctx_err: - for (k = j; k >= begin; k--) { - free((struct wd_soft_ctx *)ctx_config->ctxs[k].ctx); - ctx_config->ctxs[k].ctx = 0; + /* Release ctx_config */ + if (ctx_config) { + if (ctx_config->ctxs) { + free(ctx_config->ctxs); + ctx_config->ctxs = NULL; + } + free(ctx_config); + attrs->ctx_config = NULL; } - return -WD_ENOMEM; + attrs->ctx_config_internal = NULL; + + WD_INFO("Phase 3 reverse: ctx uninit complete\n"); } -static int wd_alg_other_init(struct wd_init_attrs *attrs) +/** + * wd_alg_ctx_init() - Allocate ctxs, scheduler, and do internal copy. + * + * Uses drivers discovered in Phase 1 (attrs->drv_array). + * Allocates ctxs via RR: ctx[i] → drv_array[i % drv_count]->alloc_ctx() + * Then allocates scheduler, registers ctx ranges, and calls alg_init + * which performs wd_init_ctx_config() (wd_ctx[] → wd_ctx_internal[] copy). + * + * On return: + * - attrs->ctx_config: user-visible ctx array (populated) + * - attrs->sched: scheduler (allocated and populated) + * - attrs->ctx_config_internal: MUST be set by alg_init callback + * + * NOTE: ctxs[i].drv is still NULL after this function — that's set in Phase 2.5. + * + * @attrs: Initialization attributes (input: drv_array, ctx_params, alg_init, etc.) + * Return: 0 on success, negative on failure + */ +int wd_alg_ctx_init(struct wd_init_attrs *attrs) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ctx_params *ctx_params = attrs->ctx_params; - __u32 ctx_set_num, op_type_num; + struct wd_ctx_config *ctx_config; + struct wd_ctx_params *ctx_params; + struct wd_drv_ctx_params dparams; + struct wd_alg_driver *drv; + handle_t ctx; + __u32 sync_num = 0, async_num = 0; + __u32 total_ctx_num = 0; + __u32 ctx_idx, drv_idx; + __u32 i, op_type; + __u16 region_num; + __u32 accum = 0; + __u32 cnt; + int ret; - WD_ERR("debug: call function: %s!\n", __func__); - op_type_num = ctx_params->op_type_num; - ctx_set_num = wd_ctxs_idx_init(attrs, 1, attrs->task_type); - if (!ctx_set_num || !op_type_num) { - WD_ERR("invalid: ctx_set_num is %u, op_type_num is %u!\n", - ctx_set_num, op_type_num); + if (!attrs || !attrs->ctx_params || !attrs->drv_array || + attrs->drv_count == 0) { + WD_ERR("invalid: attrs, ctx_params, or drv_array is NULL/empty!\n"); return -WD_EINVAL; } - ctx_config->ctx_num = ctx_set_num; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { - WD_ERR("failed to alloc ctxs!\n"); - return -WD_ENOMEM; - } - - return 0; -} + ctx_params = attrs->ctx_params; -static int wd_alg_hw_ctx_init(struct wd_init_attrs *attrs) -{ - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct bitmask *used_bmp = ctx_params->bmp; - struct uacce_dev_list *list, *used_list = NULL; - char alg_type[CRYPTO_MAX_ALG_NAME]; - __u32 ctx_set_num, op_type_num; - int numa_cnt, ret; - - WD_ERR("debug: call function: %s!\n", __func__); - wd_get_alg_type(attrs->alg, alg_type); + /* Calculate total sync/async context counts */ + for (i = 0; i < ctx_params->op_type_num; i++) { + sync_num += ctx_params->ctx_set_num[i].sync_ctx_num; + async_num += ctx_params->ctx_set_num[i].async_ctx_num; + } + total_ctx_num = sync_num + async_num; - list = wd_get_accel_list(alg_type); - if (!list) { - WD_ERR("failed to get devices for alg: %s\n", attrs->alg); - return -WD_ENODEV; + if (total_ctx_num == 0) { + WD_ERR("invalid: total_ctx_num is zero!\n"); + return -WD_EINVAL; } - /* - * Not every numa has a device. Therefore, the first thing is to - * filter the devices in the selected numa node, and the second - * thing is to obtain the distribution of devices. - */ - used_list = wd_get_usable_list(list, used_bmp); - if (WD_IS_ERR(used_list)) { - ret = WD_PTR_ERR(used_list); - WD_ERR("failed to get usable devices(%d)!\n", ret); - goto out_freelist; - } - - wd_init_device_nodemask(used_list, used_bmp); - numa_cnt = numa_bitmask_weight(used_bmp); - if (!numa_cnt) { - ret = numa_cnt; - WD_ERR("invalid: bmp is clear!\n"); - goto out_freeusedlist; - } - - op_type_num = ctx_params->op_type_num; - ctx_set_num = wd_ctxs_idx_init(attrs, numa_cnt, attrs->task_type); - WD_ERR("ctx sum num is: %u, op_type num is: %u!\n", ctx_set_num, op_type_num); - if (!ctx_set_num || !op_type_num) { - WD_ERR("invalid: ctx_set_num is %u, op_type_num is %u!\n", - ctx_set_num, op_type_num); - ret = -WD_EINVAL; - goto out_freelist; + /* Allocate ctx_config structure */ + ctx_config = calloc(1, sizeof(*ctx_config)); + if (!ctx_config) { + WD_ERR("failed to allocate ctx_config!\n"); + return -WD_ENOMEM; } - ctx_config->ctx_num = ctx_set_num; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); + /* Allocate user-visible wd_ctx array (no drv pointer — ABI safe) */ + ctx_config->ctxs = calloc(total_ctx_num, sizeof(struct wd_ctx)); if (!ctx_config->ctxs) { - ret = -WD_ENOMEM; - WD_ERR("failed to alloc ctxs!\n"); - goto out_freeusedlist; + WD_ERR("failed to allocate ctxs array!\n"); + free(ctx_config); + return -WD_ENOMEM; } + ctx_config->ctx_num = total_ctx_num; + attrs->ctx_config = ctx_config; - ret = wd_hw_ctx_and_sched(attrs, used_bmp, used_list); - if (ret) - free(ctx_config->ctxs); - -out_freeusedlist: - wd_free_list_accels(used_list); -out_freelist: - wd_free_list_accels(list); - - return ret; -} + WD_INFO("Phase 2: %u drivers, %u ctxs (sync=%u, async=%u)\n", + attrs->drv_count, total_ctx_num, sync_num, async_num); -static void wd_alg_ctxs_uninit(struct wd_ctx_config *ctx_config) -{ - __u32 i; + /* ── RR allocation loop ── */ + for (ctx_idx = 0; ctx_idx < total_ctx_num; ctx_idx++) { + drv_idx = ctx_idx % attrs->drv_count; + drv = attrs->drv_array[drv_idx]; - for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - if (ctx_config->ctxs[i].ctx_type == UADK_CTX_HW) - wd_release_ctx(ctx_config->ctxs[i].ctx); - else - free((struct wd_soft_ctx *)ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; + if (!drv || !drv->alloc_ctx) { + WD_ERR("Warning: driver-%s alloc_ctx is NULL!\n", drv->drv_name); + continue; } - } - if (ctx_config->ctxs) { - free(ctx_config->ctxs); - ctx_config->ctxs = 0; - } -} - -int wd_alg_attrs_init(struct wd_init_attrs *attrs) -{ - wd_alg_poll_ctx alg_poll_func = attrs->alg_poll_ctx; - wd_alg_init alg_init_func = attrs->alg_init; - struct wd_ctx_config *ctx_config = NULL; - struct wd_sched *alg_sched = NULL; - struct wd_alg_driver *drv = NULL; - char *alg_name = attrs->alg; - __u32 op_type_num; - int ret = 0; + WD_ERR("------driver<%s> alloc ctx<%u>\n", drv->drv_name, ctx_idx); + /* Determine op_type for this ctx index */ + op_type = 0; + accum = 0; + for (i = 0; i < ctx_params->op_type_num; i++) { + cnt = ctx_params->ctx_set_num[i].sync_ctx_num + + ctx_params->ctx_set_num[i].async_ctx_num; + if (ctx_idx < accum + cnt) { + op_type = i; + break; + } + accum += cnt; + } - if (!attrs->ctx_params) - return -WD_EINVAL; + /* Fill driver ctx params */ + memset(&dparams, 0, sizeof(dparams)); + dparams.ctx_mode = (ctx_idx < sync_num) ? + CTX_MODE_SYNC : CTX_MODE_ASYNC; + dparams.op_type = (__u16)op_type; + dparams.numa_id = 0; /* Scheduler handles NUMA at runtime */ + dparams.idx = ctx_idx; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + /* Call driver's alloc_ctx — driver owns the implementation */ + ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); + if (!ctx || ret) { + WD_ERR("driver %u (%s) alloc_ctx failed for ctx %u\n", + drv_idx, drv->drv_name, ctx_idx); + goto cleanup_ctxs; + } - WD_ERR("debug: call function: %s!\n", __func__); - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; + /* Store in user-visible ctx_config */ + ctx_config->ctxs[ctx_idx].ctx = ctx; + ctx_config->ctxs[ctx_idx].op_type = dparams.op_type; + ctx_config->ctxs[ctx_idx].ctx_mode = dparams.ctx_mode; } - attrs->ctx_config = ctx_config; - - /* Get op_type_num */ - op_type_num = attrs->ctx_params->op_type_num; - if (!op_type_num) - goto out_ctx_config; - /* Use default sched_type to alloc scheduler */ + /* ── Allocate scheduler ── */ if (attrs->sched_type == SCHED_POLICY_DEV) - alg_sched = wd_sched_rr_alloc(attrs->sched_type, attrs->ctx_params->op_type_num, - DEVICE_REGION_MAX, alg_poll_func); + region_num = DEVICE_REGION_MAX; else - alg_sched = wd_sched_rr_alloc(attrs->sched_type, attrs->ctx_params->op_type_num, - numa_max_node() + 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - goto out_ctx_config; - } - attrs->sched = alg_sched; - - /* Initialize queues according to task type */ - switch (attrs->task_type) { - case TASK_HW: - ret = wd_alg_hw_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init HW ctx\n"); - goto out_freesched; - } - - break; - case TASK_MIX: - ret = wd_alg_hw_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init mix HW ctx\n"); - goto out_freesched; - } - - WD_ERR("TASK_MIX: call function: %s!\n", __func__); - drv = wd_request_drv(alg_name, ALG_DRV_SOFT); - if (drv == NULL) { - WD_ERR("fail to find soft driver.\n"); - break; - } else if (drv->calc_type == UADK_ALG_CE_INSTR) { - ret = wd_alg_other_ctx_init(attrs, UADK_CTX_CE_INS); - if (ret) { - WD_ERR("fail to init ce ctx\n"); - goto out_ctx_init; - } - } else if (drv->calc_type == UADK_ALG_SVE_INSTR) { - ret = wd_alg_other_ctx_init(attrs, UADK_CTX_SVE_INS); - if (ret) { - WD_ERR("fail to init sve ctx\n"); - goto out_ctx_init; - } - } + region_num = numa_max_node() + 1; + + attrs->sched = wd_sched_rr_alloc(attrs->sched_type, + ctx_params->op_type_num, + region_num, + attrs->alg_poll_ctx); + if (!attrs->sched) { + WD_ERR("failed to allocate scheduler!\n"); + ret = -WD_ENOMEM; + goto cleanup_ctxs; + } - break; - /* Only pure soft queues */ - case TASK_INSTR: - ret = wd_alg_other_init(attrs); - if (ret) { - WD_ERR("fail to init other ctx.\n"); - goto out_freesched; - } + /* ── Register contexts to scheduler ── */ + ret = wd_alg_sched_instance(attrs->sched, ctx_config, ctx_params); + if (ret) { + WD_ERR("failed to register contexts to scheduler!\n"); + goto cleanup_sched; + } - WD_ERR("TASK_INSTR: call function: %s!\n", __func__); - drv = wd_request_drv(alg_name, ALG_DRV_SOFT); - if (drv == NULL) { - WD_ERR("fail to find soft driver.\n"); - goto out_ctx_init; - } else if (drv->calc_type == UADK_ALG_CE_INSTR) { - ret = wd_alg_other_ctx_init(attrs, UADK_CTX_CE_INS); - if (ret) { - WD_ERR("fail to init ce ctx\n"); - goto out_ctx_init; - } - } else if (drv->calc_type == UADK_ALG_SVE_INSTR) { - ret = wd_alg_other_ctx_init(attrs, UADK_CTX_SVE_INS); - if (ret) { - WD_ERR("fail to init sve ctx\n"); - goto out_ctx_init; - } - } - break; - default: - WD_ERR("driver type error: %d\n", drv->calc_type); - return -WD_EINVAL; + /* ── Call algorithm-specific init (does wd_init_ctx_config copy) ── */ + ctx_config->cap = ctx_params->cap; + ret = attrs->alg_init(ctx_config, attrs->sched, attrs); + if (ret) { + WD_ERR("failed to initialize algorithm!\n"); + goto cleanup_sched; } - ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_ctx_init; + /* + * IMPORTANT: attrs->ctx_config_internal must be set by alg_init. + * The alg_init callback (e.g. wd_cipher_common_init) internally calls + * wd_init_ctx_config() which creates the wd_ctx_config_internal. + * The callback must store the pointer in attrs->ctx_config_internal. + * + * If the current alg_init signature doesn't pass attrs, it needs to + * be updated. See "Required changes in other files" below. + */ - WD_ERR("---->ctx nums: %u\n", ctx_config->ctx_num); + WD_INFO("Phase 2 complete: %u ctxs from %u drivers\n", + total_ctx_num, attrs->drv_count); return 0; -out_ctx_init: - wd_alg_ctxs_uninit(ctx_config); -out_freesched: - wd_sched_rr_release(alg_sched); -out_ctx_config: - if (ctx_config) - free(ctx_config); + /* ── Error cleanup (LIFO) ── */ +cleanup_sched: + wd_sched_rr_release(attrs->sched); + attrs->sched = NULL; +cleanup_ctxs: + /* Free ctxs allocated so far using RR rule */ + for (i = 0; i < ctx_idx; i++) { + drv_idx = i % attrs->drv_count; + drv = attrs->drv_array[drv_idx]; + if (drv && drv->free_ctx && ctx_config->ctxs[i].ctx) { + drv->free_ctx(ctx_config->ctxs[i].ctx); + ctx_config->ctxs[i].ctx = 0; + } + } + free(ctx_config->ctxs); + free(ctx_config); + attrs->ctx_config = NULL; return ret; } +/** + * wd_alg_attrs_uninit() - Algorithm attribute cleanup. + * + * Releases all resources in strict reverse order of init: + * Phase 2.5 reverse: wd_ctx_unbind_drivers() + * Phase 2 reverse: wd_alg_ctx_uninit() + * Phase 1 reverse: wd_alg_drv_undiscover() + * + * @attrs: Initialization attributes + */ void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_sched *alg_sched = attrs->sched; - - if (!ctx_config) { - wd_sched_rr_release(alg_sched); + if (!attrs) return; - } - wd_alg_ctxs_uninit(ctx_config); + WD_INFO("Algorithm cleanup started: alg=%s\n", attrs->alg); - free(ctx_config); - wd_sched_rr_release(alg_sched); -} + /* Phase reverse: release ctxs, scheduler, ctx_config */ + wd_alg_ctx_uninit(attrs); -int wd_queue_is_busy(struct wd_soft_ctx *sctx) -{ - /* The queue is not used */ - if (sctx->run_num >= MAX_SOFT_QUEUE_LENGTH - 1) - return -WD_EBUSY; + /* Phase reverse: free driver array */ + wd_alg_drv_undiscover(attrs); - return 0; + WD_INFO("Algorithm cleanup complete\n"); } -int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id) +/** + * wd_alg_attrs_init() - Algorithm attribute initialization (V2 path). + * + * Orchestrates the 3-phase initialization pipeline: + * Phase 1: wd_alg_drv_discover() — discover unique drivers + * Phase 2: wd_alg_ctx_init() — allocate ctxs + scheduler + internal copy + * Phase 2.5: wd_ctx_bind_drivers() — RR bind drivers to internal ctxs + * + * After this, Phase 3 (driver init) is done by the caller via wd_alg_init_driver(). + * + * @attrs: Initialization attributes (input/output) + * Return: 0 on success, negative on failure + */ +int wd_alg_attrs_init(struct wd_init_attrs *attrs) { - struct wd_soft_sqe *sqe = NULL; - - pthread_spin_lock(&sctx->slock); - sqe = &sctx->qfifo[sctx->head]; - if (!sqe->used && !sqe->complete) { // find the next not used sqe - sctx->head++; - if (unlikely(sctx->head == MAX_SOFT_QUEUE_LENGTH)) - sctx->head = 0; + int ret; - sqe->used = 1; - sqe->complete = 1; - sqe->id = tag_id; - sqe->result = 0; - __atomic_fetch_add(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); - pthread_spin_unlock(&sctx->slock); - } else { - pthread_spin_unlock(&sctx->slock); - return -WD_EBUSY; + if (!attrs || !attrs->ctx_params) { + WD_ERR("invalid: attrs or ctx_params is NULL!\n"); + return -WD_EINVAL; } - return 0; -} + WD_INFO("Algorithm initialization started: alg=%s, task_type=%u\n", + attrs->alg, attrs->task_type); -int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result) -{ - struct wd_soft_sqe *sqe = NULL; - - /* The queue is not used */ - if (sctx->run_num < 1) - return -WD_EAGAIN; - - if (pthread_spin_trylock(&sctx->rlock)) - return -WD_EAGAIN; - sqe = &sctx->qfifo[sctx->tail]; - if (sqe->used && sqe->complete) { // find a used sqe - sctx->tail++; - if (unlikely(sctx->tail == MAX_SOFT_QUEUE_LENGTH)) - sctx->tail = 0; + /* Phase 1: Driver discovery (pure query, no side effects) */ + ret = wd_alg_drv_discover(attrs); + if (ret) { + WD_ERR("Phase 1: driver discovery failed!\n"); + return ret; + } + WD_INFO("Phase 1: discovered %u unique drivers\n", attrs->drv_count); - *tag_id = sqe->id; - *result = sqe->result; - sqe->used = 0x0; - sqe->complete = 0x0; - __atomic_fetch_sub(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); - pthread_spin_unlock(&sctx->rlock); - } else { - pthread_spin_unlock(&sctx->rlock); - return -WD_EAGAIN; + /* Phase 2: ctx allocation + internal copy + scheduler */ + ret = wd_alg_ctx_init(attrs); + if (ret) { + WD_ERR("Phase 2: ctx init failed!\n"); + goto out_undiscover; } + WD_INFO("Algorithm initialization complete: %u contexts from %u drivers\n", + attrs->ctx_config->ctx_num, attrs->drv_count); + return 0; + +out_undiscover: + wd_alg_drv_undiscover(attrs); + return ret; } + -- 2.43.0