From: Longfang Liu <liulongfang@huawei.com> In the new UADK framework, update the dynamic registration management for user-space driver algorithm-driver pairs. After decoupling the framework and driver layers, ensure service scheduling, resource initialization, and the integrity of packet send/receive functionality, while maintaining the normal operation of existing features. Signed-off-by: Longfang Liu <liulongfang@huawei.com> --- Makefile.am | 3 +- include/wd_alg.h | 148 ++++++--- include/wd_internal.h | 1 - libwd.map | 7 +- wd_alg.c | 742 +++++++++++++++++++++++++++++------------- 5 files changed, 622 insertions(+), 279 deletions(-) diff --git a/Makefile.am b/Makefile.am index df873ce..5cc8c39 100644 --- a/Makefile.am +++ b/Makefile.am @@ -89,8 +89,7 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_ecc.c wd_ecc.h wd_ecc_drv.h \ wd_digest.c wd_digest.h wd_digest_drv.h \ wd_util.c wd_util.h \ - wd_sched.c wd_sched.h \ - wd.c wd.h + wd_sched.c wd_sched.h libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ diff --git a/include/wd_alg.h b/include/wd_alg.h index 1ae1dae..9edcbf1 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,16 +64,43 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_HW = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_SOFT = 0x3, - UADK_ALG_NPU = 0x4, - UADK_ALG_GPU = 0x5, + UADK_ALG_HW, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR, + UADK_ALG_SOFT, + UADK_ALG_NPU, + UADK_ALG_GPU, UADK_ALG_TYPE_MAX, }; -/* +enum alg_drv_type { + ALG_DRV_HW = 0x0, + ALG_DRV_CE_INS, + ALG_DRV_SVE_INS, + ALG_DRV_SOFT, + ALG_DRV_INS, + ALG_DRV_FB, +}; + +/** + * struct wd_ctx_alloc_params - Minimal parameters for single context allocation. + * + * Used to pass only necessary information to driver's alloc_ctx callback. + * Keeps driver layer simple and focused. + * + * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC + * @op_type: Operation type + * @bmp: NUMA node bitmask (optional, NULL if not needed) + */ +struct wd_drv_ctx_params { + __u8 ctx_mode; + __u8 op_type; + int numa_id; + bool epoll_en; + struct bitmask *bmp; +}; + +/** * @drv_name: name of the current device driver * @alg_name: name of the algorithm supported by the driver * @priority: priority of the type of algorithm supported by the driver @@ -88,7 +115,8 @@ enum alg_dev_type { * execute the algorithm task * @op_type_num: number of modes in which the device executes the * algorithm business and requires queues to be executed separately - * @priv: pointer of priv ctx + * @priv_size: parameter memory size passed between the internal + * interfaces of the driver * @fallback: soft calculation driver handle when performing soft * calculation supplement * @init: callback interface for initializing device drivers @@ -100,6 +128,10 @@ enum alg_dev_type { * @get_usage: callback interface used to obtain the * utilization rate of devices. * @get_extend_ops: callback interface to get private operation of drivers. + * @alloc_ctx: Allocate contexts for this driver. + * HW drivers use wd_hw_alloc_ctx. + * Non-HW drivers use wd_drv_alloc_ctx_array. + * @free_ctx: Release all resources allocated by alloc_ctx. */ struct wd_alg_driver { const char *drv_name; @@ -108,15 +140,21 @@ struct wd_alg_driver { int calc_type; int queue_num; int op_type_num; - void *priv; + int priv_size; + int ops_size; + int *drv_data; + void *extend_ops; handle_t fallback; - int (*init)(struct wd_alg_driver *drv, void *conf); - void (*exit)(struct wd_alg_driver *drv); - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*init)(void *conf, void *priv); + void (*exit)(void *priv); + int (*send)(handle_t ctx, void *drv_msg); + int (*recv)(handle_t ctx, void *drv_msg); int (*get_usage)(void *param); int (*get_extend_ops)(void *ops); + + int (*alloc_ctx)(char *alg_name, void *params, handle_t *ctx); + void (*free_ctx)(handle_t ctx); }; struct hisi_dev_usage { @@ -134,68 +172,84 @@ struct hisi_dev_usage { int wd_alg_driver_register(struct wd_alg_driver *drv); void wd_alg_driver_unregister(struct wd_alg_driver *drv); -/* - * @alg_name: name of the algorithm supported by the driver - * @drv_name: name of the current device driver +#define MAX_DRV_ALG_NUM 64 +/** + * Secondary structure: Algorithm entry (only algorithm-specific attributes) + * @alg_name: Specific algorithm name, e.g., "cbc(aes)" + * @avaiblable: Availability depends on specific CE/SVE instructions + */ +struct wd_alg_entry { + char alg_name[ALG_NAME_SIZE]; + bool available; +}; + +/** + * Primary structure: Driver node (List backbone, contains driver-level shared attributes) + * @drv_name: name of the current device driver e.g., "hisi_sec" + * @alg_type: Algorithm class, e.g., "cipher" (Promoted to driver level) * @available: Indicates whether the current driver still has resources available * @priority: priority of the type of algorithm supported by the driver - * @calc_type: the calculation method of algorithm supported by the driver - * @refcnt: the number of times the algorithm driver is being cited by the task + * @calc_type: Driver calc type (HW, CE, SVE, SOFT) + * @refcnt: Driver-level global reference count * - * @drv: device Drivers Supporting Algorithms + * @drv: Pointer to driver implementation + * @algs: Static array for supported algorithms + * @alg_count: Current number of registered algorithms * @next: pointer to the next node of the algorithm linked list */ -struct wd_alg_list { - char alg_name[ALG_NAME_SIZE]; +struct wd_drv_node { char drv_name[DEV_NAME_LEN]; - bool available; - int priority; - int calc_type; - int refcnt; - - struct wd_alg_driver *drv; - struct wd_alg_list *next; char alg_type[ALG_NAME_SIZE]; + int priority; + int calc_type; + int refcnt; + struct wd_alg_driver *drv; + struct wd_alg_entry algs[MAX_DRV_ALG_NUM]; + int alg_count; + struct wd_drv_node *next; }; -/* +int wd_get_drv_array(const char *alg_type, int task_type, const char *drv_name, + struct wd_alg_driver ***drv_array, __u32 *drv_count); +void wd_put_drv_array(struct wd_alg_driver **drv_array, __u32 drv_count); + +void wd_alg_drv_ref_inc(struct wd_alg_driver **drv_array, __u32 drv_count); +void wd_alg_drv_ref_dec(struct wd_alg_driver **drv_array, __u32 drv_count); + +/** * wd_request_drv() - Apply for an algorithm driver. * @alg_name: task algorithm name. - * @hw_mask: the flag of shield hardware device drivers. + * @drv_type: the type of shield hardware device drivers. * * Returns the applied algorithm driver, non means error. */ -struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask); -void wd_release_drv(struct wd_alg_driver *drv); +struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type); -/* +/** * wd_drv_alg_support() - Check the algorithms supported by the driver. * @alg_name: task algorithm name. - * @drv: a device driver that supports an algorithm. + * @param: a device queue parameters. * * Return check result. */ -bool wd_drv_alg_support(const char *alg_name, - struct wd_alg_driver *drv); +bool wd_drv_alg_support(const char *alg_name, void *param); -/* - * wd_enable_drv() - Re-enable use of the current device driver. - * @drv: a device driver that supports an algorithm. +/** + * wd_alg_match_drv() - Check if a given algorithm matches a specific driver. + * @drv: Pointer to the driver instance + * @alg_name: Specific algorithm name to check (e.g., "cbc(aes)") + * + * Return: true if supported and available, false otherwise. */ -void wd_enable_drv(struct wd_alg_driver *drv); -void wd_disable_drv(struct wd_alg_driver *drv); +bool wd_alg_match_drv(struct wd_alg_driver *drv, const char *alg_name); -int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf); -void wd_alg_driver_exit(struct wd_alg_driver *drv); -int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg); -int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg); int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 op_type); int wd_get_alg_type(const char *alg_name, char *alg_type); -struct wd_alg_list *wd_get_alg_head(void); +struct wd_drv_node *wd_get_alg_head(void); #ifdef WD_STATIC_DRV -/* +/** * duplicate drivers will be skipped when it register to alg_list */ void hisi_sec2_probe(void); diff --git a/include/wd_internal.h b/include/wd_internal.h index 62cf1a9..3e8622c 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,7 +7,6 @@ #define WD_INTERNAL_H #include <pthread.h> -#include <stdatomic.h> #include <stdbool.h> #include "wd.h" #include "wd_alg.h" diff --git a/libwd.map b/libwd.map index 0635198..683a88f 100644 --- a/libwd.map +++ b/libwd.map @@ -45,11 +45,16 @@ global: wd_alg_driver_register; wd_alg_driver_unregister; wd_request_drv; - wd_release_drv; wd_drv_alg_support; wd_enable_drv; wd_disable_drv; wd_get_alg_head; + wd_alg_match_drv; + wd_get_drv_array; + wd_put_drv_array; + wd_alg_drv_ref_inc; + wd_alg_drv_ref_dec; + wd_alg_driver_init; wd_alg_driver_exit; wd_alg_driver_send; diff --git a/wd_alg.c b/wd_alg.c index 1e4f14a..e288bd0 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -6,23 +6,32 @@ #define _GNU_SOURCE #include <dirent.h> #include <errno.h> +#include <stdio.h> #include <stdbool.h> #include <stdlib.h> #include <pthread.h> #include <sys/auxv.h> #include "wd.h" -#include "wd_alg.h" +#include "wd_alg_common.h" #define SYS_CLASS_DIR "/sys/class/uacce" -#define SVA_FILE_NAME "flags" -#define DEV_SVA_SIZE 32 -#define STR_DECIMAL 0xA -static struct wd_alg_list alg_list_head; -static struct wd_alg_list *alg_list_tail = &alg_list_head; +/* Registry structure (List manager) */ +struct wd_alg_registry { + struct wd_drv_node *head; + struct wd_drv_node *tail; + pthread_mutex_t mutex; + int drv_type_num; /* Number of unique driver nodes in the list */ +}; -static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static struct wd_drv_node drv_list_head; +static struct wd_alg_registry alg_registry = { + .head = &drv_list_head, + .tail = &drv_list_head, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .drv_type_num = 0, +}; struct acc_alg_item { const char *name; @@ -30,25 +39,25 @@ struct acc_alg_item { }; static struct acc_alg_item alg_options[] = { - {"zlib", "zlib"}, - {"gzip", "gzip"}, - {"deflate", "deflate"}, - {"lz77_zstd", "lz77_zstd"}, - {"lz4", "lz4"}, - {"lz77_only", "lz77_only"}, + {"zlib", "comp"}, + {"gzip", "comp"}, + {"deflate", "comp"}, + {"lz77_zstd", "comp"}, + {"lz4", "comp"}, + {"lz77_only", "comp"}, {"hashagg", "hashagg"}, {"udma", "udma"}, {"hashjoin", "hashjoin"}, - {"gather", "gather"}, + {"gather", "hashjoin"}, {"join-gather", "hashjoin"}, {"rsa", "rsa"}, {"dh", "dh"}, - {"ecdh", "ecdh"}, - {"x25519", "x25519"}, - {"x448", "x448"}, - {"ecdsa", "ecdsa"}, - {"sm2", "sm2"}, + {"ecdh", "ecc"}, + {"x25519", "ecc"}, + {"x448", "ecc"}, + {"ecdsa", "ecc"}, + {"sm2", "ecc"}, {"ecb(aes)", "cipher"}, {"cbc(aes)", "cipher"}, @@ -72,6 +81,7 @@ static struct acc_alg_item alg_options[] = { {"cbc(des)", "cipher"}, {"ecb(des3_ede)", "cipher"}, {"cbc(des3_ede)", "cipher"}, + {"xts-gb(sm4)", "cipher"}, {"ccm(aes)", "aead"}, {"gcm(aes)", "aead"}, @@ -108,7 +118,7 @@ int wd_get_alg_type(const char *alg_name, char *alg_type) for (i = 0; i < ARRAY_SIZE(alg_options); i++) { if (strcmp(alg_name, alg_options[i].name) == 0) { (void)strcpy(alg_type, alg_options[i].algtype); - return 0; + return WD_SUCCESS; } } @@ -122,7 +132,7 @@ static bool wd_check_accel_dev(const char *dev_name) wd_class = opendir(SYS_CLASS_DIR); if (!wd_class) { - WD_ERR("UADK framework isn't enabled in system!\n"); + WD_ERR("invalid: UADK framework isn't enabled in system!\n"); return false; } @@ -143,20 +153,17 @@ static bool wd_check_accel_dev(const char *dev_name) static bool wd_check_ce_support(const char *alg_name) { - unsigned long support_sm3 = 0; - unsigned long support_sm4 = 0; + unsigned long hwcaps = 0; const char *alg_tail; size_t tail_len; size_t alg_len; - #if defined(__aarch64__) - unsigned long hwcaps = 0; - + #if defined(__arm__) || defined(__arm) + hwcaps = getauxval(AT_HWCAP2); + #elif defined(__aarch64__) hwcaps = getauxval(AT_HWCAP); - support_sm3 = hwcaps & HWCAP_CE_SM3; - support_sm4 = hwcaps & HWCAP_CE_SM4; #endif - if (!strcmp("sm3", alg_name) && support_sm3) + if (!strcmp("sm3", alg_name) && (hwcaps & HWCAP_CE_SM3)) return true; alg_len = strlen(alg_name); @@ -165,7 +172,7 @@ static bool wd_check_ce_support(const char *alg_name) return false; alg_tail = alg_name + (alg_len - tail_len); - if (!strcmp("(sm4)", alg_tail) && support_sm4) + if (!strcmp("(sm4)", alg_tail) && (hwcaps & HWCAP_CE_SM4)) return true; return false; @@ -177,9 +184,8 @@ static bool wd_check_sve_support(void) #if defined(__aarch64__) hwcaps = getauxval(AT_HWCAP); - hwcaps &= HWCAP_SVE; #endif - if (hwcaps) + if (hwcaps & HWCAP_SVE) return true; return false; @@ -212,47 +218,36 @@ static bool wd_alg_check_available(int calc_type, return ret; } -static bool wd_alg_driver_match(struct wd_alg_driver *drv, - struct wd_alg_list *node) +/** + * Mapping from task_type to calc_type filter: + * + * TASK_HW → calc_type == UADK_ALG_HW + * TASK_INSTR → calc_type != UADK_ALG_HW (CE_INSTR | SVE_INSTR | SOFT) + * TASK_MIX → all calc_type values + */ +static inline bool wd_alg_drv_type_match(int task_type, int drv_calc_type) { - if (strcmp(drv->alg_name, node->alg_name)) - return false; - - if (strcmp(drv->drv_name, node->drv_name)) - return false; - - if (drv->priority != node->priority) - return false; - - if (drv->calc_type != node->calc_type) + switch (task_type) { + case TASK_HW: + return drv_calc_type == UADK_ALG_HW || + drv_calc_type == UADK_ALG_NPU || + drv_calc_type == UADK_ALG_GPU; + case TASK_INSTR: + return drv_calc_type == UADK_ALG_CE_INSTR || + drv_calc_type == UADK_ALG_SVE_INSTR; + case TASK_MIX: + return true; + default: return false; - - return true; -} - -static bool wd_alg_repeat_check(struct wd_alg_driver *drv) -{ - struct wd_alg_list *npre = &alg_list_head; - struct wd_alg_list *pnext = NULL; - - pthread_mutex_lock(&mutex); - pnext = npre->next; - while (pnext) { - if (wd_alg_driver_match(drv, pnext)) { - pthread_mutex_unlock(&mutex); - return true; - } - npre = pnext; - pnext = pnext->next; } - pthread_mutex_unlock(&mutex); - - return false; } int wd_alg_driver_register(struct wd_alg_driver *drv) { - struct wd_alg_list *new_alg; + struct wd_drv_node *node; + struct wd_drv_node *target_node = NULL; + char alg_type[ALG_NAME_SIZE]; + int i, ret; if (!drv) { WD_ERR("invalid: register drv is NULL!\n"); @@ -264,150 +259,261 @@ int wd_alg_driver_register(struct wd_alg_driver *drv) return -WD_EINVAL; } - if (wd_alg_repeat_check(drv)) - return 0; - - new_alg = calloc(1, sizeof(struct wd_alg_list)); - if (!new_alg) { - WD_ERR("failed to alloc alg driver memory!\n"); - return -WD_ENOMEM; + ret = wd_get_alg_type(drv->alg_name, alg_type); + if (ret) { + WD_ERR("failed to get alg_type for %s!\n", drv->alg_name); + return -WD_EINVAL; } - (void)wd_get_alg_type(drv->alg_name, new_alg->alg_type); - strncpy(new_alg->alg_name, drv->alg_name, ALG_NAME_SIZE - 1); - strncpy(new_alg->drv_name, drv->drv_name, DEV_NAME_LEN - 1); - new_alg->priority = drv->priority; - new_alg->calc_type = drv->calc_type; - new_alg->drv = drv; - new_alg->refcnt = 0; - new_alg->next = NULL; - - new_alg->available = wd_alg_check_available(drv->calc_type, - drv->alg_name, drv->drv_name); - if (!new_alg->available) { - free(new_alg); - return -WD_ENODEV; + pthread_mutex_lock(&alg_registry.mutex); + node = alg_registry.head->next; + /* Search for an existing node with the same drv_name */ + while (node) { + if (strcmp(node->drv_name, drv->drv_name) == 0 && + strcmp(node->alg_type, alg_type) == 0) { + target_node = node; + break; + } + node = node->next; } - pthread_mutex_lock(&mutex); - alg_list_tail->next = new_alg; - alg_list_tail = new_alg; - pthread_mutex_unlock(&mutex); + if (target_node) { + /* Consistency check: a driver must strictly have uniform properties */ + if (target_node->priority != drv->priority || + target_node->calc_type != drv->calc_type) { + WD_ERR("invalid: driver %s attributes mismatch on re-register!\n", + drv->drv_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_EINVAL; + } - return 0; + /* Check if alg_name already exists in this driver's array */ + for (i = 0; i < target_node->alg_count; i++) { + if (strcmp(target_node->algs[i].alg_name, drv->alg_name) == 0) { + /* Algorithm already registered, skip duplicate */ + pthread_mutex_unlock(&alg_registry.mutex); + return WD_SUCCESS; + } + } + + /* Check array capacity */ + if (target_node->alg_count >= MAX_DRV_ALG_NUM) { + WD_ERR("invalid: driver %s alg array overflow (max %d)!\n", + drv->drv_name, MAX_DRV_ALG_NUM); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENOMEM; + } + + /* Add new algorithm to existing driver node */ + strncpy(target_node->algs[target_node->alg_count].alg_name, + drv->alg_name, ALG_NAME_SIZE - 1); + target_node->algs[target_node->alg_count].alg_name[ALG_NAME_SIZE - 1] = '\0'; + target_node->algs[target_node->alg_count].available = + wd_alg_check_available(drv->calc_type, drv->alg_name, drv->drv_name); + if (!target_node->algs[target_node->alg_count].available) { + WD_ERR("invalid: driver %s alg %s not available on current system!\n", + drv->drv_name, drv->alg_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENODEV; + } + target_node->alg_count++; + } else { + /* Create a new driver node */ + target_node = calloc(1, sizeof(struct wd_drv_node)); + if (!target_node) { + WD_ERR("failed to alloc drv node memory!\n"); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENOMEM; + } + + strncpy(target_node->drv_name, drv->drv_name, DEV_NAME_LEN - 1); + target_node->drv_name[DEV_NAME_LEN - 1] = '\0'; + snprintf(target_node->alg_type, ALG_NAME_SIZE, "%s", alg_type); + target_node->priority = drv->priority; + target_node->calc_type = drv->calc_type; + target_node->drv = drv; + target_node->refcnt = 0; + target_node->alg_count = 0; + + /* Add the first algorithm to the new node's array */ + strncpy(target_node->algs[0].alg_name, drv->alg_name, ALG_NAME_SIZE - 1); + target_node->algs[0].alg_name[ALG_NAME_SIZE - 1] = '\0'; + target_node->algs[0].available = + wd_alg_check_available(drv->calc_type, drv->alg_name, drv->drv_name); + if (!target_node->algs[0].available) { + free(target_node); + WD_INFO("info: driver %s alg %s has no device register!\n", + drv->drv_name, drv->alg_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENODEV; + } + target_node->alg_count = 1; + target_node->next = NULL; + + /* Append to list tail */ + alg_registry.tail->next = target_node; + alg_registry.tail = target_node; + __atomic_fetch_add(&alg_registry.drv_type_num, 1, __ATOMIC_RELAXED); + } + + pthread_mutex_unlock(&alg_registry.mutex); + return WD_SUCCESS; } void wd_alg_driver_unregister(struct wd_alg_driver *drv) { - struct wd_alg_list *npre = &alg_list_head; - struct wd_alg_list *pnext = npre->next; + struct wd_drv_node *npre = alg_registry.head; + struct wd_drv_node *pnext; + char alg_type[ALG_NAME_SIZE]; + int i, ret; - /* Alg driver list has no drivers */ - if (!pnext || !drv) + if (!npre || !drv) return; - pthread_mutex_lock(&mutex); + ret = wd_get_alg_type(drv->alg_name, alg_type); + if (ret) { + WD_ERR("failed to get alg_type for %s!\n", drv->alg_name); + return; + } + + pthread_mutex_lock(&alg_registry.mutex); + /* Find the driver node matching drv_name */ + pnext = npre->next; while (pnext) { - if (wd_alg_driver_match(drv, pnext)) + if (strcmp(drv->drv_name, pnext->drv_name) == 0 && + strcmp(pnext->alg_type, alg_type) == 0) break; npre = pnext; pnext = pnext->next; } - /* The current algorithm is not registered */ if (!pnext) { - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); return; } - /* Used to locate the problem and ensure symmetrical use driver */ - if (pnext->refcnt > 0) - WD_ERR("driver<%s> still in used: %d\n", pnext->drv_name, pnext->refcnt); + /* Find and remove the specific alg_name from the node's array */ + for (i = 0; i < pnext->alg_count; i++) { + if (strcmp(pnext->algs[i].alg_name, drv->alg_name) == 0) { + /* Compact the array: move the last element to the removed slot */ + if (i != pnext->alg_count - 1) + pnext->algs[i] = pnext->algs[pnext->alg_count - 1]; + pnext->alg_count--; + break; + } + } + + /* If the driver no longer supports any algorithms, remove the entire node */ + if (!pnext->alg_count) { + if (pnext->refcnt > 0) + WD_INFO("info: release driver <%s> as it is still in use: %d!\n", + pnext->drv_name, pnext->refcnt); + + if (pnext == alg_registry.tail) + alg_registry.tail = npre; - if (pnext == alg_list_tail) - alg_list_tail = npre; + npre->next = pnext->next; + free(pnext); + if (alg_registry.drv_type_num > 0) + __atomic_fetch_sub(&alg_registry.drv_type_num, 1, __ATOMIC_RELAXED); + } - npre->next = pnext->next; - free(pnext); - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); } -struct wd_alg_list *wd_get_alg_head(void) +struct wd_drv_node *wd_get_alg_head(void) { - return &alg_list_head; + return alg_registry.head; } -bool wd_drv_alg_support(const char *alg_name, - struct wd_alg_driver *drv) +/** + * wd_alg_match_drv() - Check if a given algorithm match a specific driver. + * @drv: Pointer to the driver instance + * @alg_name: Specific algorithm name to check (e.g., "cbc(aes)") + * + * Uses the new hierarchical structure: finds the driver node, then searches + * its internal static algorithm array. + * + * Return: true if supported and available, false otherwise. + */ +bool wd_alg_match_drv(struct wd_alg_driver *drv, const char *alg_name) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *node; + int i; - if (!alg_name || !drv) + if (!drv || !alg_name) return false; - while (pnext) { - if (!strcmp(alg_name, pnext->alg_name) && - !strcmp(drv->drv_name, pnext->drv_name)) { - return true; + pthread_mutex_lock(&alg_registry.mutex); + node = alg_registry.head->next; + while (node) { + if (node->drv == drv) { + /* Found the driver node, now search its algs array */ + for (i = 0; i < node->alg_count; i++) { + if (!strcmp(node->algs[i].alg_name, alg_name) && + node->algs[i].available) { + pthread_mutex_unlock(&alg_registry.mutex); + return true; + } + } + /* Driver found, but algorithm not in its array or not available */ + pthread_mutex_unlock(&alg_registry.mutex); + return false; } - pnext = pnext->next; + node = node->next; } + pthread_mutex_unlock(&alg_registry.mutex); return false; } -void wd_enable_drv(struct wd_alg_driver *drv) +bool wd_drv_alg_support(const char *alg_name, void *param) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; + struct wd_ctx_config_internal *config = param; + struct wd_drv_node *head = alg_registry.head; + struct wd_drv_node *node; + __u32 i; + int j; - if (!pnext || !drv) - return; - - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext)) - break; - pnext = pnext->next; - } - - if (pnext) - pnext->available = wd_alg_check_available(drv->calc_type, - drv->alg_name, drv->drv_name); - pthread_mutex_unlock(&mutex); -} - -void wd_disable_drv(struct wd_alg_driver *drv) -{ - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - - if (!pnext || !drv) - return; + if (!alg_name || !config || !config->ctxs) + return false; - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext) && pnext->available) - break; - pnext = pnext->next; + pthread_mutex_lock(&alg_registry.mutex); + /* Check whether the currently allocated ctxs supports the specified algorithm. */ + for (i = 0; i < config->ctx_num; i++) { + if (!config->ctxs[i].drv) + continue; + node = head->next; + while (node) { + /* Query the position of the driver matching the context in the list. */ + if (!strcmp(config->ctxs[i].drv->drv_name, node->drv_name)) { + for (j = 0; j < node->alg_count; j++) { + if (!strcmp(alg_name, node->algs[j].alg_name) && + node->algs[j].available) { + pthread_mutex_unlock(&alg_registry.mutex); + return true; + } + } + } + node = node->next; + } } + pthread_mutex_unlock(&alg_registry.mutex); - if (pnext) - pnext->available = false; - pthread_mutex_unlock(&mutex); + return false; } -struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask) +struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_list *select_node = NULL; + struct wd_drv_node *node = alg_registry.head->next; + struct wd_drv_node *drv_node = NULL; struct wd_alg_driver *drv = NULL; + bool type_match = false; int tmp_priority = -1; + int i; - if (!pnext) { - WD_ERR("invalid: requset drv pnext is NULL!\n"); + if (!node) { + WD_ERR("invalid: request drv node is NULL!\n"); return NULL; } @@ -416,106 +522,286 @@ struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask) return NULL; } - /* Check the list to get an best driver */ - pthread_mutex_lock(&mutex); - while (pnext) { - /* hw_mask true mean not to used hardware dev */ - if ((hw_mask && pnext->drv->calc_type == UADK_ALG_HW) || - (!hw_mask && pnext->drv->calc_type != UADK_ALG_HW)) { - pnext = pnext->next; - continue; - } - - if (!strcmp(alg_name, pnext->alg_name) && pnext->available && - pnext->drv->priority > tmp_priority) { - tmp_priority = pnext->drv->priority; - select_node = pnext; - drv = pnext->drv; + pthread_mutex_lock(&alg_registry.mutex); + while (node) { + type_match = false; + /* Check calc_type against requested drv_type */ + if (drv_type == ALG_DRV_FB && (node->calc_type == UADK_ALG_SOFT || + node->calc_type == UADK_ALG_CE_INSTR || + node->calc_type == UADK_ALG_SVE_INSTR)) + type_match = true; + + if (type_match && node->drv->priority > tmp_priority) { + /* Check if this driver supports the requested alg_name */ + for (i = 0; i < node->alg_count; i++) { + if (!strcmp(alg_name, node->algs[i].alg_name) && + node->algs[i].available) { + drv_node = node; + drv = node->drv; + tmp_priority = node->drv->priority; + break; + } + } } - pnext = pnext->next; + node = node->next; } - if (select_node) - select_node->refcnt++; - pthread_mutex_unlock(&mutex); + /* Increment refcnt on the selected driver node */ + if (drv) + drv_node->refcnt++; + pthread_mutex_unlock(&alg_registry.mutex); return drv; } -void wd_release_drv(struct wd_alg_driver *drv) +int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op_type) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_list *select_node = NULL; + struct wd_drv_node *node = alg_registry.head->next; + struct hisi_dev_usage dev_usage; + struct wd_alg_driver *drv; - if (!pnext || !drv) - return; + if (!dev_name || !alg_type) { + WD_ERR("invalid: dev_name or alg_type is NULL!\n"); + return -WD_EINVAL; + } - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext) && pnext->refcnt > 0) { - select_node = pnext; + while (node) { + /* Match dev_name and alg_type at the driver node level */ + if (strstr(dev_name, node->drv_name) && + !strcmp(alg_type, node->alg_type)) break; - } - pnext = pnext->next; + + node = node->next; } - if (select_node) - select_node->refcnt--; - pthread_mutex_unlock(&mutex); -} + if (!node) + return -WD_EACCES; -int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) -{ - return drv->init(drv, conf); -} + drv = node->drv; + if (!drv->get_usage) + return -WD_EINVAL; -void wd_alg_driver_exit(struct wd_alg_driver *drv) -{ - drv->exit(drv); + dev_usage.drv = drv; + dev_usage.alg_op_type = alg_op_type; + dev_usage.dev_name = dev_name; + + return drv->get_usage(&dev_usage); } -int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +/** + * wd_put_drv_array() - Release driver array allocated by wd_get_drv_array(). + * + * Frees the driver pointer array. Does NOT touch the drivers themselves + * (refcount managed separately by wd_alg_drv_ref_inc/dec). + * + * @drv_array: Driver array from wd_get_drv_array() + * @drv_count: Number of entries (unused, for API symmetry) + */ +void wd_put_drv_array(struct wd_alg_driver **drv_array, const __u32 drv_count) { - return drv->send(drv, ctx, msg); + if (drv_array) + free(drv_array); } -int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int wd_compare_drv_priority(const void *a, const void *b) { - return drv->recv(drv, ctx, msg); + struct wd_alg_driver *driver_a = *(struct wd_alg_driver **)a; + struct wd_alg_driver *driver_b = *(struct wd_alg_driver **)b; + + /* Higher priority value should come first */ + if (driver_a->priority > driver_b->priority) + return -1; + /* a has lower priority, should come after b */ + else if (driver_a->priority < driver_b->priority) + return 1; + /* equal priority */ + return 0; } -int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op_type) +/** + * wd_get_drv_array() - Discover all unique drivers matching alg_type and task_type. + * + * @alg_type: Algorithm class string ("cipher", "digest", "aead", "comp", etc.) + * @task_type: TASK_HW (hardware only), TASK_INSTR (instruction only), TASK_MIX (all) + * @drv_array: Output - newly allocated array of unique wd_alg_driver* pointers, + * caller must free with plain free() + * @drv_count: Output - number of unique drivers found + * + * Traverses wd_drv_node list once: + * 1. Matches by alg_type at node level (no need to traverse algs array for this). + * 2. Filters by task_type using wd_alg_drv_type_match(). + * 3. Deduplicates is inherently solved (each node is a unique driver). + * + * This is a PURE QUERY — no reference counting or resource allocation side effects. + * Reference counting is done separately by wd_alg_drv_ref_inc/dec(). + * + * Return: 0 on success, negative on failure. + */ +int wd_get_drv_array(const char *alg_type, int task_type, const char *drv_name, + struct wd_alg_driver ***drv_array, __u32 *drv_count) { - struct wd_alg_list *pnext = alg_list_head.next; - struct hisi_dev_usage dev_usage; - struct wd_alg_driver *drv; - size_t len; + struct wd_drv_node *head, *node; + struct wd_alg_driver **drivers; + __u32 max_driver_count; + __u32 current_count = 0; + bool has_available_alg; + int i; + + if (!alg_type || !drv_array || !drv_count) { + WD_ERR("invalid: NULL parameter!\n"); + return -WD_EINVAL; + } - if (!dev_name || !alg_type) { - WD_ERR("dev_name or alg_type is NULL!\n"); + *drv_array = NULL; + *drv_count = 0; + head = wd_get_alg_head(); + if (!head) { + WD_ERR("failed to get alg list head!\n"); return -WD_EINVAL; } - while (pnext) { - len = strlen(pnext->drv_name); - if (!strncmp(dev_name, pnext->drv_name, len) && *(dev_name + len) == '-' && - !strcmp(alg_type, pnext->alg_type) && pnext->drv->priv) - break; + max_driver_count = __atomic_load_n(&alg_registry.drv_type_num, __ATOMIC_RELAXED); + if (!max_driver_count) { + WD_ERR("invalid: no drivers registered for alg_type: %s\n", alg_type); + return -WD_EINVAL; + } - pnext = pnext->next; + drivers = calloc(max_driver_count, sizeof(struct wd_alg_driver *)); + if (!drivers) { + WD_ERR("failed to allocate drivers array!\n"); + return -WD_ENOMEM; } - if (!pnext) - return -WD_EACCES; + /* + * Single traversal of wd_drv_node list: + * - Match by alg_type at node level + * - Filter by task_type + * - Deduplication inherently solved + */ + node = head->next; + while (node) { + if (strcmp(node->alg_type, alg_type) == 0 && + wd_alg_drv_type_match(task_type, node->calc_type)) { + + if (drv_name && strcmp(node->drv_name, drv_name) != 0) { + node = node->next; + continue; + } + + /* Check if at least one algorithm in this driver is available */ + has_available_alg = false; + for (i = 0; i < node->alg_count; i++) { + if (node->algs[i].available) { + has_available_alg = true; + break; + } + } + + if (!has_available_alg) { + node = node->next; + continue; + } + + if (current_count >= max_driver_count) { + WD_ERR("failed to check driver array overflow!\n"); + goto query_failed; + } + drivers[current_count] = node->drv; + current_count++; + } + node = node->next; + } - drv = pnext->drv; - if (!drv->get_usage) - return -WD_EINVAL; + if (!current_count) { + WD_ERR("invalid: no available drivers for alg_type: %s, task_type: %d\n", + alg_type, task_type); + goto query_failed; + } - dev_usage.drv = drv; - dev_usage.alg_op_type = alg_op_type; - dev_usage.dev_name = dev_name; + /* Sort drivers by priority (higher priority first) */ + if (current_count > 1) + qsort(drivers, current_count, sizeof(struct wd_alg_driver *), + wd_compare_drv_priority); - return drv->get_usage(&dev_usage); + WD_DEBUG("Driver discovery: %u unique drivers for alg_type=%s\n", + current_count, alg_type); + *drv_array = drivers; + *drv_count = current_count; + + return WD_SUCCESS; + +query_failed: + free(drivers); + return -WD_EINVAL; +} + +/** + * wd_alg_drv_ref_inc() - Increment reference count for each unique driver. + * + * @drv_array: Array of unique driver pointers + * @drv_count: Number of drivers in the array + * + * For each unique driver, finds its node in wd_drv_node list and + * increments refcnt by exactly 1. This ensures refcnt reflects the + * number of configs using the driver, not the number of ctxs. + * + * Must be called after wd_get_drv_array() and after ctx binding. + */ +void wd_alg_drv_ref_inc(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + struct wd_drv_node *node; + __u32 i; + + if (!drv_array || !drv_count) + return; + + pthread_mutex_lock(&alg_registry.mutex); + for (i = 0; i < drv_count; i++) { + if (!drv_array[i]) + continue; + /* Directly find the unique driver node and increment refcnt */ + node = alg_registry.head->next; + while (node) { + if (node->drv == drv_array[i]) { + node->refcnt++; + break; + } + node = node->next; + } + } + pthread_mutex_unlock(&alg_registry.mutex); +} + +/** + * wd_alg_drv_ref_dec() - Decrement reference count for each unique driver. + * + * @drv_array: Array of unique driver pointers + * @drv_count: Number of drivers in the array + * + * Inverse of wd_alg_drv_ref_inc(). Decrements refcnt by 1 for each + * unique driver. Must be called during cleanup. + */ +void wd_alg_drv_ref_dec(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + struct wd_drv_node *node; + __u32 i; + + if (!drv_array || !drv_count) + return; + + pthread_mutex_lock(&alg_registry.mutex); + for (i = 0; i < drv_count; i++) { + if (!drv_array[i]) + continue; + /* Directly find the unique driver node and decrement refcnt */ + node = alg_registry.head->next; + while (node) { + if (node->drv == drv_array[i] && node->refcnt > 0) { + node->refcnt--; + break; + } + node = node->next; + } + } + pthread_mutex_unlock(&alg_registry.mutex); } -- 2.43.0