From: Longfang Liu <liulongfang@huawei.com> This represents the core modification of the new framework. It implements unified abstract processing for all device queues across the entire UADK, and unifies the channel for service requests from UADK to user devices into queues. These queues simultaneously support hardware device queues, software computation queues, and instruction queues, thereby achieving unified queue management. Additionally, the binding relationship between the algorithm layer and the driver has been decoupled, allowing a single algorithm layer to map and utilize multiple types of devices simultaneously for algorithm acceleration, thus enabling heterogeneous fusion accelerated computing. Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- Makefile.am | 15 ++- conf.sh | 19 ++-- configure.ac | 10 ++ drv/hisi_udma.c | 65 +++++++++---- drv/isa_ce_sm3.c | 13 ++- drv/wd_drv.c | 4 + include/drv/wd_aead_drv.h | 1 - include/wd.h | 5 - include/wd_alg_common.h | 35 ++++++- include/wd_bmm.h | 2 +- include/wd_internal.h | 15 +-- include/wd_util.h | 60 +++++------- wd.c | 78 ++++++++------- wd_aead.c | 146 +++++++++++++++++++--------- wd_bmm.c | 72 +++++++++++--- wd_cipher.c | 2 +- wd_comp.c | 26 ++--- wd_mempool.c | 3 +- wd_sched.c | 197 ++++++++++++++++++++++++++++++++++---- 19 files changed, 550 insertions(+), 218 deletions(-) diff --git a/Makefile.am b/Makefile.am index c08c825..d2f6cfe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,8 +1,15 @@ ACLOCAL_AMFLAGS = -I m4 -I./include AUTOMAKE_OPTIONS = foreign subdir-objects AM_CFLAGS=-Wall -Werror -fno-strict-aliasing -I$(top_srcdir)/include -AM_CFLAGS+=-fPIC -fPIE -pie -fstack-protector-strong -D_FORTIFY_SOURCE=2 \ - -O2 -ftrapv -Wl,-z,relro,-z,now -Wl,-s +AM_CFLAGS+=-fPIC -fPIE -pie -fstack-protector-strong -ftrapv -Wl,-z,relro,-z,now + +if WD_DEBUG +# Debug build: keep symbols, disable optimization and stripping for full stack traces +AM_CFLAGS+=-g -O0 -fno-omit-frame-pointer -DDEBUG +else +# Release build: optimize and strip symbols +AM_CFLAGS+=-O2 -D_FORTIFY_SOURCE=2 -Wl,-s +endif CLEANFILES = if WITH_LOG_FILE @@ -84,13 +91,13 @@ libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_aead.c wd_aead.h wd_aead_drv.h \ - wd.c wd.h wd_alg.h \ wd_rsa.c wd_rsa.h wd_rsa_drv.h \ wd_dh.c wd_dh.h wd_dh_drv.h \ wd_ecc.c wd_ecc.h wd_ecc_drv.h \ wd_digest.c wd_digest.h wd_digest_drv.h \ wd_util.c wd_util.h \ - wd_sched.c wd_sched.h + wd_sched.c wd_sched.h \ + wd.c wd.h libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ diff --git a/conf.sh b/conf.sh index af5f787..d30912c 100755 --- a/conf.sh +++ b/conf.sh @@ -4,12 +4,18 @@ # Build UADK into static library COMPILE_TYPE="--disable-static --enable-shared" +DEBUG_TYPE="" -# These two parameters could be in arbitary sequence -if [[ $1 && $1 = "--static" ]] || [[ $2 && $2 = "--static" ]]; then - echo "Configure to static compile!" - COMPILE_TYPE="--enable-static --disable-shared --with-static_drv" -fi +# These parameters could be in arbitary sequence +for arg in "$1" "$2" "$3"; do + if [[ $arg = "--static" ]]; then + echo "Configure to static compile!" + COMPILE_TYPE="--enable-static --disable-shared --with-static_drv" + elif [[ $arg = "--debug" ]]; then + echo "Configure to debug compile (with symbols, no optimization)!" + DEBUG_TYPE="--enable-debug=yes" + fi +done export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes ./configure -v \ @@ -17,4 +23,5 @@ ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes ./configure -v --host aarch64-linux-gnu \ --target aarch64-linux-gnu \ --includedir=/usr/local/include/ \ - $COMPILE_TYPE + $COMPILE_TYPE \ + $DEBUG_TYPE diff --git a/configure.ac b/configure.ac index 7261b5e..7e5d723 100644 --- a/configure.ac +++ b/configure.ac @@ -34,6 +34,16 @@ AC_ARG_ENABLE([debug-log], [debug_log=false] ) +AC_ARG_ENABLE([debug], + AS_HELP_STRING([--enable-debug], [build with -g -O0 and no symbol stripping for full stack traces]), + [ AS_IF([test "x$enable_debug" = "xyes"], + [wd_debug=true], + [wd_debug=false]) + ], + [wd_debug=false] +) +AM_CONDITIONAL([WD_DEBUG], [test "x$wd_debug" = "xtrue"]) + AC_ARG_ENABLE([perf], AS_HELP_STRING([--disble-perf], [enable measuring performance]), [ AS_IF([test "x$enable_perf" = "xyes"], diff --git a/drv/hisi_udma.c b/drv/hisi_udma.c index a9f5607..2e5aa13 100644 --- a/drv/hisi_udma.c +++ b/drv/hisi_udma.c @@ -12,6 +12,7 @@ #include <sys/types.h> #include "hisi_qm_udrv.h" #include "../include/drv/wd_udma_drv.h" +#include "wd_drv.h" #define BIT(nr) (1UL << (nr)) #define UDMA_CTX_Q_NUM_DEF 1 @@ -100,7 +101,8 @@ struct udma_internal_addr { }; struct hisi_udma_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; }; static int get_free_inter_addr(struct udma_internal_addr *inter_addr) @@ -290,7 +292,7 @@ static void fill_init_value(struct udma_sqe *sqe, struct wd_udma_msg *msg) memset(&sqe->init_val, msg->value, sizeof(__u64)); } -static int udma_send(struct wd_alg_driver *drv, handle_t ctx, void *udma_msg) +static int udma_send(handle_t ctx, void *udma_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -342,7 +344,7 @@ static void dump_udma_msg(struct udma_sqe *sqe, struct wd_udma_msg *msg) "op_type:%u addr_num:%d.\n", msg->op_type, msg->addr_num); } -static int udma_recv(struct wd_alg_driver *drv, handle_t ctx, void *udma_msg) +static int udma_recv(handle_t ctx, void *udma_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -442,36 +444,61 @@ free_inter_addr: return ret; } -static int udma_init(struct wd_alg_driver *drv, void *conf) +static int udma_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_udma_ctx *uctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_udma_ctx *priv; - handle_t h_qp = 0; - handle_t h_ctx; - __u32 i, j; + __u32 i, j, count = 0; + bool *is_match; + handle_t h_qp; int ret; if (!config || !config->ctx_num) { - WD_ERR("invalid: udma init config is null or ctx num is 0!\n"); + WD_ERR("invalid: input config or ctx num is null!\n"); return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_udma_ctx)); - if (!priv) + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; + + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && config->ctxs[i].drv && + strcmp(config->ctxs[i].drv->drv_name, "hisi_udma") == 0) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + + if (!count) { + WD_ERR("invalid: valid driver number is zero!\n"); + free(is_match); + return -WD_EINVAL; + } + + uctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!uctx->ctxs) { + free(is_match); return -WD_ENOMEM; + } + uctx->ctx_num = count; qm_priv.op_type = UDMA_ALG_TYPE; qm_priv.sqe_size = sizeof(struct udma_sqe); + count = 0; /* Allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; + if (!is_match[i]) + continue; qm_priv.qp_mode = config->ctxs[i].ctx_mode; /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv.epoll_en = (qm_priv.qp_mode == CTX_MODE_SYNC) ? config->epoll_en : 0; qm_priv.idx = i; - h_qp = hisi_qm_alloc_qp(&qm_priv, h_ctx); + h_qp = hisi_qm_alloc_qp(&qm_priv, config->ctxs[i].ctx); if (!h_qp) { ret = -WD_ENOMEM; goto out; @@ -480,20 +507,22 @@ static int udma_init(struct wd_alg_driver *drv, void *conf) ret = udma_init_qp_priv(h_qp); if (ret) goto free_h_qp; + uctx->ctxs[count++] = &config->ctxs[i]; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + free(is_match); return WD_SUCCESS; + free_h_qp: hisi_qm_free_qp(h_qp); out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(uctx->ctxs[j]->ctx); udma_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } - free(priv); + free(uctx->ctxs); + free(is_match); return ret; } diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 8d23061..627ab68 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -17,17 +17,17 @@ #include "drv/isa_ce_sm3.h" #include "drv/wd_digest_drv.h" #include "wd_digest.h" -#include "wd_util.h" +#include "wd_drv.h" #define SM3_ALIGN_MASK 63U typedef void (sm3_ce_block_fn)(__u32 word_reg[SM3_STATE_WORDS], const unsigned char *src, size_t blocks); -static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf); -static void sm3_ce_drv_exit(struct wd_alg_driver *drv); -static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); -static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); +static int sm3_ce_drv_init(void *conf, void *priv); +static void sm3_ce_drv_exit(void *priv); +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg); +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg); static int sm3_ce_get_usage(void *param); static struct wd_alg_driver sm3_ce_alg_driver = { @@ -35,6 +35,7 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .alg_name = "sm3", .calc_type = UADK_ALG_CE_INSTR, .priority = 200, + .priv_size = sizeof(struct sm3_ce_drv_ctx), .queue_num = 1, .op_type_num = 1, .fallback = 0, @@ -43,6 +44,8 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .send = sm3_ce_drv_send, .recv = sm3_ce_drv_recv, .get_usage = sm3_ce_get_usage, + .alloc_ctx = wd_soft_alloc_ctx, + .free_ctx = wd_soft_free_ctx, }; static void __attribute__((constructor)) sm3_ce_probe(void) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 83dcbb6..16dc24c 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -201,6 +201,10 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) WD_ERR("invalid: alg_name is NULL!\n"); return -WD_EINVAL; } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); dev_list = wd_get_accel_list(alg_type); if (!dev_list) { diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 2c53217..612a8f2 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -78,7 +78,6 @@ struct wd_aead_aiv_addr { }; struct wd_aead_extend_ops { - void *params; int (*eops_aiv_init)(struct wd_alg_driver *drv, struct wd_mm_ops *mm_ops, void **params); diff --git a/include/wd.h b/include/wd.h index a468047..6329528 100644 --- a/include/wd.h +++ b/include/wd.h @@ -104,11 +104,6 @@ typedef void (*wd_log)(const char *format, ...); #define WD_IS_ERR(h) ((uintptr_t)(h) > \ (uintptr_t)(-1000)) -enum wd_buff_type { - WD_FLAT_BUF, - WD_SGL_BUF, -}; - enum wd_alg_type { WD_CIPHER, WD_DIGEST, diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index ebba7f8..be4e3b5 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -56,6 +56,11 @@ enum wd_ctx_mode { CTX_MODE_MAX, }; +enum wd_buff_type { + WD_FLAT_BUF, + WD_SGL_BUF, +}; + enum wd_init_type { WD_TYPE_V1, WD_TYPE_V2, @@ -70,7 +75,8 @@ enum wd_mem_type { /* * struct wd_ctx - Define one ctx and related type. - * @ctx: The ctx itself. + * @ctx: The ctx itself, the hardware queue is wd_ctx_h. + * The soft computing queue is wd_soft_ctx. * @op_type: Define the operation type of this specific ctx. * e.g. 0: compression; 1: decompression. * @ctx_mode: Define this ctx is used for synchronization of asynchronization @@ -82,7 +88,7 @@ struct wd_ctx { __u8 ctx_mode; }; -/* +/** * struct wd_cap_config - Capabilities. * @ctx_msg_num: number of asynchronous msg pools that the user wants to allocate. * Optional, user can set ctx_msg_num based on the number of requests @@ -95,7 +101,7 @@ struct wd_cap_config { __u32 resv; }; -/* +/** * struct wd_ctx_config - Define a ctx set and its related attributes, which * will be used in the scope of current process. * @ctx_num: The ctx number in below ctx array. @@ -111,7 +117,7 @@ struct wd_ctx_config { struct wd_cap_config *cap; }; -/* +/** * struct wd_ctx_nums - Define the ctx sets numbers. * @sync_ctx_num: The ctx numbers which are used for sync mode for each * ctx sets. @@ -123,7 +129,7 @@ struct wd_ctx_nums { __u32 async_ctx_num; }; -/* +/** * struct wd_ctx_params - Define the ctx sets params which are used for init * algorithms. * @op_type_num: Used for index of ctx_set_num, the order is the same as @@ -170,6 +176,25 @@ struct wd_sched { typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); +/** + * struct wd_init_attrs - Algorithm initialization attributes. + * + * Updated: No longer contains driver field. + * Initialization path determined solely by task_type. + */ +struct wd_init_attrs { + __u32 sched_type; + __u32 task_type; + char alg[CRYPTO_MAX_ALG_NAME]; + struct wd_sched *sched; + struct wd_ctx_params *ctx_params; + struct wd_ctx_config *ctx_config; + wd_alg_init alg_init; + wd_alg_poll_ctx alg_poll_ctx; + + struct wd_ctx_config_internal *ctx_config_internal; +}; + #ifdef __cplusplus } #endif diff --git a/include/wd_bmm.h b/include/wd_bmm.h index 76b56a0..4abaa4e 100644 --- a/include/wd_bmm.h +++ b/include/wd_bmm.h @@ -33,7 +33,7 @@ int wd_get_fail_num(void *pool, __u32 *fail_num); __u32 wd_get_bufsize(void *pool); handle_t wd_find_ctx(const char *alg_name); -void wd_remove_ctx_list(void); +void wd_remove_ctx_list(handle_t h_ctx); int wd_insert_ctx_list(handle_t h_ctx, char *alg_name); __u32 wd_get_dev_id(void *pool); diff --git a/include/wd_internal.h b/include/wd_internal.h index 3e8622c..95ceaa2 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -20,6 +20,7 @@ extern "C" { #define MAX_FD_NUM 65535 struct wd_ctx_h { + __u8 ctx_type; int fd; char dev_path[MAX_DEV_NAME_LEN]; char *dev_name; @@ -30,17 +31,6 @@ struct wd_ctx_h { void *priv; }; -struct wd_soft_ctx { - int fd; - void *priv; -}; - -struct wd_ce_ctx { - int fd; - char *drv_name; - void *priv; -}; - struct wd_ctx_internal { __u8 op_type; __u8 ctx_mode; @@ -60,6 +50,9 @@ struct wd_ctx_config_internal { bool epoll_en; unsigned long *msg_cnt; char *alg_name; + + struct wd_alg_driver **drv_array; + __u32 drv_count; }; struct wd_datalist { diff --git a/include/wd_util.h b/include/wd_util.h index 56dd9ba..43b89d1 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -118,19 +118,8 @@ struct wd_ctx_attr { }; struct wd_msg_handle { - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); -}; - -struct wd_init_attrs { - __u32 sched_type; - char alg[CRYPTO_MAX_ALG_NAME]; - struct wd_alg_driver *driver; - struct wd_sched *sched; - struct wd_ctx_params *ctx_params; - struct wd_ctx_config *ctx_config; - wd_alg_init alg_init; - wd_alg_poll_ctx alg_poll_ctx; + int (*send)(handle_t sess, void *msg); + int (*recv)(handle_t sess, void *msg); }; /* @@ -345,7 +334,6 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en); /** * wd_handle_msg_sync() - recv msg from hardware - * @drv: the driver to handle msg. * @msg_handle: callback of msg handle ops. * @ctx: the handle of context. * @msg: the msg of task. @@ -354,8 +342,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en); * * Return 0 if successful or less than 0 otherwise. */ -int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en); +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en); /** * wd_init_check() - Check input parameters for wd_<alg>_init. @@ -414,7 +402,7 @@ static inline void wd_alg_clear_init(enum wd_status *status) * to the obtained queue resource and the applied driver. * @ctx_params: wd_ctx_params to be initialized. * @user_ctx_params: user input wd_ctx_params. - * @driver: device driver for the current algorithm application. + * @alg: Name of the algorithm. * @type: algorithm type. * @max_op_type: algorithm max operation type. * @@ -422,8 +410,8 @@ static inline void wd_alg_clear_init(enum wd_status *status) */ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type); + char *alg, enum wd_type type, + int max_op_type); void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); @@ -437,29 +425,17 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); int wd_alg_attrs_init(struct wd_init_attrs *attrs); void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); -/** - * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain - * with the given devices list, ctxs number and numa mask. - * @task_type: the type of task specified by the current algorithm. - * @alg_name: the name of the algorithm specified by the task. - * - * Return device driver if succeed and other NULL if fail. - */ -struct wd_alg_driver *wd_alg_drv_bind(int task_type, const char *alg_name); -void wd_alg_drv_unbind(struct wd_alg_driver *drv); - /** * wd_alg_init_driver() - Initialize the current device driver according * to the obtained queue resource and the applied driver. * @config: device resources requested by the current algorithm. * @driver: device driver for the current algorithm application. + * @drv_priv: the parameter pointer of the current device driver. * * Return 0 if succeed and other error number if fail. */ -int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); -void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); +int wd_alg_init_driver(struct wd_ctx_config_internal *config); +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config); /** * wd_dlopen_drv() - Open the dynamic library file of the device driver. @@ -490,10 +466,16 @@ static inline void wd_dfx_msg_cnt(struct wd_ctx_config_internal *config, bool ret; ret = wd_need_info(); - if (idx > numsize || !ret) + if (idx >= numsize || !ret) + return; + + if (!config->msg_cnt) return; sqn = config->ctxs[idx].sqn; + if (sqn >= numsize) + return; + config->msg_cnt[sqn]++; } @@ -522,6 +504,14 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type) int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type); +int wd_alg_config_init(struct wd_init_attrs *attrs); +void wd_alg_config_uninit(struct wd_init_attrs *attrs); +int wd_alg_ctx_init(struct wd_init_attrs *attrs); +void wd_alg_ctx_uninit(struct wd_init_attrs *attrs); +int wd_ctx_bind_drivers(struct wd_ctx_config_internal *config_api, + struct wd_ctx_config_internal *config_in, int init_type); +void wd_ctx_unbind_drivers(struct wd_ctx_config_internal *config); + #ifdef __cplusplus } #endif diff --git a/wd.c b/wd.c index 9cdc70f..ede0bf2 100644 --- a/wd.c +++ b/wd.c @@ -21,9 +21,11 @@ #include "wd.h" #include "wd_alg.h" #include "wd_internal.h" +#include "wd_bmm.h" #define SYS_CLASS_DIR "/sys/class/uacce" #define FILE_MAX_SIZE (8 << 20) #define WD_DEV_USAGE_SIZE 256 +#define WD_DEFAULT_NUMA_DISTANCE 1024 enum UADK_LOG_LEVEL { WD_LOG_NONE = 0, @@ -77,15 +79,10 @@ int memcmp_consttime(const void *s1, const void *s2, size_t n) static int wd_check_ctx_type(handle_t h_ctx) { - struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; - - /* A simple and efficient method to check the queue type */ - if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) { - WD_INFO("Invalid: this ctx not HW ctx.\n"); - return -WD_HW_EACCESS; - } + if (h_ctx && (*(__u8 *)h_ctx == UADK_ALG_HW)) + return 0; - return 0; + return -WD_HW_EACCESS; } static void wd_parse_log_level(void) @@ -193,6 +190,7 @@ static int get_int_attr(struct uacce_dev *dev, const char *attr, int *val) if (ret < 0) return ret; + errno = 0; *val = strtol(buf, NULL, 10); if (errno == ERANGE) { WD_ERR("failed to strtol %s, out of range!\n", buf); @@ -205,7 +203,7 @@ static int get_int_attr(struct uacce_dev *dev, const char *attr, int *val) static int get_str_attr(struct uacce_dev *dev, const char *attr, char *buf, size_t buf_sz) { - __u32 ret; + int ret; int size; size = get_raw_attr(dev->dev_root, attr, buf, buf_sz); @@ -489,6 +487,7 @@ void wd_release_ctx(handle_t h_ctx) if (!ctx || wd_check_ctx_type(h_ctx)) return; + wd_remove_ctx_list(h_ctx); close(ctx->fd); free(ctx->dev); free(ctx->drv_name); @@ -847,9 +846,9 @@ struct uacce_dev *wd_get_accel_dev(const char *alg_name) { struct uacce_dev_list *list, *head; struct uacce_dev *dev = NULL, *target = NULL; + int dis = WD_DEFAULT_NUMA_DISTANCE; unsigned int node; int ctx_num, tmp; - int dis = 1024; int max = 0; /* Under default conditions in a VM, the node value is 0 */ @@ -915,7 +914,8 @@ struct bitmask *wd_create_device_nodemask(struct uacce_dev_list *list) p = list; while (p) { - numa_bitmask_setbit(bmp, p->dev->numa_id); + if (p->dev) + numa_bitmask_setbit(bmp, p->dev->numa_id); p = p->next; } @@ -997,33 +997,43 @@ void wd_release_alg_cap(struct wd_capability *head) struct wd_capability *wd_get_alg_cap(void) { - struct wd_alg_list *head = wd_get_alg_head(); - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *head = wd_get_alg_head(); + struct wd_drv_node *drv_node = head->next; struct wd_capability *cap_head = NULL; struct wd_capability *cap_pnext = NULL; struct wd_capability *cap_node; - - while (pnext) { - cap_node = calloc(1, sizeof(struct wd_capability)); - if (!cap_node) { - WD_ERR("fail to alloc wd capability head\n"); - goto alloc_err; - } - - (void)strcpy(cap_node->alg_name, pnext->alg_name); - (void)strcpy(cap_node->drv_name, pnext->drv_name); - cap_node->available = pnext->available; - cap_node->priority = pnext->priority; - cap_node->calc_type = pnext->calc_type; - cap_node->next = NULL; - - pnext = pnext->next; - if (!cap_pnext) { - cap_head = cap_node; - cap_pnext = cap_node; + int i; + + while (drv_node) { + /* Traverse the static algorithm array inside each driver node */ + for (i = 0; i < drv_node->alg_count; i++) { + cap_node = calloc(1, sizeof(struct wd_capability)); + if (!cap_node) { + WD_ERR("fail to alloc wd capability head\n"); + goto alloc_err; + } + /* Flatten the secondary structure into the original binary-tuple format */ + strncpy(cap_node->alg_name, drv_node->algs[i].alg_name, + CRYPTO_MAX_ALG_NAME - 1); + cap_node->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + strncpy(cap_node->drv_name, drv_node->drv_name, + CRYPTO_MAX_ALG_NAME - 1); + cap_node->drv_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + cap_node->available = drv_node->algs[i].available; + cap_node->priority = drv_node->priority; + cap_node->calc_type = drv_node->calc_type; + cap_node->next = NULL; + + /* Append to the capability linked list */ + if (!cap_head) { + cap_head = cap_node; + cap_pnext = cap_node; + } else { + cap_pnext->next = cap_node; + cap_pnext = cap_node; + } } - cap_pnext->next = cap_node; - cap_pnext = cap_node; + drv_node = drv_node->next; } return cap_head; diff --git a/wd_aead.c b/wd_aead.c index a9e6279..eb3597e 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -32,7 +32,6 @@ struct wd_aead_setting { enum wd_status status; struct wd_ctx_config_internal config; struct wd_sched sched; - struct wd_alg_driver *driver; struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; @@ -51,7 +50,7 @@ struct wd_aead_sess { __u16 ckey_bytes; __u16 akey_bytes; __u16 auth_bytes; - void *priv; + void **priv; void *sched_key; /* Stored the counter for gcm stream mode */ __u8 *iv; @@ -59,7 +58,6 @@ struct wd_aead_sess { __u64 long_data_len; struct wd_mm_ops mm_ops; enum wd_mem_type mm_type; - struct wd_aead_extend_ops eops; }; struct wd_env_config wd_aead_env_config; @@ -75,20 +73,16 @@ static void wd_aead_close_driver(int init_type) } if (wd_aead_setting.dlhandle) { - wd_release_drv(wd_aead_setting.driver); dlclose(wd_aead_setting.dlhandle); wd_aead_setting.dlhandle = NULL; } #else - wd_release_drv(wd_aead_setting.driver); hisi_sec2_remove(); #endif } static int wd_aead_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "gcm(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -122,14 +116,6 @@ static int wd_aead_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_aead_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_aead_setting.driver = driver; return WD_SUCCESS; } @@ -334,7 +320,7 @@ static struct wd_aead_sess *check_and_init_sess(struct wd_aead_sess_setup *setup sess->dalg = setup->dalg; sess->dmode = setup->dmode; - ret = wd_drv_alg_support(sess->alg_name, wd_aead_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, &wd_aead_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); free(sess); @@ -368,7 +354,7 @@ static int aead_setup_memory_and_buffers(struct wd_aead_sess *sess, sess->mac_bak = aead_alloc_func(mempool, WD_AEAD_CCM_GCM_MAX); if (!sess->mac_bak) { - WD_ERR("aead failed to calloc mac_bak memory!\n"); + WD_ERR("failed to calloc aead mac_bak memory!\n"); return -WD_ENOMEM; } memset(sess->mac_bak, 0, WD_AEAD_CCM_GCM_MAX); @@ -418,35 +404,99 @@ static void cleanup_session(struct wd_aead_sess *sess) static int wd_aead_sess_eops_init(struct wd_aead_sess *sess) { - int ret; - - if (sess->eops.eops_aiv_init) { - if (!sess->eops.eops_aiv_uninit) { - WD_ERR("failed to get aead extend ops free in session!\n"); - return -WD_EINVAL; + struct wd_ctx_config_internal *config = &wd_aead_setting.config; + struct wd_aead_extend_ops *eops; + struct wd_alg_driver *drv; + int ret, prev_idx; + int valid = 0; + __u32 i, j; + + /* Allocate priv array like wd_agg does */ + sess->priv = malloc(sizeof(void *) * config->ctx_num); + if (!sess->priv) + return -WD_ENOMEM; + memset(sess->priv, 0, sizeof(void *) * config->ctx_num); + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + /* the drv initial assignment is not NULL. */ + if (!drv->get_extend_ops) + continue; + + /* check if same driver was already initialized. */ + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } } - ret = sess->eops.eops_aiv_init(wd_aead_setting.driver, &sess->mm_ops, - &sess->eops.params); - if (ret) { - WD_ERR("failed to init aead extend ops params in session!\n"); - return ret; + if (prev_idx >= 0) { + /* Reuse existing priv */ + sess->priv[i] = sess->priv[prev_idx]; + continue; + } + + ret = drv->get_extend_ops(config->ctxs[i].drv->extend_ops); + if (!ret && config->ctxs[i].drv->extend_ops) { + eops = config->ctxs[i].drv->extend_ops; + + if (!eops->eops_aiv_init || !eops->eops_aiv_uninit) + continue; + /* call eops_aiv_init and store in priv[i]. */ + ret = eops->eops_aiv_init(drv, &sess->mm_ops, &sess->priv[i]); + if (ret) { + WD_ERR("failed to init aead extend ops params in session!\n"); + return ret; + } + valid++; } } + if (!valid) { + WD_DEBUG("failed to get aead extend ops!\n"); + return -WD_EINVAL; + } + return WD_SUCCESS; } static void wd_aead_sess_eops_uninit(struct wd_aead_sess *sess) { - if (sess->eops.eops_aiv_uninit) { - sess->eops.eops_aiv_uninit(wd_aead_setting.driver, &sess->mm_ops, - sess->eops.params); - sess->eops.params = NULL; + struct wd_ctx_config_internal *config = &wd_aead_setting.config; + struct wd_aead_extend_ops *eops; + struct wd_alg_driver *drv; + int prev_idx; + __u32 i, j; + + if (!sess->priv) + return; + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + /* check if this is a duplicate driver that was skipped. */ + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } + } + if (prev_idx >= 0 || !sess->priv[i] || !drv->extend_ops) + continue; + + eops = drv->extend_ops; + if (eops->eops_aiv_uninit) + eops->eops_aiv_uninit(drv, &sess->mm_ops, sess->priv[i]); } + + free(sess->priv); + sess->priv = NULL; } handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) { + struct wd_sched_params params; struct wd_aead_sess *sess; int ret; @@ -459,32 +509,30 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) return (handle_t)0; } - if (wd_aead_setting.driver->get_extend_ops) { - ret = wd_aead_setting.driver->get_extend_ops(&sess->eops); - if (ret) { - WD_ERR("failed to get aead sess extend ops!\n"); - goto sess_err; - } - } - ret = wd_aead_sess_eops_init(sess); if (ret) { WD_ERR("failed to init aead sess extend eops!\n"); - goto sess_err; + goto clean_up; } sess->sched_key = (void *)wd_aead_setting.sched.sched_init( wd_aead_setting.sched.h_sched_ctx, setup->sched_param); if (WD_IS_ERR(sess->sched_key)) { WD_ERR("failed to init session schedule key!\n"); - goto sched_key_err; + goto clean_up; } + /* Set compat filtering parameters for session-ctx matching */ + memset(¶ms, 0, sizeof(params)); + params.alg_name = sess->alg_name; + params.ctxs = wd_aead_setting.config.ctxs; + wd_aead_setting.sched.set_param(wd_aead_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; -sched_key_err: +clean_up: wd_aead_sess_eops_uninit(sess); -sess_err: cleanup_session(sess); return (handle_t)0; } @@ -501,8 +549,14 @@ void wd_aead_free_sess(handle_t h_sess) wd_memset_zero(sess->ckey, sess->ckey_bytes); wd_memset_zero(sess->akey, sess->akey_bytes); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_aead_setting.sched.sched_uninit) + wd_aead_setting.sched.sched_uninit( + wd_aead_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } wd_aead_sess_eops_uninit(sess); cleanup_session(sess); } diff --git a/wd_bmm.c b/wd_bmm.c index 462a638..811afa2 100644 --- a/wd_bmm.c +++ b/wd_bmm.c @@ -95,6 +95,46 @@ struct mem_ctx_node { static TAILQ_HEAD(, mem_ctx_node) g_mem_ctx_list = TAILQ_HEAD_INITIALIZER(g_mem_ctx_list); static pthread_mutex_t g_mem_ctx_mutex = PTHREAD_MUTEX_INITIALIZER; +/* + * Fork child handler for the global ctx pool. + * + * After fork() the child is single-threaded; the inherited mutex state is + * unreliable (the holding thread, if any, does not exist in the child). + * POSIX allows reinitializing a statically-initialized mutex in the child + * fork handler, so do that unconditionally and skip lock/unlock below. + * + * Discard all inherited ctx nodes. Each node's h_ctx points to a wd_ctx_h + * whose fd and mmap regions are shared with the parent (fd via dup, + * MAP_SHARED mmap via shared physical pages). Using these ctx in the child + * would cause parent/child to contend on the same hardware queue. Free only + * the node shells (COW copies, safe to free in the child). Do NOT call + * wd_release_ctx: closing fd or munmapping regions in the child could + * interfere with the parent's shared resources. The kernel reclaims child + * fds and VMAs on exit. + */ +static void wd_bmm_atfork_child(void) +{ + struct mem_ctx_node *node; + + pthread_mutex_init(&g_mem_ctx_mutex, NULL); + + while ((node = TAILQ_FIRST(&g_mem_ctx_list)) != NULL) { + TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); + free(node); + } + TAILQ_INIT(&g_mem_ctx_list); +} + +/* + * Register the atfork handler once when libwd is loaded, before any user + * code (including fork) runs. The constructor attribute ensures single + * registration regardless of how many alg init calls happen later. + */ +static void __attribute__((constructor)) wd_bmm_ctor(void) +{ + pthread_atfork(NULL, NULL, wd_bmm_atfork_child); +} + handle_t wd_find_ctx(const char *alg_name) { struct mem_ctx_node *close_node = NULL; @@ -146,21 +186,29 @@ handle_t wd_find_ctx(const char *alg_name) return h_ctx; } -void wd_remove_ctx_list(void) +void wd_remove_ctx_list(handle_t h_ctx) { struct mem_ctx_node *node; + if (!h_ctx) + return; + pthread_mutex_lock(&g_mem_ctx_mutex); - /* Free all list node */ - while ((node = TAILQ_FIRST(&g_mem_ctx_list)) != NULL) { - /* Use TAILQ_REMOVE to remove list node */ - TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); - free(node); + TAILQ_FOREACH(node, &g_mem_ctx_list, list_node) { + if (node->h_ctx == h_ctx) { + TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); + free(node); + break; + } } - pthread_mutex_unlock(&g_mem_ctx_mutex); } +static inline bool wd_ctx_is_hw(handle_t h_ctx) +{ + return h_ctx && (*(__u8 *)h_ctx == UADK_ALG_HW); +} + int wd_insert_ctx_list(handle_t h_ctx, char *alg_name) { struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; @@ -173,8 +221,8 @@ int wd_insert_ctx_list(handle_t h_ctx, char *alg_name) } /* A simple and efficient method to check the queue type */ - if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) { - WD_INFO("Invalid ctx: this ctx not HW ctx.\n"); + if (!wd_ctx_is_hw(h_ctx)) { + WD_INFO("Notes: this ctx not HW ctx.\n"); return 0; } @@ -568,7 +616,7 @@ static int wd_pool_pre_layout(handle_t h_ctx, * When IOMMU is disabled, the PA refers to the kernel's physical address, which * must be physically contiguous to be allocated by the kernel. * Therefore, the PA address can be obtained from the offset of the VA. - * + * */ static void *wd_iova_map(struct ctx_info *cinfo, void *va, size_t sz) { @@ -918,7 +966,7 @@ void wd_mem_free(void *pool, void *buf) struct wd_blkpool *p = pool; struct wd_blk_hd *current_hd; struct wd_blk_hd *hd; - unsigned int current_idx; + unsigned int current_idx; unsigned int blk_idx; unsigned long offset; unsigned int i, num; @@ -946,7 +994,7 @@ void wd_mem_free(void *pool, void *buf) } /* Calculate the block index. */ - offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start); + offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start); blk_idx = offset / sz; /* Check if the index is valid. */ diff --git a/wd_cipher.c b/wd_cipher.c index 5258191..3bfead8 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -40,7 +40,7 @@ static const unsigned char des_weak_keys[DES_WEAK_KEY_NUM][DES_KEY_SIZE] = { static const char *wd_cipher_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] = { {"ecb(sm4)", "cbc(sm4)", "ctr(sm4)", "xts(sm4)", "ofb(sm4)", "cfb(sm4)", "cbc-cs1(sm4)", "cbc-cs2(sm4)", "cbc-cs3(sm4)", - "", "", "xts(sm4)"}, + "", "", "xts-gb(sm4)"}, {"ecb(aes)", "cbc(aes)", "ctr(aes)", "xts(aes)", "ofb(aes)", "cfb(aes)", "cbc-cs1(aes)", "cbc-cs2(aes)", "cbc-cs3(aes)"}, {"ecb(des)", "cbc(des)",}, diff --git a/wd_comp.c b/wd_comp.c index 7a2b0bc..df32ea5 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -19,9 +19,10 @@ #define STREAM_CHUNK (128 * 1024) #define WD_ZLIB_HEADER_SZ 2 #define WD_GZIP_HEADER_SZ 10 +#define WD_COMP_ALG_TYPE_V1 "comp" static const char *wd_comp_alg_name[WD_COMP_ALG_MAX] = { - "zlib", "gzip", "deflate", "lz77_zstd", "lz4", "lz77_only" + "deflate", "zlib", "gzip", "lz77_zstd", "lz4", "lz77_only" }; struct wd_comp_sess { @@ -35,14 +36,17 @@ struct wd_comp_sess { void *sched_key; struct wd_mm_ops mm_ops; enum wd_mem_type mm_type; + bool strm_ctx_fixed; + __u32 strm_sync_ctx_idx; + }; struct wd_comp_setting { enum wd_status status; + enum wd_init_type init_type; struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_comp_setting; @@ -60,20 +64,16 @@ static void wd_comp_close_driver(int init_type) } if (wd_comp_setting.dlhandle) { - wd_release_drv(wd_comp_setting.driver); dlclose(wd_comp_setting.dlhandle); wd_comp_setting.dlhandle = NULL; } #else - wd_release_drv(wd_comp_setting.driver); hisi_zip_remove(); #endif } static int wd_comp_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "zlib"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -107,18 +107,11 @@ static int wd_comp_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_comp_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_comp_setting.driver = driver; - return WD_SUCCESS; } +static bool wd_comp_atfork_registered; + static void wd_comp_clear_status(void) { wd_alg_clear_init(&wd_comp_setting.status); @@ -163,8 +156,7 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + ret = wd_alg_init_driver(&wd_comp_setting.config); if (ret) goto out_clear_pool; diff --git a/wd_mempool.c b/wd_mempool.c index 2f49764..3ce9ef0 100644 --- a/wd_mempool.c +++ b/wd_mempool.c @@ -943,7 +943,8 @@ void wd_mempool_destroy(handle_t mempool) } wd_atomic_sub(&mp->ref, 1); - while(wd_atomic_load(&mp->ref)); + while (wd_atomic_load(&mp->ref)) + sched_yield(); uninit_mempool(mp); free_hugepage_mem(mp); pthread_spin_destroy(&mp->lock); diff --git a/wd_sched.c b/wd_sched.c index 2efa2aa..f704d59 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -28,21 +28,17 @@ #include "wd_internal.h" #define MAX_POLL_TIMES 1000 -#define HUNGRY_LOAD_THRESHOLD 256 #define SKEY_CTX_MAX_NUM 16 #define SKEY_MAX_THREAD_NUM 64 #define SKEY_LOAD_UPDATE_INTERVAL 1 #define HW_QUEUE_FULL_DEPTH 1024 -#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) - /* ============================================================================ * Hash Table Configuration * ============================================================================ */ #define WD_SCHED_MAX_BUCKETS 512 #define WD_SCHED_MIN_BUCKETS 32 -#define WD_SCHED_LOAD_FACTOR 0.75f #define HASH_PRIME1 73 #define HASH_PRIME2 13 #define HASH_PRIME3 7 @@ -239,6 +235,10 @@ struct wd_sched_ctx { __u32 skey_num; pthread_mutex_t skey_lock; struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; + + /* First ctx index per mode, used by SINGLE/NONE. */ + __u32 sync_idx; + __u32 async_idx; }; /* ============================================================================ @@ -904,6 +904,130 @@ static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) wd_sched_skey_cache_uninit(&key_domain->idx_cache); } +static __u32 wd_sched_find_compatible_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int region, int sched_mode) +{ + struct wd_sched_ctx_domain *domain; + __u32 i, ctx_idx, prop; + + for (prop = 0; prop < UADK_ALG_TYPE_MAX; prop++) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region, sched_mode, skey->type, prop); + if (!domain || !domain->valid) + continue; + for (i = 0; i < domain->total_ctx_count; i++) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx == INVALID_POS) + continue; + if (skey->ctxs[ctx_idx].drv && + wd_alg_match_drv(skey->ctxs[ctx_idx].drv, skey->alg_name)) + return ctx_idx; + } + } + + return INVALID_POS; +} + +static __u32 wd_sched_get_new_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + int region_id = skey->region_id; + __u8 ctx_prop = skey->ctx_prop; + __u32 op_type = skey->type; + __u32 ctx_idx; + int r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, ctx_prop); + return INVALID_POS; + } + + if (region_id < 0 || + (sched_ctx->policy != SCHED_POLICY_DEV && + region_id >= sched_ctx->region_num)) { + WD_ERR("invalid: region_id is %d, region_num is %u!\n", + region_id, sched_ctx->region_num); + return INVALID_POS; + } + + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + ctx_idx = wd_sched_find_compatible_ctx(sched_ctx, skey, region_id, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) + return INVALID_POS; + + /* Cross-region fallback: try all other regions */ + for (r = 0; r < sched_ctx->region_num; r++) { + if (r == region_id) + continue; + ctx_idx = wd_sched_find_compatible_ctx(sched_ctx, skey, r, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } + + return INVALID_POS; +} + +/** + * wd_sched_skey_compat_filter - Filter and replace incompatible ctxs in domain cache + * @sched_ctx: Scheduler context + * @skey: Session key with alg_name and ctxs + * @domain: Target domain (sync or async) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * For each ctx in domain cache, check if it supports alg_name. + * If not, find a compatible replacement from the global domain. + */ +static void wd_sched_skey_compat_filter(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, struct wd_sched_key_domain *domain, int sched_mode) +{ + __u32 ctx_idx, new_ctx; + __u32 i; + + if (!skey || !skey->alg_name || !skey->ctxs || !domain) + return; + + /* Skip uninitialized domains (no ctxs cached) */ + if (!domain->idx_cache.valid_count) + return; + + pthread_mutex_lock(&domain->lock); + + for (i = 0; i < domain->idx_cache.valid_count; i++) { + ctx_idx = domain->idx_cache.idx_list[i]; + + /* Check if current ctx is compatible */ + if (skey->ctxs[ctx_idx].drv && + wd_alg_match_drv(skey->ctxs[ctx_idx].drv, skey->alg_name)) { + /* Compatible, keep unchanged */ + continue; + } + + /* Not compatible, find a replacement from domain */ + new_ctx = wd_sched_get_new_ctx(sched_ctx, skey, sched_mode); + if (new_ctx != INVALID_POS && new_ctx != ctx_idx) { + /* Found compatible ctx, replace */ + domain->idx_cache.idx_list[i] = new_ctx; + __atomic_store_n(&domain->idx_cache.load_values[i], 0, __ATOMIC_RELAXED); + } else { + /* No compatible ctx found, mark as invalid */ + domain->idx_cache.idx_list[i] = INVALID_POS; + WD_INFO("info: no compatible ctx found for alg %s!\n", skey->alg_name); + } + } + + pthread_mutex_unlock(&domain->lock); +} /** * wd_sched_poll_skey - Poll contexts for scheduler session @@ -1444,10 +1568,17 @@ static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) return (handle_t)0; } -static __u32 sched_none_pick_next_ctx(handle_t sched_ctx, +static __u32 sched_none_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, const int sched_mode) { - return 0; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + + if (!sched_ctx) { + WD_ERR("invalid: sched ctx is NULL!\n"); + return INVALID_POS; + } + + return sched_mode == SCHED_MODE_SYNC ? sched_ctx->sync_idx : sched_ctx->async_idx; } static int sched_none_poll_policy(handle_t h_sched_ctx, @@ -1455,7 +1586,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; __u32 loop_times = MAX_POLL_TIMES + expect; - __u32 poll_num = 0; + __u32 poll_num = 0, poll_idx; int ret; if (!sched_ctx || !sched_ctx->poll_func) { @@ -1463,9 +1594,15 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } + poll_idx = sched_ctx->async_idx; + if (poll_idx == INVALID_POS) { + WD_ERR("invalid: no async ctx available to poll!\n"); + return -WD_EINVAL; + } + while (loop_times > 0) { loop_times--; - ret = sched_ctx->poll_func(0, 1, &poll_num); + ret = sched_ctx->poll_func(poll_idx, 1, &poll_num); if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; else if (ret == -WD_EAGAIN) @@ -1484,13 +1621,17 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) return (handle_t)0; } -static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, +static __u32 sched_single_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, const int sched_mode) { - if (sched_mode) - return 1; - else - return 0; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + + if (!sched_ctx) { + WD_ERR("invalid: sched ctx is NULL!\n"); + return INVALID_POS; + } + + return sched_mode == SCHED_MODE_SYNC ? sched_ctx->sync_idx : sched_ctx->async_idx; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -1498,7 +1639,7 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; __u32 loop_times = MAX_POLL_TIMES + expect; - __u32 poll_num = 0; + __u32 poll_num = 0, poll_idx; int ret; if (!sched_ctx || !sched_ctx->poll_func) { @@ -1506,9 +1647,15 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } + poll_idx = sched_ctx->async_idx; + if (poll_idx == INVALID_POS) { + WD_ERR("invalid: no async ctx available to poll!\n"); + return -WD_EINVAL; + } + while (loop_times > 0) { loop_times--; - ret = sched_ctx->poll_func(1, 1, &poll_num); + ret = sched_ctx->poll_func(poll_idx, 1, &poll_num); if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; else if (ret == -WD_EAGAIN) @@ -1890,7 +2037,7 @@ static void wd_sched_set_param(handle_t h_sched_ctx, struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; if (unlikely(!params || !skey)) { - WD_ERR("invalid: sched parmas or skey is NULL!\n"); + WD_INFO("info: sched parmas or skey is NULL!\n"); return; } @@ -1902,6 +2049,13 @@ static void wd_sched_set_param(handle_t h_sched_ctx, skey->alg_name = params->alg_name; skey->ctxs = params->ctxs; + /* If compat info provided, fix up pre-fetched ctxs */ + if (skey->alg_name && skey->ctxs) { + wd_sched_skey_compat_filter(h_sched_ctx, skey, + &skey->sync_domain, SCHED_MODE_SYNC); + wd_sched_skey_compat_filter(sched_ctx, skey, + &skey->async_domain, SCHED_MODE_ASYNC); + } } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { @@ -2055,6 +2209,15 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para } domain->valid = true; + /* SINGLE/NONE: record first ctx index per mode. */ + if (sched_ctx->policy == SCHED_POLICY_SINGLE || + sched_ctx->policy == SCHED_POLICY_NONE) { + if (mode == SCHED_MODE_SYNC && sched_ctx->sync_idx == INVALID_POS) + sched_ctx->sync_idx = param->begin; + else if (mode == SCHED_MODE_ASYNC && sched_ctx->async_idx == INVALID_POS) + sched_ctx->async_idx = param->begin; + } + return WD_SUCCESS; } @@ -2172,6 +2335,8 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, goto err_destroy_hash; } sched_ctx->skey_num = 0; + sched_ctx->sync_idx = INVALID_POS; + sched_ctx->async_idx = INVALID_POS; sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; -- 2.43.0