From: lizhi <lizhi206@huawei.com> In the updated UADK framework, for asymmetric encryption algorithms such as RSA, DH, and ECC, the API layer needs to be adapted to the new UADK heterogeneous hybrid acceleration framework to ensure these algorithms can achieve heterogeneous hybrid acceleration. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/hisi_hpre.c | 213 ++++++++++++++----------- drv/wd_drv.c | 19 ++- include/drv/wd_ecc_drv.h | 7 +- wd_dh.c | 186 +++++++++++++--------- wd_ecc.c | 325 +++++++++++++++++++++++++++------------ wd_rsa.c | 192 ++++++++++++++--------- 6 files changed, 599 insertions(+), 343 deletions(-) diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index fae8315..9e46d77 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -10,6 +10,7 @@ #include <sys/mman.h> #include <sys/types.h> #include "hisi_qm_udrv.h" +#include "wd_drv.h" #include "../include/wd_ecc_curve.h" #include "../include/drv/wd_rsa_drv.h" #include "../include/drv/wd_dh_drv.h" @@ -130,12 +131,13 @@ struct hisi_hpre_sqe { }; struct hisi_hpre_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; struct wd_mm_ops *mm_ops; handle_t rsv_mem_ctx; }; -struct hpre_ecc_ctx { +struct hisi_hpre_eops_ctx { __u32 enable_hpcore; }; @@ -647,44 +649,76 @@ static int hpre_init_qm_priv(struct wd_ctx_config_internal *config, struct hisi_hpre_ctx *hpre_ctx, struct hisi_qm_priv *qm_priv) { - handle_t h_ctx, h_qp; - __u32 i, j; + __u32 i, j, count; + bool *is_match; + handle_t h_qp; - memcpy(&hpre_ctx->config, config, sizeof(*config)); + /* First pass: traverse and count the number of contexts supported by this driver. */ + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; - /* allocate qp for each context */ - qm_priv->sqe_size = sizeof(struct hisi_hpre_sqe); + count = 0; + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && + strcmp(config->ctxs[i].drv->drv_name, "hisi_hpre") == 0) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + if (!count) { + free(is_match); + return -WD_EINVAL; + } + + hpre_ctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!hpre_ctx->ctxs) { + free(is_match); + return -WD_ENOMEM; + } + hpre_ctx->ctx_num = count; + + /* Second pass: allocate QP and store context mirror. */ + qm_priv->sqe_size = sizeof(struct hisi_hpre_sqe); + count = 0; for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; + if (!is_match[i]) + continue; + qm_priv->qp_mode = config->ctxs[i].ctx_mode; - /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv->epoll_en = (qm_priv->qp_mode == CTX_MODE_SYNC) ? - config->epoll_en : 0; + config->epoll_en : 0; qm_priv->idx = i; - h_qp = hisi_qm_alloc_qp(qm_priv, h_ctx); - if (!h_qp) { - WD_ERR("failed to alloc qp!\n"); + h_qp = hisi_qm_alloc_qp(qm_priv, config->ctxs[i].ctx); + if (!h_qp) goto out; - } + config->ctxs[i].sqn = qm_priv->sqn; + /* Store the queues allocated by your own driver. */ + hpre_ctx->ctxs[count++] = &config->ctxs[i]; } + free(is_match); return WD_SUCCESS; + out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(hpre_ctx->ctxs[j]->ctx); hisi_qm_free_qp(h_qp); } - + free(hpre_ctx->ctxs); + free(is_match); return -WD_EINVAL; } -static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) +static int hpre_rsa_dh_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret; if (!config->ctx_num) { @@ -692,27 +726,19 @@ static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V2_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv; return WD_SUCCESS; } -static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) +static int hpre_ecc_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret; if (!config->ctx_num) { @@ -720,44 +746,33 @@ static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V3_ECC_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv; return WD_SUCCESS; } -static void hpre_exit(struct wd_alg_driver *drv) +static void hpre_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_hpre_ctx *priv; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) - return; - - priv = (struct hisi_hpre_ctx *)drv->priv; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + /* Only release the queues allocated by your own driver. */ + for (i = 0; i < hpre_ctx->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(hpre_ctx->ctxs[i]->ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; + if (hpre_ctx->ctxs) { + free(hpre_ctx->ctxs); + hpre_ctx->ctxs = NULL; + } } -static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_send(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_rsa_msg *msg = rsa_msg; @@ -829,7 +844,7 @@ static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, } } -static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_recv(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -949,7 +964,7 @@ static int dh_out_transfer(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg, return WD_SUCCESS; } -static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_send(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct map_info_cache cache = {0}; @@ -1020,7 +1035,7 @@ dh_fail: return ret; } -static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_recv(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -1168,7 +1183,7 @@ static bool big_than_one(const char *data, __u32 data_sz) static bool less_than_latter(struct wd_dtb *d, struct wd_dtb *n) { - unsigned char *d_data, *n_data; + char *d_data, *n_data; __u32 shift, i; if (d->dsize > n->dsize) @@ -1608,7 +1623,7 @@ static int u_is_in_p(struct wd_ecc_msg *msg) static int ecc_prepare_in(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, void **data) { - struct hpre_ecc_ctx *ecc_ctx = msg->drv_cfg; + struct hisi_hpre_eops_ctx *eops_ctx = (struct hisi_hpre_eops_ctx *)msg->priv; int ret = -WD_EINVAL; switch (msg->req.op_type) { @@ -1621,11 +1636,11 @@ static int ecc_prepare_in(struct wd_ecc_msg *msg, ret = ecc_prepare_dh_gen_in(msg, hw_msg, data); break; case WD_ECXDH_GEN_KEY: - hw_msg->bd_rsv2 = ecc_ctx->enable_hpcore; + hw_msg->bd_rsv2 = eops_ctx->enable_hpcore; ret = ecc_prepare_dh_gen_in(msg, hw_msg, data); break; case WD_ECXDH_COMPUTE_KEY: - hw_msg->bd_rsv2 = ecc_ctx->enable_hpcore; + hw_msg->bd_rsv2 = eops_ctx->enable_hpcore; ret = ecc_prepare_dh_compute_in(msg, hw_msg, data); if (!ret && (msg->curve_id == WD_X25519 || msg->curve_id == WD_X448)) @@ -2154,7 +2169,7 @@ free_dst: return ret; } -static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_send(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -2750,7 +2765,7 @@ fail: return ret; } -static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_recv(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -2786,22 +2801,20 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) static handle_t hpre_find_dev_qp(struct wd_alg_driver *drv, const char *dev_name) { - struct wd_ctx_config_internal *config; struct hisi_hpre_ctx *priv; char *ctx_dev_name; handle_t ctx = 0; handle_t qp = 0; __u32 i; - priv = (struct hisi_hpre_ctx *)drv->priv; + priv = (struct hisi_hpre_ctx *)drv->drv_data; if (!priv) return 0; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - ctx_dev_name = wd_ctx_get_dev_name(config->ctxs[i].ctx); + for (i = 0; i < priv->ctx_num; i++) { + ctx_dev_name = wd_ctx_get_dev_name(priv->ctxs[i]->ctx); if (!strcmp(ctx_dev_name, dev_name)) { - ctx = config->ctxs[i].ctx; + ctx = priv->ctxs[i]->ctx; break; } } @@ -2848,9 +2861,24 @@ static int hpre_rsa_get_usage(void *param) return -WD_EACCES; } +static bool is_valid_hw_type(struct wd_alg_driver *drv) +{ + struct hisi_hpre_ctx *hpre_ctx; + struct hisi_qp *qp; + + if (unlikely(!drv || !drv->drv_data)) + return false; + + hpre_ctx = (struct hisi_hpre_ctx *)drv->drv_data; + qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->ctxs[0]->ctx); + if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE) + return false; + return true; +} + static int ecc_sess_eops_init(struct wd_alg_driver *drv, void **params) { - struct hpre_ecc_ctx *ecc_ctx; + struct hisi_hpre_eops_ctx *eops_ctx; if (!params) { WD_ERR("invalid: extend ops init params address is NULL!\n"); @@ -2862,11 +2890,11 @@ static int ecc_sess_eops_init(struct wd_alg_driver *drv, void **params) return -WD_EINVAL; } - ecc_ctx = calloc(1, sizeof(struct hpre_ecc_ctx)); - if (!ecc_ctx) + eops_ctx = calloc(1, sizeof(struct hisi_hpre_eops_ctx)); + if (!eops_ctx) return -WD_ENOMEM; - *params = ecc_ctx; + *params = eops_ctx; return WD_SUCCESS; } @@ -2881,34 +2909,19 @@ static void ecc_sess_eops_uninit(struct wd_alg_driver *drv, void *params) free(params); } -static bool is_valid_hw_type(struct wd_alg_driver *drv) -{ - struct hisi_hpre_ctx *hpre_ctx; - struct hisi_qp *qp; - - if (unlikely(!drv || !drv->priv)) - return false; - - hpre_ctx = (struct hisi_hpre_ctx *)drv->priv; - qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->config.ctxs[0].ctx); - if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE) - return false; - return true; -} - static void ecc_sess_eops_params_cfg(struct wd_alg_driver *drv, struct wd_ecc_sess_setup *setup, struct wd_ecc_curve *cv, void *params) { __u8 data[SECP256R1_PARAM_SIZE] = SECG_P256_R1_PARAM; - struct hpre_ecc_ctx *ecc_ctx = params; + struct hisi_hpre_eops_ctx *eops_ctx = params; __u32 key_size; int ret; if (!is_valid_hw_type(drv)) return; - if (!ecc_ctx) { + if (!eops_ctx) { WD_INFO("Info: eops config exits, but params is NULL!\n"); return; } @@ -2922,7 +2935,7 @@ static void ecc_sess_eops_params_cfg(struct wd_alg_driver *drv, ret = memcmp_consttime(data, cv->p.data, SECP256R1_PARAM_SIZE); if (!ret) - ecc_ctx->enable_hpcore = 1; + eops_ctx->enable_hpcore = 1; } static int hpre_ecc_get_extend_ops(void *ops) @@ -2932,7 +2945,6 @@ static int hpre_ecc_get_extend_ops(void *ops) if (!ecc_ops) return -WD_EINVAL; - ecc_ops->params = NULL; ecc_ops->sess_init = ecc_sess_eops_init; ecc_ops->eops_params_cfg = ecc_sess_eops_params_cfg; ecc_ops->sess_uninit = ecc_sess_eops_uninit; @@ -2945,15 +2957,22 @@ static int hpre_ecc_get_extend_ops(void *ops) .alg_name = (hpre_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_hpre_ctx),\ + .ops_size = sizeof(struct wd_ecc_extend_ops),\ .queue_num = HPRE_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ + .drv_data = NULL, \ + .extend_ops = NULL, \ .fallback = 0,\ + .init_state = 0,\ .init = hpre_ecc_init,\ .exit = hpre_exit,\ .send = ecc_send,\ .recv = ecc_recv,\ .get_usage = hpre_ecc_get_usage,\ .get_extend_ops = hpre_ecc_get_extend_ops,\ + .alloc_ctx = wd_hw_alloc_ctx, \ + .free_ctx = wd_hw_free_ctx, \ } static struct wd_alg_driver hpre_ecc_driver[] = { @@ -2969,14 +2988,19 @@ static struct wd_alg_driver hpre_rsa_driver = { .alg_name = "rsa", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, + .drv_data = NULL, .fallback = 0, + .init_state = 0, .init = hpre_rsa_dh_init, .exit = hpre_exit, .send = rsa_send, .recv = rsa_recv, .get_usage = hpre_rsa_get_usage, + .alloc_ctx = wd_hw_alloc_ctx, + .free_ctx = wd_hw_free_ctx, }; static struct wd_alg_driver hpre_dh_driver = { @@ -2984,14 +3008,19 @@ static struct wd_alg_driver hpre_dh_driver = { .alg_name = "dh", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, + .drv_data = NULL, .fallback = 0, + .init_state = 0, .init = hpre_rsa_dh_init, .exit = hpre_exit, .send = dh_send, .recv = dh_recv, .get_usage = hpre_rsa_get_usage, + .alloc_ctx = wd_hw_alloc_ctx, + .free_ctx = wd_hw_free_ctx, }; #ifdef WD_STATIC_DRV diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 16dc24c..d6a7626 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -163,6 +163,21 @@ out_free_list: return ret; } +static int wd_get_alg_class_type(const char *alg_name, char *alg_type) +{ + int ret; + + ret = wd_get_alg_type(alg_name, alg_type); + if (ret) + return -WD_EINVAL; + + /* all ECC types use sm2 to find device. */ + if (!strcmp(alg_type, "ecc")) + strcpy(alg_type, "sm2"); + + return WD_SUCCESS; +} + /** * wd_hw_alloc_ctx() - HW driver's alloc_ctx callback. * @@ -195,8 +210,8 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) } target_numa = ctx_params->numa_id; - /* Get algorithm type and device list */ - ret = wd_get_alg_type(alg_name, alg_type); + /* Get algorithm class type and device list */ + ret = wd_get_alg_class_type(alg_name, alg_type); if (ret) { WD_ERR("invalid: alg_name is NULL!\n"); return -WD_EINVAL; diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index 48c422f..585811e 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -56,7 +56,7 @@ struct wd_ecc_msg { __u16 key_bytes; /* key bytes */ __u8 curve_id; /* Ec curve denoted by enum wd_ecc_curve_type */ __u8 result; /* alg op error code */ - void *drv_cfg; /* internal driver configuration */ + void *priv; /* internal driver configuration */ __u8 *rsv_out; /* reserved output data pointer */ }; @@ -180,10 +180,9 @@ struct wd_ecc_out { }; struct wd_ecc_extend_ops { - void *params; /* the params are passed to the following ops */ void (*eops_params_cfg)(struct wd_alg_driver *drv, - struct wd_ecc_sess_setup *setup, - struct wd_ecc_curve *cv, void *params); + struct wd_ecc_sess_setup *setup, struct wd_ecc_curve *cv, + void *params); int (*sess_init)(struct wd_alg_driver *drv, void **params); void (*sess_uninit)(struct wd_alg_driver *drv, void *params); }; diff --git a/wd_dh.c b/wd_dh.c index 9a0176b..612e15f 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -17,6 +17,7 @@ #define DH_MAX_KEY_SIZE 512 #define WD_DH_G2 2 +#define WD_DH_OP_TYPE 1 static __thread __u64 balance; @@ -35,7 +36,6 @@ static struct wd_dh_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_dh_setting; @@ -55,19 +55,15 @@ static void wd_dh_close_driver(int init_type) if (!wd_dh_setting.dlhandle) return; - wd_release_drv(wd_dh_setting.driver); dlclose(wd_dh_setting.dlhandle); wd_dh_setting.dlhandle = NULL; #else - wd_release_drv(wd_dh_setting.driver); hisi_hpre_remove(); #endif } static int wd_dh_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "dh"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -101,18 +97,12 @@ static int wd_dh_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_dh_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_dh_setting.driver = driver; return WD_SUCCESS; } +static bool wd_dh_atfork_registered; + static void wd_dh_clear_status(void) { wd_alg_clear_init(&wd_dh_setting.status); @@ -143,15 +133,8 @@ static int wd_dh_common_init(struct wd_ctx_config *config, struct wd_sched *sche if (ret) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_dh_setting.config, - wd_dh_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_dh_setting.pool); out_clear_sched: wd_clear_sched(&wd_dh_setting.sched); out_clear_ctx_config: @@ -159,30 +142,24 @@ out_clear_ctx_config: return ret; } -static int wd_dh_common_uninit(void) +static void wd_dh_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_dh_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_dh_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_dh_setting.sched); - wd_alg_uninit_driver(&wd_dh_setting.config, - wd_dh_setting.driver); - - return WD_SUCCESS; } int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_dh_clear_status); + if (!wd_dh_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_dh_clear_status) == 0) + wd_dh_atfork_registered = true; + } ret = wd_alg_try_init(&wd_dh_setting.status); if (ret) @@ -200,10 +177,38 @@ int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array("dh", TASK_HW, "hisi_hpre", + &wd_dh_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_common_uninit; + } + + wd_dh_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_dh_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_dh_setting.config); + if (ret) { + WD_ERR("dh driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_dh_setting.status); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_dh_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_dh_setting.config.drv_array, drv_count); + wd_dh_setting.config.drv_array = NULL; + wd_dh_setting.config.drv_count = 0; +out_common_uninit: + wd_dh_common_uninit(); out_close_driver: wd_dh_close_driver(WD_TYPE_V1); out_clear_init: @@ -213,12 +218,19 @@ out_clear_init: void wd_dh_uninit(void) { - int ret; + enum wd_status status; - ret = wd_dh_common_uninit(); - if (ret) + wd_alg_get_init(&wd_dh_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_dh_setting.config); + wd_ctx_unbind_drivers(&wd_dh_setting.config); + wd_put_drv_array(wd_dh_setting.config.drv_array, wd_dh_setting.config.drv_count); + wd_dh_setting.config.drv_array = NULL; + wd_dh_setting.config.drv_count = 0; + wd_dh_common_uninit(); + wd_dh_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_dh_setting.status); } @@ -229,7 +241,10 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param struct wd_ctx_params dh_ctx_params = {0}; int state, ret = -WD_EINVAL; - pthread_atfork(NULL, NULL, wd_dh_clear_status); + if (!wd_dh_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_dh_clear_status) == 0) + wd_dh_atfork_registered = true; + } state = wd_alg_try_init(&wd_dh_setting.status); if (state) @@ -252,38 +267,25 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param while (ret) { memset(&wd_dh_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_dh_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_dh_setting.driver) { - WD_ERR("fail to bind a valid driver.\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ dh_ctx_params.ctx_set_num = dh_ctx_num; ret = wd_ctx_param_init(&dh_ctx_params, ctx_params, - wd_dh_setting.driver, WD_DH_TYPE, WD_DH_PHASE2); + alg, WD_DH_TYPE, WD_DH_OP_TYPE); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); + if (ret == -WD_EAGAIN) continue; - } goto out_driver; } (void)strcpy(wd_dh_init_attrs.alg, alg); wd_dh_init_attrs.sched_type = sched_type; - wd_dh_init_attrs.driver = wd_dh_setting.driver; + wd_dh_init_attrs.task_type = task_type; wd_dh_init_attrs.ctx_params = &dh_ctx_params; wd_dh_init_attrs.alg_init = wd_dh_common_init; wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx; ret = wd_alg_attrs_init(&wd_dh_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); wd_ctx_param_uninit(&dh_ctx_params); continue; } @@ -292,16 +294,33 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param } } + ret = wd_ctx_bind_drivers(&wd_dh_setting.config, + wd_dh_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_dh_setting.config); + if (ret) { + WD_ERR("driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_dh_setting.status); wd_ctx_param_uninit(&dh_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_dh_setting.config); +out_common_uninit: + wd_dh_common_uninit(); + wd_alg_attrs_uninit(&wd_dh_init_attrs); out_params_uninit: wd_ctx_param_uninit(&dh_ctx_params); out_driver: - wd_alg_drv_unbind(wd_dh_setting.driver); -out_dlopen: wd_dh_close_driver(WD_TYPE_V2); out_clear_init: wd_alg_clear_init(&wd_dh_setting.status); @@ -310,14 +329,19 @@ out_clear_init: void wd_dh_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_dh_common_uninit(); - if (ret) + wd_alg_get_init(&wd_dh_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_dh_setting.config); + wd_ctx_unbind_drivers(&wd_dh_setting.config); + wd_dh_setting.config.drv_array = NULL; + wd_dh_setting.config.drv_count = 0; + wd_dh_common_uninit(); + wd_alg_attrs_uninit(&wd_dh_init_attrs); - wd_alg_drv_unbind(wd_dh_setting.driver); wd_dh_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_dh_setting.status); } @@ -386,13 +410,13 @@ int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) if (unlikely(ret)) return ret; - msg_handle.send = wd_dh_setting.driver->send; - msg_handle.recv = wd_dh_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_dh_setting.driver, &msg_handle, ctx->ctx, - &msg, &balance, wd_dh_setting.config.epoll_en); - pthread_spin_unlock(&ctx->lock); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, &balance, + wd_dh_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -428,8 +452,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) mid = wd_get_msg_from_pool(&wd_dh_setting.pool, idx, (void **)&msg); if (unlikely(mid < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return mid; + return -WD_EBUSY; } ret = fill_dh_msg(msg, req, (struct wd_dh_sess *)sess); @@ -437,7 +460,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_dh_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send dh BD, hw is err!\n"); @@ -485,7 +508,7 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_dh_setting.driver, ctx->ctx, &rcv_msg); + ret = ctx->drv->recv(ctx->ctx, &rcv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret)) { @@ -581,6 +604,7 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g) handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) { + struct wd_sched_params params = {0}; struct wd_dh_sess *sess; int ret; @@ -589,6 +613,12 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) return (handle_t)0; } + ret = wd_drv_alg_support("dh", &wd_dh_setting.config); + if (!ret) { + WD_ERR("failed to support this algorithm: rsa!\n"); + return (handle_t)0; + } + /* key width check */ if (setup->key_bits != 768 && setup->key_bits != 1024 && @@ -630,6 +660,13 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = "dh"; + params.ctxs = wd_dh_setting.config.ctxs; + wd_dh_setting.sched.set_param( + wd_dh_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -651,8 +688,15 @@ void wd_dh_free_sess(handle_t sess) if (sess_t->g.data) sess_t->mm_ops.free(sess_t->mm_ops.usr, sess_t->g.data); - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_dh_setting.sched.sched_uninit) + wd_dh_setting.sched.sched_uninit( + wd_dh_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } + free(sess_t); } diff --git a/wd_ecc.c b/wd_ecc.c index dae8a05..2d1aa77 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -29,6 +29,9 @@ #define GET_NEGATIVE(val) (0 - (val)) #define ZA_PARAM_NUM 6 #define WD_SECP256R1 0x18 /* consistent with enum wd_ecc_curve_id */ +#define WD_ECC_OP_TYPE 1 +/* Default ECC algorithm for wd_ecc_init v1 */ +#define ECC_ALG_DEFAULT_V1 "ecc" static __thread __u64 balance; @@ -50,10 +53,10 @@ struct wd_ecc_sess { __u32 key_size; struct wd_ecc_key key; struct wd_ecc_sess_setup setup; - struct wd_ecc_extend_ops eops; void *sched_key; struct wd_mm_ops mm_ops; enum wd_mem_type mm_type; + void **priv; }; struct wd_ecc_curve_list { @@ -68,7 +71,7 @@ static struct wd_ecc_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; + enum wd_init_type init_type; void *dlhandle; void *dlh_list; } wd_ecc_setting; @@ -112,19 +115,15 @@ static void wd_ecc_close_driver(int init_type) if (!wd_ecc_setting.dlhandle) return; - wd_release_drv(wd_ecc_setting.driver); dlclose(wd_ecc_setting.dlhandle); wd_ecc_setting.dlhandle = NULL; #else - wd_release_drv(wd_ecc_setting.driver); hisi_hpre_remove(); #endif } static int wd_ecc_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm2"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -158,14 +157,6 @@ static int wd_ecc_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_ecc_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_ecc_setting.driver = driver; return WD_SUCCESS; } @@ -180,6 +171,8 @@ static bool is_alg_support(const char *alg) return true; } +static bool wd_ecc_atfork_registered; + static void wd_ecc_clear_status(void) { wd_alg_clear_init(&wd_ecc_setting.status); @@ -209,15 +202,8 @@ static int wd_ecc_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_ecc_setting.pool); out_clear_sched: wd_clear_sched(&wd_ecc_setting.sched); out_clear_ctx_config: @@ -225,30 +211,24 @@ out_clear_ctx_config: return ret; } -static int wd_ecc_common_uninit(void) +static void wd_ecc_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_ecc_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_ecc_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_ecc_setting.sched); - wd_alg_uninit_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); - - return WD_SUCCESS; } int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_ecc_clear_status); + if (!wd_ecc_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_ecc_clear_status) == 0) + wd_ecc_atfork_registered = true; + } ret = wd_alg_try_init(&wd_ecc_setting.status); if (ret) @@ -266,10 +246,39 @@ int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array(ECC_ALG_DEFAULT_V1, TASK_HW, "hisi_hpre", + &wd_ecc_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_common_uninit; + } + + wd_ecc_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_ecc_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_ecc_setting.config); + if (ret) { + WD_ERR("ecc driver init failed!\n"); + goto out_unbind_drivers; + } + wd_ecc_setting.init_type = WD_TYPE_V1; wd_alg_set_init(&wd_ecc_setting.status); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_ecc_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_ecc_setting.config.drv_array, + wd_ecc_setting.config.drv_count); + wd_ecc_setting.config.drv_array = NULL; + wd_ecc_setting.config.drv_count = 0; +out_common_uninit: + wd_ecc_common_uninit(); out_close_driver: wd_ecc_close_driver(WD_TYPE_V1); out_clear_init: @@ -279,12 +288,20 @@ out_clear_init: void wd_ecc_uninit(void) { - int ret; + enum wd_status status; - ret = wd_ecc_common_uninit(); - if (ret) + wd_alg_get_init(&wd_ecc_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_ecc_setting.config); + wd_ctx_unbind_drivers(&wd_ecc_setting.config); + wd_put_drv_array(wd_ecc_setting.config.drv_array, + wd_ecc_setting.config.drv_count); + wd_ecc_setting.config.drv_array = NULL; + wd_ecc_setting.config.drv_count = 0; + wd_ecc_common_uninit(); + wd_ecc_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_ecc_setting.status); } @@ -296,7 +313,10 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para int state, ret = -WD_EINVAL; bool flag; - pthread_atfork(NULL, NULL, wd_ecc_clear_status); + if (!wd_ecc_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_ecc_clear_status) == 0) + wd_ecc_atfork_registered = true; + } state = wd_alg_try_init(&wd_ecc_setting.status); if (state) @@ -321,37 +341,26 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para while (ret) { memset(&wd_ecc_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - /* Get alg driver and dev name */ - wd_ecc_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_ecc_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ ecc_ctx_params.ctx_set_num = ecc_ctx_num; ret = wd_ctx_param_init(&ecc_ctx_params, ctx_params, - wd_ecc_setting.driver, WD_ECC_TYPE, WD_EC_OP_MAX); + alg, WD_ECC_TYPE, WD_ECC_OP_TYPE); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); + if (ret == -WD_EAGAIN) continue; - } + goto out_driver; } (void)strcpy(wd_ecc_init_attrs.alg, alg); wd_ecc_init_attrs.sched_type = sched_type; - wd_ecc_init_attrs.driver = wd_ecc_setting.driver; + wd_ecc_init_attrs.task_type = task_type; wd_ecc_init_attrs.ctx_params = &ecc_ctx_params; wd_ecc_init_attrs.alg_init = wd_ecc_common_init; wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx; ret = wd_alg_attrs_init(&wd_ecc_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); wd_ctx_param_uninit(&ecc_ctx_params); continue; } @@ -359,17 +368,32 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_params_uninit; } } + ret = wd_ctx_bind_drivers(&wd_ecc_setting.config, + wd_ecc_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + ret = wd_alg_init_driver(&wd_ecc_setting.config); + if (ret) + goto out_unbind_drivers; + + wd_ecc_setting.init_type = WD_TYPE_V2; wd_alg_set_init(&wd_ecc_setting.status); wd_ctx_param_uninit(&ecc_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_ecc_setting.config); +out_common_uninit: + wd_ecc_common_uninit(); + wd_alg_attrs_uninit(&wd_ecc_init_attrs); out_params_uninit: wd_ctx_param_uninit(&ecc_ctx_params); out_driver: - wd_alg_drv_unbind(wd_ecc_setting.driver); -out_dlopen: wd_ecc_close_driver(WD_TYPE_V2); out_clear_init: wd_alg_clear_init(&wd_ecc_setting.status); @@ -378,14 +402,20 @@ out_clear_init: void wd_ecc_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_ecc_common_uninit(); - if (ret) + wd_alg_get_init(&wd_ecc_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_ecc_setting.config); + wd_ctx_unbind_drivers(&wd_ecc_setting.config); + wd_ecc_setting.config.drv_array = NULL; + wd_ecc_setting.config.drv_count = 0; + wd_ecc_common_uninit(); + wd_alg_attrs_uninit(&wd_ecc_init_attrs); - wd_alg_drv_unbind(wd_ecc_setting.driver); + wd_ecc_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_ecc_setting.status); } @@ -1174,49 +1204,135 @@ static void del_sess_key(struct wd_ecc_sess *sess) static int wd_ecc_sess_eops_init(struct wd_ecc_sess *sess) { - int ret; - - if (sess->eops.sess_init) { - if (!sess->eops.sess_uninit) { - WD_ERR("failed to get extend ops in session!\n"); - return -WD_EINVAL; + struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct wd_ecc_extend_ops *eops; + struct wd_alg_driver *drv; + int ret, valid = 0; + int prev_idx; + __u32 i, j; + + sess->priv = malloc(sizeof(void *) * config->ctx_num); + if (!sess->priv) + return -WD_ENOMEM; + memset(sess->priv, 0, sizeof(void *) * config->ctx_num); + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + if (!drv->extend_ops) + continue; + + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } } - ret = sess->eops.sess_init(wd_ecc_setting.driver, &sess->eops.params); - if (ret) { - WD_ERR("failed to init extend ops params in session!\n"); - return ret; + if (prev_idx >= 0) { + sess->priv[i] = sess->priv[prev_idx]; + continue; + } + + ret = drv->get_extend_ops(config->ctxs[i].drv->extend_ops); + if (!ret && config->ctxs[i].drv->extend_ops) { + valid++; + eops = config->ctxs[i].drv->extend_ops; + + if (eops->sess_init) { + if (!eops->sess_uninit) { + WD_ERR("failed to get session uninit ops!\n"); + return -WD_EINVAL; + } + ret = eops->sess_init(drv, &sess->priv[i]); + if (ret) { + WD_ERR("failed to init session priv!\n"); + return ret; + } + } } } + + if (!valid) + WD_DEBUG("no ecc extend ops found!\n"); + return WD_SUCCESS; } static void wd_ecc_sess_eops_uninit(struct wd_ecc_sess *sess) { - if (sess->eops.sess_uninit) { - sess->eops.sess_uninit(wd_ecc_setting.driver, sess->eops.params); - sess->eops.params = NULL; + struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct wd_ecc_extend_ops *eops; + struct wd_alg_driver *drv; + int prev_idx; + __u32 i, j; + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + eops = drv->extend_ops; + if (!eops) + continue; + + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } + } + if (prev_idx >= 0) + continue; + + if (eops->sess_uninit) + eops->sess_uninit(drv, sess->priv[i]); } + + if (sess->priv) { + free(sess->priv); + sess->priv = NULL; + } + + return; } -static void wd_ecc_sess_eops_cfg(struct wd_ecc_sess_setup *setup, - struct wd_ecc_sess *sess) + +static void wd_ecc_sess_eops_cfg(struct wd_ecc_sess *sess, struct wd_ecc_sess_setup *setup) { - if (sess->eops.sess_init && sess->eops.eops_params_cfg) { - /* the config result does not impact task sucesss or failure */ - sess->eops.eops_params_cfg(wd_ecc_setting.driver, setup, sess->key.cv, - sess->eops.params); + struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct wd_ecc_extend_ops *eops; + struct wd_alg_driver *drv; + int prev_idx; + __u32 i, j; + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + eops = drv->extend_ops; + if (!eops || !eops->eops_params_cfg) + continue; + + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } + } + if (prev_idx >= 0) + continue; + + eops->eops_params_cfg(drv, setup, sess->key.cv, sess->priv[i]); } } handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) { + struct wd_sched_params params = {0}; struct wd_ecc_sess *sess; int ret; if (setup_param_check(setup)) return (handle_t)0; - ret = wd_drv_alg_support(setup->alg, wd_ecc_setting.driver); + ret = wd_drv_alg_support(setup->alg, &wd_ecc_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", setup->alg); return (handle_t)0; @@ -1238,14 +1354,6 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); sess->mm_type = setup->mm_type; - if (wd_ecc_setting.driver->get_extend_ops) { - ret = wd_ecc_setting.driver->get_extend_ops(&sess->eops); - if (ret) { - WD_ERR("failed to get ecc sess extend ops!\n"); - goto sess_err; - } - } - ret = wd_ecc_sess_eops_init(sess); if (ret) { WD_ERR("failed to init ecc sess extend eops!\n"); @@ -1258,7 +1366,7 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) goto eops_err; } - wd_ecc_sess_eops_cfg(setup, sess); + wd_ecc_sess_eops_cfg(sess, setup); /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_ecc_setting.sched.sched_init( @@ -1268,6 +1376,13 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = sess->setup.alg; + params.ctxs = wd_ecc_setting.config.ctxs; + wd_ecc_setting.sched.set_param( + wd_ecc_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -1288,8 +1403,15 @@ void wd_ecc_free_sess(handle_t sess) return; } - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_ecc_setting.sched.sched_uninit) + wd_ecc_setting.sched.sched_uninit( + wd_ecc_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } + del_sess_key(sess_t); wd_ecc_sess_eops_uninit(sess_t); free(sess_t); @@ -1568,7 +1690,6 @@ static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req, msg->mm_type = sess->mm_type; msg->key_bytes = sess->key_size; msg->curve_id = sess->setup.cv.cfg.id; - msg->drv_cfg = sess->eops.params; msg->result = WD_EINVAL; switch (req->op_type) { @@ -1648,13 +1769,14 @@ int wd_do_ecc_sync(handle_t h_sess, struct wd_ecc_req *req) if (unlikely(ret)) return ret; - msg_handle.send = wd_ecc_setting.driver->send; - msg_handle.recv = wd_ecc_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; + msg.priv = sess->priv[idx]; - pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_ecc_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_ecc_setting.config.epoll_en); - pthread_spin_unlock(&ctx->lock); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, &balance, + wd_ecc_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -2327,19 +2449,18 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return ret; ctx = config->ctxs + idx; - mid = wd_get_msg_from_pool(&wd_ecc_setting.pool, idx, (void **)&msg); if (unlikely(mid < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return mid; + return -WD_EBUSY; } ret = fill_ecc_msg(msg, req, (struct wd_ecc_sess *)sess); if (ret) goto fail_with_msg; msg->tag = mid; + msg->priv = sess_t->priv[idx]; - ret = wd_alg_driver_send(wd_ecc_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send ecc BD, hw is err!\n"); @@ -2386,7 +2507,7 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_ecc_setting.driver, ctx->ctx, &recv_msg); + ret = ctx->drv->recv(ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { diff --git a/wd_rsa.c b/wd_rsa.c index 93ec336..754c28a 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -16,6 +16,7 @@ #include "wd_rsa.h" #define RSA_MAX_KEY_SIZE 512 +#define WD_RSA_OP_TYPE 1 static __thread __u64 balance; @@ -76,7 +77,6 @@ static struct wd_rsa_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_rsa_setting; @@ -96,19 +96,15 @@ static void wd_rsa_close_driver(int init_type) if (!wd_rsa_setting.dlhandle) return; - wd_release_drv(wd_rsa_setting.driver); dlclose(wd_rsa_setting.dlhandle); wd_rsa_setting.dlhandle = NULL; #else - wd_release_drv(wd_rsa_setting.driver); hisi_hpre_remove(); #endif } static int wd_rsa_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "rsa"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -142,18 +138,12 @@ static int wd_rsa_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_rsa_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support!\n", alg_name); - return -WD_EINVAL; - } - - wd_rsa_setting.driver = driver; return WD_SUCCESS; } +static bool wd_rsa_atfork_registered; + static void wd_rsa_clear_status(void) { wd_alg_clear_init(&wd_rsa_setting.status); @@ -183,15 +173,8 @@ static int wd_rsa_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_rsa_setting.pool); out_clear_sched: wd_clear_sched(&wd_rsa_setting.sched); out_clear_ctx_config: @@ -199,30 +182,24 @@ out_clear_ctx_config: return ret; } -static int wd_rsa_common_uninit(void) +static void wd_rsa_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_rsa_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_rsa_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_rsa_setting.sched); - wd_alg_uninit_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); - - return WD_SUCCESS; } int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_rsa_clear_status); + if (!wd_rsa_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_rsa_clear_status) == 0) + wd_rsa_atfork_registered = true; + } ret = wd_alg_try_init(&wd_rsa_setting.status); if (ret) @@ -240,10 +217,39 @@ int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array("rsa", TASK_HW, "hisi_hpre", + &wd_rsa_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_common_uninit; + } + + wd_rsa_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_rsa_setting.config, + NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_rsa_setting.config); + if (ret) { + WD_ERR("rsa driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_rsa_setting.status); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_rsa_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_rsa_setting.config.drv_array, drv_count); + wd_rsa_setting.config.drv_array = NULL; + wd_rsa_setting.config.drv_count = 0; +out_common_uninit: + wd_rsa_common_uninit(); out_close_driver: wd_rsa_close_driver(WD_TYPE_V1); out_clear_init: @@ -253,12 +259,20 @@ out_clear_init: void wd_rsa_uninit(void) { - int ret; + enum wd_status status; - ret = wd_rsa_common_uninit(); - if (ret) + wd_alg_get_init(&wd_rsa_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_rsa_setting.config); + wd_ctx_unbind_drivers(&wd_rsa_setting.config); + wd_put_drv_array(wd_rsa_setting.config.drv_array, + wd_rsa_setting.config.drv_count); + wd_rsa_setting.config.drv_array = NULL; + wd_rsa_setting.config.drv_count = 0; + wd_rsa_common_uninit(); + wd_rsa_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_rsa_setting.status); } @@ -269,7 +283,10 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para struct wd_ctx_params rsa_ctx_params = {0}; int state, ret = -WD_EINVAL; - pthread_atfork(NULL, NULL, wd_rsa_clear_status); + if (!wd_rsa_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_rsa_clear_status) == 0) + wd_rsa_atfork_registered = true; + } state = wd_alg_try_init(&wd_rsa_setting.status); if (state) @@ -293,37 +310,26 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para while (ret) { memset(&wd_rsa_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - /* Get alg driver and dev name */ - wd_rsa_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_rsa_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ rsa_ctx_params.ctx_set_num = rsa_ctx_num; ret = wd_ctx_param_init(&rsa_ctx_params, ctx_params, - wd_rsa_setting.driver, WD_RSA_TYPE, WD_RSA_GENKEY); + alg, WD_RSA_TYPE, WD_RSA_OP_TYPE); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); + if (ret == -WD_EAGAIN) continue; - } - goto out_driver; + + goto out_dlclose; } (void)strcpy(wd_rsa_init_attrs.alg, alg); wd_rsa_init_attrs.sched_type = sched_type; - wd_rsa_init_attrs.driver = wd_rsa_setting.driver; + wd_rsa_init_attrs.task_type = task_type; wd_rsa_init_attrs.ctx_params = &rsa_ctx_params; wd_rsa_init_attrs.alg_init = wd_rsa_common_init; wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx; ret = wd_alg_attrs_init(&wd_rsa_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); wd_ctx_param_uninit(&rsa_ctx_params); continue; } @@ -332,16 +338,33 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para } } + ret = wd_ctx_bind_drivers(&wd_rsa_setting.config, + wd_rsa_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_rsa_setting.config); + if (ret) { + WD_ERR("driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_rsa_setting.status); wd_ctx_param_uninit(&rsa_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_rsa_setting.config); +out_common_uninit: + wd_alg_attrs_uninit(&wd_rsa_init_attrs); + wd_rsa_common_uninit(); out_params_uninit: wd_ctx_param_uninit(&rsa_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_rsa_setting.driver); -out_dlopen: +out_dlclose: wd_rsa_close_driver(WD_TYPE_V2); out_clear_init: wd_alg_clear_init(&wd_rsa_setting.status); @@ -350,14 +373,19 @@ out_clear_init: void wd_rsa_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_rsa_common_uninit(); - if (ret) + wd_alg_get_init(&wd_rsa_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_rsa_setting.config); + wd_ctx_unbind_drivers(&wd_rsa_setting.config); + wd_rsa_setting.config.drv_array = NULL; + wd_rsa_setting.config.drv_count = 0; + wd_rsa_common_uninit(); + wd_alg_attrs_uninit(&wd_rsa_init_attrs); - wd_alg_drv_unbind(wd_rsa_setting.driver); wd_rsa_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_rsa_setting.status); } @@ -447,13 +475,13 @@ int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) if (unlikely(ret)) return ret; - msg_handle.send = wd_rsa_setting.driver->send; - msg_handle.recv = wd_rsa_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_rsa_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_rsa_setting.config.epoll_en); - pthread_spin_unlock(&ctx->lock); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, &balance, + wd_rsa_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -489,8 +517,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) mid = wd_get_msg_from_pool(&wd_rsa_setting.pool, idx, (void **)&msg); if (unlikely(mid < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return mid; + return -WD_EBUSY; } ret = fill_rsa_msg(msg, req, (struct wd_rsa_sess *)sess); @@ -498,7 +525,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_rsa_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send rsa BD, hw is err!\n"); @@ -545,7 +572,7 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_rsa_setting.driver, ctx->ctx, &recv_msg); + ret = ctx->drv->recv(ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -935,6 +962,7 @@ static void del_sess(struct wd_rsa_sess *c) /* Before initiate this context, we should get a queue from WD */ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) { + struct wd_sched_params params = {0}; struct wd_rsa_sess *sess; int ret; @@ -943,6 +971,12 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) return(handle_t)0; } + ret = wd_drv_alg_support("rsa", &wd_rsa_setting.config); + if (!ret) { + WD_ERR("failed to support this algorithm: rsa!\n"); + return (handle_t)0; + } + if (setup->key_bits != 1024 && setup->key_bits != 2048 && setup->key_bits != 3072 && @@ -981,6 +1015,13 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = "rsa"; + params.ctxs = wd_rsa_setting.config.ctxs; + wd_rsa_setting.sched.set_param( + wd_rsa_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -999,8 +1040,15 @@ void wd_rsa_free_sess(handle_t sess) return; } - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_rsa_setting.sched.sched_uninit) + wd_rsa_setting.sched.sched_uninit( + wd_rsa_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } + del_sess_key(sess_t); del_sess(sess_t); } -- 2.43.0