From: Longfang Liu <liulongfang@huawei.com> After adapting the new heterogeneous hybrid acceleration function. The initialization of the device driver requires adaptation updates. In addition, the instruction acceleration algorithm driver needs to fully adapt to the synchronous and asynchronous mode of the uadk framework. Signed-off-by: Longfang Liu <liulongfang@huawei.com> --- Makefile.am | 2 +- drv/hisi_comp.c | 3 ++ drv/hisi_dae_common.c | 3 ++ drv/hisi_hpre.c | 65 +++++++++++-------------------------- drv/hisi_sec.c | 3 ++ drv/hisi_udma.c | 3 ++ drv/isa_ce_sm3.c | 18 +++++++++- drv/isa_ce_sm4.c | 19 ++++++++++- include/drv/wd_digest_drv.h | 2 ++ include/drv/wd_ecc_drv.h | 2 +- include/wd_util.h | 4 +++ wd_ecc.c | 3 +- wd_util.c | 64 ++++++++++++++++++++++++++++++++++++ 13 files changed, 139 insertions(+), 52 deletions(-) diff --git a/Makefile.am b/Makefile.am index b38c5ce..e59fdb9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -101,7 +101,7 @@ libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ if ARCH_ARM64 libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ - drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h + drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ drv/hash_mb/sm3_sve_common.S drv/hash_mb/sm3_mb_asimd_x1.S \ diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index be16b83..50643d0 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -1431,6 +1431,9 @@ static int hisi_zip_init(void *conf, void *priv) memcpy(&zip_ctx->config, config, sizeof(struct wd_ctx_config_internal)); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW || + !config->ctxs[i].ctx) + continue; h_ctx = config->ctxs[i].ctx; qm_priv.sqe_size = sizeof(struct hisi_zip_sqe); qm_priv.op_type = config->ctxs[i].op_type; diff --git a/drv/hisi_dae_common.c b/drv/hisi_dae_common.c index 216e424..74dc84f 100644 --- a/drv/hisi_dae_common.c +++ b/drv/hisi_dae_common.c @@ -327,6 +327,9 @@ int dae_init(void *conf, void *priv) qm_priv.sqe_size = sizeof(struct dae_sqe); /* Allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW || + !config->ctxs[i].ctx) + continue; h_ctx = config->ctxs[i].ctx; qm_priv.qp_mode = config->ctxs[i].ctx_mode; /* Setting the epoll en to 0 for ASYNC ctx */ diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 7da8407..d8f3b53 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -133,9 +133,6 @@ struct hisi_hpre_ctx { struct wd_ctx_config_internal config; struct wd_mm_ops *mm_ops; handle_t rsv_mem_ctx; -}; - -struct hpre_ecc_ctx { __u32 enable_hpcore; }; @@ -656,6 +653,9 @@ static int hpre_init_qm_priv(struct wd_ctx_config_internal *config, qm_priv->sqe_size = sizeof(struct hisi_hpre_sqe); for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW || + !config->ctxs[i].ctx) + continue; h_ctx = config->ctxs[i].ctx; qm_priv->qp_mode = config->ctxs[i].ctx_mode; /* Setting the epoll en to 0 for ASYNC ctx */ @@ -1577,7 +1577,7 @@ static int u_is_in_p(struct wd_ecc_msg *msg) static int ecc_prepare_in(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, void **data) { - struct hpre_ecc_ctx *ecc_ctx = msg->drv_cfg; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)msg->drv_cfg; int ret = -WD_EINVAL; switch (msg->req.op_type) { @@ -1590,11 +1590,11 @@ static int ecc_prepare_in(struct wd_ecc_msg *msg, ret = ecc_prepare_dh_gen_in(msg, hw_msg, data); break; case WD_ECXDH_GEN_KEY: - hw_msg->bd_rsv2 = ecc_ctx->enable_hpcore; + hw_msg->bd_rsv2 = hpre_ctx->enable_hpcore; ret = ecc_prepare_dh_gen_in(msg, hw_msg, data); break; case WD_ECXDH_COMPUTE_KEY: - hw_msg->bd_rsv2 = ecc_ctx->enable_hpcore; + hw_msg->bd_rsv2 = hpre_ctx->enable_hpcore; ret = ecc_prepare_dh_compute_in(msg, hw_msg, data); if (!ret && (msg->curve_id == WD_X25519 || msg->curve_id == WD_X448)) @@ -2817,39 +2817,6 @@ static int hpre_rsa_get_usage(void *param) return -WD_EACCES; } -static int ecc_sess_eops_init(struct wd_alg_driver *drv, void **params) -{ - struct hpre_ecc_ctx *ecc_ctx; - - if (!params) { - WD_ERR("invalid: extend ops init params address is NULL!\n"); - return -WD_EINVAL; - } - - if (*params) { - WD_ERR("invalid: extend ops init params repeatedly!\n"); - return -WD_EINVAL; - } - - ecc_ctx = calloc(1, sizeof(struct hpre_ecc_ctx)); - if (!ecc_ctx) - return -WD_ENOMEM; - - *params = ecc_ctx; - - return WD_SUCCESS; -} - -static void ecc_sess_eops_uninit(struct wd_alg_driver *drv, void *params) -{ - if (!params) { - WD_ERR("invalid: extend ops uninit params is NULL!\n"); - return; - } - - free(params); -} - static bool is_valid_hw_type(void *drv_priv) { struct hisi_hpre_ctx *hpre_ctx; @@ -2866,21 +2833,27 @@ static bool is_valid_hw_type(void *drv_priv) } static void ecc_sess_eops_params_cfg(struct wd_ecc_sess_setup *setup, - struct wd_ecc_curve *cv, void *drv_priv, void *params) + struct wd_ecc_curve *cv, void *priv) { __u8 data[SECP256R1_PARAM_SIZE] = SECG_P256_R1_PARAM; - struct hpre_ecc_ctx *ecc_ctx = params; + struct hisi_hpre_ctx *hpre_ctx; + struct hisi_qp *qp; __u32 key_size; int ret; - if (!is_valid_hw_type(drv_priv)) + if (unlikely(!priv)) return; - if (!ecc_ctx) { + hpre_ctx = (struct hisi_hpre_ctx *)priv; + if (!hpre_ctx) { WD_INFO("Info: eops config exits, but params is NULL!\n"); return; } + qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->config.ctxs[0].ctx); + if (qp->q_info.hw_type < HISI_QM_API_VER3_BASE) + return; + if (strcmp(setup->alg, "ecdh")) return; @@ -2890,7 +2863,7 @@ static void ecc_sess_eops_params_cfg(struct wd_ecc_sess_setup *setup, ret = memcmp(data, cv->p.data, SECP256R1_PARAM_SIZE); if (!ret) - ecc_ctx->enable_hpcore = 1; + hpre_ctx->enable_hpcore = 1; } static int hpre_ecc_get_extend_ops(void *ops) @@ -2901,9 +2874,9 @@ static int hpre_ecc_get_extend_ops(void *ops) return -WD_EINVAL; ecc_ops->params = NULL; - ecc_ops->sess_init = ecc_sess_eops_init; + ecc_ops->sess_init = NULL; ecc_ops->eops_params_cfg = ecc_sess_eops_params_cfg; - ecc_ops->sess_uninit = ecc_sess_eops_uninit; + ecc_ops->sess_uninit = NULL; return WD_SUCCESS; } diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index fb9de2c..c2f73b1 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -3906,6 +3906,9 @@ static int hisi_sec_init(void *conf, void *priv) qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW || + !config->ctxs[i].ctx) + continue; h_ctx = config->ctxs[i].ctx; /* setting the type is 0 for sqc_type */ qm_priv.op_type = 0; diff --git a/drv/hisi_udma.c b/drv/hisi_udma.c index 5c8a743..62b83e7 100644 --- a/drv/hisi_udma.c +++ b/drv/hisi_udma.c @@ -461,6 +461,9 @@ static int udma_init(void *conf, void *priv) qm_priv.sqe_size = sizeof(struct udma_sqe); /* Allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx_type != UADK_CTX_HW || + !config->ctxs[i].ctx) + continue; h_ctx = config->ctxs[i].ctx; qm_priv.qp_mode = config->ctxs[i].ctx_mode; /* Setting the epoll en to 0 for ASYNC ctx */ diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 0ebaba5..9dbd0a8 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -17,7 +17,6 @@ #include "drv/isa_ce_sm3.h" #include "drv/wd_digest_drv.h" #include "wd_digest.h" -#include "wd_util.h" #define SM3_ALIGN_MASK 63U @@ -338,6 +337,7 @@ static int do_hmac_sm3_ce(struct wd_digest_msg *msg, __u8 *out_hmac) static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; __u8 digest[SM3_DIGEST_SIZE] = {0}; int ret; @@ -347,6 +347,10 @@ static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) return -WD_EINVAL; } + ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM3 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -368,11 +372,23 @@ static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) ret = -WD_EINVAL; } + ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; } static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return WD_SUCCESS; } diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 504ef73..4c42693 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -324,15 +324,20 @@ static int sm4_xts_decrypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_cipher_msg *msg = wd_msg; struct SM4_KEY rkey; int ret = 0; - if (!msg) { + if (!msg || !ctx) { WD_ERR("invalid: input sm4 msg is NULL!\n"); return -WD_EINVAL; } + ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM4 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -385,11 +390,23 @@ static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) return -WD_EINVAL; } + ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; } static int isa_ce_cipher_recv(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_cipher_msg *msg = wd_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return 0; } diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index 12398f2..2711ab1 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -3,6 +3,8 @@ #ifndef __WD_DIGEST_DRV_H #define __WD_DIGEST_DRV_H +#include <asm/types.h> + #include "../wd_digest.h" #include "../wd_util.h" diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index 98ff28c..899f593 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -182,7 +182,7 @@ struct wd_ecc_out { struct wd_ecc_extend_ops { void *params; /* the params are passed to the following ops */ void (*eops_params_cfg)(struct wd_ecc_sess_setup *setup, - struct wd_ecc_curve *cv, void *drv_priv, void *params); + struct wd_ecc_curve *cv, void *priv); int (*sess_init)(struct wd_alg_driver *drv, void **params); void (*sess_uninit)(struct wd_alg_driver *drv, void *params); }; diff --git a/include/wd_util.h b/include/wd_util.h index 3be75d3..f226122 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -565,6 +565,10 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type) int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type); +int wd_queue_is_busy(struct wd_soft_ctx *sctx); +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id); +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result); + #ifdef __cplusplus } #endif diff --git a/wd_ecc.c b/wd_ecc.c index 900f36c..b7e5481 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -1205,8 +1205,7 @@ static void wd_ecc_sess_eops_cfg(struct wd_ecc_sess_setup *setup, { if (sess->eops.sess_init && sess->eops.eops_params_cfg) { /* the config result does not impact task sucesss or failure */ - sess->eops.eops_params_cfg(setup, sess->key.cv, - wd_ecc_setting.priv, sess->eops.params); + sess->eops.eops_params_cfg(setup, sess->key.cv, wd_ecc_setting.priv); } } diff --git a/wd_util.c b/wd_util.c index 8e8260e..19bfcce 100644 --- a/wd_util.c +++ b/wd_util.c @@ -3363,3 +3363,67 @@ void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) free(ctx_config); wd_sched_rr_release(alg_sched); } + +int wd_queue_is_busy(struct wd_soft_ctx *sctx) +{ + /* The queue is not used */ + if (sctx->run_num >= MAX_SOFT_QUEUE_LENGTH - 1) + return -WD_EBUSY; + + return 0; +} + +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id) +{ + struct wd_soft_sqe *sqe = NULL; + + pthread_spin_lock(&sctx->slock); + sqe = &sctx->qfifo[sctx->head]; + if (!sqe->used && !sqe->complete) { // find the next not used sqe + sctx->head++; + if (unlikely(sctx->head == MAX_SOFT_QUEUE_LENGTH)) + sctx->head = 0; + + sqe->used = 1; + sqe->complete = 1; + sqe->id = tag_id; + sqe->result = 0; + __atomic_fetch_add(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->slock); + } else { + pthread_spin_unlock(&sctx->slock); + return -WD_EBUSY; + } + + return 0; +} + +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result) +{ + struct wd_soft_sqe *sqe = NULL; + + /* The queue is not used */ + if (sctx->run_num < 1) + return -WD_EAGAIN; + + if (pthread_spin_trylock(&sctx->rlock)) + return -WD_EAGAIN; + sqe = &sctx->qfifo[sctx->tail]; + if (sqe->used && sqe->complete) { // find a used sqe + sctx->tail++; + if (unlikely(sctx->tail == MAX_SOFT_QUEUE_LENGTH)) + sctx->tail = 0; + + *tag_id = sqe->id; + *result = sqe->result; + sqe->used = 0x0; + sqe->complete = 0x0; + __atomic_fetch_sub(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->rlock); + } else { + pthread_spin_unlock(&sctx->rlock); + return -WD_EAGAIN; + } + + return 0; +} -- 2.43.0