From: Zhushuai Yin <yinzhushuai@huawei.com> This modification primarily aims to enable the UADK Cipher algorithm to support the new heterogeneous scheduling framework, while the corresponding user-space driver must also be adapted accordingly. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/hisi_sec.c | 232 +++++++++++++++++++++++++++-------------------- drv/isa_ce_sm4.c | 86 +++++++++++------- wd_cipher.c | 221 +++++++++++++++++++++++++++----------------- 3 files changed, 326 insertions(+), 213 deletions(-) diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 7d472b8..91fa4ce 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -10,6 +10,7 @@ #include "crypto/aes.h" #include "crypto/galois.h" #include "hisi_qm_udrv.h" +#include "wd_drv.h" #define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 @@ -205,7 +206,8 @@ enum sec_c_width { }; struct hisi_sec_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; }; struct hisi_sec_sqe_type2 { @@ -526,86 +528,84 @@ static __u32 g_sec_hmac_full_len[WD_DIGEST_TYPE_MAX] = { SEC_HMAC_SHA512_MAC_LEN, SEC_HMAC_SHA512_224_MAC_LEN, SEC_HMAC_SHA512_256_MAC_LEN }; -static int hisi_sec_init(struct wd_alg_driver *drv, void *conf); -static void hisi_sec_exit(struct wd_alg_driver *drv); +static int hisi_sec_init(void *conf, void *priv); +static void hisi_sec_exit(void *priv); -static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg); -static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg); -static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg); -static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_send(drv, ctx, msg); - return hisi_sec_cipher_send_v3(drv, ctx, msg); + return hisi_sec_cipher_send(ctx, msg); + return hisi_sec_cipher_send_v3(ctx, msg); } -static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_recv(drv, ctx, msg); - return hisi_sec_cipher_recv_v3(drv, ctx, msg); + return hisi_sec_cipher_recv(ctx, msg); + return hisi_sec_cipher_recv_v3(ctx, msg); } -static int digest_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_send(drv, ctx, msg); - return hisi_sec_digest_send_v3(drv, ctx, msg); + return hisi_sec_digest_send(ctx, msg); + return hisi_sec_digest_send_v3(ctx, msg); } -static int digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_recv(drv, ctx, msg); - return hisi_sec_digest_recv_v3(drv, ctx, msg); + return hisi_sec_digest_recv(ctx, msg); + return hisi_sec_digest_recv_v3(ctx, msg); } -static int aead_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_send(drv, ctx, msg); - return hisi_sec_aead_send_v3(drv, ctx, msg); + return hisi_sec_aead_send(ctx, msg); + return hisi_sec_aead_send_v3(ctx, msg); } -static int aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_recv(drv, ctx, msg); - return hisi_sec_aead_recv_v3(drv, ctx, msg); + return hisi_sec_aead_recv(ctx, msg); + return hisi_sec_aead_recv_v3(ctx, msg); } static int hisi_sec_get_usage(void *param) { struct hisi_dev_usage *sec_usage = (struct hisi_dev_usage *)param; struct wd_alg_driver *drv = sec_usage->drv; - struct wd_ctx_config_internal *config; - struct hisi_sec_ctx *priv; + struct hisi_sec_ctx *sec_ctx; char *ctx_dev_name; - handle_t ctx = 0; handle_t qp = 0; __u32 i; @@ -614,32 +614,27 @@ static int hisi_sec_get_usage(void *param) return -WD_EINVAL; } - priv = (struct hisi_sec_ctx *)drv->priv; - if (!priv) + sec_ctx = (struct hisi_sec_ctx *)drv->drv_data; + if (!sec_ctx) return -WD_EACCES; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - ctx_dev_name = wd_ctx_get_dev_name(config->ctxs[i].ctx); + /* Only process the queues used by your own driver. */ + for (i = 0; i < sec_ctx->ctx_num; i++) { + ctx_dev_name = wd_ctx_get_dev_name(sec_ctx->ctxs[i]->ctx); if (!strcmp(sec_usage->dev_name, ctx_dev_name)) { - ctx = config->ctxs[i].ctx; - break; + qp = (handle_t)wd_ctx_get_priv(sec_ctx->ctxs[i]->ctx); + if (qp) + return hisi_qm_get_usage(qp, 0); } } - if (ctx) - qp = (handle_t)wd_ctx_get_priv(ctx); - - if (qp) - return hisi_qm_get_usage(qp, 0); - return -WD_EACCES; } static int eops_param_check(struct wd_alg_driver *drv, struct wd_mm_ops *mm_ops) { - if (!drv || !drv->priv) { - WD_ERR("invalid: aead drv or priv is NULL!\n"); + if (!drv || !drv->drv_data) { + WD_ERR("invalid: aead drv or data is NULL!\n"); return -WD_EINVAL; } @@ -680,8 +675,8 @@ static int aead_sess_eops_init(struct wd_alg_driver *drv, return -WD_ENOMEM; } - sec_ctx = (struct hisi_sec_ctx *)drv->priv; - qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->config.ctxs[0].ctx); + sec_ctx = (struct hisi_sec_ctx *)drv->drv_data; + qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->ctxs[0]->ctx); sq_depth = qp->q_info.sq_depth; aiv_addr->aiv = mm_ops->alloc(mm_ops->usr, (__u32)sq_depth << AEAD_AIV_OFFSET); if (!aiv_addr->aiv) { @@ -735,8 +730,8 @@ static void aead_sess_eops_uninit(struct wd_alg_driver *drv, return; } - sec_ctx = (struct hisi_sec_ctx *)drv->priv; - qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->config.ctxs[0].ctx); + sec_ctx = (struct hisi_sec_ctx *)drv->drv_data; + qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->ctxs[0]->ctx); sq_depth = qp->q_info.sq_depth; aiv_addr = (struct wd_aead_aiv_addr *)params; @@ -780,7 +775,6 @@ static int sec_aead_get_extend_ops(void *ops) if (!aead_ops) return -WD_EINVAL; - aead_ops->params = NULL; aead_ops->eops_aiv_init = aead_sess_eops_init; aead_ops->eops_aiv_uninit = aead_sess_eops_uninit; @@ -793,15 +787,21 @@ static int sec_aead_get_extend_ops(void *ops) .alg_name = (sec_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_sec_ctx),\ + .ops_size = sizeof(struct wd_aead_extend_ops),\ .queue_num = SEC_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ + .drv_data = NULL,\ .fallback = 0,\ + .init_state = 0,\ .init = hisi_sec_init,\ .exit = hisi_sec_exit,\ .send = alg_type##_send,\ .recv = alg_type##_recv,\ .get_usage = hisi_sec_get_usage,\ .get_extend_ops = sec_aead_get_extend_ops,\ + .alloc_ctx = wd_hw_alloc_ctx, \ + .free_ctx = wd_hw_free_ctx, \ } static struct wd_alg_driver cipher_alg_driver[] = { @@ -812,6 +812,8 @@ static struct wd_alg_driver cipher_alg_driver[] = { GEN_SEC_ALG_DRIVER("cbc(sm4)", cipher), GEN_SEC_ALG_DRIVER("ctr(sm4)", cipher), GEN_SEC_ALG_DRIVER("xts(sm4)", cipher), + GEN_SEC_ALG_DRIVER("xts-gb(sm4)", cipher), + GEN_SEC_ALG_DRIVER("ecb(des)", cipher), GEN_SEC_ALG_DRIVER("cbc(des)", cipher), GEN_SEC_ALG_DRIVER("ecb(des3_ede)", cipher), @@ -1400,7 +1402,7 @@ static int fill_cipher_bd2(struct wd_cipher_msg *msg, struct hisi_sec_sqe *sqe) return 0; } -static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; @@ -1456,7 +1458,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w return 0; } -static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -1693,7 +1695,7 @@ static void fill_sec_prefetch(__u8 data_fmt, __u32 len, __u16 hw_type, struct hi sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; } -static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -1800,7 +1802,7 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "cipher"); } -static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -2132,7 +2134,7 @@ static int digest_len_check(struct wd_digest_msg *msg, enum sec_bd_type type) return 0; } -static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; @@ -2209,7 +2211,7 @@ put_sgl: return ret; } -static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -2473,7 +2475,7 @@ map_err: return -WD_ENOMEM; } -static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -2587,7 +2589,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "digest"); } -static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -3216,7 +3218,7 @@ static int fill_aead_bd2_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe, return aead_mem_nosva_map(msg, sqe, idx); } -static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -3347,7 +3349,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, dump_sec_msg(temp_msg, "aead"); } -static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -3744,7 +3746,7 @@ static int fill_aead_bd3_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe return aead_mem_nosva_map_v3(msg, sqe, idx); } -static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -3863,7 +3865,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "aead"); } -static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -3888,73 +3890,105 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * return 0; } -static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) +static int hisi_sec_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_sec_ctx *sec_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_sec_ctx *priv; - handle_t h_qp = 0; - handle_t h_ctx; - __u32 i, j; + __u32 i, j, count = 0; + bool *is_match; + handle_t h_qp; - if (!config->ctx_num) { - WD_ERR("invalid: sec init config ctx num is 0!\n"); + if (!config || !config->ctx_num) { + WD_ERR("invalid: input config or ctx num is null!\n"); return -WD_EINVAL; } + /* traverse and count the number of contexts supported by this driver. */ + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; - priv = malloc(sizeof(struct hisi_sec_ctx)); - if (!priv) + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && config->ctxs[i].drv && + !strcmp(config->ctxs[i].drv->drv_name, "hisi_sec2")) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + + if (!count) { + WD_ERR("invalid: valid driver number is zero!\n"); + free(is_match); return -WD_EINVAL; + } + sec_ctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!sec_ctx->ctxs) { + free(is_match); + return -WD_ENOMEM; + } + sec_ctx->ctx_num = count; + + /* allocate QP and store context mirror. */ qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); - /* allocate qp for each context */ + count = 0; for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; - /* setting the type is 0 for sqc_type */ + if (!is_match[i]) + continue; + qm_priv.op_type = 0; qm_priv.qp_mode = config->ctxs[i].ctx_mode; - /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv.epoll_en = (qm_priv.qp_mode == CTX_MODE_SYNC) ? config->epoll_en : 0; qm_priv.idx = i; - h_qp = hisi_qm_alloc_qp(&qm_priv, h_ctx); + h_qp = hisi_qm_alloc_qp(&qm_priv, config->ctxs[i].ctx); if (!h_qp) goto out; + config->ctxs[i].sqn = qm_priv.sqn; + /* Store the queues allocated by your own driver. */ + sec_ctx->ctxs[count++] = &config->ctxs[i]; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; - return 0; + free(is_match); + + return WD_SUCCESS; out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(sec_ctx->ctxs[j]->ctx); hisi_qm_free_qp(h_qp); } - free(priv); + free(sec_ctx->ctxs); + free(is_match); return -WD_EINVAL; } -static void hisi_sec_exit(struct wd_alg_driver *drv) +static void hisi_sec_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_sec_ctx *priv; + struct hisi_sec_ctx *sec_ctx = priv; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + } - priv = (struct hisi_sec_ctx *)drv->priv; - config = &priv->config; - - for (i = 0; i < config->ctx_num; i++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + /* Only release the queues allocated by your own driver. */ + for (i = 0; i < sec_ctx->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(sec_ctx->ctxs[i]->ctx); + if (!h_qp) + continue; hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; + + if (sec_ctx->ctxs) { + free(sec_ctx->ctxs); + sec_ctx->ctxs = NULL; + } } #ifdef WD_STATIC_DRV diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 52dca1f..863b99e 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -12,14 +12,17 @@ */ #include "drv/wd_cipher_drv.h" -#include "wd_cipher.h" #include "isa_ce_sm4.h" +#include "wd_cipher.h" +#include "wd_drv.h" #define SM4_ENCRYPT 1 #define SM4_DECRYPT 0 #define MSG_Q_DEPTH 1024 #define INCREASE_BYTES 12 #define SM4_BLOCK_SIZE 16 +/* CTS tail: last full block + partial block */ +#define SM4_CTS_TAIL_SIZE 32 #define MAX_BLOCK_NUM (1U << 28) #define CTR96_SHIFT_BITS 8 #define SM4_BYTES2BLKS(nbytes) ((nbytes) >> 4) @@ -31,36 +34,23 @@ ((p)[0] = (__u8)((v) >> 24), (p)[1] = (__u8)((v) >> 16), \ (p)[2] = (__u8)((v) >> 8), (p)[3] = (__u8)(v)) -static int isa_ce_init(struct wd_alg_driver *drv, void *conf) +static int isa_ce_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct sm4_ce_drv_ctx *priv; + struct sm4_ce_drv_ctx *sctx = priv; /* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0; - priv = malloc(sizeof(struct sm4_ce_drv_ctx)); - if (!priv) - return -WD_EINVAL; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal)); - return WD_SUCCESS; + return 0; } -static void isa_ce_exit(struct wd_alg_driver *drv) +static void isa_ce_exit(void *priv) { - struct sm4_ce_drv_ctx *sctx; - - if (!drv || !drv->priv) - return; - - sctx = (struct sm4_ce_drv_ctx *)drv->priv; - free(sctx); - drv->priv = NULL; } /* increment upper 96 bits of 128-bit counter by 1 */ @@ -179,6 +169,8 @@ static void sm4_cts_cs1_mode_adapt(__u8 *cts_in, __u8 *cts_out, static void sm4_cts_cbc_crypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey_enc, const int enc) { + /* Stack buffer for CS1 decrypt to avoid modifying caller's msg->in */ + __u8 cts_in_buf[SM4_CTS_TAIL_SIZE] = {0}; enum wd_cipher_mode mode = msg->mode; __u32 in_bytes = msg->in_bytes; __u8 *cts_in, *cts_out; @@ -204,10 +196,21 @@ static void sm4_cts_cbc_crypt(struct wd_cipher_msg *msg, if (mode == WD_CIPHER_CBC_CS1) sm4_cts_cs1_mode_adapt(cts_in, cts_out, cts_bytes, enc); } else { - if (mode == WD_CIPHER_CBC_CS1) - sm4_cts_cs1_mode_adapt(cts_in, cts_out, cts_bytes, enc); - - sm4_v8_cbc_cts_decrypt(cts_in, cts_out, cts_bytes, rkey_enc, msg->iv); + if (mode == WD_CIPHER_CBC_CS1) { + /* + * CS1 decrypt: sm4_cts_cs1_mode_adapt swaps the CTS tail + * from CS1 layout to CS3 layout so sm4_v8_cbc_cts_decrypt + * can process it. Copy to a stack buffer first to avoid + * modifying the caller's input buffer (msg->in), which + * breaks verification tools that re-encrypt the decrypted + * output and compare against the original ciphertext. + */ + memcpy(cts_in_buf, cts_in, cts_bytes); + sm4_cts_cs1_mode_adapt(cts_in_buf, cts_out, cts_bytes, enc); + sm4_v8_cbc_cts_decrypt(cts_in_buf, cts_out, cts_bytes, rkey_enc, msg->iv); + } else { + sm4_v8_cbc_cts_decrypt(cts_in, cts_out, cts_bytes, rkey_enc, msg->iv); + } } } @@ -334,17 +337,22 @@ static int sm4_xts_decrypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey return 0; } -static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_cipher_msg *msg = wd_msg; struct SM4_KEY rkey; int ret = 0; - if (!msg) { + if (!msg || !ctx) { WD_ERR("invalid: input sm4 msg is NULL!\n"); return -WD_EINVAL; } + ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM4 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -397,22 +405,34 @@ static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_ return -WD_EINVAL; } + ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; } -static int isa_ce_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_recv(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_cipher_msg *msg = wd_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return 0; } -static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { - return isa_ce_cipher_send(drv, ctx, msg); + return isa_ce_cipher_send(ctx, msg); } -static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { - return isa_ce_cipher_recv(drv, ctx, msg); + return isa_ce_cipher_recv(ctx, msg); } #define GEN_CE_ALG_DRIVER(ce_alg_name, alg_type) \ @@ -421,12 +441,16 @@ static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) .alg_name = (ce_alg_name),\ .calc_type = UADK_ALG_CE_INSTR,\ .priority = 200,\ + .priv_size = sizeof(struct sm4_ce_drv_ctx),\ + .queue_num = 1,\ .op_type_num = 1,\ .fallback = 0,\ .init = isa_ce_init,\ .exit = isa_ce_exit,\ .send = alg_type##_send,\ .recv = alg_type##_recv,\ + .alloc_ctx = wd_soft_alloc_ctx, \ + .free_ctx = wd_soft_free_ctx, \ } static struct wd_alg_driver cipher_alg_driver[] = { diff --git a/wd_cipher.c b/wd_cipher.c index 3bfead8..e0670f5 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -52,7 +52,6 @@ struct wd_cipher_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_cipher_setting; @@ -83,20 +82,16 @@ static void wd_cipher_close_driver(int init_type) } if (wd_cipher_setting.dlhandle) { - wd_release_drv(wd_cipher_setting.driver); dlclose(wd_cipher_setting.dlhandle); wd_cipher_setting.dlhandle = NULL; } #else - wd_release_drv(wd_cipher_setting.driver); hisi_sec2_remove(); #endif } static int wd_cipher_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "cbc(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -130,15 +125,6 @@ static int wd_cipher_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_cipher_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_cipher_setting.driver = driver; - return WD_SUCCESS; } @@ -201,7 +187,7 @@ static int cipher_key_len_check(struct wd_cipher_sess *sess, __u32 length) ret = -WD_EINVAL; break; default: - WD_ERR("cipher input alg err, alg = %u\n", sess->alg); + WD_ERR("invalid: unsupported cipher input alg, alg = %u\n", sess->alg); return -WD_EINVAL; } @@ -260,7 +246,7 @@ static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, ret = wd_mem_ops_init(wd_cipher_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); if (ret) { - WD_ERR("cipher failed to init memory ops!\n"); + WD_ERR("failed to init cipher memory ops!\n"); return ret; } @@ -269,7 +255,7 @@ static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, sess->key = sess->mm_ops.alloc(sess->mm_ops.usr, MAX_CIPHER_KEY_SIZE); if (!sess->key) { - WD_ERR("cipher failed to alloc key memory!\n"); + WD_ERR("failed to alloc cipher key memory!\n"); return -WD_ENOMEM; } memset(sess->key, 0, MAX_CIPHER_KEY_SIZE); @@ -280,6 +266,7 @@ static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) { struct wd_cipher_sess *sess = NULL; + struct wd_sched_params params; bool ret; if (unlikely(!setup)) { @@ -295,13 +282,13 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) memset(sess, 0, sizeof(struct wd_cipher_sess)); if (setup->alg >= WD_CIPHER_ALG_TYPE_MAX || - setup->mode >= WD_CIPHER_MODE_TYPE_MAX) { + setup->mode >= WD_CIPHER_MODE_TYPE_MAX) { WD_ERR("failed to check algorithm!\n"); goto free_sess; } sess->alg_name = wd_cipher_alg_name[setup->alg][setup->mode]; - ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, &wd_cipher_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto free_sess; @@ -321,6 +308,13 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) goto free_key; } + /* Set compat filtering parameters for session-ctx matching */ + memset(¶ms, 0, sizeof(params)); + params.alg_name = sess->alg_name; + params.ctxs = wd_cipher_setting.config.ctxs; + wd_cipher_setting.sched.set_param(wd_cipher_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; free_key: @@ -342,11 +336,19 @@ void wd_cipher_free_sess(handle_t h_sess) wd_memset_zero(sess->key, sess->key_bytes); sess->mm_ops.free(sess->mm_ops.usr, sess->key); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_cipher_setting.sched.sched_uninit) + wd_cipher_setting.sched.sched_uninit( + wd_cipher_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free(sess); } +static bool wd_cipher_atfork_registered; + static void wd_cipher_clear_status(void) { wd_alg_clear_init(&wd_cipher_setting.status); @@ -378,15 +380,8 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); - if (ret) - goto out_clear_pool; - return 0; -out_clear_pool: - wd_uninit_async_request_pool(&wd_cipher_setting.pool); out_clear_sched: wd_clear_sched(&wd_cipher_setting.sched); out_clear_ctx_config: @@ -394,31 +389,24 @@ out_clear_ctx_config: return ret; } -static int wd_cipher_common_uninit(void) +static void wd_cipher_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_cipher_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_cipher_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_cipher_setting.sched); - - wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); - - return 0; } int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count; int ret; - pthread_atfork(NULL, NULL, wd_cipher_clear_status); + if (!wd_cipher_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_cipher_clear_status) == 0) + wd_cipher_atfork_registered = true; + } ret = wd_alg_try_init(&wd_cipher_setting.status); if (ret) @@ -428,18 +416,58 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + /* init1 path is HW-only; CE/SVE drivers require init2 */ + if (sched->sched_policy == SCHED_POLICY_NONE || + sched->sched_policy == SCHED_POLICY_SINGLE) { + WD_ERR("init1 does not support NONE/SINGLE schedulers, use init2\n"); + ret = -WD_EINVAL; + goto out_clear_init; + } + ret = wd_cipher_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; + /* Internal copy (existing common_init) */ ret = wd_cipher_common_init(config, sched); if (ret) goto out_close_driver; + /* Driver discovery */ + ret = wd_get_drv_array("cipher", TASK_HW, "hisi_sec2", + &wd_cipher_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("failed to get driver array!\n"); + goto out_common_uninit; + } + + /* RR bind drivers to internal ctxs */ + wd_cipher_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_cipher_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("failed to bind driver!\n"); + goto out_free_drv_array; + } + + /* Driver initialization */ + ret = wd_alg_init_driver(&wd_cipher_setting.config); + if (ret) { + WD_ERR("failed to init cipher driver!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_cipher_setting.status); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_cipher_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_cipher_setting.config.drv_array, drv_count); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_setting.config.drv_count = 0; +out_common_uninit: + wd_cipher_common_uninit(); out_close_driver: wd_cipher_close_driver(WD_TYPE_V1); out_clear_init: @@ -449,24 +477,37 @@ out_clear_init: void wd_cipher_uninit(void) { - int ret; + enum wd_status status; - ret = wd_cipher_common_uninit(); - if (ret) + wd_alg_get_init(&wd_cipher_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_cipher_setting.config); + wd_ctx_unbind_drivers(&wd_cipher_setting.config); + wd_put_drv_array(wd_cipher_setting.config.drv_array, + wd_cipher_setting.config.drv_count); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_setting.config.drv_count = 0; + + wd_cipher_common_uninit(); + wd_cipher_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_cipher_setting.status); } -int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) +int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, + struct wd_ctx_params *ctx_params) { struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; int state, ret = -WD_EINVAL; bool flag; - pthread_atfork(NULL, NULL, wd_cipher_clear_status); + if (!wd_cipher_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_cipher_clear_status) == 0) + wd_cipher_atfork_registered = true; + } state = wd_alg_try_init(&wd_cipher_setting.status); if (state) @@ -489,39 +530,30 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p goto out_uninit; while (ret != 0) { - memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_cipher_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_cipher_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + memset(&wd_cipher_setting.config, 0, + sizeof(struct wd_ctx_config_internal)); + /* Init ctx param and prepare for ctx request */ cipher_ctx_params.ctx_set_num = cipher_ctx_num; ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, - wd_cipher_setting.driver, - WD_CIPHER_TYPE, WD_CIPHER_DECRYPTION + 1); + alg, WD_CIPHER_TYPE, + WD_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); + if (ret == -WD_EAGAIN) continue; - } - goto out_driver; + goto out_dlclose; } (void)strcpy(wd_cipher_init_attrs.alg, alg); wd_cipher_init_attrs.sched_type = sched_type; - wd_cipher_init_attrs.driver = wd_cipher_setting.driver; + wd_cipher_init_attrs.task_type = task_type; wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; + ret = wd_alg_attrs_init(&wd_cipher_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); wd_ctx_param_uninit(&cipher_ctx_params); continue; } @@ -530,16 +562,35 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p } } + /* RR bind drivers */ + ret = wd_ctx_bind_drivers(&wd_cipher_setting.config, + wd_cipher_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver!\n"); + goto out_common_uninit; + } + + /* Driver initialization */ + ret = wd_alg_init_driver(&wd_cipher_setting.config); + if (ret) { + WD_ERR("failed to init driver!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_cipher_setting.status); wd_ctx_param_uninit(&cipher_ctx_params); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_cipher_setting.config); +out_common_uninit: + wd_cipher_common_uninit(); + wd_alg_attrs_uninit(&wd_cipher_init_attrs); out_params_uninit: wd_ctx_param_uninit(&cipher_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_cipher_setting.driver); -out_dlopen: +out_dlclose: wd_cipher_close_driver(WD_TYPE_V2); out_uninit: wd_alg_clear_init(&wd_cipher_setting.status); @@ -548,14 +599,20 @@ out_uninit: void wd_cipher_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_cipher_common_uninit(); - if (ret) + wd_alg_get_init(&wd_cipher_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_cipher_setting.config); + wd_ctx_unbind_drivers(&wd_cipher_setting.config); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_setting.config.drv_count = 0; + wd_cipher_common_uninit(); + wd_alg_attrs_uninit(&wd_cipher_init_attrs); - wd_alg_drv_unbind(wd_cipher_setting.driver); + wd_cipher_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_cipher_setting.status); } @@ -716,13 +773,13 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_cipher_setting.driver->send; - msg_handle.recv = wd_cipher_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_cipher_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_cipher_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, + wd_cipher_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); return ret; } @@ -788,15 +845,13 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, (void **)&msg); - if (unlikely(msg_id < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return msg_id; - } + if (unlikely(msg_id < 0)) + return -WD_EBUSY; fill_request_msg(msg, req, sess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_cipher_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -843,7 +898,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_cipher_setting.driver, ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { -- 2.43.0