The per-cipher hot path allocated the hardware session, set the key and freed the session on every operation, which added avoidable latency. Move these one-shot operations into the einit/dinit/freectx callbacks so the session is allocated once at context initialization, the key is set only when it changes, and the session is freed when the context is destroyed. Introduce alloc_sess/free_sess/set_key/copy_sess helpers for this, and drop the unused data bounce buffer. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_prov_aead.c | 339 ++++++++++++++++++++++++------------------- 1 file changed, 186 insertions(+), 153 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index 623bc54..bb5f6a4 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -91,9 +91,7 @@ struct aead_priv_ctx { unsigned char iv[MAX_IV_LEN]; unsigned char key[MAX_KEY_LEN]; unsigned char buf[AES_GCM_TAG_LEN]; /* mac buffers */ - unsigned char *data; /* store input and output when block mode */ - struct wd_aead_sess_setup setup; struct wd_aead_req req; enum uadk_aead_mode mode; handle_t sess; @@ -218,37 +216,37 @@ static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv, const unsigned c } static int uadk_aead_soft_update(struct aead_priv_ctx *priv, unsigned char *out, - int *outl, const unsigned char *in, size_t len) + size_t *outl, const unsigned char *in, size_t len) { - int ret; + int ret, outsize = 0; if (!priv->sw_aead) return UADK_AEAD_FAIL; if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) - ret = EVP_EncryptUpdate(priv->sw_ctx, out, outl, in, len); + ret = EVP_EncryptUpdate(priv->sw_ctx, out, &outsize, in, len); else - ret = EVP_DecryptUpdate(priv->sw_ctx, out, outl, in, len); + ret = EVP_DecryptUpdate(priv->sw_ctx, out, &outsize, in, len); if (!ret) { UADK_ERR("aead soft update error.\n"); return UADK_AEAD_FAIL; } - priv->stream_switch_flag = UADK_DO_SOFT; + *outl = outsize; return UADK_AEAD_SUCCESS; } static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *digest, size_t *outl) { - int ret; + int ret, outsize = 0; if (!priv->sw_aead) goto error; if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) { - ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, &outsize); if (!ret) goto error; @@ -262,11 +260,12 @@ static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *diges if (!ret) goto error; - ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, &outsize); if (!ret) goto error; } + *outl = 0; priv->stream_switch_flag = 0; return UADK_AEAD_SUCCESS; @@ -276,19 +275,6 @@ error: return UADK_AEAD_FAIL; } -static void uadk_aead_soft_cleanup(struct aead_priv_ctx *priv) -{ - if (priv->sw_ctx) { - EVP_CIPHER_CTX_free(priv->sw_ctx); - priv->sw_ctx = NULL; - } - - if (priv->sw_aead) { - EVP_CIPHER_free(priv->sw_aead); - priv->sw_aead = NULL; - } -} - static int uadk_prov_aead_dev_init(struct aead_priv_ctx *priv) { struct wd_ctx_nums ctx_set_num; @@ -335,8 +321,6 @@ free_nodemask: static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) { - struct wd_aead_sess_setup setup = {0}; - struct sched_params params = {0}; int ret; if (!priv->key_set || !priv->iv_set) { @@ -350,44 +334,13 @@ static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) priv->req.mac = priv->buf; priv->req.mac_bytes = priv->taglen; - ret = uadk_prov_aead_dev_init(priv); - if (unlikely(ret < 0)) + ret = wd_aead_set_authsize(priv->sess, priv->taglen); + if (ret) { + UADK_ERR("uadk failed to set authsize!\n"); return UADK_AEAD_FAIL; - - /* dec and enc use the same op */ - params.type = 0; - /* Use the default numa parameters */ - params.numa_id = -1; - setup.sched_param = ¶ms; - setup.calg = priv->setup.calg; - setup.cmode = priv->setup.cmode; - - if (!priv->sess) { - priv->sess = wd_aead_alloc_sess(&setup); - if (!priv->sess) { - UADK_ERR("uadk failed to alloc session!\n"); - return UADK_AEAD_FAIL; - } - - ret = wd_aead_set_authsize(priv->sess, priv->taglen); - if (ret) { - UADK_ERR("uadk failed to set authsize!\n"); - goto free_sess; - } - - ret = wd_aead_set_ckey(priv->sess, priv->key, priv->keylen); - if (ret) { - UADK_ERR("uadk failed to set key!\n"); - goto free_sess; - } } return UADK_AEAD_SUCCESS; - -free_sess: - wd_aead_free_sess(priv->sess); - priv->sess = 0; - return UADK_AEAD_FAIL; } static void *uadk_prov_aead_cb(struct wd_aead_req *req, void *data) @@ -558,7 +511,7 @@ static int uadk_do_aead_async_inner(struct aead_priv_ctx *priv, struct async_op } static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { struct async_op op; int ret; @@ -589,6 +542,8 @@ static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char if (unlikely(ret < 0)) goto soft; + *outl = 0; + return UADK_AEAD_SUCCESS; free_notification: @@ -653,11 +608,13 @@ free_notification: } static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { if (priv->stream_switch_flag == UADK_DO_SOFT) return SWITCH_TO_SOFT; + *outl = inlen; + if (priv->mode == ASYNC_MODE) return uadk_do_aead_async(priv, out, in, inlen); @@ -665,7 +622,7 @@ static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char } static int uadk_prov_do_aes_gcm_final(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { struct async_op op; int ret; @@ -702,16 +659,35 @@ out: priv->tag_set = INIT_TAG; priv->mode = UNINIT_MODE; - + *outl = 0; return UADK_AEAD_SUCCESS; } +static int uadk_prov_sw_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, + size_t *outl, const unsigned char *in, size_t inlen) +{ + int ret; + + if (priv->stream_switch_flag != UADK_DO_SOFT) { + ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); + if (ret <= 0) + return UADK_OSSL_FAIL; + } + + if (in) + return uadk_aead_soft_update(priv, out, outl, in, inlen); + + return uadk_aead_soft_final(priv, out, outl); +} + static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, - size_t *outl, size_t outsize, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { int ret; + if (priv->stream_switch_flag == UADK_DO_SOFT || !priv->sess) + return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); + ret = uadk_prov_aead_ctx_init(priv); if (ret != UADK_AEAD_SUCCESS) return UADK_AEAD_FAIL; @@ -722,12 +698,16 @@ static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, if (in) { if (!out) - return uadk_prov_do_aes_gcm_first(priv, out, in, inlen); - - return uadk_prov_do_aes_gcm_update(priv, out, in, inlen); + ret = uadk_prov_do_aes_gcm_first(priv, out, outl, in, inlen); + else + ret = uadk_prov_do_aes_gcm_update(priv, out, outl, in, inlen); + } else { + ret = uadk_prov_do_aes_gcm_final(priv, out, outl, NULL, 0); } + if (ret == SWITCH_TO_SOFT) + return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); - return uadk_prov_do_aes_gcm_final(priv, out, NULL, 0); + return ret; } void uadk_prov_destroy_aead(void) @@ -764,11 +744,12 @@ static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, return UADK_OSSL_FAIL; } - ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, in, inl); - if (ret < 0) + ret = uadk_prov_do_aes_gcm(priv, out, outl, in, inl); + if (ret <= 0) { + *outl = 0; return UADK_OSSL_FAIL; + } - *outl = inl; return UADK_AEAD_SUCCESS; } @@ -777,7 +758,7 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, const unsigned char *in, size_t inl) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; - int ret, outlen; + int ret; if (!vctx) return UADK_OSSL_FAIL; @@ -787,31 +768,12 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, return UADK_OSSL_FAIL; } - if (priv->stream_switch_flag == UADK_DO_SOFT) - goto do_soft; - ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, in, inl); - if (ret == SWITCH_TO_SOFT) - goto do_soft; - else if (ret < 0) { - UADK_ERR("stream data update failed.\n"); + ret = uadk_prov_do_aes_gcm(priv, out, outl, in, inl); + if (ret <= 0) { + *outl = 0; return UADK_OSSL_FAIL; - } else { - *outl = inl; - return UADK_AEAD_SUCCESS; - } - -do_soft: - if (priv->stream_switch_flag != UADK_DO_SOFT) { - ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); - if (ret <= 0) - return UADK_OSSL_FAIL; } - ret = uadk_aead_soft_update(priv, out, &outlen, in, inl); - if (ret <= 0) - return UADK_OSSL_FAIL; - - *outl = outlen; return UADK_AEAD_SUCCESS; } @@ -824,37 +786,24 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, if (!vctx || !out || !outl) return UADK_OSSL_FAIL; - if (priv->stream_switch_flag == UADK_DO_SOFT) - goto do_soft; - - ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, NULL, 0); - if (ret < 0) { - UADK_ERR("stream data final failed, ret = %d\n", ret); + ret = uadk_prov_do_aes_gcm(priv, out, outl, NULL, 0); + if (ret <= 0) { + *outl = 0; return UADK_OSSL_FAIL; } - *outl = 0; return UADK_AEAD_SUCCESS; - -do_soft: - ret = uadk_aead_soft_final(priv, out, outl); - if (ret) { - *outl = 0; - return UADK_AEAD_SUCCESS; - } - - return UADK_OSSL_FAIL; } -static int uadk_get_aead_info(struct aead_priv_ctx *priv) +static int uadk_get_aead_info(struct wd_aead_sess_setup *setup, int nid) { int aead_counts = ARRAY_SIZE(aead_info_table); int i; for (i = 0; i < aead_counts; i++) { - if (priv->nid == aead_info_table[i].nid) { - priv->setup.calg = aead_info_table[i].alg; - priv->setup.cmode = aead_info_table[i].mode; + if (nid == aead_info_table[i].nid) { + setup->calg = aead_info_table[i].alg; + setup->cmode = aead_info_table[i].mode; break; } } @@ -867,6 +816,72 @@ static int uadk_get_aead_info(struct aead_priv_ctx *priv) return UADK_AEAD_SUCCESS; } +static int uadk_prov_aead_alloc_sess(struct aead_priv_ctx *priv) +{ + struct wd_aead_sess_setup setup = {0}; + struct sched_params params = {0}; + int ret; + + if (priv->sess) + return UADK_AEAD_SUCCESS; + + ret = uadk_prov_aead_dev_init(priv); + if (unlikely(ret < 0)) + return SWITCH_TO_SOFT; + + ret = uadk_get_aead_info(&setup, priv->nid); + if (unlikely(ret < 0)) + return UADK_OSSL_FAIL; + + /* dec and enc use the same op */ + params.type = 0; + /* Use the default numa parameters */ + params.numa_id = -1; + setup.sched_param = ¶ms; + priv->sess = wd_aead_alloc_sess(&setup); + if (!priv->sess) { + UADK_ERR("uadk failed to alloc session, switch to soft\n"); + return SWITCH_TO_SOFT; + } + + if (priv->key_set == KEY_STATE_SET) { + ret = wd_aead_set_ckey(priv->sess, priv->key, priv->keylen); + if (ret) { + UADK_ERR("uadk failed to set key!\n"); + return UADK_OSSL_FAIL; + } + } + + return UADK_AEAD_SUCCESS; +} + +static int uadk_prov_aead_set_key(struct aead_priv_ctx *priv, + const unsigned char *key, + size_t keylen) +{ + int ret; + + if (keylen != priv->keylen) { + UADK_ERR("invalid keylen %zu!\n", keylen); + return UADK_OSSL_FAIL; + } + + memcpy(priv->key, key, keylen); + priv->key_set = KEY_STATE_SET; + + /* use default provider */ + if (!priv->sess) + return UADK_AEAD_SUCCESS; + + ret = wd_aead_set_ckey(priv->sess, priv->key, priv->keylen); + if (ret) { + UADK_ERR("uadk failed to set key!\n"); + return UADK_OSSL_FAIL; + } + + return UADK_AEAD_SUCCESS; +} + static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char *key, size_t keylen, const unsigned char *iv, size_t ivlen, const OSSL_PARAM *params) { @@ -877,32 +892,26 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char * return UADK_OSSL_FAIL; } + /* will free in freectx */ + ret = uadk_prov_aead_alloc_sess(priv); + if (ret == UADK_OSSL_FAIL) + return UADK_OSSL_FAIL; + if (iv) { memcpy(priv->iv, iv, ivlen); priv->iv_set = IV_STATE_SET; } - ret = uadk_get_aead_info(priv); - if (unlikely(ret < 0)) - return UADK_OSSL_FAIL; - if (key) { - memcpy(priv->key, key, keylen); - priv->key_set = KEY_STATE_SET; + ret = uadk_prov_aead_set_key(priv, key, keylen); + if (ret == UADK_OSSL_FAIL) + return UADK_OSSL_FAIL; } priv->stream_switch_flag = 0; + priv->req.msg_state = AEAD_MSG_INVALID; - if (uadk_get_sw_offload_state()) - uadk_create_aead_soft_ctx(priv); - - ret = uadk_prov_aead_dev_init(priv); - if (unlikely(ret < 0)) { - UADK_ERR("aead switch to soft init.!\n"); - return uadk_prov_aead_soft_init(priv, key, iv, params); - } - - return UADK_AEAD_SUCCESS; + return uadk_prov_aead_set_ctx_params(priv, params); } static int uadk_prov_aead_einit(void *vctx, const unsigned char *key, size_t keylen, @@ -1152,6 +1161,30 @@ static int uadk_cipher_aead_get_params(OSSL_PARAM params[], unsigned int md, return UADK_AEAD_SUCCESS; } +static void uadk_prov_aead_free_sess(struct aead_priv_ctx *priv) +{ + if (priv->sess) + wd_aead_free_sess(priv->sess); +} + +static int uadk_prov_aead_copy_sess(struct aead_priv_ctx *priv) +{ + if (!priv->sess) + return UADK_AEAD_SUCCESS; + priv->sess = 0; + + /* + * Encryption and decryption have already started, so it cannot + * switch to software calculation, hence it returns a failure. + */ + if (priv->req.msg_state != AEAD_MSG_INVALID) { + UADK_ERR("invalid: The data has been processed by hardware, cannot be copied.\n"); + return UADK_OSSL_FAIL; + } + + return uadk_prov_aead_alloc_sess(priv); +} + static void *uadk_prov_aead_dupctx(void *ctx) { struct aead_priv_ctx *dst_ctx, *src_ctx; @@ -1165,16 +1198,15 @@ static void *uadk_prov_aead_dupctx(void *ctx) if (!dst_ctx) return NULL; - dst_ctx->sess = 0; - dst_ctx->data = OPENSSL_memdup(src_ctx->data, AEAD_BLOCK_SIZE << 1); - if (!dst_ctx->data) + ret = uadk_prov_aead_copy_sess(dst_ctx); + if (ret == UADK_OSSL_FAIL) goto free_ctx; if (dst_ctx->sw_ctx) { dst_ctx->sw_ctx = EVP_CIPHER_CTX_dup(src_ctx->sw_ctx); if (!dst_ctx->sw_ctx) { UADK_ERR("EVP_CIPHER_CTX_dup failed in ctx copy.\n"); - goto free_data; + goto free_sess; } ret = EVP_CIPHER_up_ref(dst_ctx->sw_aead); @@ -1187,13 +1219,22 @@ static void *uadk_prov_aead_dupctx(void *ctx) free_dup: if (dst_ctx->sw_ctx) EVP_CIPHER_CTX_free(dst_ctx->sw_ctx); -free_data: - OPENSSL_clear_free(dst_ctx->data, AEAD_BLOCK_SIZE << 1); +free_sess: + uadk_prov_aead_free_sess(dst_ctx); free_ctx: OPENSSL_clear_free(dst_ctx, sizeof(*dst_ctx)); return NULL; } +static void uadk_aead_soft_cleanup(struct aead_priv_ctx *priv) +{ + if (priv->sw_ctx) + EVP_CIPHER_CTX_free(priv->sw_ctx); + + if (priv->sw_aead) + EVP_CIPHER_free(priv->sw_aead); +} + static void uadk_prov_aead_freectx(void *ctx) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)ctx; @@ -1201,15 +1242,8 @@ static void uadk_prov_aead_freectx(void *ctx) if (!ctx) return; - if (priv->sess) - wd_aead_free_sess(priv->sess); - - if (priv->data) - OPENSSL_clear_free(priv->data, AEAD_BLOCK_SIZE << 1); - - if (priv->sw_ctx) - uadk_aead_soft_cleanup(priv); - + uadk_prov_aead_free_sess(priv); + uadk_aead_soft_cleanup(priv); OPENSSL_clear_free(priv, sizeof(*priv)); } @@ -1218,15 +1252,11 @@ static void uadk_prov_aead_freectx(void *ctx) static OSSL_FUNC_cipher_newctx_fn uadk_##nm##_newctx; \ static void *uadk_##nm##_newctx(void *provctx) \ { \ - struct aead_priv_ctx *ctx = OPENSSL_zalloc(sizeof(*ctx)); \ - if (!ctx) \ - return NULL; \ + struct aead_priv_ctx *ctx; \ \ - ctx->data = OPENSSL_zalloc(AEAD_BLOCK_SIZE << 1); \ - if (!ctx->data) { \ - OPENSSL_free(ctx); \ + ctx = OPENSSL_zalloc(sizeof(*ctx)); \ + if (!ctx) \ return NULL; \ - } \ \ ctx->keylen = key_len; \ ctx->ivlen = iv_len; \ @@ -1234,6 +1264,9 @@ static void *uadk_##nm##_newctx(void *provctx) \ ctx->taglen = tag_len; \ strncpy(ctx->alg_name, #algnm, ALG_NAME_SIZE - 1); \ \ + if (uadk_get_sw_offload_state()) \ + uadk_create_aead_soft_ctx(ctx); \ + \ return ctx; \ } \ static OSSL_FUNC_cipher_get_params_fn uadk_##nm##_get_params; \ -- 2.53.0.windows.2