[PATCH 00/13] uadk_engine/uadk_provider: bug fixes and AEAD GCM enhancements
From: qianweili <qianweili@huawei.com> This series fixes several bugs in the UADK engine and provider, and adds non-block-aligned data handling and TLS 1.x GCM record offload to the provider AES-GCM path. Engine fixes: - Correct error log format and a misleading async macro name (01) - Balance the async recv loop count (02) - Fix wrong return values and missing NULL checks in error paths (03) - Securely release DH/ECDH/SM2 private keys and fix an SM2 order leak (04) - Fall back to software when digest/cipher session allocation fails (05) - Copy md_nid and md_update_status in SM2 ctx copy (06) Provider fixes: - Fix wrong parameter order in wd_xxx_init2_ calls (07) - Split session/key management into the init stage to avoid per-operation alloc/free overhead (08) - Handle non-block-aligned GCM data via an AES-CTR tail soft path (09) - Add TLS 1.x GCM record offload and an IV state machine (10) - Widen the IV buffer and fall back to software for non-default IV lengths (11) - Fix session allocation failure handling in multiple threads (12) - Fix a double-free in digest cleanup under multiple threads (13) Weili Qian (9): uadk_engine: fix error log format coding style and misleading async macro name uadk_engine: fix return value checks in error paths uadk_engine: securely release DH/ECDH/SM2 private key and fix SM2 order leak uadk_engine: copy md_nid and md_update_status in SM2 ctx copy uadk_provider: fix wrong parameter order in wd_xxx_init2_ call uadk_provider: split session/key management into init stage uadk_provider: handle non-block-aligned data via AES-CTR tail soft path uadk_provider: add TLS 1.x GCM record offload and IV state machine uadk_provider: widen IV buffer and fall back to software for non-default IV lengths Wenkai Lin (2): uadk_engine: fall back to software when digest/cipher session alloc fails uadk_provider: fix the double free problem in multiple threads Zhushuai Yin (1): uadk_provider: fix the error of allocating sessions in multiple threads lizhi (1): uadk_engine: adjust async recv loop count to balance async mode behavior src/uadk.h | 2 +- src/uadk_cipher.c | 3 +- src/uadk_dh.c | 8 +- src/uadk_digest.c | 40 +- src/uadk_ec.c | 52 +- src/uadk_ecx.c | 4 +- src/uadk_prov.h | 2 +- src/uadk_prov_aead.c | 1337 +++++++++++++++++++++++++------------ src/uadk_prov_cipher.c | 2 +- src/uadk_prov_dh.c | 6 +- src/uadk_prov_digest.c | 97 ++- src/uadk_prov_ecdh_exch.c | 23 +- src/uadk_prov_ecx.c | 10 +- src/uadk_prov_hmac.c | 4 +- src/uadk_prov_pkey.c | 8 +- src/uadk_prov_rsa_enc.c | 2 +- src/uadk_prov_sm2_enc.c | 2 +- src/uadk_prov_sm2_kmgmt.c | 11 +- src/uadk_prov_sm2_sign.c | 4 +- src/uadk_sm2.c | 18 +- 20 files changed, 1135 insertions(+), 500 deletions(-) -- 2.53.0.windows.2
Add missing trailing newlines to the dlen fprintf messages in the EC sign/verify checks, and collapse the temporary variable in ecx_keygen_set_pkey to a direct return. No behavioural change. Use %zu to log the size_t outsize in uadk_asym_cipher_rsa_encrypt instead of %d, which read only half the value on 64-bit. Also update the provider async retry checks to use PROV_SEND_MAX_CNT from the shared provider header instead of ENGINE_SEND_MAX_CNT, since the ENGINE_ prefix is misleading for provider code. The ENGINE_SEND_MAX_CNT macro itself is unchanged and still used by the engine side. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_ec.c | 4 ++-- src/uadk_ecx.c | 4 +--- src/uadk_prov_aead.c | 2 +- src/uadk_prov_digest.c | 2 +- src/uadk_prov_hmac.c | 2 +- src/uadk_prov_rsa_enc.c | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/uadk_ec.c b/src/uadk_ec.c index 08ff7a3..1965461 100644 --- a/src/uadk_ec.c +++ b/src/uadk_ec.c @@ -297,7 +297,7 @@ static int ecdsa_do_sign_check(EC_KEY *eckey, } if (dlen <= 0) { - fprintf(stderr, "dlen error, dlen = %d", dlen); + fprintf(stderr, "dlen error, dlen = %d\n", dlen); return -1; } @@ -586,7 +586,7 @@ static int ecdsa_do_verify_check(EC_KEY *eckey, } if (dlen <= 0) { - fprintf(stderr, "digest len error, dlen = %d", dlen); + fprintf(stderr, "digest len error, dlen = %d\n", dlen); return -1; } diff --git a/src/uadk_ecx.c b/src/uadk_ecx.c index 3e79a54..fbab680 100644 --- a/src/uadk_ecx.c +++ b/src/uadk_ecx.c @@ -508,9 +508,7 @@ static int ecx_keygen_set_pkey(EVP_PKEY *pkey, struct ecx_ctx *ecx_ctx, ecx_key->privkey[X448_KEYLEN - 1] |= 0x80; } - ret = EVP_PKEY_assign(pkey, ecx_ctx->nid, ecx_key); - - return ret; + return EVP_PKEY_assign(pkey, ecx_ctx->nid, ecx_key); } static int openssl_do_derive(EVP_PKEY_CTX *ctx, unsigned char *key, diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index 88ec8e4..93929a0 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -537,7 +537,7 @@ static int uadk_do_aead_async_inner(struct aead_priv_ctx *priv, struct async_op if (unlikely(ret < 0)) { if (unlikely(ret != -EBUSY)) UADK_ERR("do aead async operation failed ret = %d.\n", ret); - else if (unlikely(cnt++ > ENGINE_SEND_MAX_CNT)) + else if (unlikely(cnt++ > PROV_SEND_MAX_CNT)) UADK_ERR("do aead async operation timeout.\n"); else continue; diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index 0f4cb87..0b24a26 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -628,7 +628,7 @@ static int uadk_do_digest_async(struct digest_priv_ctx *priv, struct async_op *o goto free_poll_task; } - if (unlikely(++cnt > ENGINE_SEND_MAX_CNT)) { + if (unlikely(++cnt > PROV_SEND_MAX_CNT)) { UADK_ERR("do digest async operation timeout.\n"); goto free_poll_task; } diff --git a/src/uadk_prov_hmac.c b/src/uadk_prov_hmac.c index 5a88fe4..ad1aa2e 100644 --- a/src/uadk_prov_hmac.c +++ b/src/uadk_prov_hmac.c @@ -599,7 +599,7 @@ static int uadk_do_hmac_async(struct hmac_priv_ctx *priv, struct async_op *op) goto free_poll_task; } - if (unlikely(++cnt > ENGINE_SEND_MAX_CNT)) { + if (unlikely(++cnt > PROV_SEND_MAX_CNT)) { UADK_ERR("do hmac async operation timeout.\n"); goto free_poll_task; } diff --git a/src/uadk_prov_rsa_enc.c b/src/uadk_prov_rsa_enc.c index 3b7ce40..80a40d5 100644 --- a/src/uadk_prov_rsa_enc.c +++ b/src/uadk_prov_rsa_enc.c @@ -491,7 +491,7 @@ static int uadk_asym_cipher_rsa_encrypt(void *vprsactx, unsigned char *out, } if (outsize < len) { - UADK_ERR("invalid: outsize %d is too small.\n", outsize); + UADK_ERR("invalid: outsize %zu is too small.\n", outsize); return UADK_P_FAIL; } -- 2.53.0.windows.2
From: lizhi <lizhi206@huawei.com> Increase ENGINE_ENV_RECV_MAX_CNT in uadk.h and PROV_SCH_RECV_MAX_CNT in uadk_prov.h from 60000 to 60000000 to offset the loop count decrease made by uadk in its async mode, ensuring balanced behavior under heavy asynchronous load. Signed-off-by: lizhi <lizhi206@huawei.com> --- src/uadk.h | 2 +- src/uadk_prov.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uadk.h b/src/uadk.h index 6baa968..2c52643 100644 --- a/src/uadk.h +++ b/src/uadk.h @@ -23,7 +23,7 @@ #define ENV_STRING_LEN 256 #define ENGINE_SEND_MAX_CNT 90000000 #define ENGINE_RECV_MAX_CNT 60000000 -#define ENGINE_ENV_RECV_MAX_CNT 60000 +#define ENGINE_ENV_RECV_MAX_CNT 60000000 #define UADK_UNINIT 0 #define UADK_INIT_SUCCESS 1 #define UADK_INIT_FAIL 2 diff --git a/src/uadk_prov.h b/src/uadk_prov.h index c802c71..41c61b6 100644 --- a/src/uadk_prov.h +++ b/src/uadk_prov.h @@ -30,7 +30,7 @@ #define POLL_ERROR (-1) #define PROV_SEND_MAX_CNT 90000000 #define PROV_RECV_MAX_CNT 60000000 -#define PROV_SCH_RECV_MAX_CNT 60000 +#define PROV_SCH_RECV_MAX_CNT 60000000 #define UADK_P_SUCCESS 1 #define UADK_P_FAIL 0 #define UADK_DO_SOFT (-0xE0) -- 2.53.0.windows.2
Several error paths in the engine DH/EC and provider DH/ECDH/SM2/X25519 code returned the wrong code or an uninitialized value: the DH soft generate/compute key check treated 0 as success, set_sess_setup_cv fell through with an uninitialized ret on a NULL generator/order, ecdh_init_req/ecdh_compkey_init_iot returned -ENOMEM and ignored affine/BN_bn2binpad failures, and several provider alloc-sess branches returned UADK_P_FAIL instead of UADK_DO_SOFT. The x25519 keygen path also returned NULL directly instead of falling back to software. Also add NULL checks for BN_bin2bn in sm2_set_key_to_ec_key. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_dh.c | 4 ++-- src/uadk_ec.c | 42 +++++++++++++++++++++++++++++++++------ src/uadk_prov_dh.c | 2 +- src/uadk_prov_ecdh_exch.c | 23 +++++++++++++++++---- src/uadk_prov_ecx.c | 10 +++++----- src/uadk_prov_pkey.c | 8 ++++++-- src/uadk_prov_sm2_enc.c | 2 +- src/uadk_prov_sm2_kmgmt.c | 9 +++++++++ src/uadk_prov_sm2_sign.c | 4 ++-- 9 files changed, 81 insertions(+), 23 deletions(-) diff --git a/src/uadk_dh.c b/src/uadk_dh.c index ce00953..fbb6dc9 100644 --- a/src/uadk_dh.c +++ b/src/uadk_dh.c @@ -108,7 +108,7 @@ static int uadk_e_dh_soft_generate_key(DH *dh) } ret = dh_soft_generate_key(dh); - if (ret < 0) { + if (ret <= 0) { fprintf(stderr, "failed to do dh soft generate key\n"); return UADK_E_FAIL; } @@ -137,7 +137,7 @@ static int uadk_e_dh_soft_compute_key(unsigned char *key, } ret = dh_soft_compute_key(key, pub_key, dh); - if (ret < 0) { + if (ret <= 0) { fprintf(stderr, "failed to do dh soft compute key\n"); return UADK_E_FAIL; } diff --git a/src/uadk_ec.c b/src/uadk_ec.c index 1965461..c7d50ac 100644 --- a/src/uadk_ec.c +++ b/src/uadk_ec.c @@ -151,16 +151,20 @@ static int set_sess_setup_cv(const EC_GROUP *group, goto free_cv; cv_param->g = EC_GROUP_get0_generator(group); - if (!cv_param->g) + if (!cv_param->g) { + ret = -1; goto free_cv; + } ret = uadk_get_affine_coordinates(group, cv_param->g, g_x, g_y, ctx); if (ret) goto free_cv; cv_param->order = EC_GROUP_get0_order(group); - if (!cv_param->order) + if (!cv_param->order) { + ret = -1; goto free_cv; + } fill_ecc_cv_param(pparam, cv_param, g_x, g_y); cv->type = WD_CV_CFG_PARAM; @@ -791,6 +795,10 @@ static int sm2_set_key_to_ec_key(EC_KEY *ec, struct wd_ecc_req *req) } tmp = BN_bin2bn((unsigned char *)privkey->data, privkey->dsize, NULL); + if (!tmp) { + fprintf(stderr, "failed to BN_bin2bn privkey\n"); + return -EINVAL; + } ret = EC_KEY_set_private_key(ec, tmp); BN_free(tmp); if (!ret) { @@ -812,6 +820,12 @@ static int sm2_set_key_to_ec_key(EC_KEY *ec, struct wd_ecc_req *req) memcpy(buff + x_offset, pubkey->x.data, pubkey->x.dsize); memcpy(buff + y_offset, pubkey->y.data, pubkey->y.dsize); tmp = BN_bin2bn(buff, ECC_POINT_SIZE(SM2_KEY_BYTES) + 1, NULL); + if (!tmp) { + fprintf(stderr, "failed to BN_bin2bn pubkey\n"); + EC_POINT_free(point); + return -EINVAL; + } + ptr = EC_POINT_bn2point(group, tmp, point, NULL); BN_free(tmp); if (!ptr) { @@ -1034,13 +1048,14 @@ static int ecdh_compkey_init_iot(handle_t sess, struct wd_ecc_req *req, struct wd_ecc_in *ecdh_in; BIGNUM *pkey_x, *pkey_y; const EC_GROUP *group; + int xlen, ylen; size_t ec_size; BN_CTX *ctx; int ret = 0; ctx = BN_CTX_new(); if (!ctx) - return -ENOMEM; + return 0; BN_CTX_start(ctx); pkey_x = BN_CTX_get(ctx); @@ -1056,16 +1071,30 @@ static int ecdh_compkey_init_iot(handle_t sess, struct wd_ecc_req *req, goto free_ctx; ec_size = ecdh_get_ec_size(group); - uadk_get_affine_coordinates(group, pubkey, pkey_x, pkey_y, ctx); + ret = uadk_get_affine_coordinates(group, pubkey, pkey_x, pkey_y, ctx); + if (ret) { + ret = 0; + goto free_ctx; + } + in_pkey.x.data = buf_x; in_pkey.y.data = buf_y; - in_pkey.x.dsize = BN_bn2binpad(pkey_x, (unsigned char *)in_pkey.x.data, ec_size); - in_pkey.y.dsize = BN_bn2binpad(pkey_y, (unsigned char *)in_pkey.y.data, ec_size); + xlen = BN_bn2binpad(pkey_x, (unsigned char *)in_pkey.x.data, ec_size); + ylen = BN_bn2binpad(pkey_y, (unsigned char *)in_pkey.y.data, ec_size); + if (xlen < 0 || ylen < 0) { + fprintf(stderr, "failed to BN_bn2binpad, xlen = %d, ylen = %d\n", + xlen, ylen); + ret = 0; + goto free_ctx; + } + in_pkey.x.dsize = xlen; + in_pkey.y.dsize = ylen; /* Set public key */ ecdh_in = wd_ecxdh_new_in(sess, &in_pkey); if (!ecdh_in) { fprintf(stderr, "failed to new ecxdh in\n"); + ret = 0; goto free_ctx; } @@ -1073,6 +1102,7 @@ static int ecdh_compkey_init_iot(handle_t sess, struct wd_ecc_req *req, if (!ecdh_out) { fprintf(stderr, "failed to new ecxdh out\n"); wd_ecc_del_in(sess, ecdh_in); + ret = 0; goto free_ctx; } diff --git a/src/uadk_prov_dh.c b/src/uadk_prov_dh.c index cf31971..3f49a6f 100644 --- a/src/uadk_prov_dh.c +++ b/src/uadk_prov_dh.c @@ -782,7 +782,7 @@ static int uadk_prov_dh_prepare_data(const BIGNUM *g, DH *dh, struct uadk_dh_ses *dh_sess = uadk_prov_dh_new_session(dh, bits, is_g2); if (*dh_sess == NULL) { UADK_ERR("failed to get session\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } ret = uadk_prov_dh_prepare_prikey(*dh_sess, dh, prikey); diff --git a/src/uadk_prov_ecdh_exch.c b/src/uadk_prov_ecdh_exch.c index f2d09fb..01473a8 100644 --- a/src/uadk_prov_ecdh_exch.c +++ b/src/uadk_prov_ecdh_exch.c @@ -216,12 +216,13 @@ static int ecdh_init_req(struct ecdh_sess_ctx *sess_ctx, struct wd_ecc_in *ecdh_in; BIGNUM *pkey_x, *pkey_y; int ret = UADK_P_FAIL; + int xlen, ylen; size_t ec_size; BN_CTX *ctx; ctx = BN_CTX_new(); if (!ctx) - return -ENOMEM; + return UADK_P_FAIL; BN_CTX_start(ctx); pkey_x = BN_CTX_get(ctx); @@ -233,16 +234,29 @@ static int ecdh_init_req(struct ecdh_sess_ctx *sess_ctx, goto free_ctx; ec_size = ecdh_get_ec_size(sess_ctx->group); - uadk_prov_get_affine_coordinates(sess_ctx->group, sess_ctx->pub_key, pkey_x, pkey_y, ctx); + ret = uadk_prov_get_affine_coordinates(sess_ctx->group, + sess_ctx->pub_key, pkey_x, pkey_y, ctx); + if (ret != UADK_P_SUCCESS) + goto free_ctx; + in_pkey.x.data = buf_x; in_pkey.y.data = buf_y; - in_pkey.x.dsize = BN_bn2binpad(pkey_x, (unsigned char *)in_pkey.x.data, ec_size); - in_pkey.y.dsize = BN_bn2binpad(pkey_y, (unsigned char *)in_pkey.y.data, ec_size); + xlen = BN_bn2binpad(pkey_x, (unsigned char *)in_pkey.x.data, ec_size); + ylen = BN_bn2binpad(pkey_y, (unsigned char *)in_pkey.y.data, ec_size); + if (xlen < 0 || ylen < 0) { + UADK_ERR("failed to BN_bn2binpad, xlen = %d, ylen = %d\n", + xlen, ylen); + ret = UADK_P_FAIL; + goto free_ctx; + } + in_pkey.x.dsize = xlen; + in_pkey.y.dsize = ylen; /* Set public key */ ecdh_in = wd_ecxdh_new_in(sess, &in_pkey); if (!ecdh_in) { UADK_ERR("failed to new ecxdh in\n"); + ret = UADK_P_FAIL; goto free_ctx; } @@ -250,6 +264,7 @@ static int ecdh_init_req(struct ecdh_sess_ctx *sess_ctx, if (!ecdh_out) { UADK_ERR("failed to new ecxdh out\n"); wd_ecc_del_in(sess, ecdh_in); + ret = UADK_P_FAIL; goto free_ctx; } diff --git a/src/uadk_prov_ecx.c b/src/uadk_prov_ecx.c index 69494cc..5550af8 100644 --- a/src/uadk_prov_ecx.c +++ b/src/uadk_prov_ecx.c @@ -888,7 +888,7 @@ static void *uadk_keymgmt_x448_gen(void *genctx, OSSL_CALLBACK *cb, void *cb_par gctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X448); if (gctx->sess == (handle_t)0) { UADK_ERR("failed to alloc x448 sess\n"); - ret = UADK_P_FAIL; + ret = UADK_DO_SOFT; goto exe_soft; } @@ -1273,7 +1273,7 @@ static int uadk_keyexch_x448_derive(void *vecxctx, unsigned char *secret, size_t ecxctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X448); if (ecxctx->sess == (handle_t)0) { UADK_ERR("failed to alloc sess\n"); - ret = UADK_P_FAIL; + ret = UADK_DO_SOFT; goto exe_soft; } @@ -1532,8 +1532,8 @@ static void *uadk_keymgmt_x25519_gen(void *genctx, OSSL_CALLBACK *cb, void *cb_p gctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X25519); if (gctx->sess == (handle_t)0) { UADK_ERR("failed to alloc x25519 sess\n"); - ret = UADK_P_FAIL; - return NULL; + ret = UADK_DO_SOFT; + goto exe_soft; } ret = uadk_prov_ecx_keygen(gctx, &ecx_key); @@ -1675,7 +1675,7 @@ static int uadk_keyexch_x25519_derive(void *vecxctx, unsigned char *secret, size ecxctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X25519); if (ecxctx->sess == (handle_t)0) { UADK_ERR("failed to alloc sess\n"); - ret = UADK_P_FAIL; + ret = UADK_DO_SOFT; goto exe_soft; } diff --git a/src/uadk_prov_pkey.c b/src/uadk_prov_pkey.c index 2e2ca0b..41d0102 100644 --- a/src/uadk_prov_pkey.c +++ b/src/uadk_prov_pkey.c @@ -271,16 +271,20 @@ static int uadk_prov_set_sess_setup_cv(const EC_GROUP *group, goto free_cv; cv_param->g = EC_GROUP_get0_generator(group); - if (cv_param->g == NULL) + if (cv_param->g == NULL) { + ret = UADK_P_FAIL; goto free_cv; + } ret = uadk_prov_get_affine_coordinates(group, cv_param->g, g_x, g_y, bn_ctx); if (ret == 0) goto free_cv; cv_param->order = EC_GROUP_get0_order(group); - if (cv_param->order == NULL) + if (cv_param->order == NULL) { + ret = UADK_P_FAIL; goto free_cv; + } uadk_prov_fill_ecc_cv_param(ecc_param, cv_param, g_x, g_y); cv->type = WD_CV_CFG_PARAM; diff --git a/src/uadk_prov_sm2_enc.c b/src/uadk_prov_sm2_enc.c index e987202..16e85ae 100644 --- a/src/uadk_prov_sm2_enc.c +++ b/src/uadk_prov_sm2_enc.c @@ -348,7 +348,7 @@ static int sm2_prov_alloc_sess(PROV_SM2_ASYM_CTX *vpsm2ctx, handle_t *sess) *sess = wd_ecc_alloc_sess(&setup); if (*sess == (handle_t)0) { UADK_ERR("failed to alloc sess\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } return UADK_P_SUCCESS; diff --git a/src/uadk_prov_sm2_kmgmt.c b/src/uadk_prov_sm2_kmgmt.c index 1791cdd..8f36dfa 100644 --- a/src/uadk_prov_sm2_kmgmt.c +++ b/src/uadk_prov_sm2_kmgmt.c @@ -337,6 +337,10 @@ static int uadk_prov_sm2_set_key_to_ec_key(EC_KEY *ec, struct wd_ecc_req *req) } bn_key = BN_bin2bn((unsigned char *)privkey->data, privkey->dsize, NULL); + if (!bn_key) { + UADK_ERR("failed to BN_bin2bn privkey\n"); + return UADK_P_FAIL; + } ret = EC_KEY_set_private_key(ec, bn_key); BN_free(bn_key); if (ret == 0) { @@ -358,6 +362,11 @@ static int uadk_prov_sm2_set_key_to_ec_key(EC_KEY *ec, struct wd_ecc_req *req) memcpy(key_buff + x_offset, pubkey->x.data, pubkey->x.dsize); memcpy(key_buff + y_offset, pubkey->y.data, pubkey->y.dsize); bn_key = BN_bin2bn(key_buff, ECC_POINT_SIZE(SM2_KEY_BYTES) + 1, NULL); + if (!bn_key) { + UADK_ERR("failed to BN_bin2bn pubkey\n"); + EC_POINT_free(point); + return UADK_P_FAIL; + } ptr = EC_POINT_bn2point(group, bn_key, point, NULL); BN_free(bn_key); if (ptr == NULL) { diff --git a/src/uadk_prov_sm2_sign.c b/src/uadk_prov_sm2_sign.c index aa7049e..aa94875 100644 --- a/src/uadk_prov_sm2_sign.c +++ b/src/uadk_prov_sm2_sign.c @@ -538,7 +538,7 @@ static int sm2_sign_hw(PROV_SM2_SIGN_CTX *psm2ctx, sess = sm2_alloc_sess(psm2ctx->key); if (sess == (handle_t)0) { UADK_ERR("failed to alloc sess in sign\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } ret = sm2_sign_init_iot(sess, &req, (void *)tbs, tbslen); @@ -687,7 +687,7 @@ static int sm2_verify_hw(PROV_SM2_SIGN_CTX *psm2ctx, sess = sm2_alloc_sess(psm2ctx->key); if (sess == (handle_t)0) { UADK_ERR("failed to alloc sess in verify\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } ret = sm2_verify_init_iot(sess, &req, sig, siglen, tbs, tbslen); -- 2.53.0.windows.2
Release the engine DH/ECDH and provider DH/SM2 private keys with BN_clear_free instead of BN_free, and allocate the engine ECDH private key with BN_secure_new so the secret material is wiped. Also fix a memory leak in sm2_update_sess where the SM2 curve order was re-allocated on every call; reuse smctx->order across calls instead. Remove the erroneous BN_free on the DH-internal private key in the compute_key error path. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_dh.c | 4 +--- src/uadk_ec.c | 6 +++--- src/uadk_prov_dh.c | 4 ++-- src/uadk_prov_sm2_kmgmt.c | 2 +- src/uadk_sm2.c | 16 +++++++++++----- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/uadk_dh.c b/src/uadk_dh.c index fbb6dc9..be3b41c 100644 --- a/src/uadk_dh.c +++ b/src/uadk_dh.c @@ -183,7 +183,7 @@ static int dh_try_get_priv_key(struct uadk_dh_sess *dh_sess, const DH *dh, BIGNU return UADK_E_SUCCESS; err: - BN_free(*priv_key); + BN_clear_free(*priv_key); return UADK_E_FAIL; } @@ -934,8 +934,6 @@ static int uadk_e_dh_compute_key(unsigned char *key, const BIGNUM *pub_key, return ret; free_data: - if (dh_sess->key_flag == KEY_GEN_BY_ENGINE) - BN_free(priv_key); dh_free_eng_session(dh_sess); soft_log: fprintf(stderr, "switch to execute openssl software calculation.\n"); diff --git a/src/uadk_ec.c b/src/uadk_ec.c index c7d50ac..d241e79 100644 --- a/src/uadk_ec.c +++ b/src/uadk_ec.c @@ -800,7 +800,7 @@ static int sm2_set_key_to_ec_key(EC_KEY *ec, struct wd_ecc_req *req) return -EINVAL; } ret = EC_KEY_set_private_key(ec, tmp); - BN_free(tmp); + BN_clear_free(tmp); if (!ret) { fprintf(stderr, "failed to EC KEY set private key\n"); return -EINVAL; @@ -899,7 +899,7 @@ static int ecdh_set_private_key(EC_KEY *eckey, BIGNUM *order) BIGNUM *priv_key; int ret; - priv_key = BN_new(); + priv_key = BN_secure_new(); if (!priv_key) { fprintf(stderr, "failed to BN_new priv_key\n"); return 0; @@ -918,7 +918,7 @@ static int ecdh_set_private_key(EC_KEY *eckey, BIGNUM *order) fprintf(stderr, "failed to set private key\n"); free_priv_key: - BN_free(priv_key); + BN_clear_free(priv_key); return ret; } diff --git a/src/uadk_prov_dh.c b/src/uadk_prov_dh.c index 3f49a6f..004b60e 100644 --- a/src/uadk_prov_dh.c +++ b/src/uadk_prov_dh.c @@ -581,7 +581,7 @@ static int uadk_prov_dh_prepare_prikey(struct uadk_dh_sess *dh_sess, const DH *d free_prikey: /* Free the prikey generated by uadk provider */ - BN_free(*prikey); + BN_clear_free(*prikey); *prikey = NULL; return UADK_P_FAIL; @@ -594,7 +594,7 @@ static void uadk_prov_dh_free_prikey(struct uadk_dh_sess *dh_sess, BIGNUM *prike /* User generated key will be freed by user, not uadk provider */ if (prikey && dh_sess->key_flag == KEY_GEN_BY_PROV) - BN_free(prikey); + BN_clear_free(prikey); } static void uadk_prov_dh_cb(void *req_t) diff --git a/src/uadk_prov_sm2_kmgmt.c b/src/uadk_prov_sm2_kmgmt.c index 8f36dfa..0085251 100644 --- a/src/uadk_prov_sm2_kmgmt.c +++ b/src/uadk_prov_sm2_kmgmt.c @@ -342,7 +342,7 @@ static int uadk_prov_sm2_set_key_to_ec_key(EC_KEY *ec, struct wd_ecc_req *req) return UADK_P_FAIL; } ret = EC_KEY_set_private_key(ec, bn_key); - BN_free(bn_key); + BN_clear_free(bn_key); if (ret == 0) { UADK_ERR("failed to EC KEY set private key\n"); return UADK_P_FAIL; diff --git a/src/uadk_sm2.c b/src/uadk_sm2.c index 9eb9f7b..6472abc 100644 --- a/src/uadk_sm2.c +++ b/src/uadk_sm2.c @@ -183,7 +183,6 @@ static int sm2_update_sess(struct sm2_ctx *smctx) }; struct wd_ecc_sess_setup setup; handle_t sess; - BIGNUM *order; int type; memset(&setup, 0, sizeof(setup)); @@ -202,13 +201,21 @@ static int sm2_update_sess(struct sm2_ctx *smctx) setup.hash.type = type; } - order = BN_bin2bn((void *)sm2_order, sizeof(sm2_order), NULL); + /* order free in sm2_cleanup() */ + if (!smctx->order) { + smctx->order = BN_bin2bn((void *)sm2_order, sizeof(sm2_order), NULL); + if (!smctx->order) { + fprintf(stderr, "failed to alloc order\n"); + smctx->init_status = CTX_INIT_FAIL; + return -ENOMEM; + } + } + setup.rand.cb = uadk_ecc_get_rand; - setup.rand.usr = (void *)order; + setup.rand.usr = (void *)smctx->order; sess = wd_ecc_alloc_sess(&setup); if (!sess) { fprintf(stderr, "failed to alloc sess\n"); - BN_free(order); smctx->init_status = CTX_INIT_FAIL; return -EINVAL; } @@ -220,7 +227,6 @@ static int sm2_update_sess(struct sm2_ctx *smctx) smctx->prikey = NULL; smctx->pubkey = NULL; - smctx->order = order; return 0; } -- 2.53.0.windows.2
From: Wenkai Lin <linwenkai6@hisilicon.com> When hardware session allocation fails the digest and cipher paths abort the operation even though a software implementation is available. Switch to UADK_DO_SOFT and initialise the soft context instead. For digest ctx copy, migrate the buffered/soft state so streamed data is not lost, and drop the stale check that skipped the soft path when an async job was active. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- src/uadk_cipher.c | 3 ++- src/uadk_digest.c | 40 +++++++++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/uadk_cipher.c b/src/uadk_cipher.c index 4067fb6..deceaee 100644 --- a/src/uadk_cipher.c +++ b/src/uadk_cipher.c @@ -685,7 +685,8 @@ static void uadk_e_ctx_init(EVP_CIPHER_CTX *ctx, struct cipher_priv_ctx *priv) priv->sess = wd_cipher_alloc_sess(&priv->setup); if (!priv->sess) { - fprintf(stderr, "uadk failed to alloc session!\n"); + priv->switch_flag = UADK_DO_SOFT; + fprintf(stderr, "uadk failed to alloc session, switch to soft\n"); return; } } diff --git a/src/uadk_digest.c b/src/uadk_digest.c index c8372b4..efdc9b8 100644 --- a/src/uadk_digest.c +++ b/src/uadk_digest.c @@ -629,8 +629,13 @@ static int uadk_e_digest_init(EVP_MD_CTX *ctx) if (!priv->sess) { priv->sess = wd_digest_alloc_sess(&priv->setup); - if (unlikely(!priv->sess)) - return 0; + if (unlikely(!priv->sess)) { + priv->switch_flag = UADK_DO_SOFT; + priv->data = NULL; + priv->soft_md = NULL; + priv->soft_ctx = NULL; + return digest_soft_init(priv); + } priv->data = malloc(DIGEST_BLOCK_SIZE); if (unlikely(!priv->data)) @@ -901,10 +906,6 @@ static int uadk_e_digest_final(EVP_MD_CTX *ctx, unsigned char *digest) priv->req.out_bytes = priv->out_bytes; if (unlikely(priv->switch_flag == UADK_DO_SOFT)) { - if (async_get_async_job()) - goto hw_err; - - /* Synchronous, only the synchronous mode supports soft computing */ ret = digest_soft_final(priv, digest); digest_soft_cleanup(priv); return ret; @@ -950,7 +951,7 @@ hw_err: ret = 0; fprintf(stderr, "do sec digest stream mode failed.\n"); } - + if (op) { (void)async_clear_async_event_notification(); free(op); @@ -1004,8 +1005,29 @@ static int uadk_e_digest_copy(EVP_MD_CTX *to, const EVP_MD_CTX *from) t->setup.sched_param = ¶ms; t->sess = wd_digest_alloc_sess(&t->setup); if (!t->sess) { - fprintf(stderr, "failed to alloc session for digest ctx copy.\n"); - return 0; + if (f->state != SEC_DIGEST_INIT && !f->soft_ctx) { + fprintf(stderr, "HW stream state cannot migrate to soft.\n"); + return 0; + } + t->switch_flag = UADK_DO_SOFT; + t->data = NULL; + t->soft_md = NULL; + t->soft_ctx = NULL; + if (!digest_soft_init(t)) { + fprintf(stderr, "failed to init soft for digest ctx copy.\n"); + return 0; + } + if (f->soft_ctx) { + memcpy(t->soft_ctx->md_data, f->soft_ctx->md_data, + t->app_datasize); + } else if (f->last_update_bufflen) { + if (!digest_soft_update(t, f->data, f->last_update_bufflen)) { + fprintf(stderr, "failed to update for digest ctx copy.\n"); + return 0; + } + t->last_update_bufflen = 0; + } + return 1; } t->data = malloc(DIGEST_BLOCK_SIZE); -- 2.53.0.windows.2
Copy md_nid and md_update_status in sm2_copy so a duplicated EVP_PKEY_CTX keeps the message-digest NID and update-status flag of the source context. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_sm2.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/uadk_sm2.c b/src/uadk_sm2.c index 6472abc..1878ed4 100644 --- a/src/uadk_sm2.c +++ b/src/uadk_sm2.c @@ -1668,6 +1668,8 @@ static int sm2_copy(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src) dctx->ctx.id_len = sctx->ctx.id_len; dctx->ctx.id_set = sctx->ctx.id_set; dctx->ctx.md = sctx->ctx.md; + dctx->md_nid = sctx->md_nid; + dctx->md_update_status = sctx->md_update_status; return 1; } -- 2.53.0.windows.2
The sched_policy and task_type parameters were passed to wd_xxx_init2_ in the wrong order, which caused the initialization to use incorrect scheduling and task type values. Swap them to the correct order so the hardware queue is initialized with the round-robin policy and the expected task type. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_prov_aead.c | 2 +- src/uadk_prov_cipher.c | 2 +- src/uadk_prov_digest.c | 2 +- src/uadk_prov_hmac.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index 93929a0..623bc54 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -316,7 +316,7 @@ static int uadk_prov_aead_dev_init(struct aead_priv_ctx *priv) if (aprov.pid == getpid()) goto free_nodemask; - ret = wd_aead_init2_(priv->alg_name, TASK_MIX, SCHED_POLICY_RR, &cparams); + ret = wd_aead_init2_(priv->alg_name, SCHED_POLICY_RR, TASK_MIX, &cparams); if (unlikely(ret)) { ret = UADK_AEAD_FAIL; UADK_ERR("failed to init aead!\n"); diff --git a/src/uadk_prov_cipher.c b/src/uadk_prov_cipher.c index 2606c2f..caa0cba 100644 --- a/src/uadk_prov_cipher.c +++ b/src/uadk_prov_cipher.c @@ -567,7 +567,7 @@ static int uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) if (prov.pid == getpid()) goto init_err; - ret = wd_cipher_init2_(priv->alg_name, TASK_MIX, SCHED_POLICY_RR, &cparams); + ret = wd_cipher_init2_(priv->alg_name, SCHED_POLICY_RR, TASK_MIX, &cparams); if (unlikely(ret)) { UADK_ERR("failed to init cipher!\n"); ret = UADK_P_FAIL; diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index 0b24a26..2da3a36 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -368,7 +368,7 @@ static int uadk_prov_digest_dev_init(struct digest_priv_ctx *priv) if (dprov.pid == getpid()) goto free_nodemask; - ret = wd_digest_init2_(priv->alg_name, TASK_MIX, SCHED_POLICY_RR, &cparams); + ret = wd_digest_init2_(priv->alg_name, SCHED_POLICY_RR, TASK_MIX, &cparams); if (unlikely(ret && ret != -WD_EEXIST)) { UADK_ERR("uadk failed to initialize digest dev, ret = %d\n", ret); goto free_nodemask; diff --git a/src/uadk_prov_hmac.c b/src/uadk_prov_hmac.c index ad1aa2e..3f7de9e 100644 --- a/src/uadk_prov_hmac.c +++ b/src/uadk_prov_hmac.c @@ -437,7 +437,7 @@ static int uadk_prov_hmac_dev_init(struct hmac_priv_ctx *priv) if (hprov.pid == getpid()) goto free_nodemask; - ret = wd_digest_init2_((char *)alg_name, TASK_MIX, SCHED_POLICY_RR, &cparams); + ret = wd_digest_init2_((char *)alg_name, SCHED_POLICY_RR, TASK_MIX, &cparams); if (unlikely(ret && ret != -WD_EEXIST)) { UADK_ERR("uadk failed to initialize hmac, ret = %d\n", ret); goto free_nodemask; -- 2.53.0.windows.2
The per-cipher hot path allocated the hardware session, set the key and freed the session on every operation, which added avoidable latency. Move these one-shot operations into the einit/dinit/freectx callbacks so the session is allocated once at context initialization, the key is set only when it changes, and the session is freed when the context is destroyed. Introduce alloc_sess/free_sess/set_key/copy_sess helpers for this, and drop the unused data bounce buffer. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_prov_aead.c | 339 ++++++++++++++++++++++++------------------- 1 file changed, 186 insertions(+), 153 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index 623bc54..bb5f6a4 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -91,9 +91,7 @@ struct aead_priv_ctx { unsigned char iv[MAX_IV_LEN]; unsigned char key[MAX_KEY_LEN]; unsigned char buf[AES_GCM_TAG_LEN]; /* mac buffers */ - unsigned char *data; /* store input and output when block mode */ - struct wd_aead_sess_setup setup; struct wd_aead_req req; enum uadk_aead_mode mode; handle_t sess; @@ -218,37 +216,37 @@ static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv, const unsigned c } static int uadk_aead_soft_update(struct aead_priv_ctx *priv, unsigned char *out, - int *outl, const unsigned char *in, size_t len) + size_t *outl, const unsigned char *in, size_t len) { - int ret; + int ret, outsize = 0; if (!priv->sw_aead) return UADK_AEAD_FAIL; if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) - ret = EVP_EncryptUpdate(priv->sw_ctx, out, outl, in, len); + ret = EVP_EncryptUpdate(priv->sw_ctx, out, &outsize, in, len); else - ret = EVP_DecryptUpdate(priv->sw_ctx, out, outl, in, len); + ret = EVP_DecryptUpdate(priv->sw_ctx, out, &outsize, in, len); if (!ret) { UADK_ERR("aead soft update error.\n"); return UADK_AEAD_FAIL; } - priv->stream_switch_flag = UADK_DO_SOFT; + *outl = outsize; return UADK_AEAD_SUCCESS; } static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *digest, size_t *outl) { - int ret; + int ret, outsize = 0; if (!priv->sw_aead) goto error; if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) { - ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, &outsize); if (!ret) goto error; @@ -262,11 +260,12 @@ static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *diges if (!ret) goto error; - ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, &outsize); if (!ret) goto error; } + *outl = 0; priv->stream_switch_flag = 0; return UADK_AEAD_SUCCESS; @@ -276,19 +275,6 @@ error: return UADK_AEAD_FAIL; } -static void uadk_aead_soft_cleanup(struct aead_priv_ctx *priv) -{ - if (priv->sw_ctx) { - EVP_CIPHER_CTX_free(priv->sw_ctx); - priv->sw_ctx = NULL; - } - - if (priv->sw_aead) { - EVP_CIPHER_free(priv->sw_aead); - priv->sw_aead = NULL; - } -} - static int uadk_prov_aead_dev_init(struct aead_priv_ctx *priv) { struct wd_ctx_nums ctx_set_num; @@ -335,8 +321,6 @@ free_nodemask: static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) { - struct wd_aead_sess_setup setup = {0}; - struct sched_params params = {0}; int ret; if (!priv->key_set || !priv->iv_set) { @@ -350,44 +334,13 @@ static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) priv->req.mac = priv->buf; priv->req.mac_bytes = priv->taglen; - ret = uadk_prov_aead_dev_init(priv); - if (unlikely(ret < 0)) + ret = wd_aead_set_authsize(priv->sess, priv->taglen); + if (ret) { + UADK_ERR("uadk failed to set authsize!\n"); return UADK_AEAD_FAIL; - - /* dec and enc use the same op */ - params.type = 0; - /* Use the default numa parameters */ - params.numa_id = -1; - setup.sched_param = ¶ms; - setup.calg = priv->setup.calg; - setup.cmode = priv->setup.cmode; - - if (!priv->sess) { - priv->sess = wd_aead_alloc_sess(&setup); - if (!priv->sess) { - UADK_ERR("uadk failed to alloc session!\n"); - return UADK_AEAD_FAIL; - } - - ret = wd_aead_set_authsize(priv->sess, priv->taglen); - if (ret) { - UADK_ERR("uadk failed to set authsize!\n"); - goto free_sess; - } - - ret = wd_aead_set_ckey(priv->sess, priv->key, priv->keylen); - if (ret) { - UADK_ERR("uadk failed to set key!\n"); - goto free_sess; - } } return UADK_AEAD_SUCCESS; - -free_sess: - wd_aead_free_sess(priv->sess); - priv->sess = 0; - return UADK_AEAD_FAIL; } static void *uadk_prov_aead_cb(struct wd_aead_req *req, void *data) @@ -558,7 +511,7 @@ static int uadk_do_aead_async_inner(struct aead_priv_ctx *priv, struct async_op } static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { struct async_op op; int ret; @@ -589,6 +542,8 @@ static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char if (unlikely(ret < 0)) goto soft; + *outl = 0; + return UADK_AEAD_SUCCESS; free_notification: @@ -653,11 +608,13 @@ free_notification: } static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { if (priv->stream_switch_flag == UADK_DO_SOFT) return SWITCH_TO_SOFT; + *outl = inlen; + if (priv->mode == ASYNC_MODE) return uadk_do_aead_async(priv, out, in, inlen); @@ -665,7 +622,7 @@ static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char } static int uadk_prov_do_aes_gcm_final(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { struct async_op op; int ret; @@ -702,16 +659,35 @@ out: priv->tag_set = INIT_TAG; priv->mode = UNINIT_MODE; - + *outl = 0; return UADK_AEAD_SUCCESS; } +static int uadk_prov_sw_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, + size_t *outl, const unsigned char *in, size_t inlen) +{ + int ret; + + if (priv->stream_switch_flag != UADK_DO_SOFT) { + ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); + if (ret <= 0) + return UADK_OSSL_FAIL; + } + + if (in) + return uadk_aead_soft_update(priv, out, outl, in, inlen); + + return uadk_aead_soft_final(priv, out, outl); +} + static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, - size_t *outl, size_t outsize, - const unsigned char *in, size_t inlen) + size_t *outl, const unsigned char *in, size_t inlen) { int ret; + if (priv->stream_switch_flag == UADK_DO_SOFT || !priv->sess) + return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); + ret = uadk_prov_aead_ctx_init(priv); if (ret != UADK_AEAD_SUCCESS) return UADK_AEAD_FAIL; @@ -722,12 +698,16 @@ static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, if (in) { if (!out) - return uadk_prov_do_aes_gcm_first(priv, out, in, inlen); - - return uadk_prov_do_aes_gcm_update(priv, out, in, inlen); + ret = uadk_prov_do_aes_gcm_first(priv, out, outl, in, inlen); + else + ret = uadk_prov_do_aes_gcm_update(priv, out, outl, in, inlen); + } else { + ret = uadk_prov_do_aes_gcm_final(priv, out, outl, NULL, 0); } + if (ret == SWITCH_TO_SOFT) + return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); - return uadk_prov_do_aes_gcm_final(priv, out, NULL, 0); + return ret; } void uadk_prov_destroy_aead(void) @@ -764,11 +744,12 @@ static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, return UADK_OSSL_FAIL; } - ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, in, inl); - if (ret < 0) + ret = uadk_prov_do_aes_gcm(priv, out, outl, in, inl); + if (ret <= 0) { + *outl = 0; return UADK_OSSL_FAIL; + } - *outl = inl; return UADK_AEAD_SUCCESS; } @@ -777,7 +758,7 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, const unsigned char *in, size_t inl) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; - int ret, outlen; + int ret; if (!vctx) return UADK_OSSL_FAIL; @@ -787,31 +768,12 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, return UADK_OSSL_FAIL; } - if (priv->stream_switch_flag == UADK_DO_SOFT) - goto do_soft; - ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, in, inl); - if (ret == SWITCH_TO_SOFT) - goto do_soft; - else if (ret < 0) { - UADK_ERR("stream data update failed.\n"); + ret = uadk_prov_do_aes_gcm(priv, out, outl, in, inl); + if (ret <= 0) { + *outl = 0; return UADK_OSSL_FAIL; - } else { - *outl = inl; - return UADK_AEAD_SUCCESS; - } - -do_soft: - if (priv->stream_switch_flag != UADK_DO_SOFT) { - ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); - if (ret <= 0) - return UADK_OSSL_FAIL; } - ret = uadk_aead_soft_update(priv, out, &outlen, in, inl); - if (ret <= 0) - return UADK_OSSL_FAIL; - - *outl = outlen; return UADK_AEAD_SUCCESS; } @@ -824,37 +786,24 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, if (!vctx || !out || !outl) return UADK_OSSL_FAIL; - if (priv->stream_switch_flag == UADK_DO_SOFT) - goto do_soft; - - ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, NULL, 0); - if (ret < 0) { - UADK_ERR("stream data final failed, ret = %d\n", ret); + ret = uadk_prov_do_aes_gcm(priv, out, outl, NULL, 0); + if (ret <= 0) { + *outl = 0; return UADK_OSSL_FAIL; } - *outl = 0; return UADK_AEAD_SUCCESS; - -do_soft: - ret = uadk_aead_soft_final(priv, out, outl); - if (ret) { - *outl = 0; - return UADK_AEAD_SUCCESS; - } - - return UADK_OSSL_FAIL; } -static int uadk_get_aead_info(struct aead_priv_ctx *priv) +static int uadk_get_aead_info(struct wd_aead_sess_setup *setup, int nid) { int aead_counts = ARRAY_SIZE(aead_info_table); int i; for (i = 0; i < aead_counts; i++) { - if (priv->nid == aead_info_table[i].nid) { - priv->setup.calg = aead_info_table[i].alg; - priv->setup.cmode = aead_info_table[i].mode; + if (nid == aead_info_table[i].nid) { + setup->calg = aead_info_table[i].alg; + setup->cmode = aead_info_table[i].mode; break; } } @@ -867,6 +816,72 @@ static int uadk_get_aead_info(struct aead_priv_ctx *priv) return UADK_AEAD_SUCCESS; } +static int uadk_prov_aead_alloc_sess(struct aead_priv_ctx *priv) +{ + struct wd_aead_sess_setup setup = {0}; + struct sched_params params = {0}; + int ret; + + if (priv->sess) + return UADK_AEAD_SUCCESS; + + ret = uadk_prov_aead_dev_init(priv); + if (unlikely(ret < 0)) + return SWITCH_TO_SOFT; + + ret = uadk_get_aead_info(&setup, priv->nid); + if (unlikely(ret < 0)) + return UADK_OSSL_FAIL; + + /* dec and enc use the same op */ + params.type = 0; + /* Use the default numa parameters */ + params.numa_id = -1; + setup.sched_param = ¶ms; + priv->sess = wd_aead_alloc_sess(&setup); + if (!priv->sess) { + UADK_ERR("uadk failed to alloc session, switch to soft\n"); + return SWITCH_TO_SOFT; + } + + if (priv->key_set == KEY_STATE_SET) { + ret = wd_aead_set_ckey(priv->sess, priv->key, priv->keylen); + if (ret) { + UADK_ERR("uadk failed to set key!\n"); + return UADK_OSSL_FAIL; + } + } + + return UADK_AEAD_SUCCESS; +} + +static int uadk_prov_aead_set_key(struct aead_priv_ctx *priv, + const unsigned char *key, + size_t keylen) +{ + int ret; + + if (keylen != priv->keylen) { + UADK_ERR("invalid keylen %zu!\n", keylen); + return UADK_OSSL_FAIL; + } + + memcpy(priv->key, key, keylen); + priv->key_set = KEY_STATE_SET; + + /* use default provider */ + if (!priv->sess) + return UADK_AEAD_SUCCESS; + + ret = wd_aead_set_ckey(priv->sess, priv->key, priv->keylen); + if (ret) { + UADK_ERR("uadk failed to set key!\n"); + return UADK_OSSL_FAIL; + } + + return UADK_AEAD_SUCCESS; +} + static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char *key, size_t keylen, const unsigned char *iv, size_t ivlen, const OSSL_PARAM *params) { @@ -877,32 +892,26 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char * return UADK_OSSL_FAIL; } + /* will free in freectx */ + ret = uadk_prov_aead_alloc_sess(priv); + if (ret == UADK_OSSL_FAIL) + return UADK_OSSL_FAIL; + if (iv) { memcpy(priv->iv, iv, ivlen); priv->iv_set = IV_STATE_SET; } - ret = uadk_get_aead_info(priv); - if (unlikely(ret < 0)) - return UADK_OSSL_FAIL; - if (key) { - memcpy(priv->key, key, keylen); - priv->key_set = KEY_STATE_SET; + ret = uadk_prov_aead_set_key(priv, key, keylen); + if (ret == UADK_OSSL_FAIL) + return UADK_OSSL_FAIL; } priv->stream_switch_flag = 0; + priv->req.msg_state = AEAD_MSG_INVALID; - if (uadk_get_sw_offload_state()) - uadk_create_aead_soft_ctx(priv); - - ret = uadk_prov_aead_dev_init(priv); - if (unlikely(ret < 0)) { - UADK_ERR("aead switch to soft init.!\n"); - return uadk_prov_aead_soft_init(priv, key, iv, params); - } - - return UADK_AEAD_SUCCESS; + return uadk_prov_aead_set_ctx_params(priv, params); } static int uadk_prov_aead_einit(void *vctx, const unsigned char *key, size_t keylen, @@ -1152,6 +1161,30 @@ static int uadk_cipher_aead_get_params(OSSL_PARAM params[], unsigned int md, return UADK_AEAD_SUCCESS; } +static void uadk_prov_aead_free_sess(struct aead_priv_ctx *priv) +{ + if (priv->sess) + wd_aead_free_sess(priv->sess); +} + +static int uadk_prov_aead_copy_sess(struct aead_priv_ctx *priv) +{ + if (!priv->sess) + return UADK_AEAD_SUCCESS; + priv->sess = 0; + + /* + * Encryption and decryption have already started, so it cannot + * switch to software calculation, hence it returns a failure. + */ + if (priv->req.msg_state != AEAD_MSG_INVALID) { + UADK_ERR("invalid: The data has been processed by hardware, cannot be copied.\n"); + return UADK_OSSL_FAIL; + } + + return uadk_prov_aead_alloc_sess(priv); +} + static void *uadk_prov_aead_dupctx(void *ctx) { struct aead_priv_ctx *dst_ctx, *src_ctx; @@ -1165,16 +1198,15 @@ static void *uadk_prov_aead_dupctx(void *ctx) if (!dst_ctx) return NULL; - dst_ctx->sess = 0; - dst_ctx->data = OPENSSL_memdup(src_ctx->data, AEAD_BLOCK_SIZE << 1); - if (!dst_ctx->data) + ret = uadk_prov_aead_copy_sess(dst_ctx); + if (ret == UADK_OSSL_FAIL) goto free_ctx; if (dst_ctx->sw_ctx) { dst_ctx->sw_ctx = EVP_CIPHER_CTX_dup(src_ctx->sw_ctx); if (!dst_ctx->sw_ctx) { UADK_ERR("EVP_CIPHER_CTX_dup failed in ctx copy.\n"); - goto free_data; + goto free_sess; } ret = EVP_CIPHER_up_ref(dst_ctx->sw_aead); @@ -1187,13 +1219,22 @@ static void *uadk_prov_aead_dupctx(void *ctx) free_dup: if (dst_ctx->sw_ctx) EVP_CIPHER_CTX_free(dst_ctx->sw_ctx); -free_data: - OPENSSL_clear_free(dst_ctx->data, AEAD_BLOCK_SIZE << 1); +free_sess: + uadk_prov_aead_free_sess(dst_ctx); free_ctx: OPENSSL_clear_free(dst_ctx, sizeof(*dst_ctx)); return NULL; } +static void uadk_aead_soft_cleanup(struct aead_priv_ctx *priv) +{ + if (priv->sw_ctx) + EVP_CIPHER_CTX_free(priv->sw_ctx); + + if (priv->sw_aead) + EVP_CIPHER_free(priv->sw_aead); +} + static void uadk_prov_aead_freectx(void *ctx) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)ctx; @@ -1201,15 +1242,8 @@ static void uadk_prov_aead_freectx(void *ctx) if (!ctx) return; - if (priv->sess) - wd_aead_free_sess(priv->sess); - - if (priv->data) - OPENSSL_clear_free(priv->data, AEAD_BLOCK_SIZE << 1); - - if (priv->sw_ctx) - uadk_aead_soft_cleanup(priv); - + uadk_prov_aead_free_sess(priv); + uadk_aead_soft_cleanup(priv); OPENSSL_clear_free(priv, sizeof(*priv)); } @@ -1218,15 +1252,11 @@ static void uadk_prov_aead_freectx(void *ctx) static OSSL_FUNC_cipher_newctx_fn uadk_##nm##_newctx; \ static void *uadk_##nm##_newctx(void *provctx) \ { \ - struct aead_priv_ctx *ctx = OPENSSL_zalloc(sizeof(*ctx)); \ - if (!ctx) \ - return NULL; \ + struct aead_priv_ctx *ctx; \ \ - ctx->data = OPENSSL_zalloc(AEAD_BLOCK_SIZE << 1); \ - if (!ctx->data) { \ - OPENSSL_free(ctx); \ + ctx = OPENSSL_zalloc(sizeof(*ctx)); \ + if (!ctx) \ return NULL; \ - } \ \ ctx->keylen = key_len; \ ctx->ivlen = iv_len; \ @@ -1234,6 +1264,9 @@ static void *uadk_##nm##_newctx(void *provctx) \ ctx->taglen = tag_len; \ strncpy(ctx->alg_name, #algnm, ALG_NAME_SIZE - 1); \ \ + if (uadk_get_sw_offload_state()) \ + uadk_create_aead_soft_ctx(ctx); \ + \ return ctx; \ } \ static OSSL_FUNC_cipher_get_params_fn uadk_##nm##_get_params; \ -- 2.53.0.windows.2
UADK hardware requires middle GCM packets to be 16-byte aligned. Buffer the sub-block remainder and process it with AES-CTR through the default provider, then fold the result back into the GCM stream. Rework the first/update/final flow and add ctr_iv_inc/reset_ctx helpers to support this. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_prov_aead.c | 524 ++++++++++++++++++++++++------------------- 1 file changed, 290 insertions(+), 234 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index bb5f6a4..ad990c2 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -58,6 +58,15 @@ #define UADK_AEAD_DEF_CTXS 2 #define UADK_AEAD_OP_NUM 1 +#define AES_CTR_IV_LEN 16 +#define GCM_IV_DEFAULT_SIZE 12 +#define AES_GCM_COUNTER_SIZE 4 + +#define AES_BLOCK_OFFSET 4 +#define AES_CTR_COUNTER_SIZE 8 +#define BYTE_TO_BITS 8 +#define ALIGN_DOWN(x, align) ((x) & ~((align) - 1)) + struct aead_prov { int pid; }; @@ -99,6 +108,9 @@ struct aead_priv_ctx { int stream_switch_flag; /* soft calculation switch flag for stream mode */ EVP_CIPHER_CTX *sw_ctx; EVP_CIPHER *sw_aead; + unsigned char partial_data[AES_BLOCK_SIZE]; + OSSL_LIB_CTX *libctx; + size_t partial_len; }; struct aead_info { @@ -160,13 +172,13 @@ static int uadk_create_aead_soft_ctx(struct aead_priv_ctx *priv) switch (priv->nid) { case NID_aes_128_gcm: - priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-128-GCM", "provider=default"); + priv->sw_aead = EVP_CIPHER_fetch(priv->libctx, "AES-128-GCM", "provider=default"); break; case NID_aes_192_gcm: - priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-192-GCM", "provider=default"); + priv->sw_aead = EVP_CIPHER_fetch(priv->libctx, "AES-192-GCM", "provider=default"); break; case NID_aes_256_gcm: - priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-256-GCM", "provider=default"); + priv->sw_aead = EVP_CIPHER_fetch(priv->libctx, "AES-256-GCM", "provider=default"); break; default: break; @@ -192,8 +204,7 @@ free: return UADK_AEAD_FAIL; } -static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv, const unsigned char *key, - const unsigned char *iv, const OSSL_PARAM *params) +static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv) { int ret; @@ -201,9 +212,11 @@ static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv, const unsigned c return UADK_AEAD_FAIL; if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) - ret = EVP_EncryptInit_ex2(priv->sw_ctx, priv->sw_aead, key, iv, params); + ret = EVP_EncryptInit_ex2(priv->sw_ctx, priv->sw_aead, + priv->key, priv->iv, NULL); else - ret = EVP_DecryptInit_ex2(priv->sw_ctx, priv->sw_aead, key, iv, params); + ret = EVP_DecryptInit_ex2(priv->sw_ctx, priv->sw_aead, + priv->key, priv->iv, NULL); if (!ret) { UADK_ERR("aead soft init error!\n"); @@ -238,12 +251,21 @@ static int uadk_aead_soft_update(struct aead_priv_ctx *priv, unsigned char *out, return UADK_AEAD_SUCCESS; } +static void uadk_prov_aead_reset_ctx(struct aead_priv_ctx *priv) +{ + priv->stream_switch_flag = 0; + priv->req.assoc_bytes = 0; + priv->partial_len = 0; + priv->mode = UNINIT_MODE; + priv->req.msg_state = AEAD_MSG_INVALID; +} + static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *digest, size_t *outl) { int ret, outsize = 0; if (!priv->sw_aead) - goto error; + return UADK_OSSL_FAIL; if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) { ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, &outsize); @@ -252,8 +274,8 @@ static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *diges ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_GET_TAG, priv->taglen, priv->buf); - if (!ret) - goto error; + if (ret == UADK_AEAD_SUCCESS) + priv->tag_set = SET_TAG; } else { ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_SET_TAG, priv->taglen, priv->buf); @@ -261,18 +283,14 @@ static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *diges goto error; ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, &outsize); - if (!ret) - goto error; } - *outl = 0; - priv->stream_switch_flag = 0; - - return UADK_AEAD_SUCCESS; - error: - UADK_ERR("aead soft final failed.\n"); - return UADK_AEAD_FAIL; + if (!ret) + UADK_ERR("aead soft final failed.\n"); + *outl = 0; + uadk_prov_aead_reset_ctx(priv); + return ret; } static int uadk_prov_aead_dev_init(struct aead_priv_ctx *priv) @@ -330,11 +348,15 @@ static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) priv->req.iv_bytes = priv->ivlen; priv->req.iv = priv->iv; + /* Initialize the counter value for CTR (Counter) mode encryption. */ + memset(priv->iv + GCM_IV_DEFAULT_SIZE, 0, AES_GCM_COUNTER_SIZE); + priv->iv[AES_CTR_IV_LEN - 1] = 0x2; + priv->req.out_bytes = 0; priv->req.mac = priv->buf; - priv->req.mac_bytes = priv->taglen; + priv->req.mac_bytes = AES_GCM_TAG_LEN; - ret = wd_aead_set_authsize(priv->sess, priv->taglen); + ret = wd_aead_set_authsize(priv->sess, AES_GCM_TAG_LEN); if (ret) { UADK_ERR("uadk failed to set authsize!\n"); return UADK_AEAD_FAIL; @@ -365,125 +387,41 @@ static void *uadk_prov_aead_cb(struct wd_aead_req *req, void *data) return NULL; } -static int do_aes_gcm_prepare(struct aead_priv_ctx *priv) -{ - if (priv->mode == UNINIT_MODE) { - if (ASYNC_get_current_job()) - priv->mode = ASYNC_MODE; - else - priv->mode = SYNC_MODE; - } - - if (!priv->enc && priv->tag_set == READ_TAG) { - if (likely(priv->taglen == AES_GCM_TAG_LEN)) { - memcpy(priv->req.mac, priv->buf, AES_GCM_TAG_LEN); - priv->tag_set = SET_TAG; - } else { - UADK_ERR("invalid: aead gcm mac length only support 16B.\n"); - return UADK_AEAD_FAIL; - } - } - - return UADK_AEAD_SUCCESS; -} - -static int uadk_do_aead_sync_inner(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen, - enum wd_aead_msg_state state) +static int uadk_do_aead_sync_inner(struct aead_priv_ctx *priv) { int ret; - if ((state == AEAD_MSG_BLOCK || state == AEAD_MSG_END) - && !priv->enc && priv->tag_set != SET_TAG) { - UADK_ERR("The tag for synchronous decryption is not set.\n"); - return UADK_AEAD_FAIL; - } - - priv->req.msg_state = state; - priv->req.src = (unsigned char *)in; - priv->req.dst = out; - priv->req.in_bytes = inlen; - priv->req.state = 0; ret = wd_do_aead_sync(priv->sess, &priv->req); if (unlikely(ret < 0 || priv->req.state)) { - UADK_ERR("do aead task failed, msg state: %u, ret: %d, state: %u!\n", - state, ret, priv->req.state); + UADK_ERR("do aead sync task failed, ret: %d, state: %u!\n", + ret, priv->req.state); return UADK_AEAD_FAIL; } return UADK_AEAD_SUCCESS; } -static int uadk_do_aead_sync(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) -{ - size_t nbytes, tail, processing_len, max_mid_len; - const unsigned char *in_block = in; - unsigned char *out_block = out; - int ret; - - tail = inlen % AES_BLOCK_SIZE; - nbytes = inlen - tail; - max_mid_len = AEAD_BLOCK_SIZE - priv->req.assoc_bytes; - - /* If the data length is not 16-byte aligned, it is split according to the protocol. */ - while (nbytes > 0) { - processing_len = nbytes > max_mid_len ? max_mid_len : nbytes; - processing_len -= (processing_len % AES_BLOCK_SIZE); - - ret = uadk_do_aead_sync_inner(priv, out_block, in_block, - processing_len, AEAD_MSG_MIDDLE); - if (ret < 0) - return UADK_AEAD_FAIL; - nbytes -= processing_len; - in_block = in_block + processing_len; - out_block = out_block + processing_len; - } - - if (tail) { - ret = uadk_do_aead_sync_inner(priv, out_block, in_block, tail, AEAD_MSG_END); - if (ret < 0) - return UADK_AEAD_FAIL; - } - - return UADK_AEAD_SUCCESS; -} - -static int uadk_do_aead_async_inner(struct aead_priv_ctx *priv, struct async_op *op, - unsigned char *out, const unsigned char *in, size_t inlen) +static int uadk_do_aead_async_inner(struct aead_priv_ctx *priv) { struct uadk_e_cb_info cb_param; + struct async_op op; int cnt = 0; int ret; - if ((priv->req.msg_state == AEAD_MSG_BLOCK || priv->req.msg_state == AEAD_MSG_END) - && !priv->enc && priv->tag_set != SET_TAG) { - UADK_ERR("The tag for asynchronous decryption is not set.\n"); - return UADK_AEAD_FAIL; - } - - if (unlikely(priv->req.assoc_bytes + inlen > AEAD_BLOCK_SIZE)) { - UADK_ERR("aead input data length is too long!\n"); + ret = async_setup_async_event_notification(&op); + if (unlikely(!ret)) { + UADK_ERR("failed to setup async event notification.\n"); return UADK_AEAD_FAIL; } - cb_param.op = op; + cb_param.op = &op; cb_param.priv = &priv->req; priv->req.cb = uadk_prov_aead_cb; priv->req.cb_param = &cb_param; - priv->req.state = POLL_ERROR; - priv->req.src = (unsigned char *)in; - priv->req.dst = out; - priv->req.in_bytes = inlen; - if (unlikely(!priv->sess)) { - UADK_ERR("uadk session is NULL!\n"); - return UADK_AEAD_FAIL; - } - - ret = async_get_free_task(&op->idx); + ret = async_get_free_task(&op.idx); if (unlikely(!ret)) - return UADK_AEAD_FAIL; + goto free_notification; do { ret = wd_do_aead_async(priv->sess, &priv->req); @@ -495,172 +433,295 @@ static int uadk_do_aead_async_inner(struct aead_priv_ctx *priv, struct async_op else continue; - async_free_poll_task(op->idx, 0); - return UADK_AEAD_FAIL; + async_free_poll_task(op.idx, 0); + goto free_notification; } } while (ret == -EBUSY); - ret = async_pause_job(priv, op, ASYNC_TASK_AEAD); + ret = async_pause_job(priv, &op, ASYNC_TASK_AEAD); if (unlikely(!ret || priv->req.state)) { UADK_ERR("do aead async job failed, ret: %d, state: %u!\n", ret, priv->req.state); - return UADK_AEAD_FAIL; + goto free_notification; } - return ret; + return UADK_AEAD_SUCCESS; + +free_notification: + (void)async_clear_async_event_notification(); + return UADK_AEAD_FAIL; +} + +static int uadk_do_aes_gcm_inner(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t inlen, + enum wd_aead_msg_state state) +{ + priv->req.msg_state = state; + priv->req.src = (unsigned char *)in; + priv->req.dst = out; + priv->req.in_bytes = inlen; + priv->req.state = POLL_ERROR; + + if (priv->mode == ASYNC_MODE) + return uadk_do_aead_async_inner(priv); + + return uadk_do_aead_sync_inner(priv); } static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char *out, size_t *outl, const unsigned char *in, size_t inlen) { - struct async_op op; int ret; if (inlen > MAX_AAD_LEN || !inlen) - goto soft; - - priv->req.assoc_bytes = inlen; + return SWITCH_TO_SOFT; - if (priv->mode == ASYNC_MODE) { - ret = async_setup_async_event_notification(&op); - if (unlikely(!ret)) { - UADK_ERR("failed to setup async event notification.\n"); - goto soft; - } + ret = uadk_prov_aead_ctx_init(priv); + if (ret != UADK_AEAD_SUCCESS) + return UADK_AEAD_FAIL; - priv->req.msg_state = AEAD_MSG_FIRST; - ret = uadk_do_aead_async_inner(priv, &op, out, in, inlen); - if (unlikely(ret < 0)) { - UADK_ERR("aead async first failed, switch to soft.\n"); - goto free_notification; - } + if (ASYNC_get_current_job()) + priv->mode = ASYNC_MODE; + else + priv->mode = SYNC_MODE; - return UADK_AEAD_SUCCESS; + priv->req.assoc_bytes = inlen; + ret = uadk_do_aes_gcm_inner(priv, out, in, inlen, AEAD_MSG_FIRST); + if (ret == UADK_AEAD_FAIL) { + priv->req.msg_state = AEAD_MSG_INVALID; + priv->req.assoc_bytes = 0; + UADK_ERR("aead failed to update aad, switch to soft.\n"); + return SWITCH_TO_SOFT; } - ret = uadk_do_aead_sync_inner(priv, out, in, inlen, AEAD_MSG_FIRST); - if (unlikely(ret < 0)) - goto soft; - *outl = 0; return UADK_AEAD_SUCCESS; +} -free_notification: - (void)async_clear_async_event_notification(); -soft: - UADK_ERR("aead failed to update aad, switch to soft.\n"); - return SWITCH_TO_SOFT; +/* + * Increment counter (128-bit int) by software, + * in CTR mode, the last 8 bytes are the counter. + */ +static void ctr_iv_inc(__u8 *counter, __u32 len) +{ + __u32 n = AES_CTR_COUNTER_SIZE; + __u32 c = len; + + do { + --n; + c += counter[n]; + counter[n] = (__u8)c; + c >>= BYTE_TO_BITS; + } while (n); } -static int uadk_do_aead_async(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) +static int uadk_prov_process_partial_data(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t inlen, + size_t *processed_len) { - size_t nbytes, tail, processing_len, max_mid_len; - const unsigned char *in_block = in; - unsigned char *out_block = out; - struct async_op op; + size_t processing_len = AES_BLOCK_SIZE - priv->partial_len; + unsigned char block_out[AES_BLOCK_SIZE]; int ret; - ret = async_setup_async_event_notification(&op); - if (unlikely(!ret)) { - UADK_ERR("failed to setup async event notification.\n"); + if (!priv->partial_len) + return UADK_AEAD_SUCCESS; + + /* If input can't complete the partial block, switch to soft */ + if (inlen < processing_len) + return SWITCH_TO_SOFT; + + memcpy(priv->partial_data + priv->partial_len, in, processing_len); + ret = uadk_do_aes_gcm_inner(priv, block_out, priv->partial_data, + AES_BLOCK_SIZE, AEAD_MSG_MIDDLE); + if (unlikely(ret == UADK_AEAD_FAIL)) { + UADK_ERR("failed to process partial block.\n"); return UADK_AEAD_FAIL; } - tail = inlen % AES_BLOCK_SIZE; - nbytes = inlen - tail; - max_mid_len = AEAD_BLOCK_SIZE - priv->req.assoc_bytes; + memcpy(out, block_out + priv->partial_len, processing_len); + priv->partial_len = 0; + ctr_iv_inc(priv->iv + AES_CTR_COUNTER_SIZE, 1); + *processed_len = processing_len; - /* Middle packets processing */ - while (nbytes > 0) { - processing_len = nbytes > max_mid_len ? max_mid_len : nbytes; - processing_len -= (processing_len % AES_BLOCK_SIZE); + return UADK_AEAD_SUCCESS; +} - priv->req.msg_state = AEAD_MSG_MIDDLE; - ret = uadk_do_aead_async_inner(priv, &op, out_block, in_block, - processing_len); - if (unlikely(ret < 0)) { - UADK_ERR("aead async middle failed!\n"); - goto free_notification; - } - nbytes -= processing_len; - in_block = in_block + processing_len; - out_block = out_block + processing_len; +/* Process last incomplete block, encrypt/decrypt using OpenSSL software implementation */ +static int uadk_prov_process_tail_data(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t inlen) +{ + unsigned char block_out[AES_BLOCK_SIZE]; + EVP_CIPHER *cipher = NULL; + int ret = UADK_AEAD_FAIL; + EVP_CIPHER_CTX *ctx; + int outsize = 0; + + if (!inlen) + return UADK_AEAD_SUCCESS; + + /* Buffer the tail data */ + memcpy(priv->partial_data + priv->partial_len, in, inlen); + + ctx = EVP_CIPHER_CTX_new(); + if (!ctx) + return UADK_AEAD_FAIL; + + switch (priv->nid) { + case NID_aes_128_gcm: + cipher = EVP_CIPHER_fetch(priv->libctx, "AES-128-CTR", "provider=default"); + break; + case NID_aes_192_gcm: + cipher = EVP_CIPHER_fetch(priv->libctx, "AES-192-CTR", "provider=default"); + break; + case NID_aes_256_gcm: + cipher = EVP_CIPHER_fetch(priv->libctx, "AES-256-CTR", "provider=default"); + break; + default: + break; } + if (!cipher) + goto free_ctx; - /* Tail packet processing */ - if (tail) { - priv->req.msg_state = AEAD_MSG_END; - ret = uadk_do_aead_async_inner(priv, &op, out_block, in_block, tail); - if (unlikely(ret < 0)) { - UADK_ERR("aead async tail failed!\n"); - goto free_notification; + ret = EVP_CipherInit_ex2(ctx, cipher, priv->key, priv->iv, priv->enc, NULL); + if (!ret) + goto free_cipher; + + ret = EVP_CipherUpdate(ctx, block_out, &outsize, priv->partial_data, + priv->partial_len + inlen); + if (!ret) + goto free_cipher; + + ret = EVP_CipherFinal_ex(ctx, block_out + priv->partial_len + inlen, &outsize); + if (!ret) + goto free_cipher; + + memcpy(out, block_out + priv->partial_len, inlen); + priv->partial_len += inlen; + +free_cipher: + EVP_CIPHER_free(cipher); +free_ctx: + EVP_CIPHER_CTX_free(ctx); + return ret; +} + +/* Process complete blocks in bulk */ +static int uadk_process_complete_blocks(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t len) +{ + size_t max_mid_len = AEAD_BLOCK_SIZE - priv->req.assoc_bytes; + size_t remain_len = len; + size_t chunk; + int ret; + + while (remain_len > 0) { + chunk = (remain_len > max_mid_len) ? max_mid_len : remain_len; + chunk = ALIGN_DOWN(chunk, AES_BLOCK_SIZE); + + ret = uadk_do_aes_gcm_inner(priv, out, in, chunk, AEAD_MSG_MIDDLE); + if (unlikely(ret == UADK_AEAD_FAIL)) { + UADK_ERR("failed to process complete block.\n"); + return UADK_AEAD_FAIL; } - } - return UADK_AEAD_SUCCESS; + remain_len -= chunk; + out += chunk; + in += chunk; + } -free_notification: - (void)async_clear_async_event_notification(); + ctr_iv_inc(priv->iv + AES_CTR_COUNTER_SIZE, len >> AES_BLOCK_OFFSET); - return UADK_AEAD_FAIL; + return UADK_AEAD_SUCCESS; } +/* + * The uadk does not support the scenario where the length of the intermediate + * packet is not 16-byte aligned. To avoid task failures, AES-CTR is used for + * encryption and decryption, and the data and the next task are combined to make the + * length 16-byte aligned. Then, the uadk calculates the hash value. However, it is + * recommended that the packet length be aligned to ensure that the performance is not + * affected by this problem. + */ static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char *out, size_t *outl, const unsigned char *in, size_t inlen) { - if (priv->stream_switch_flag == UADK_DO_SOFT) + size_t remain_len = inlen; + size_t processed_len = 0; + int ret; + + if (!priv->req.assoc_bytes) return SWITCH_TO_SOFT; *outl = inlen; + /* Process buffered partial data */ + ret = uadk_prov_process_partial_data(priv, out, in, remain_len, &processed_len); + if (ret == SWITCH_TO_SOFT) + goto soft_fallback; + else if (ret < 0) + return UADK_AEAD_FAIL; - if (priv->mode == ASYNC_MODE) - return uadk_do_aead_async(priv, out, in, inlen); + out += processed_len; + in += processed_len; + remain_len -= processed_len; - return uadk_do_aead_sync(priv, out, in, inlen); + if (remain_len >= AES_BLOCK_SIZE) { + processed_len = ALIGN_DOWN(remain_len, AES_BLOCK_SIZE); + ret = uadk_process_complete_blocks(priv, out, in, processed_len); + if (ret != UADK_AEAD_SUCCESS) + return UADK_AEAD_FAIL; + remain_len -= processed_len; + out += processed_len; + in += processed_len; + } + +soft_fallback: + return uadk_prov_process_tail_data(priv, out, in, remain_len); } static int uadk_prov_do_aes_gcm_final(struct aead_priv_ctx *priv, unsigned char *out, size_t *outl, const unsigned char *in, size_t inlen) { - struct async_op op; + unsigned char block_out[AES_BLOCK_SIZE]; int ret; - if (!priv->req.assoc_bytes || priv->req.msg_state == AEAD_MSG_END) - goto out; + if (!priv->req.assoc_bytes) + return SWITCH_TO_SOFT; - if (priv->mode == ASYNC_MODE) { - ret = async_setup_async_event_notification(&op); - if (unlikely(!ret)) { - UADK_ERR("failed to setup async event notification.\n"); - return UADK_AEAD_FAIL; + if (!priv->enc) { + if (priv->tag_set != READ_TAG) { + UADK_ERR("decrypt tag not set.\n"); + ret = UADK_OSSL_FAIL; + goto out; } - priv->req.msg_state = AEAD_MSG_END; - ret = uadk_do_aead_async_inner(priv, &op, out, in, inlen); - if (unlikely(ret < 0)) { - UADK_ERR("aead async final failed!\n"); - (void)async_clear_async_event_notification(); - return UADK_AEAD_FAIL; + if (priv->taglen != AES_GCM_TAG_LEN) { + ret = wd_aead_set_authsize(priv->sess, priv->taglen); + if (ret) { + ret = UADK_OSSL_FAIL; + goto out; + } } + } + if (priv->partial_len) + ret = uadk_do_aes_gcm_inner(priv, block_out, priv->partial_data, + priv->partial_len, AEAD_MSG_END); + else + ret = uadk_do_aes_gcm_inner(priv, out, in, inlen, AEAD_MSG_END); + if (unlikely(ret == UADK_AEAD_FAIL)) { + UADK_ERR("uadk_prov_do_aes_gcm_final failed.\n"); goto out; } - ret = uadk_do_aead_sync_inner(priv, out, in, inlen, AEAD_MSG_END); - if (unlikely(ret < 0)) - return UADK_AEAD_FAIL; - -out: if (priv->enc) - memcpy(priv->buf, priv->req.mac, priv->taglen); - else - priv->tag_set = INIT_TAG; + priv->tag_set = SET_TAG; - priv->mode = UNINIT_MODE; +out: + uadk_prov_aead_reset_ctx(priv); *outl = 0; - return UADK_AEAD_SUCCESS; + + return ret; } static int uadk_prov_sw_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, @@ -669,7 +730,7 @@ static int uadk_prov_sw_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, int ret; if (priv->stream_switch_flag != UADK_DO_SOFT) { - ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); + ret = uadk_prov_aead_soft_init(priv); if (ret <= 0) return UADK_OSSL_FAIL; } @@ -688,14 +749,6 @@ static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, if (priv->stream_switch_flag == UADK_DO_SOFT || !priv->sess) return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); - ret = uadk_prov_aead_ctx_init(priv); - if (ret != UADK_AEAD_SUCCESS) - return UADK_AEAD_FAIL; - - ret = do_aes_gcm_prepare(priv); - if (unlikely(ret < 0)) - return UADK_AEAD_FAIL; - if (in) { if (!out) ret = uadk_prov_do_aes_gcm_first(priv, out, outl, in, inlen); @@ -736,10 +789,10 @@ static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; - if (!vctx || !out || !outl) + if (!vctx || !outl) return UADK_OSSL_FAIL; - if (outsize < inl) { + if (out && outsize < inl) { UADK_ERR("invalid: aead cipher outsize is too small.\n"); return UADK_OSSL_FAIL; } @@ -760,10 +813,15 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; - if (!vctx) + if (!vctx || !outl) return UADK_OSSL_FAIL; - if (outsize < inl) { + if (!inl) { + *outl = 0; + return UADK_AEAD_SUCCESS; + } + + if (out && outsize < inl) { UADK_ERR("invalid: input param outsize is too small.\n"); return UADK_OSSL_FAIL; } @@ -783,7 +841,7 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; - if (!vctx || !out || !outl) + if (!vctx || !outl) return UADK_OSSL_FAIL; ret = uadk_prov_do_aes_gcm(priv, out, outl, NULL, 0); @@ -887,11 +945,6 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char * { int ret; - if (ivlen > MAX_IV_LEN || keylen > MAX_KEY_LEN) { - UADK_ERR("invalid keylen or ivlen.\n"); - return UADK_OSSL_FAIL; - } - /* will free in freectx */ ret = uadk_prov_aead_alloc_sess(priv); if (ret == UADK_OSSL_FAIL) @@ -909,6 +962,8 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char * } priv->stream_switch_flag = 0; + priv->tag_set = INIT_TAG; + priv->partial_len = 0; priv->req.msg_state = AEAD_MSG_INVALID; return uadk_prov_aead_set_ctx_params(priv, params); @@ -1004,7 +1059,7 @@ static int uadk_prov_aead_set_ctx_params(void *vctx, const OSSL_PARAM params[]) return UADK_OSSL_FAIL; } if (sz == 0 || sz > priv->ivlen) { - UADK_ERR("invalid sz or ivlen.\n"); + UADK_ERR("invalid ivlen %zu.\n", sz); return UADK_OSSL_FAIL; } priv->ivlen = sz; @@ -1092,7 +1147,7 @@ static int uadk_prov_aead_get_ctx_params(void *vctx, OSSL_PARAM params[]) size_t sz = p->data_size; if (sz == 0 || sz > EVP_GCM_TLS_TAG_LEN || !priv->enc - || priv->taglen == UNINITIALISED_SIZET) { + || priv->tag_set != SET_TAG) { UADK_ERR("invalid size enc or taglen.\n"); return UADK_OSSL_FAIL; } @@ -1263,6 +1318,7 @@ static void *uadk_##nm##_newctx(void *provctx) \ ctx->nid = e_nid; \ ctx->taglen = tag_len; \ strncpy(ctx->alg_name, #algnm, ALG_NAME_SIZE - 1); \ + ctx->libctx = prov_libctx_of(provctx); \ \ if (uadk_get_sw_offload_state()) \ uadk_create_aead_soft_ctx(ctx); \ -- 2.53.0.windows.2
Add full TLS 1.x AES-GCM record processing that operates in place on records carrying an explicit IV, payload and trailing tag. Introduce TLS AAD, IV-fixed and IV-invert parameters, implement hardware and software TLS cipher paths, and enforce the SP800-38D record limit. Also replace the iv_set boolean with the aead_iv_state enum and report the block size as 1 so unaligned data is accepted upstream. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_prov_aead.c | 454 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 420 insertions(+), 34 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index ad990c2..499d5dd 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -24,6 +24,7 @@ #include <numa.h> #include <openssl/core_names.h> #include <openssl/proverr.h> +#include <openssl/rand.h> #include <uadk/wd_aead.h> #include <uadk/wd_sched.h> #include "uadk.h" @@ -45,8 +46,6 @@ #define UADK_AEAD_FAIL (-1) #define UNINITIALISED_SIZET ((size_t)-1) -#define IV_STATE_UNINITIALISED 0 -#define IV_STATE_SET 1 #define KEY_STATE_SET 1 /* Internal flags that can be queried */ @@ -67,6 +66,11 @@ #define BYTE_TO_BITS 8 #define ALIGN_DOWN(x, align) ((x) & ~((align) - 1)) +#define AEAD_ADD_LAST_BYTE1 1 +#define AEAD_ADD_LAST_BYTE2 2 +#define AEAD_ADD_BYTE_OFFSET 8 +#define AEAD_ADD_LEN_MASK 0xff + struct aead_prov { int pid; }; @@ -85,6 +89,13 @@ enum aead_tag_status { SET_TAG /* The MAC has been set to req. */ }; +enum aead_iv_state { + IV_STATE_UNINITIALISED = 0, /* initial state is not initialized */ + IV_STATE_BUFFERED, /* iv has been copied to the iv buffer */ + IV_STATE_COPIED, /* iv has been copied from the iv buffer */ + IV_STATE_FINISHED, /* the iv has been used - so don't reuse it */ +}; + struct aead_priv_ctx { int nid; char alg_name[ALG_NAME_SIZE]; @@ -94,7 +105,6 @@ struct aead_priv_ctx { unsigned int enc : 1; unsigned int key_set : 1; /* Whether key is copied to priv key buffers */ - unsigned int iv_set : 1; /* Whether iv is copied to priv iv buffers */ enum aead_tag_status tag_set; /* Whether mac is copied to priv mac buffers */ unsigned char iv[MAX_IV_LEN]; @@ -109,8 +119,16 @@ struct aead_priv_ctx { EVP_CIPHER_CTX *sw_ctx; EVP_CIPHER *sw_aead; unsigned char partial_data[AES_BLOCK_SIZE]; - OSSL_LIB_CTX *libctx; + unsigned char iv_copied[MAX_IV_LEN]; + enum aead_iv_state iv_state; size_t partial_len; + size_t tls_aad_len; /* saved TLS AAD length, UNINITIALISED_SIZET if not TLS */ + size_t tls_aad_pad_sz; /* padded length reported back via AEAD_TLS1_AAD_PAD */ + uint64_t tls_enc_records; /* number of TLS records encrypted (overflow check) */ + unsigned int iv_gen; /* OK to generate/retrieve explicit IV */ + unsigned char tls_aad[EVP_AEAD_TLS1_AAD_LEN]; /* saved TLS AAD bytes */ + OSSL_LIB_CTX *libctx; + unsigned int iv_gen_rand; }; struct aead_info { @@ -125,6 +143,9 @@ static struct aead_info aead_info_table[] = { { NID_aes_256_gcm, WD_CIPHER_AES, WD_CIPHER_GCM } }; +static int uadk_prov_aead_alloc_sess(struct aead_priv_ctx *priv); +static int uadk_prov_get_iv_gen(struct aead_priv_ctx *priv, unsigned char *out, size_t out_size); + # if OPENSSL_VERSION_NUMBER <= 0x30200000L static EVP_CIPHER_CTX *EVP_CIPHER_CTX_dup(const EVP_CIPHER_CTX *in) { @@ -206,17 +227,24 @@ free: static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv) { + unsigned char *iv = priv->iv_copied; int ret; + if (priv->stream_switch_flag == UADK_DO_SOFT) + return UADK_AEAD_SUCCESS; + if (!priv->sw_aead) return UADK_AEAD_FAIL; + if (priv->iv_state == IV_STATE_BUFFERED) + iv = priv->iv; + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) ret = EVP_EncryptInit_ex2(priv->sw_ctx, priv->sw_aead, - priv->key, priv->iv, NULL); + priv->key, iv, NULL); else ret = EVP_DecryptInit_ex2(priv->sw_ctx, priv->sw_aead, - priv->key, priv->iv, NULL); + priv->key, iv, NULL); if (!ret) { UADK_ERR("aead soft init error!\n"); @@ -224,6 +252,8 @@ static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv) } priv->stream_switch_flag = UADK_DO_SOFT; + if (!priv->req.mac) + priv->req.mac = priv->buf; return UADK_AEAD_SUCCESS; } @@ -254,10 +284,13 @@ static int uadk_aead_soft_update(struct aead_priv_ctx *priv, unsigned char *out, static void uadk_prov_aead_reset_ctx(struct aead_priv_ctx *priv) { priv->stream_switch_flag = 0; + priv->iv_state = IV_STATE_FINISHED; priv->req.assoc_bytes = 0; priv->partial_len = 0; priv->mode = UNINIT_MODE; + priv->req.mac = NULL; priv->req.msg_state = AEAD_MSG_INVALID; + priv->tls_aad_len = UNINITIALISED_SIZET; } static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *digest, size_t *outl) @@ -273,12 +306,12 @@ static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *diges goto error; ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_GET_TAG, - priv->taglen, priv->buf); + priv->taglen, priv->req.mac); if (ret == UADK_AEAD_SUCCESS) priv->tag_set = SET_TAG; } else { ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_SET_TAG, - priv->taglen, priv->buf); + priv->taglen, priv->req.mac); if (!ret) goto error; @@ -341,19 +374,26 @@ static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) { int ret; - if (!priv->key_set || !priv->iv_set) { + if (!priv->key_set || priv->iv_state == IV_STATE_UNINITIALISED || + priv->iv_state == IV_STATE_FINISHED) { UADK_ERR("key or iv is not set yet!\n"); return UADK_AEAD_FAIL; } + if (priv->iv_state == IV_STATE_BUFFERED) { + memcpy(priv->iv_copied, priv->iv, priv->ivlen); + priv->iv_state = IV_STATE_COPIED; + } + priv->req.iv_bytes = priv->ivlen; - priv->req.iv = priv->iv; + priv->req.iv = priv->iv_copied; /* Initialize the counter value for CTR (Counter) mode encryption. */ - memset(priv->iv + GCM_IV_DEFAULT_SIZE, 0, AES_GCM_COUNTER_SIZE); - priv->iv[AES_CTR_IV_LEN - 1] = 0x2; + memset(priv->iv_copied + GCM_IV_DEFAULT_SIZE, 0, AES_GCM_COUNTER_SIZE); + priv->iv_copied[AES_CTR_IV_LEN - 1] = 0x2; priv->req.out_bytes = 0; - priv->req.mac = priv->buf; + if (!priv->req.mac) + priv->req.mac = priv->buf; priv->req.mac_bytes = AES_GCM_TAG_LEN; ret = wd_aead_set_authsize(priv->sess, AES_GCM_TAG_LEN); @@ -541,7 +581,7 @@ static int uadk_prov_process_partial_data(struct aead_priv_ctx *priv, unsigned c memcpy(out, block_out + priv->partial_len, processing_len); priv->partial_len = 0; - ctr_iv_inc(priv->iv + AES_CTR_COUNTER_SIZE, 1); + ctr_iv_inc(priv->iv_copied + AES_CTR_COUNTER_SIZE, 1); *processed_len = processing_len; return UADK_AEAD_SUCCESS; @@ -583,7 +623,7 @@ static int uadk_prov_process_tail_data(struct aead_priv_ctx *priv, unsigned char if (!cipher) goto free_ctx; - ret = EVP_CipherInit_ex2(ctx, cipher, priv->key, priv->iv, priv->enc, NULL); + ret = EVP_CipherInit_ex2(ctx, cipher, priv->key, priv->iv_copied, priv->enc, NULL); if (!ret) goto free_cipher; @@ -630,7 +670,7 @@ static int uadk_process_complete_blocks(struct aead_priv_ctx *priv, unsigned cha in += chunk; } - ctr_iv_inc(priv->iv + AES_CTR_COUNTER_SIZE, len >> AES_BLOCK_OFFSET); + ctr_iv_inc(priv->iv_copied + AES_CTR_COUNTER_SIZE, len >> AES_BLOCK_OFFSET); return UADK_AEAD_SUCCESS; } @@ -741,11 +781,172 @@ static int uadk_prov_sw_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, return uadk_aead_soft_final(priv, out, outl); } +static int uadk_prov_aead_hw_tls_cipher(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t inlen) +{ + size_t processed_len, remain_len; + size_t outsize; + int ret; + + ret = uadk_prov_do_aes_gcm_first(priv, NULL, &outsize, priv->tls_aad, priv->tls_aad_len); + if (ret != UADK_AEAD_SUCCESS) + return ret; + + if (priv->tls_aad_len + inlen <= AEAD_BLOCK_SIZE) { + ret = uadk_prov_do_aes_gcm_final(priv, out, &outsize, in, inlen); + } else { + processed_len = ALIGN_DOWN(inlen, AES_BLOCK_SIZE); + remain_len = inlen - processed_len; + ret = uadk_prov_do_aes_gcm_update(priv, out, &outsize, in, processed_len); + if (ret != UADK_AEAD_SUCCESS) + return ret; + + if (remain_len) + ret = uadk_prov_do_aes_gcm_final(priv, out + processed_len, &outsize, + in + processed_len, remain_len); + else + ret = uadk_prov_do_aes_gcm_final(priv, NULL, &outsize, NULL, 0); + } + + return ret; +} + +static int uadk_prov_aead_sw_tls_cipher(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t inlen) +{ + size_t outsize = 0; + int ret; + + ret = uadk_prov_aead_soft_init(priv); + if (ret != UADK_AEAD_SUCCESS) + return UADK_OSSL_FAIL; + + ret = uadk_aead_soft_update(priv, NULL, &outsize, priv->tls_aad, priv->tls_aad_len); + if (ret != UADK_AEAD_SUCCESS) + return UADK_OSSL_FAIL; + + ret = uadk_aead_soft_update(priv, out, &outsize, in, inlen); + if (ret != UADK_AEAD_SUCCESS) + return UADK_OSSL_FAIL; + + ret = uadk_aead_soft_final(priv, out + outsize, &outsize); + if (ret != UADK_AEAD_SUCCESS) + return UADK_OSSL_FAIL; + + return UADK_AEAD_SUCCESS; +} + +static int uadk_prov_aead_set_tls_iv(struct aead_priv_ctx *priv, unsigned char *out, + const unsigned char *in, size_t inlen) +{ + int ret; + + if (!priv->iv_gen || !priv->key_set) { + UADK_ERR("invalid: aead tls iv or key not set.\n"); + return UADK_OSSL_FAIL; + } + + /* + * priv->ivlen may have been overridden via OSSL_CIPHER_PARAM_AEAD_IVLEN; + * the explicit-IV layout requires at least EVP_GCM_TLS_EXPLICIT_IV_LEN + * bytes at the tail of priv->iv, otherwise the pointer arithmetic below + * would underflow and produce an out-of-bounds access. + */ + if (priv->ivlen < EVP_GCM_TLS_EXPLICIT_IV_LEN) { + UADK_ERR("aead tls: ivlen too short %zu.\n", priv->ivlen); + return UADK_OSSL_FAIL; + } + + /* TLS GCM is always in-place and always carries explicit IV + tag. */ + if (out != in || inlen < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN)) { + UADK_ERR("aead tls: invalid in-place buffer.\n"); + return UADK_OSSL_FAIL; + } + + if (priv->enc) { + ret = uadk_prov_get_iv_gen(priv, out, EVP_GCM_TLS_EXPLICIT_IV_LEN); + if (ret == UADK_OSSL_FAIL) + return UADK_OSSL_FAIL; + } else { + memcpy(priv->iv + priv->ivlen - EVP_GCM_TLS_EXPLICIT_IV_LEN, in, + EVP_GCM_TLS_EXPLICIT_IV_LEN); + memcpy(priv->iv_copied, priv->iv, priv->ivlen); + priv->iv_state = IV_STATE_COPIED; + } + + return UADK_AEAD_SUCCESS; +} + +/* + * Process a TLS 1.x GCM record entirely in software. + * + * The on-wire buffer layout is: explicit_iv(8) || payload || tag(16), + * and the operation must be performed in place (out == in). This mirrors + * gcm_tls_cipher() from the OpenSSL default provider, but every cryptographic + * step is delegated to the default provider's AES-GCM through the EVP API. + * + * Encryption: write a freshly generated explicit IV to out[0..8), feed the + * saved 13-byte AAD, encrypt the payload in place, then append the tag. + * Decryption: read the explicit IV from in[0..8), feed the AAD, decrypt the + * payload in place and verify the trailing tag. + */ +static int uadk_prov_aead_tls_cipher(struct aead_priv_ctx *priv, unsigned char *out, + size_t *outl, const unsigned char *in, size_t inlen) +{ + size_t payload_len; + int ret; + + ret = uadk_prov_aead_set_tls_iv(priv, out, in, inlen); + if (ret != UADK_AEAD_SUCCESS) + goto out; + + /* + * SP800-38D requires the encrypting side to fail after 2^64 - 1 keys. + */ + if (priv->enc && ++priv->tls_enc_records == 0) { + UADK_ERR("aead tls: too many records.\n"); + ret = UADK_OSSL_FAIL; + goto out; + } + + payload_len = inlen - EVP_GCM_TLS_EXPLICIT_IV_LEN - EVP_GCM_TLS_TAG_LEN; + in += EVP_GCM_TLS_EXPLICIT_IV_LEN; + out += EVP_GCM_TLS_EXPLICIT_IV_LEN; + priv->req.mac = out + payload_len; + priv->tag_set = READ_TAG; + if (priv->stream_switch_flag == UADK_DO_SOFT || !priv->sess || + priv->ivlen != GCM_IV_DEFAULT_SIZE) + goto do_soft; + + ret = uadk_prov_aead_hw_tls_cipher(priv, out, in, payload_len); + if (ret == SWITCH_TO_SOFT) + goto do_soft; + else + goto out; +do_soft: + ret = uadk_prov_aead_sw_tls_cipher(priv, out, in, payload_len); +out: + if (ret == UADK_AEAD_SUCCESS) { + if (priv->enc) + *outl = inlen; + else + *outl = payload_len; + } else { + *outl = 0; + } + uadk_prov_aead_reset_ctx(priv); + priv->tag_set = INIT_TAG; + return ret; +} + static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, size_t *outl, const unsigned char *in, size_t inlen) { int ret; + if (priv->tls_aad_len != UNINITIALISED_SIZET) + return uadk_prov_aead_tls_cipher(priv, out, outl, in, inlen); + if (priv->stream_switch_flag == UADK_DO_SOFT || !priv->sess) return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); @@ -926,6 +1127,7 @@ static int uadk_prov_aead_set_key(struct aead_priv_ctx *priv, memcpy(priv->key, key, keylen); priv->key_set = KEY_STATE_SET; + priv->tls_enc_records = 0; /* use default provider */ if (!priv->sess) @@ -951,8 +1153,13 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char * return UADK_OSSL_FAIL; if (iv) { + if (!ivlen || ivlen > MAX_IV_LEN) { + UADK_ERR("invalid ivlen %zu.\n", ivlen); + return UADK_OSSL_FAIL; + } memcpy(priv->iv, iv, ivlen); - priv->iv_set = IV_STATE_SET; + priv->ivlen = ivlen; + priv->iv_state = IV_STATE_BUFFERED; } if (key) { @@ -1000,9 +1207,11 @@ static int uadk_prov_aead_dinit(void *vctx, const unsigned char *key, size_t key } static const OSSL_PARAM uadk_prov_settable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_AEAD_IVLEN, NULL), OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TAG, NULL, 0), + OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TLS1_AAD, NULL, 0), + OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TLS1_IV_FIXED, NULL, 0), + OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV, NULL, 0), OSSL_PARAM_END }; @@ -1012,12 +1221,119 @@ const OSSL_PARAM *uadk_prov_aead_settable_ctx_params(ossl_unused void *cctx, return uadk_prov_settable_ctx_params; } +/* + * Save the TLS 1.x AAD (13 bytes) and compute the padded plaintext length + * that the TLS record carries in its header. Mirrors gcm_tls_init() from the + * OpenSSL default provider: the on-wire record length is rewritten to the + * payload length (excluding explicit IV and, for decryption, the trailing + * tag), and the tag length is returned as the padding size. + */ +static int uadk_prov_aead_tls_aad_init(struct aead_priv_ctx *priv, + const unsigned char *aad, size_t aad_len) +{ + size_t len; + + if (aad_len != EVP_AEAD_TLS1_AAD_LEN) + return UADK_OSSL_FAIL; + + memcpy(priv->tls_aad, aad, aad_len); + priv->tls_aad_len = aad_len; + + len = priv->tls_aad[aad_len - AEAD_ADD_LAST_BYTE2] << AEAD_ADD_BYTE_OFFSET | + priv->tls_aad[aad_len - AEAD_ADD_LAST_BYTE1]; + if (len < EVP_GCM_TLS_EXPLICIT_IV_LEN) + return UADK_OSSL_FAIL; + len -= EVP_GCM_TLS_EXPLICIT_IV_LEN; + + if (!priv->enc) { + if (len < EVP_GCM_TLS_TAG_LEN) + return UADK_OSSL_FAIL; + len -= EVP_GCM_TLS_TAG_LEN; + } + + priv->tls_aad[aad_len - AEAD_ADD_LAST_BYTE2] = (unsigned char)(len >> AEAD_ADD_BYTE_OFFSET); + priv->tls_aad[aad_len - AEAD_ADD_LAST_BYTE1] = (unsigned char)(len & AEAD_ADD_LEN_MASK); + priv->tls_aad_pad_sz = EVP_GCM_TLS_TAG_LEN; + + return UADK_AEAD_SUCCESS; +} + +/* + * Set the fixed part of the TLS GCM IV. With len == (size_t)-1 the whole IV + * is restored (used when the caller already holds a full IV). Otherwise the + * first 'len' bytes are taken from 'iv' and, for encryption, the remaining + * (explicit) bytes are randomly generated. + */ +static int uadk_prov_aead_tls_iv_set_fixed(struct aead_priv_ctx *priv, + const unsigned char *iv, size_t len) +{ + if (len == UNINITIALISED_SIZET) { + /* + * Sentinel: restore the whole IV. The caller (set_ctx_params) + * validates p->data_size before reaching here, and the EVP + * layer guarantees the supplied buffer holds at least + * priv->ivlen bytes when the sentinel is used. + */ + memcpy(priv->iv, iv, priv->ivlen); + priv->iv_gen = 1; + priv->iv_state = IV_STATE_BUFFERED; + return UADK_AEAD_SUCCESS; + } + + /* Fixed field must be at least 4 bytes and invocation field at least 8 */ + if (len < EVP_GCM_TLS_FIXED_IV_LEN) + return UADK_OSSL_FAIL; + + if (priv->ivlen < len || priv->ivlen - len < EVP_GCM_TLS_EXPLICIT_IV_LEN) + return UADK_OSSL_FAIL; + + if (len > 0) + memcpy(priv->iv, iv, len); + + if (priv->enc) { + if (RAND_bytes_ex(priv->libctx, priv->iv + len, priv->ivlen - len, 0) <= 0) + return UADK_OSSL_FAIL; + + priv->iv_gen_rand = 1; + } + + priv->iv_gen = 1; + priv->iv_state = IV_STATE_BUFFERED; + + return UADK_AEAD_SUCCESS; +} + +/* Retrieve the explicit IV bytes from the start of a TLS record on decrypt. */ +static int uadk_prov_aead_tls_set_iv_inv(struct aead_priv_ctx *priv, + const unsigned char *iv, size_t ivlen) +{ + if (!priv->iv_gen || priv->enc) + return UADK_OSSL_FAIL; + + /* + * The explicit IV is written to the tail of priv->iv; ivlen must not + * exceed priv->ivlen, otherwise the subtraction below underflows and + * the memcpy would write out of bounds. + */ + if (ivlen > priv->ivlen) + return UADK_OSSL_FAIL; + + memcpy(priv->iv + priv->ivlen - ivlen, iv, ivlen); + priv->iv_state = IV_STATE_BUFFERED; + + return UADK_AEAD_SUCCESS; +} + static int uadk_prov_aead_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; const OSSL_PARAM *p; size_t sz = 0; void *vp; + int ret; + + if (!params) + return UADK_AEAD_SUCCESS; if (!vctx) return UADK_OSSL_FAIL; @@ -1038,31 +1354,56 @@ static int uadk_prov_aead_set_ctx_params(void *vctx, const OSSL_PARAM params[]) priv->taglen = sz; } - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_IVLEN); if (p) { - size_t keylen; + if (!OSSL_PARAM_get_size_t(p, &sz)) { + UADK_ERR("failed to get size parameter: sz.\n"); + return UADK_OSSL_FAIL; + } + if (sz == 0 || sz > sizeof(priv->iv)) { + UADK_ERR("invalid ivlen %zu.\n", sz); + return UADK_OSSL_FAIL; + } + priv->ivlen = sz; + } - if (!OSSL_PARAM_get_size_t(p, &keylen)) { - UADK_ERR("failed to get parameter: keylen.\n"); + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_AAD); + if (p) { + if (p->data_type != OSSL_PARAM_OCTET_STRING || !p->data) { + UADK_ERR("invalid tls aad parameter.\n"); return UADK_OSSL_FAIL; } - if (priv->keylen != keylen) { - UADK_ERR("keylen is invalid.\n"); + ret = uadk_prov_aead_tls_aad_init(priv, p->data, p->data_size); + if (ret != UADK_AEAD_SUCCESS) { + UADK_ERR("failed to init tls aad.\n"); return UADK_OSSL_FAIL; } } - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_IVLEN); + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_IV_FIXED); if (p) { - if (!OSSL_PARAM_get_size_t(p, &sz)) { - UADK_ERR("failed to get size parameter: sz.\n"); + if (p->data_type != OSSL_PARAM_OCTET_STRING || !p->data) { + UADK_ERR("invalid tls iv fixed parameter.\n"); return UADK_OSSL_FAIL; } - if (sz == 0 || sz > priv->ivlen) { - UADK_ERR("invalid ivlen %zu.\n", sz); + ret = uadk_prov_aead_tls_iv_set_fixed(priv, p->data, p->data_size); + if (ret != UADK_AEAD_SUCCESS) { + UADK_ERR("failed to set tls iv fixed.\n"); + return UADK_OSSL_FAIL; + } + } + + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV); + if (p) { + if (p->data_type != OSSL_PARAM_OCTET_STRING || !p->data) { + UADK_ERR("invalid tls iv inv parameter.\n"); + return UADK_OSSL_FAIL; + } + ret = uadk_prov_aead_tls_set_iv_inv(priv, p->data, p->data_size); + if (ret != UADK_AEAD_SUCCESS) { + UADK_ERR("failed to set tls iv inv.\n"); return UADK_OSSL_FAIL; } - priv->ivlen = sz; } return UADK_AEAD_SUCCESS; @@ -1075,6 +1416,8 @@ static const OSSL_PARAM uadk_prov_aead_ctx_params[] = { OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_IV, NULL, 0), OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_UPDATED_IV, NULL, 0), OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TAG, NULL, 0), + OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_AEAD_TLS1_AAD_PAD, NULL), + OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN, NULL, 0), OSSL_PARAM_END }; @@ -1086,7 +1429,7 @@ static const OSSL_PARAM *uadk_prov_aead_gettable_ctx_params(ossl_unused void *cc static int uadk_prov_aead_get_ctx_iv(OSSL_PARAM *p, struct aead_priv_ctx *priv) { - if (priv->iv_set == IV_STATE_UNINITIALISED) + if (priv->iv_state == IV_STATE_UNINITIALISED) return UADK_OSSL_FAIL; if (priv->ivlen > p->data_size) { @@ -1103,6 +1446,30 @@ static int uadk_prov_aead_get_ctx_iv(OSSL_PARAM *p, struct aead_priv_ctx *priv) return UADK_AEAD_SUCCESS; } +static int uadk_prov_get_iv_gen(struct aead_priv_ctx *priv, unsigned char *out, size_t out_size) +{ + size_t len = out_size; + + if (priv->iv_state == IV_STATE_UNINITIALISED || !priv->iv_gen || + priv->ivlen < EVP_GCM_TLS_EXPLICIT_IV_LEN) + return UADK_OSSL_FAIL; + + memcpy(priv->iv_copied, priv->iv, priv->ivlen); + priv->iv_state = IV_STATE_COPIED; + + if (!len || len > priv->ivlen) + len = priv->ivlen; + memcpy(out, priv->iv + priv->ivlen - len, len); + + /* + * Invocation field will be at least 8 bytes in size and so no need + * to check wrap around or increment more than last 8 bytes. + */ + ctr_iv_inc(priv->iv + priv->ivlen - EVP_GCM_TLS_EXPLICIT_IV_LEN, 1); + + return UADK_AEAD_SUCCESS; +} + static int uadk_prov_aead_get_ctx_params(void *vctx, OSSL_PARAM params[]) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; @@ -1158,6 +1525,24 @@ static int uadk_prov_aead_get_ctx_params(void *vctx, OSSL_PARAM params[]) } } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TLS1_AAD_PAD); + if (p) { + if (!OSSL_PARAM_set_size_t(p, priv->tls_aad_pad_sz)) { + UADK_ERR("failed to set size parameter: tls aad pad %zu.\n", + priv->tls_aad_pad_sz); + return UADK_OSSL_FAIL; + } + } + + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN); + if (p) { + if (!p->data || p->data_type != OSSL_PARAM_OCTET_STRING) + return UADK_OSSL_FAIL; + + if (!uadk_prov_get_iv_gen(priv, p->data, p->data_size)) + return UADK_OSSL_FAIL; + } + return UADK_AEAD_SUCCESS; } @@ -1317,6 +1702,7 @@ static void *uadk_##nm##_newctx(void *provctx) \ ctx->ivlen = iv_len; \ ctx->nid = e_nid; \ ctx->taglen = tag_len; \ + ctx->tls_aad_len = UNINITIALISED_SIZET; \ strncpy(ctx->alg_name, #algnm, ALG_NAME_SIZE - 1); \ ctx->libctx = prov_libctx_of(provctx); \ \ @@ -1359,9 +1745,9 @@ const OSSL_DISPATCH uadk_##nm##_functions[] = { \ { 0, NULL } \ } -UADK_AEAD_DESCR(aes_128_gcm, AES_GCM_TAG_LEN, 16, 12, 8, AEAD_FLAGS, NID_aes_128_gcm, gcm(aes), +UADK_AEAD_DESCR(aes_128_gcm, AES_GCM_TAG_LEN, 16, 12, 1, AEAD_FLAGS, NID_aes_128_gcm, gcm(aes), EVP_CIPH_GCM_MODE); -UADK_AEAD_DESCR(aes_192_gcm, AES_GCM_TAG_LEN, 24, 12, 8, AEAD_FLAGS, NID_aes_192_gcm, gcm(aes), +UADK_AEAD_DESCR(aes_192_gcm, AES_GCM_TAG_LEN, 24, 12, 1, AEAD_FLAGS, NID_aes_192_gcm, gcm(aes), EVP_CIPH_GCM_MODE); -UADK_AEAD_DESCR(aes_256_gcm, AES_GCM_TAG_LEN, 32, 12, 8, AEAD_FLAGS, NID_aes_256_gcm, gcm(aes), +UADK_AEAD_DESCR(aes_256_gcm, AES_GCM_TAG_LEN, 32, 12, 1, AEAD_FLAGS, NID_aes_256_gcm, gcm(aes), EVP_CIPH_GCM_MODE); -- 2.53.0.windows.2
UADK hardware only supports the default 12-byte GCM IV. Widen the IV storage from MAX_IV_LEN to GCM_IV_MAX_SIZE (128 bytes) so non-default IV lengths do not overflow the buffer, and update the ivlen bounds checks accordingly. When the IV length is not GCM_IV_DEFAULT_SIZE, route the request through the software AES-GCM path instead of failing or producing incorrect output: add the ivlen check to the hardware entry point so non-default IV lengths switch to uadk_prov_sw_aes_gcm, and pass the custom IV length to the software cipher via the OSSL_CIPHER_PARAM_IVLEN parameter in uadk_prov_aead_soft_init. Signed-off-by: Weili Qian <qianweili@huawei.com> --- src/uadk_prov_aead.c | 82 ++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index 499d5dd..1919d47 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -32,7 +32,6 @@ #include "uadk_prov.h" #include "uadk_utils.h" -#define MAX_IV_LEN 16 #define MAX_KEY_LEN 64 #define MAX_AAD_LEN 0xFFFF #define ALG_NAME_SIZE 128 @@ -58,6 +57,7 @@ #define UADK_AEAD_OP_NUM 1 #define AES_CTR_IV_LEN 16 +#define GCM_IV_MAX_SIZE 128 #define GCM_IV_DEFAULT_SIZE 12 #define AES_GCM_COUNTER_SIZE 4 @@ -107,7 +107,7 @@ struct aead_priv_ctx { unsigned int key_set : 1; /* Whether key is copied to priv key buffers */ enum aead_tag_status tag_set; /* Whether mac is copied to priv mac buffers */ - unsigned char iv[MAX_IV_LEN]; + unsigned char iv[GCM_IV_MAX_SIZE]; /* Buffer to use for IV's */ unsigned char key[MAX_KEY_LEN]; unsigned char buf[AES_GCM_TAG_LEN]; /* mac buffers */ @@ -119,16 +119,16 @@ struct aead_priv_ctx { EVP_CIPHER_CTX *sw_ctx; EVP_CIPHER *sw_aead; unsigned char partial_data[AES_BLOCK_SIZE]; - unsigned char iv_copied[MAX_IV_LEN]; + OSSL_LIB_CTX *libctx; + unsigned char iv_copied[GCM_IV_MAX_SIZE]; enum aead_iv_state iv_state; size_t partial_len; size_t tls_aad_len; /* saved TLS AAD length, UNINITIALISED_SIZET if not TLS */ size_t tls_aad_pad_sz; /* padded length reported back via AEAD_TLS1_AAD_PAD */ uint64_t tls_enc_records; /* number of TLS records encrypted (overflow check) */ unsigned int iv_gen; /* OK to generate/retrieve explicit IV */ - unsigned char tls_aad[EVP_AEAD_TLS1_AAD_LEN]; /* saved TLS AAD bytes */ - OSSL_LIB_CTX *libctx; unsigned int iv_gen_rand; + unsigned char tls_aad[EVP_AEAD_TLS1_AAD_LEN]; /* saved TLS AAD bytes */ }; struct aead_info { @@ -143,7 +143,6 @@ static struct aead_info aead_info_table[] = { { NID_aes_256_gcm, WD_CIPHER_AES, WD_CIPHER_GCM } }; -static int uadk_prov_aead_alloc_sess(struct aead_priv_ctx *priv); static int uadk_prov_get_iv_gen(struct aead_priv_ctx *priv, unsigned char *out, size_t out_size); # if OPENSSL_VERSION_NUMBER <= 0x30200000L @@ -228,6 +227,8 @@ free: static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv) { unsigned char *iv = priv->iv_copied; + OSSL_PARAM params_ivlen[2]; + OSSL_PARAM *params = NULL; int ret; if (priv->stream_switch_flag == UADK_DO_SOFT) @@ -239,12 +240,19 @@ static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv) if (priv->iv_state == IV_STATE_BUFFERED) iv = priv->iv; + if (priv->ivlen != GCM_IV_DEFAULT_SIZE) { + params_ivlen[0] = OSSL_PARAM_construct_size_t( + OSSL_CIPHER_PARAM_IVLEN, &priv->ivlen); + params_ivlen[1] = OSSL_PARAM_construct_end(); + params = params_ivlen; + } + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) ret = EVP_EncryptInit_ex2(priv->sw_ctx, priv->sw_aead, - priv->key, iv, NULL); + priv->key, iv, params); else ret = EVP_DecryptInit_ex2(priv->sw_ctx, priv->sw_aead, - priv->key, iv, NULL); + priv->key, iv, params); if (!ret) { UADK_ERR("aead soft init error!\n"); @@ -374,12 +382,6 @@ static int uadk_prov_aead_ctx_init(struct aead_priv_ctx *priv) { int ret; - if (!priv->key_set || priv->iv_state == IV_STATE_UNINITIALISED || - priv->iv_state == IV_STATE_FINISHED) { - UADK_ERR("key or iv is not set yet!\n"); - return UADK_AEAD_FAIL; - } - if (priv->iv_state == IV_STATE_BUFFERED) { memcpy(priv->iv_copied, priv->iv, priv->ivlen); priv->iv_state = IV_STATE_COPIED; @@ -769,11 +771,9 @@ static int uadk_prov_sw_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, { int ret; - if (priv->stream_switch_flag != UADK_DO_SOFT) { - ret = uadk_prov_aead_soft_init(priv); - if (ret <= 0) - return UADK_OSSL_FAIL; - } + ret = uadk_prov_aead_soft_init(priv); + if (ret <= 0) + return UADK_OSSL_FAIL; if (in) return uadk_aead_soft_update(priv, out, outl, in, inlen); @@ -939,6 +939,40 @@ out: return ret; } +static int uadk_prov_aead_iv_generate(struct aead_priv_ctx *priv) +{ + /* Must be at least 96 bits */ + if (priv->ivlen < GCM_IV_DEFAULT_SIZE) + return UADK_OSSL_FAIL; + + /* Use DRBG to generate random iv */ + if (RAND_bytes_ex(priv->libctx, priv->iv, priv->ivlen, 0) <= 0) + return UADK_OSSL_FAIL; + + priv->iv_state = IV_STATE_BUFFERED; + priv->iv_gen_rand = 1; + + return UADK_AEAD_SUCCESS; +} + +static int uadk_prov_aead_check_params(struct aead_priv_ctx *priv) +{ + int ret; + + if (!priv->key_set || priv->iv_state == IV_STATE_FINISHED) + return UADK_OSSL_FAIL; + + if (priv->iv_state == IV_STATE_UNINITIALISED) { + if (!priv->enc) + return UADK_OSSL_FAIL; + ret = uadk_prov_aead_iv_generate(priv); + if (ret != UADK_AEAD_SUCCESS) + return UADK_OSSL_FAIL; + } + + return UADK_AEAD_SUCCESS; +} + static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, size_t *outl, const unsigned char *in, size_t inlen) { @@ -947,7 +981,13 @@ static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, if (priv->tls_aad_len != UNINITIALISED_SIZET) return uadk_prov_aead_tls_cipher(priv, out, outl, in, inlen); - if (priv->stream_switch_flag == UADK_DO_SOFT || !priv->sess) + ret = uadk_prov_aead_check_params(priv); + if (ret != UADK_AEAD_SUCCESS) + return UADK_OSSL_FAIL; + + if (priv->stream_switch_flag == UADK_DO_SOFT || + priv->ivlen != GCM_IV_DEFAULT_SIZE || + !priv->sess) return uadk_prov_sw_aes_gcm(priv, out, outl, in, inlen); if (in) { @@ -1153,7 +1193,7 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char * return UADK_OSSL_FAIL; if (iv) { - if (!ivlen || ivlen > MAX_IV_LEN) { + if (!ivlen || ivlen > GCM_IV_MAX_SIZE) { UADK_ERR("invalid ivlen %zu.\n", ivlen); return UADK_OSSL_FAIL; } -- 2.53.0.windows.2
From: Zhushuai Yin <yinzhushuai@huawei.com> If the session allocation process fails to allocate resources properly, the system should automatically transition to a software computation to ensure continued functionality. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- src/uadk_prov_digest.c | 85 ++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index 2da3a36..3894920 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -39,6 +39,7 @@ #define DIGEST_END 0 #define UADK_DIGEST_SUCCESS 1 #define UADK_DIGEST_FAIL 0 +#define UADK_DIGEST_SWITCH_SOFT 2 /* The max BD data length is 16M-512B */ #define BUF_LEN 0xFFFE00 @@ -393,7 +394,7 @@ static int uadk_digest_ctx_init(struct digest_priv_ctx *priv) ret = uadk_prov_digest_dev_init(priv); if (unlikely(ret <= 0)) - return UADK_DIGEST_FAIL; + return UADK_DIGEST_SWITCH_SOFT; /* Use the default numa parameters */ params.numa_id = -1; @@ -404,8 +405,8 @@ static int uadk_digest_ctx_init(struct digest_priv_ctx *priv) if (!priv->sess) { priv->sess = wd_digest_alloc_sess(&setup); if (unlikely(!priv->sess)) { - UADK_ERR("uadk failed to alloc sess.\n"); - return UADK_DIGEST_FAIL; + UADK_ERR("uadk failed to alloc sess, switch to soft.\n"); + return UADK_DIGEST_SWITCH_SOFT; } } @@ -445,15 +446,61 @@ static void uadk_digest_set_msg_state(struct digest_priv_ctx *priv, bool is_end) } } +static int uadk_digest_switch_soft(struct digest_priv_ctx *priv, + unsigned char *input_data, + size_t remain_len, + bool has_hw_partial_data, + size_t hw_processing_len) +{ + int ret; + + if (unlikely(!priv->soft_md)) { + UADK_ERR("digest soft ctx not available, soft offload not enabled.\n"); + return UADK_DIGEST_FAIL; + } + + ret = uadk_digest_soft_init(priv); + if (!ret) + goto out; + + if (has_hw_partial_data) { + ret = uadk_digest_soft_update(priv, priv->data, DIGEST_BLOCK_SIZE); + if (!ret) + goto out; + + remain_len -= hw_processing_len; + input_data += hw_processing_len; + } else if (priv->last_update_bufflen > 0) { + ret = uadk_digest_soft_update(priv, priv->data, + priv->last_update_bufflen); + if (!ret) + goto out; + } + + ret = uadk_digest_soft_update(priv, input_data, remain_len); + if (!ret) + goto out; + + priv->last_update_bufflen = 0; + return UADK_DIGEST_SUCCESS; + +out: + digest_soft_cleanup(priv); + return UADK_DIGEST_FAIL; +} + static int uadk_digest_update_inner(struct digest_priv_ctx *priv, const void *data, size_t data_len) { unsigned char *input_data = (unsigned char *)data; size_t remain_len = data_len; size_t processing_len; + bool has_partial; int ret; ret = uadk_digest_ctx_init(priv); - if (ret != UADK_DIGEST_SUCCESS) + if (ret == UADK_DIGEST_SWITCH_SOFT) + goto do_soft_digest; + else if (ret != UADK_DIGEST_SUCCESS) return UADK_DIGEST_FAIL; uadk_digest_set_msg_state(priv, false); @@ -506,36 +553,20 @@ static int uadk_digest_update_inner(struct digest_priv_ctx *priv, const void *da return UADK_DIGEST_SUCCESS; do_soft_digest: - if (priv->state == SEC_DIGEST_FIRST_UPDATING) { - ret = uadk_digest_soft_init(priv); - if (!ret) - return ret; - - /* filling buf has been executed */ - if (processing_len < DIGEST_BLOCK_SIZE) { - ret = uadk_digest_soft_update(priv, priv->data, DIGEST_BLOCK_SIZE); - if (!ret) - goto out; + if (priv->state == SEC_DIGEST_INIT) + return uadk_digest_switch_soft(priv, input_data, remain_len, + false, 0); - remain_len -= processing_len; - input_data += processing_len; - } - - ret = uadk_digest_soft_update(priv, input_data, remain_len); - if (!ret) - goto out; + if (priv->state == SEC_DIGEST_FIRST_UPDATING) { + has_partial = (processing_len < DIGEST_BLOCK_SIZE); - /* the soft ctx will be free in the final stage. */ - return ret; + return uadk_digest_switch_soft(priv, input_data, remain_len, + has_partial, processing_len); } UADK_ERR("do soft digest failed during updating!\n"); return UADK_DIGEST_FAIL; - -out: - digest_soft_cleanup(priv); - return ret; } static int uadk_digest_update(struct digest_priv_ctx *priv, const void *data, size_t data_len) -- 2.53.0.windows.2
From: Wenkai Lin <linwenkai6@hisilicon.com> Zero out the session and data pointers in uadk_digest_cleanup after freeing them to prevent double-free on repeated cleanup calls. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- src/uadk_prov_digest.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index 3894920..7488fc6 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -775,11 +775,15 @@ static int uadk_digest_digest(struct digest_priv_ctx *priv, const void *data, static void uadk_digest_cleanup(struct digest_priv_ctx *priv) { - if (priv->sess) + if (priv->sess) { wd_digest_free_sess(priv->sess); + priv->sess = 0; + } - if (priv->data) + if (priv->data) { OPENSSL_clear_free(priv->data, DIGEST_BLOCK_SIZE); + priv->data = NULL; + } digest_soft_cleanup(priv); } -- 2.53.0.windows.2
participants (1)
-
Weili Qian