From: Zhiqi Song <songzhiqi1@huawei.com> Use explicit initialization values in __atomic_compare_exchange_n(). Signed-off-by: Zhiqi Song <songzhiqi1@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_init.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/uadk_prov_init.c b/src/uadk_prov_init.c index f3f360f..54e3fcb 100644 --- a/src/uadk_prov_init.c +++ b/src/uadk_prov_init.c @@ -209,24 +209,29 @@ static const OSSL_ALGORITHM uadk_prov_keyexch[] = { static const OSSL_ALGORITHM *uadk_query(void *provctx, int operation_id, int *no_cache) { + OSSL_LIB_CTX *libctx; static int prov_init; int ver; - prov = OSSL_PROVIDER_load(NULL, "default"); - if (!prov_init) { - prov_init = 1; + if (__atomic_compare_exchange_n(&prov_init, &(int){0}, 1, false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST)) { + libctx = prov_libctx_of(provctx); + prov = OSSL_PROVIDER_load(libctx, "default"); + if (!prov) { + fprintf(stderr, "failed to load default provider\n"); + return NULL; + } /* * uadk_provider takes the highest priority * and overwrite the openssl.cnf property. */ - EVP_set_default_properties(NULL, "?provider=uadk_provider"); + EVP_set_default_properties(libctx, "?provider=uadk_provider"); /* * In asynchronous scenarios, if random numbers are obtained using * uadk provider cipher, deadlocks may occur. Therefore, random numbers are * obtained using default provider cipher. */ - (void)RAND_set_DRBG_type(prov_libctx_of(provctx), NULL, - "provider=default", NULL, NULL); + (void)RAND_set_DRBG_type(libctx, NULL, "provider=default", NULL, NULL); } *no_cache = 0; -- 2.33.0

Async packet reception timeout without timely failure return can lead to issues such as software calculation failure. First initialize the value of req.state to -1. If the value of req.state is not 0 after async_pause_job(), it indicates an exception in packet reception, and the function can directly return an error. Signed-off-by: Qi Tao <taoqi10@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov.h | 1 + src/uadk_prov_aead.c | 26 ++++++++------------------ src/uadk_prov_cipher.c | 8 ++++++-- src/uadk_prov_dh.c | 2 +- src/uadk_prov_digest.c | 6 +++++- src/uadk_prov_pkey.c | 2 +- src/uadk_prov_rsa.c | 2 +- 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/uadk_prov.h b/src/uadk_prov.h index 47bfb43..b6c236a 100644 --- a/src/uadk_prov.h +++ b/src/uadk_prov.h @@ -28,6 +28,7 @@ #define UADK_DEVICE_ERROR 3 #define HW_SEC_V2 2 #define HW_SEC_V3 3 +#define POLL_ERROR (-1) /* Copy openssl/providers/implementations/include/prov/names.h */ #define PROV_NAMES_SHA2_224 "SHA2-224:SHA-224:SHA224:2.16.840.1.101.3.4.2.4" diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index 1163328..6f33f43 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -360,7 +360,7 @@ static int uadk_do_aead_sync(struct aead_priv_ctx *priv, unsigned char *out, static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, unsigned char *out, const unsigned char *in, size_t inlen) { - struct uadk_e_cb_info *cb_param; + struct uadk_e_cb_info cb_param; int cnt = 0; int ret; @@ -376,22 +376,16 @@ static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, uadk_do_aead_async_prepare(priv, out, in, inlen); - cb_param = malloc(sizeof(struct uadk_e_cb_info)); - if (unlikely(!cb_param)) { - fprintf(stderr, "failed to alloc cb_param.\n"); - return UADK_AEAD_FAIL; - } - - cb_param->op = op; - cb_param->priv = &priv->req; + cb_param.op = op; + cb_param.priv = &priv->req; priv->req.cb = uadk_prov_aead_cb; - priv->req.cb_param = cb_param; + priv->req.cb_param = &cb_param; priv->req.msg_state = AEAD_MSG_BLOCK; - priv->req.state = UADK_AEAD_FAIL; + priv->req.state = POLL_ERROR; ret = async_get_free_task(&op->idx); if (unlikely(!ret)) - goto free_cb_param; + return UADK_AEAD_FAIL; do { ret = wd_do_aead_async(priv->sess, &priv->req); @@ -404,8 +398,7 @@ static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, continue; async_free_poll_task(op->idx, 0); - ret = UADK_AEAD_FAIL; - goto free_cb_param; + return UADK_AEAD_FAIL; } } while (ret == -EBUSY); @@ -413,15 +406,12 @@ static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, if (unlikely(!ret || priv->req.state)) { fprintf(stderr, "do aead async job failed, ret: %d, state: %u!\n", ret, priv->req.state); - ret = UADK_AEAD_FAIL; - goto free_cb_param; + return UADK_AEAD_FAIL; } if (priv->req.assoc_bytes) memcpy(out, priv->req.dst + priv->req.assoc_bytes, inlen); -free_cb_param: - free(cb_param); return ret; } diff --git a/src/uadk_prov_cipher.c b/src/uadk_prov_cipher.c index f63288f..eb31f2d 100644 --- a/src/uadk_prov_cipher.c +++ b/src/uadk_prov_cipher.c @@ -418,12 +418,15 @@ static int uadk_prov_cipher_init(struct cipher_priv_ctx *priv, static void async_cb(struct wd_cipher_req *req, void *data) { struct uadk_e_cb_info *cipher_cb_param; + struct wd_cipher_req *req_origin; struct async_op *op; if (!req || !req->cb_param) return; cipher_cb_param = req->cb_param; + req_origin = cipher_cb_param->priv; + req_origin->state = req->state; op = cipher_cb_param->op; if (op && op->job && !op->done) { op->done = 1; @@ -454,9 +457,10 @@ static int uadk_do_cipher_async(struct cipher_priv_ctx *priv, struct async_op *o } cb_param.op = op; - cb_param.priv = priv; + cb_param.priv = &priv->req; priv->req.cb = (void *)async_cb; priv->req.cb_param = &cb_param; + priv->req.state = POLL_ERROR; ret = async_get_free_task(&idx); if (!ret) return UADK_P_FAIL; @@ -472,7 +476,7 @@ static int uadk_do_cipher_async(struct cipher_priv_ctx *priv, struct async_op *o } while (ret == -EBUSY); ret = async_pause_job(priv, op, ASYNC_TASK_CIPHER); - if (!ret) + if (!ret || priv->req.state) return UADK_P_FAIL; return UADK_P_SUCCESS; diff --git a/src/uadk_prov_dh.c b/src/uadk_prov_dh.c index f3724ac..f6262d4 100644 --- a/src/uadk_prov_dh.c +++ b/src/uadk_prov_dh.c @@ -845,7 +845,7 @@ static int uadk_prov_dh_do_crypto(struct uadk_dh_sess *dh_sess) cb_param.priv = &dh_sess->req; dh_sess->req.cb = uadk_prov_dh_cb; dh_sess->req.cb_param = &cb_param; - dh_sess->req.status = -1; + dh_sess->req.status = POLL_ERROR; ret = async_get_free_task(&idx); if (!ret) goto err; diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index 921c3db..d6af557 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -497,12 +497,15 @@ soft_update: static void uadk_async_cb(struct wd_digest_req *req, void *data) { struct uadk_e_cb_info *digest_cb_param; + struct wd_digest_req *req_origin; struct async_op *op; if (!req || !req->cb_param) return; digest_cb_param = req->cb_param; + req_origin = digest_cb_param->priv; + req_origin->state = req->state; op = digest_cb_param->op; if (op && op->job && !op->done) { op->done = 1; @@ -540,9 +543,10 @@ static int uadk_do_digest_async(struct digest_priv_ctx *priv, struct async_op *o } cb_param.op = op; - cb_param.priv = priv; + cb_param.priv = &priv->req; priv->req.cb = (void *)uadk_async_cb; priv->req.cb_param = &cb_param; + priv->req.state = POLL_ERROR; ret = async_get_free_task(&idx); if (!ret) diff --git a/src/uadk_prov_pkey.c b/src/uadk_prov_pkey.c index 1323cfd..ca853ae 100644 --- a/src/uadk_prov_pkey.c +++ b/src/uadk_prov_pkey.c @@ -390,7 +390,7 @@ int uadk_prov_ecc_crypto(handle_t sess, struct wd_ecc_req *req, void *usr) cb_param.priv = req; req->cb = uadk_prov_ecc_cb; req->cb_param = &cb_param; - req->status = -1; + req->status = POLL_ERROR; ret = async_get_free_task(&idx); if (ret == 0) diff --git a/src/uadk_prov_rsa.c b/src/uadk_prov_rsa.c index eac4e46..8f7d734 100644 --- a/src/uadk_prov_rsa.c +++ b/src/uadk_prov_rsa.c @@ -1297,7 +1297,7 @@ static int rsa_do_crypto(struct uadk_rsa_sess *rsa_sess) cb_param.priv = &(rsa_sess->req); rsa_sess->req.cb = uadk_e_rsa_cb; rsa_sess->req.cb_param = &cb_param; - rsa_sess->req.status = -1; + rsa_sess->req.status = POLL_ERROR; ret = async_get_free_task(&idx); if (ret == 0) -- 2.33.0

From: Chenghai Huang <huangchenghai2@huawei.com> When HW hmac processing failed, uadk_provider could switch to soft mac to keep the work success. When HMAC switches to software, if the new libctx is not used, the digest of hardware is used for software computing. For the uadk, hmac and digest are the same module. When any fault occurs, do not try to use hardware again. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_hmac.c | 333 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 307 insertions(+), 26 deletions(-) diff --git a/src/uadk_prov_hmac.c b/src/uadk_prov_hmac.c index 3c70c5d..3e57012 100644 --- a/src/uadk_prov_hmac.c +++ b/src/uadk_prov_hmac.c @@ -31,6 +31,7 @@ #include "uadk_prov.h" #include "uadk_utils.h" +#define UADK_DO_SOFT (-0xE0) #define UADK_HMAC_SUCCESS 1 #define UADK_HMAC_FAIL 0 @@ -41,8 +42,14 @@ #define MAX_KEY_LEN 144 #define HMAC_BLOCK_SIZE (512 * 1024) #define ALG_NAME_SIZE 128 +#define PARAMS_SIZE 2 #define KEY_4BYTE_ALIGN(keylen) ((keylen + 3) & ~3) +#define SW_SWITCH_PRINT_ENABLE(SW) (SW ? ", switch to soft hmac" : "") + +#define SM3_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT (512) +#define MD5_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT (8 * 1024) +#define SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT (512) #define UADK_DIGEST_DEF_CTXS 1 #define UADK_DIGEST_OP_NUM 1 @@ -64,12 +71,17 @@ static pthread_mutex_t hmac_mutex = PTHREAD_MUTEX_INITIALIZER; struct hmac_priv_ctx { __u32 alg_id; __u32 state; + int switch_flag; size_t out_len; size_t blk_size; size_t keylen; size_t last_update_bufflen; size_t total_data_len; + size_t switch_threshold; OSSL_LIB_CTX *libctx; + OSSL_LIB_CTX *soft_libctx; + EVP_MAC_CTX *soft_ctx; + EVP_MAC *soft_md; handle_t sess; struct wd_digest_sess_setup setup; struct wd_digest_req req; @@ -83,23 +95,185 @@ struct hmac_priv_ctx { struct hmac_info { enum wd_digest_type alg; __u32 alg_id; + __u32 threshold; size_t out_len; size_t blk_size; const char ossl_alg_name[ALG_NAME_SIZE]; }; static struct hmac_info hmac_info_table[] = { - {WD_DIGEST_MD5, NID_md5, 16, 64, PROV_NAMES_MD5}, - {WD_DIGEST_SM3, NID_sm3, 32, 64, PROV_NAMES_SM3}, - {WD_DIGEST_SHA1, NID_sha1, 20, 64, PROV_NAMES_SHA1}, - {WD_DIGEST_SHA224, NID_sha224, 28, 64, PROV_NAMES_SHA2_224}, - {WD_DIGEST_SHA256, NID_sha256, 32, 64, PROV_NAMES_SHA2_256}, - {WD_DIGEST_SHA384, NID_sha384, 48, 64, PROV_NAMES_SHA2_384}, - {WD_DIGEST_SHA512, NID_sha512, 64, 128, PROV_NAMES_SHA2_512}, - {WD_DIGEST_SHA512_224, NID_sha512_224, 28, 128, PROV_NAMES_SHA2_512_224}, - {WD_DIGEST_SHA512_256, NID_sha512_256, 32, 128, PROV_NAMES_SHA2_512_256} + {WD_DIGEST_MD5, NID_md5, MD5_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 16, 64, PROV_NAMES_MD5}, + {WD_DIGEST_SM3, NID_sm3, SM3_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 32, 64, PROV_NAMES_SM3}, + {WD_DIGEST_SHA1, NID_sha1, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 20, 64, PROV_NAMES_SHA1}, + {WD_DIGEST_SHA224, NID_sha224, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 28, 64, PROV_NAMES_SHA2_224}, + {WD_DIGEST_SHA256, NID_sha256, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 32, 64, PROV_NAMES_SHA2_256}, + {WD_DIGEST_SHA384, NID_sha384, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 48, 64, PROV_NAMES_SHA2_384}, + {WD_DIGEST_SHA512, NID_sha512, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 64, 128, PROV_NAMES_SHA2_512}, + {WD_DIGEST_SHA512_224, NID_sha512_224, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 28, 128, PROV_NAMES_SHA2_512_224}, + {WD_DIGEST_SHA512_256, NID_sha512_256, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT, + 32, 128, PROV_NAMES_SHA2_512_256} }; +static int uadk_create_hmac_soft_ctx(struct hmac_priv_ctx *priv) +{ + if (priv->soft_md) + return UADK_HMAC_SUCCESS; + + priv->soft_libctx = OSSL_LIB_CTX_new(); + if (!priv->soft_libctx) { + fprintf(stderr, "new soft libctx failed.\n"); + return UADK_HMAC_FAIL; + } + + switch (priv->alg_id) { + case NID_md5: + case NID_sm3: + case NID_sha1: + case NID_sha224: + case NID_sha256: + case NID_sha384: + case NID_sha512: + case NID_sha512_224: + case NID_sha512_256: + priv->soft_md = EVP_MAC_fetch(priv->soft_libctx, "HMAC", NULL); + break; + default: + break; + } + + if (unlikely(!priv->soft_md)) { + fprintf(stderr, "hmac soft fetch failed.\n"); + goto free_libctx; + } + + priv->soft_ctx = EVP_MAC_CTX_new(priv->soft_md); + if (!priv->soft_ctx) { + fprintf(stderr, "hmac soft new ctx failed.\n"); + goto free_mac_md; + } + + return UADK_HMAC_SUCCESS; + +free_mac_md: + EVP_MAC_free(priv->soft_md); + priv->soft_md = NULL; +free_libctx: + OSSL_LIB_CTX_free(priv->soft_libctx); + priv->soft_libctx = NULL; + + return UADK_HMAC_FAIL; +} + +static int uadk_hmac_soft_init(struct hmac_priv_ctx *priv) +{ + OSSL_PARAM params[PARAMS_SIZE]; + OSSL_PARAM *p = params; + int ret; + + if (!priv->soft_md) + return UADK_HMAC_FAIL; + + /* The underlying digest to be used */ + *p++ = OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_DIGEST, priv->alg_name, + sizeof(priv->alg_name)); + *p = OSSL_PARAM_construct_end(); + + ret = EVP_MAC_init(priv->soft_ctx, priv->key, priv->keylen, params); + if (!ret) { + fprintf(stderr, "do soft hmac init failed!\n"); + return UADK_HMAC_FAIL; + } + + priv->switch_flag = UADK_DO_SOFT; + + return ret; +} + +static int uadk_hmac_soft_update(struct hmac_priv_ctx *priv, + const void *data, size_t len) +{ + int ret; + + if (!priv->soft_md) + return UADK_HMAC_FAIL; + + ret = EVP_MAC_update(priv->soft_ctx, data, len); + if (!ret) + fprintf(stderr, "do soft hmac update failed!\n"); + + return ret; +} + +static int uadk_hmac_soft_final(struct hmac_priv_ctx *priv, unsigned char *out) +{ + size_t hmac_length; + int ret; + + if (!priv->soft_md) + return UADK_HMAC_FAIL; + + ret = EVP_MAC_final(priv->soft_ctx, out, &hmac_length, priv->out_len); + if (!ret) + fprintf(stderr, "do soft hmac final failed!\n"); + + return ret; +} + +static void hmac_soft_cleanup(struct hmac_priv_ctx *priv) +{ + if (priv->soft_ctx) { + EVP_MAC_CTX_free(priv->soft_ctx); + priv->soft_ctx = NULL; + } + + if (priv->soft_md) { + EVP_MAC_free(priv->soft_md); + priv->soft_md = NULL; + } + + if (priv->soft_libctx) { + OSSL_LIB_CTX_free(priv->soft_libctx); + priv->soft_libctx = NULL; + } + + priv->switch_flag = 0; +} + +static int uadk_hmac_soft_work(struct hmac_priv_ctx *priv, int inl, + unsigned char *out) +{ + int ret; + + if (!priv->soft_md) + return UADK_HMAC_FAIL; + + if (!priv->switch_flag) { + ret = uadk_hmac_soft_init(priv); + if (unlikely(!ret)) + return UADK_HMAC_FAIL; + } + + if (inl) { + ret = uadk_hmac_soft_update(priv, priv->data, inl); + if (unlikely(!ret)) + goto out; + } + + ret = uadk_hmac_soft_final(priv, out); + +out: + hmac_soft_cleanup(priv); + return ret; +} + static OSSL_FUNC_mac_newctx_fn uadk_prov_hmac_newctx; static OSSL_FUNC_mac_dupctx_fn uadk_prov_hmac_dupctx; static OSSL_FUNC_mac_freectx_fn uadk_prov_hmac_freectx; @@ -180,6 +354,7 @@ static int uadk_get_hmac_info(struct hmac_priv_ctx *priv) priv->setup.mode = WD_DIGEST_HMAC; priv->req.out_buf_bytes = MAX_DIGEST_LENGTH; priv->req.out_bytes = hmac_info_table[i].out_len; + priv->switch_threshold = hmac_info_table[i].threshold; return UADK_HMAC_SUCCESS; } @@ -310,9 +485,15 @@ static int uadk_hmac_ctx_init(struct hmac_priv_ctx *priv) struct sched_params params = {0}; int ret; + if (enable_sw_offload) + uadk_create_hmac_soft_ctx(priv); + ret = uadk_prov_hmac_dev_init(priv); - if (unlikely(ret <= 0)) - return UADK_HMAC_FAIL; + if (unlikely(ret <= 0)) { + fprintf(stderr, "uadk failed to initialize hmac%s.\n", + SW_SWITCH_PRINT_ENABLE(enable_sw_offload)); + goto soft_init; + } /* Use the default numa parameters */ params.numa_id = -1; @@ -321,13 +502,15 @@ static int uadk_hmac_ctx_init(struct hmac_priv_ctx *priv) if (!priv->sess) { priv->sess = wd_digest_alloc_sess(&priv->setup); if (unlikely(!priv->sess)) { - fprintf(stderr, "uadk failed to alloc hmac sess.\n"); - return UADK_HMAC_FAIL; + fprintf(stderr, "uadk failed to alloc hmac sess%s.\n", + SW_SWITCH_PRINT_ENABLE(enable_sw_offload)); + goto soft_init; } ret = wd_digest_set_key(priv->sess, priv->key, priv->keylen); if (ret) { - fprintf(stderr, "uadk failed to set hmac key!\n"); + fprintf(stderr, "uadk failed to set hmac key%s.\n", + SW_SWITCH_PRINT_ENABLE(enable_sw_offload)); goto free_sess; } } @@ -337,7 +520,9 @@ static int uadk_hmac_ctx_init(struct hmac_priv_ctx *priv) free_sess: wd_digest_free_sess(priv->sess); priv->sess = 0; - return UADK_HMAC_FAIL; + +soft_init: + return uadk_hmac_soft_init(priv); } static void uadk_hmac_async_cb(struct wd_digest_req *req, void *data) @@ -364,6 +549,11 @@ static int uadk_do_hmac_sync(struct hmac_priv_ctx *priv) { int ret; + if (priv->soft_md && + priv->req.in_bytes <= priv->switch_threshold && + priv->state == SEC_DIGEST_INIT) + return UADK_HMAC_FAIL; + ret = wd_do_digest_sync(priv->sess, &priv->req); if (ret) { fprintf(stderr, "do sec hmac sync failed.\n"); @@ -379,6 +569,11 @@ static int uadk_do_hmac_async(struct hmac_priv_ctx *priv, struct async_op *op) int idx, ret; int cnt = 0; + if (unlikely(priv->switch_flag == UADK_DO_SOFT)) { + fprintf(stderr, "soft switching is not supported in asynchronous mode.\n"); + return UADK_HMAC_FAIL; + } + cb_param.op = op; cb_param.priv = &priv->req; priv->req.cb = (void *)uadk_hmac_async_cb; @@ -461,8 +656,9 @@ static int uadk_hmac_update_inner(struct hmac_priv_ctx *priv, const void *data, ret = uadk_do_hmac_sync(priv); if (!ret) { - fprintf(stderr, "do sec digest update failed.\n"); - return UADK_HMAC_FAIL; + fprintf(stderr, "do sec hmac update failed%s.\n", + SW_SWITCH_PRINT_ENABLE(enable_sw_offload)); + goto do_soft_hmac; } remain_len -= processing_len; @@ -473,6 +669,39 @@ static int uadk_hmac_update_inner(struct hmac_priv_ctx *priv, const void *data, uadk_memcpy(priv->data, input_data, priv->last_update_bufflen); return UADK_HMAC_SUCCESS; + +do_soft_hmac: + if (priv->state == SEC_DIGEST_FIRST_UPDATING) { + ret = uadk_hmac_soft_init(priv); + if (!ret) + return ret; + + /* + * If the hardware fails to process the data in the cache, + * the software computing needs to finish the cached data first. + */ + if (processing_len < HMAC_BLOCK_SIZE) { + ret = uadk_hmac_soft_update(priv, priv->data, HMAC_BLOCK_SIZE); + if (!ret) + goto err_out; + + remain_len -= processing_len; + input_data += processing_len; + } + + ret = uadk_hmac_soft_update(priv, input_data, remain_len); + if (!ret) + goto err_out; + + /* the soft ctx will be free in the final stage. */ + return ret; + } + + return UADK_HMAC_FAIL; + +err_out: + hmac_soft_cleanup(priv); + return ret; } static int uadk_hmac_update(struct hmac_priv_ctx *priv, const void *data, size_t data_len) @@ -482,6 +711,9 @@ static int uadk_hmac_update(struct hmac_priv_ctx *priv, const void *data, size_t return UADK_HMAC_FAIL; } + if (unlikely(priv->switch_flag == UADK_DO_SOFT)) + goto soft_update; + priv->total_data_len += data_len; if (priv->last_update_bufflen + data_len <= HMAC_BLOCK_SIZE) { @@ -491,6 +723,9 @@ static int uadk_hmac_update(struct hmac_priv_ctx *priv, const void *data, size_t } return uadk_hmac_update_inner(priv, data, data_len); + +soft_update: + return uadk_hmac_soft_update(priv, data, data_len); } static int uadk_hmac_final(struct hmac_priv_ctx *priv, unsigned char *digest) @@ -503,9 +738,12 @@ static int uadk_hmac_final(struct hmac_priv_ctx *priv, unsigned char *digest) return UADK_HMAC_FAIL; } - ret = uadk_hmac_ctx_init(priv); - if (!ret) - return UADK_HMAC_FAIL; + /* It dose not need to be initialized again if the software calculation is applied. */ + if (priv->switch_flag != UADK_DO_SOFT) { + ret = uadk_hmac_ctx_init(priv); + if (!ret) + return UADK_HMAC_FAIL; + } priv->req.in = priv->data; priv->req.out = priv->state == SEC_DIGEST_INIT ? digest : priv->out; @@ -520,19 +758,37 @@ static int uadk_hmac_final(struct hmac_priv_ctx *priv, unsigned char *digest) return UADK_HMAC_FAIL; } - if (!op.job) + if (!op.job) { + /* Synchronous, only the synchronous mode supports soft computing */ + if (unlikely(priv->switch_flag == UADK_DO_SOFT)) { + ret = uadk_hmac_soft_final(priv, digest); + hmac_soft_cleanup(priv); + goto clear; + } + ret = uadk_do_hmac_sync(priv); - else + if (!ret) + goto do_hmac_err; + } else { ret = uadk_do_hmac_async(priv, &op); - - if (!ret) - goto clear; + if (!ret) + goto clear; + } if (priv->state != SEC_DIGEST_INIT) memcpy(digest, priv->req.out, priv->req.out_bytes); return UADK_HMAC_SUCCESS; +do_hmac_err: + if (priv->state == SEC_DIGEST_INIT) { + fprintf(stderr, "do sec digest final failed%s.\n", + SW_SWITCH_PRINT_ENABLE(enable_sw_offload)); + ret = uadk_hmac_soft_work(priv, priv->req.in_bytes, digest); + } else { + ret = UADK_HMAC_FAIL; + fprintf(stderr, "do sec digest final failed.\n"); + } clear: async_clear_async_event_notification(); return ret; @@ -541,6 +797,7 @@ clear: static void *uadk_prov_hmac_dupctx(void *hctx) { struct hmac_priv_ctx *dst_ctx, *src_ctx; + int ret; if (!hctx) return NULL; @@ -569,8 +826,23 @@ static void *uadk_prov_hmac_dupctx(void *hctx) if (!dst_ctx->data) goto free_ctx; + if (dst_ctx->soft_ctx) { + dst_ctx->soft_libctx = NULL; + dst_ctx->soft_ctx = EVP_MAC_CTX_dup(src_ctx->soft_ctx); + if (!dst_ctx->soft_ctx) { + fprintf(stderr, "create soft_ctx failed in ctx copy.\n"); + goto free_data; + } + + ret = EVP_MAC_up_ref(dst_ctx->soft_md); + if (!ret) + goto free_dup; + } + return dst_ctx; +free_dup: + EVP_MAC_CTX_free(dst_ctx->soft_ctx); free_data: OPENSSL_clear_free(dst_ctx->data, HMAC_BLOCK_SIZE); free_ctx: @@ -596,6 +868,7 @@ static void uadk_prov_hmac_freectx(void *hctx) return; } + hmac_soft_cleanup(priv); uadk_hmac_cleanup(priv); OPENSSL_clear_free(priv, sizeof(*priv)); } @@ -637,6 +910,9 @@ static int uadk_prov_hmac_init(void *hctx, const unsigned char *key, if (unlikely(!ret)) return UADK_HMAC_FAIL; + if (enable_sw_offload) + uadk_create_hmac_soft_ctx(hctx); + if (key) { ret = uadk_prov_hmac_setkey(priv, key, keylen); if (!ret) @@ -645,9 +921,14 @@ static int uadk_prov_hmac_init(void *hctx, const unsigned char *key, ret = uadk_prov_hmac_dev_init(priv); if (unlikely(ret <= 0)) - return UADK_HMAC_FAIL; + goto soft_init; return UADK_HMAC_SUCCESS; + +soft_init: + fprintf(stderr, "uadk failed to initialize dev%s.\n", + SW_SWITCH_PRINT_ENABLE(enable_sw_offload)); + return uadk_hmac_soft_init(priv); } static int uadk_prov_hmac_update(void *hctx, const unsigned char *data, size_t datalen) -- 2.33.0

From: lizhi <lizhi206@huawei.com> When hardware failures occur, or in cases of insufficient queue resources or unsupported algorithm specifications, the uadk provider can switch rsa to openssl soft computation. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_rsa.c | 180 ++++++++++++++++++++++++++------------------ 1 file changed, 105 insertions(+), 75 deletions(-) diff --git a/src/uadk_prov_rsa.c b/src/uadk_prov_rsa.c index 8f7d734..65e78c3 100644 --- a/src/uadk_prov_rsa.c +++ b/src/uadk_prov_rsa.c @@ -62,7 +62,6 @@ #define GENCB_NEXT 2 #define GENCB_RETRY 3 #define PRIME_CHECK_BIT_NUM 4 -#define SOFT_SWITCH 0 UADK_PKEY_KEYMGMT_DESCR(rsa, RSA); UADK_PKEY_SIGNATURE_DESCR(rsa, RSA); @@ -284,11 +283,14 @@ enum { MAX_CODE, }; +static pthread_mutex_t sig_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t asym_mutex = PTHREAD_MUTEX_INITIALIZER; + static UADK_PKEY_SIGNATURE get_default_rsa_signature(void) { static UADK_PKEY_SIGNATURE s_signature; static int initilazed; - + pthread_mutex_lock(&sig_mutex); if (!initilazed) { UADK_PKEY_SIGNATURE *signature = (UADK_PKEY_SIGNATURE *)EVP_SIGNATURE_fetch(NULL, "RSA", "provider=default"); @@ -301,6 +303,7 @@ static UADK_PKEY_SIGNATURE get_default_rsa_signature(void) fprintf(stderr, "failed to EVP_SIGNATURE_fetch default RSA provider\n"); } } + pthread_mutex_unlock(&sig_mutex); return s_signature; } @@ -308,7 +311,7 @@ static UADK_PKEY_ASYM_CIPHER get_default_rsa_asym_cipher(void) { static UADK_PKEY_ASYM_CIPHER s_asym_cipher; static int initilazed; - + pthread_mutex_lock(&asym_mutex); if (!initilazed) { UADK_PKEY_ASYM_CIPHER *asym_cipher = (UADK_PKEY_ASYM_CIPHER *)EVP_ASYM_CIPHER_fetch(NULL, "RSA", @@ -322,6 +325,7 @@ static UADK_PKEY_ASYM_CIPHER get_default_rsa_asym_cipher(void) fprintf(stderr, "failed to EVP_ASYM_CIPHER_fetch default RSA provider\n"); } } + pthread_mutex_unlock(&asym_mutex); return s_asym_cipher; } @@ -2001,6 +2005,17 @@ static int uadk_signature_rsa_verify_init(void *vprsactx, void *vrsa, return uadk_rsa_init(vprsactx, vrsa, params, EVP_PKEY_OP_VERIFY); } +static int uadk_rsa_sw_verify(void *vprsactx, const unsigned char *sig, + size_t siglen, const unsigned char *tbs, + size_t tbslen) +{ + if (!enable_sw_offload || !get_default_rsa_signature().verify) + return UADK_E_FAIL; + + fprintf(stderr, "switch to openssl software calculation in verifaction.\n"); + return get_default_rsa_signature().verify(vprsactx, sig, siglen, tbs, tbslen); +} + static int uadk_signature_rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen, const unsigned char *tbs, size_t tbslen) @@ -2008,10 +2023,10 @@ static int uadk_signature_rsa_verify(void *vprsactx, const unsigned char *sig, PROV_RSA_SIG_CTX *priv = (PROV_RSA_SIG_CTX *)vprsactx; size_t rslen = 0; - if (priv->soft && SOFT_SWITCH) - goto soft; - else if (priv->soft && !SOFT_SWITCH) - return UADK_E_FAIL; + if (priv->soft) { + rslen = UADK_DO_SOFT; + goto exe_soft; + } /* todo call public_verify */ if (priv->md != NULL) { @@ -2021,12 +2036,8 @@ static int uadk_signature_rsa_verify(void *vprsactx, const unsigned char *sig, return UADK_E_FAIL; rslen = uadk_prov_rsa_public_verify(siglen, sig, priv->tbuf, priv->rsa, priv->pad_mode); - if (rslen == UADK_DO_SOFT || rslen == UADK_E_FAIL) { - if (rslen == UADK_DO_SOFT && SOFT_SWITCH) - goto soft; - else - return UADK_E_FAIL; - } + if (rslen == UADK_DO_SOFT || rslen == UADK_E_FAIL) + goto exe_soft; } if ((rslen != tbslen) || memcmp(tbs, priv->tbuf, rslen)) @@ -2034,12 +2045,21 @@ static int uadk_signature_rsa_verify(void *vprsactx, const unsigned char *sig, return UADK_E_SUCCESS; -soft: - fprintf(stderr, "switch to execute openssl software calculation.\n"); - if (!get_default_rsa_signature().verify) +exe_soft: + if (rslen == UADK_DO_SOFT) + return uadk_rsa_sw_verify(vprsactx, sig, siglen, tbs, tbslen); + return UADK_E_FAIL; +} + +static int uadk_rsa_sw_sign(void *vprsactx, unsigned char *sig, + size_t *siglen, size_t sigsize, + const unsigned char *tbs, size_t tbslen) +{ + if (!enable_sw_offload || !get_default_rsa_signature().sign) return UADK_E_FAIL; - return get_default_rsa_signature().verify(vprsactx, sig, siglen, tbs, tbslen); + fprintf(stderr, "switch to openssl software calculation in rsa signature.\n"); + return get_default_rsa_signature().sign(vprsactx, sig, siglen, sigsize, tbs, tbslen); } static int uadk_signature_rsa_sign(void *vprsactx, unsigned char *sig, @@ -2050,10 +2070,10 @@ static int uadk_signature_rsa_sign(void *vprsactx, unsigned char *sig, size_t rsasize = uadk_rsa_size(priv->rsa); int ret; - if (priv->soft && SOFT_SWITCH) - goto soft; - else if (priv->soft && !SOFT_SWITCH) - return UADK_E_FAIL; + if (priv->soft) { + ret = UADK_DO_SOFT; + goto exe_soft; + } if (sig == NULL) { *siglen = rsasize; @@ -2067,12 +2087,8 @@ static int uadk_signature_rsa_sign(void *vprsactx, unsigned char *sig, } ret = uadk_prov_rsa_private_sign(tbslen, tbs, sig, priv->rsa, priv->pad_mode); - if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) { - if (ret == UADK_DO_SOFT && SOFT_SWITCH) - goto soft; - else - return UADK_E_FAIL; - } + if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) + goto exe_soft; if (ret < 0) return ret; @@ -2080,12 +2096,10 @@ static int uadk_signature_rsa_sign(void *vprsactx, unsigned char *sig, *siglen = ret; return UADK_E_SUCCESS; -soft: - fprintf(stderr, "switch to execute openssl software calculation.\n"); - if (!get_default_rsa_signature().sign) - return UADK_E_FAIL; - - return get_default_rsa_signature().sign(vprsactx, sig, siglen, sigsize, tbs, tbslen); +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_rsa_sw_sign(vprsactx, sig, siglen, sigsize, tbs, tbslen); + return UADK_E_FAIL; } static int uadk_signature_rsa_sign_init(void *vprsactx, void *vrsa, const OSSL_PARAM params[]) @@ -2355,6 +2369,17 @@ static int uadk_asym_cipher_rsa_decrypt_init(void *vprsactx, void *vrsa, return uadk_rsa_asym_init(vprsactx, vrsa, params, EVP_PKEY_OP_DECRYPT); } +static int uadk_rsa_sw_encrypt(void *vprsactx, unsigned char *out, + size_t *outlen, size_t outsize, + const unsigned char *in, size_t inlen) +{ + if (!enable_sw_offload || !get_default_rsa_asym_cipher().encrypt) + return UADK_E_FAIL; + + fprintf(stderr, "switch to openssl software calculation in rsa encryption.\n"); + return get_default_rsa_asym_cipher().encrypt(vprsactx, out, outlen, outsize, in, inlen); +} + static int uadk_asym_cipher_rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen, size_t outsize, const unsigned char *in, size_t inlen) @@ -2363,10 +2388,10 @@ static int uadk_asym_cipher_rsa_encrypt(void *vprsactx, unsigned char *out, size_t len; int ret; - if (priv->soft && SOFT_SWITCH) - goto soft; - else if (priv->soft && !SOFT_SWITCH) - return UADK_E_FAIL; + if (priv->soft) { + ret = UADK_DO_SOFT; + goto exe_soft; + } if (out == NULL) { len = uadk_rsa_size(priv->rsa); @@ -2379,22 +2404,27 @@ static int uadk_asym_cipher_rsa_encrypt(void *vprsactx, unsigned char *out, } ret = uadk_prov_rsa_public_encrypt(inlen, in, out, priv->rsa, priv->pad_mode); - if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) { - if (ret == UADK_DO_SOFT && SOFT_SWITCH) - goto soft; - else - return UADK_E_FAIL; - } + if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) + goto exe_soft; *outlen = ret; return UADK_E_SUCCESS; -soft: - fprintf(stderr, "switch to execute openssl software calculation.\n"); - if (!get_default_rsa_asym_cipher().encrypt) +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_rsa_sw_encrypt(vprsactx, out, outlen, outsize, in, inlen); + return UADK_E_FAIL; +} + +static int uadk_rsa_sw_decrypt(void *vprsactx, unsigned char *out, + size_t *outlen, size_t outsize, + const unsigned char *in, size_t inlen) +{ + if (!enable_sw_offload || !get_default_rsa_asym_cipher().decrypt) return UADK_E_FAIL; - return get_default_rsa_asym_cipher().encrypt(vprsactx, out, outlen, outsize, in, inlen); + fprintf(stderr, "switch to openssl software calculation in rsa decryption.\n"); + return get_default_rsa_asym_cipher().decrypt(vprsactx, out, outlen, outsize, in, inlen); } static int uadk_asym_cipher_rsa_decrypt(void *vprsactx, unsigned char *out, @@ -2405,10 +2435,10 @@ static int uadk_asym_cipher_rsa_decrypt(void *vprsactx, unsigned char *out, size_t len = uadk_rsa_size(priv->rsa); int ret; - if (priv->soft && SOFT_SWITCH) - goto soft; - else if (priv->soft && !SOFT_SWITCH) - return UADK_E_FAIL; + if (priv->soft) { + ret = UADK_DO_SOFT; + goto exe_soft; + } if (out == NULL) { if (len == 0) { @@ -2425,21 +2455,16 @@ static int uadk_asym_cipher_rsa_decrypt(void *vprsactx, unsigned char *out, } ret = uadk_prov_rsa_private_decrypt(inlen, in, out, priv->rsa, priv->pad_mode); - if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) { - if (ret == UADK_DO_SOFT && SOFT_SWITCH) - goto soft; - else - return UADK_E_FAIL; - } + if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) + goto exe_soft; + *outlen = ret; return UADK_E_SUCCESS; -soft: - fprintf(stderr, "switch to execute openssl software calculation.\n"); - if (!get_default_rsa_asym_cipher().decrypt) - return UADK_E_FAIL; - - return get_default_rsa_asym_cipher().decrypt(vprsactx, out, outlen, outsize, in, inlen); +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_rsa_sw_decrypt(vprsactx, out, outlen, outsize, in, inlen); + return UADK_E_FAIL; } static int uadk_asym_cipher_rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params) @@ -2588,10 +2613,20 @@ static RSA *ossl_rsa_new_with_ctx(OSSL_LIB_CTX *libctx) } ret->libctx = libctx; + ret->meth = RSA_get_default_method(); return ret; } +static void *uadk_rsa_sw_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) +{ + if (!enable_sw_offload || !get_default_rsa_keymgmt().gen) + return NULL; + + fprintf(stderr, "switch to openssl software calculation in rsa key generation.\n"); + return get_default_rsa_keymgmt().gen(genctx, osslcb, cbarg); +} + static void *uadk_keymgmt_rsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) { struct rsa_gen_ctx *gctx = genctx; @@ -2603,10 +2638,10 @@ static void *uadk_keymgmt_rsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cba return NULL; ret = uadk_prov_rsa_init(); - if (ret && SOFT_SWITCH) + if (ret) { + ret = UADK_DO_SOFT; goto exe_soft; - else if (ret && !SOFT_SWITCH) - return NULL; + } rsa = ossl_rsa_new_with_ctx(gctx->libctx); if (rsa == NULL) @@ -2622,11 +2657,7 @@ static void *uadk_keymgmt_rsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cba if (ret == UADK_DO_SOFT || ret == UADK_E_FAIL) { BN_GENCB_free(gencb); uadk_keymgmt_rsa_free(rsa); - - if (ret == UADK_DO_SOFT && SOFT_SWITCH) - goto exe_soft; - else - return NULL; + goto exe_soft; } uadk_rsa_clear_flags(rsa, RSA_FLAG_TYPE_MASK); @@ -2636,10 +2667,9 @@ static void *uadk_keymgmt_rsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cba return rsa; exe_soft: - fprintf(stderr, "switch to execute openssl software calculation.\n"); - if (!get_default_rsa_keymgmt().gen) - return NULL; - return get_default_rsa_keymgmt().gen(genctx, osslcb, cbarg); + if (ret == UADK_DO_SOFT) + return uadk_rsa_sw_gen(genctx, osslcb, cbarg); + return NULL; } static void uadk_keymgmt_rsa_gen_cleanup(void *genctx) -- 2.33.0

From: lizhi <lizhi206@huawei.com> When hardware failures occur, or in cases of insufficient queue resources or unsupported algorithm specifications, the uadk provider can switch ecx to openssl soft computation. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_ecx.c | 242 ++++++++++++++++++++++++++++++-------------- 1 file changed, 166 insertions(+), 76 deletions(-) diff --git a/src/uadk_prov_ecx.c b/src/uadk_prov_ecx.c index 47fe1a9..3cc7fa1 100644 --- a/src/uadk_prov_ecx.c +++ b/src/uadk_prov_ecx.c @@ -50,14 +50,56 @@ static inline int UADK_CRYPTO_DOWN_REF(int *val, int *ret, __atomic_thread_fence(__ATOMIC_ACQUIRE); return 1; } +static pthread_mutex_t x25519_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t x448_mutex = PTHREAD_MUTEX_INITIALIZER; UADK_PKEY_KEYMGMT_DESCR(x448, X448); UADK_PKEY_KEYEXCH_DESCR(x448, X448); +static UADK_PKEY_KEYEXCH get_default_x448_keyexch(void) +{ + static UADK_PKEY_KEYEXCH s_keyexch; + static int initilazed; + + pthread_mutex_lock(&x448_mutex); + if (!initilazed) { + UADK_PKEY_KEYEXCH *keyexch = + (UADK_PKEY_KEYEXCH *)EVP_KEYEXCH_fetch(NULL, "X448", "provider=default"); + if (keyexch) { + s_keyexch = *keyexch; + EVP_KEYEXCH_free((EVP_KEYEXCH *)keyexch); + initilazed = 1; + } else { + fprintf(stderr, "failed to EVP_KEYEXCH_fetch default X448 provider\n"); + } + } + pthread_mutex_unlock(&x448_mutex); + + return s_keyexch; +} + UADK_PKEY_KEYMGMT_DESCR(x25519, X25519); UADK_PKEY_KEYEXCH_DESCR(x25519, X25519); +static UADK_PKEY_KEYEXCH get_default_x25519_keyexch(void) +{ + static UADK_PKEY_KEYEXCH s_keyexch; + static int initilazed; -static pthread_mutex_t x25519_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_mutex_t x448_mutex = PTHREAD_MUTEX_INITIALIZER; + pthread_mutex_lock(&x25519_mutex); + if (!initilazed) { + UADK_PKEY_KEYEXCH *keyexch = + (UADK_PKEY_KEYEXCH *)EVP_KEYEXCH_fetch(NULL, "X25519", "provider=default"); + if (keyexch) { + s_keyexch = *keyexch; + EVP_KEYEXCH_free((EVP_KEYEXCH *)keyexch); + initilazed = 1; + } else { + fprintf(stderr, "failed to EVP_KEYEXCH_fetch default X25519 provider\n"); + } + } + pthread_mutex_unlock(&x25519_mutex); + + return s_keyexch; +} typedef enum { ECX_KEY_TYPE_X25519 = 0x0, @@ -706,44 +748,89 @@ static int uadk_prov_ecx_keygen_set_prikey(PROV_ECX_KEYMGMT_CTX *gctx, ECX_KEY * return UADK_P_SUCCESS; } -static void *uadk_prov_ecx_keygen(PROV_ECX_KEYMGMT_CTX *gctx) +static int uadk_prov_ecx_keygen(PROV_ECX_KEYMGMT_CTX *gctx, ECX_KEY **ecx_key) { struct wd_ecc_req req = {0}; - ECX_KEY *ecx_key = NULL; int ret; - ecx_key = uadk_prov_ecx_create_prikey(gctx); if (ecx_key == NULL) - return NULL; + return UADK_P_FAIL; + + *ecx_key = uadk_prov_ecx_create_prikey(gctx); + if (*ecx_key == NULL) + return UADK_P_FAIL; ret = uadk_prov_ecx_keygen_init_iot(gctx->sess, &req); if (ret == UADK_P_FAIL) goto free_prikey; - ret = uadk_prov_ecx_keygen_set_prikey(gctx, ecx_key); + ret = uadk_prov_ecx_keygen_set_prikey(gctx, *ecx_key); if (ret == UADK_P_FAIL) goto uninit_iot; ret = uadk_prov_ecc_crypto(gctx->sess, &req, (void *)gctx->sess); - if (ret == UADK_P_FAIL) + if (ret != UADK_P_SUCCESS) { + ret = UADK_DO_SOFT; goto uninit_iot; - - ret = uadk_prov_ecx_set_pkey(gctx, &req, ecx_key); + } + ret = uadk_prov_ecx_set_pkey(gctx, &req, *ecx_key); if (ret == UADK_P_FAIL) goto uninit_iot; - ecx_key->haspubkey = 1; + (*ecx_key)->haspubkey = 1; uadk_prov_ecx_keygen_uninit_iot(gctx->sess, &req); - return ecx_key; + return UADK_P_SUCCESS; uninit_iot: uadk_prov_ecx_keygen_uninit_iot(gctx->sess, &req); free_prikey: - uadk_prov_ecx_free_prikey(ecx_key); + uadk_prov_ecx_free_prikey(*ecx_key); - return NULL; + return ret; +} + +static void *uadk_ecx_sw_gen(void *genctx, OSSL_CALLBACK *cb, void *cb_params, + ECX_KEY_TYPE alg_type) +{ + if (!enable_sw_offload) + return NULL; + + fprintf(stderr, "switch to openssl software calculation in ecx generation.\n"); + switch (alg_type) { + case ECX_KEY_TYPE_X448: + if (!get_default_x448_keymgmt().gen) + return NULL; + return get_default_x448_keymgmt().gen(genctx, cb, cb_params); + case ECX_KEY_TYPE_X25519: + if (!get_default_x25519_keymgmt().gen) + return NULL; + return get_default_x25519_keymgmt().gen(genctx, cb, cb_params); + default: + return NULL; + } +} + +static int uadk_ecx_sw_derive(void *vecxctx, unsigned char *secret, size_t *secretlen, + size_t outlen, ECX_KEY_TYPE alg_type) +{ + if (!enable_sw_offload) + return UADK_P_FAIL; + + fprintf(stderr, "switch to openssl software calculation in ecx derivation.\n"); + switch (alg_type) { + case ECX_KEY_TYPE_X448: + if (!get_default_x448_keyexch().derive) + return UADK_P_FAIL; + return get_default_x448_keyexch().derive(vecxctx, secret, secretlen, outlen); + case ECX_KEY_TYPE_X25519: + if (!get_default_x25519_keyexch().derive) + return UADK_P_FAIL; + return get_default_x25519_keyexch().derive(vecxctx, secret, secretlen, outlen); + default: + return UADK_P_FAIL; + } } static void *uadk_keymgmt_x448_gen(void *genctx, OSSL_CALLBACK *cb, void *cb_params) @@ -765,50 +852,39 @@ static void *uadk_keymgmt_x448_gen(void *genctx, OSSL_CALLBACK *cb, void *cb_par ret = uadk_prov_keymgmt_get_support_state(KEYMGMT_X448); if (ret == UADK_P_FAIL) { fprintf(stderr, "failed to get hardware x448 keygen support\n"); - return NULL; + ret = UADK_DO_SOFT; + goto exe_soft; } ret = uadk_prov_ecc_init("x448"); if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to init x448\n"); - return NULL; + ret = UADK_DO_SOFT; + goto exe_soft; } gctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X448); if (gctx->sess == (handle_t)0) { fprintf(stderr, "failed to alloc x448 sess\n"); - return NULL; + ret = UADK_P_FAIL; + goto exe_soft; } - ecx_key = uadk_prov_ecx_keygen(gctx); - if (ecx_key == NULL) + ret = uadk_prov_ecx_keygen(gctx, &ecx_key); + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to generate x448 key\n"); + uadk_prov_ecx_free_sess(gctx->sess); + goto exe_soft; + } uadk_prov_ecx_free_sess(gctx->sess); return ecx_key; -} - -static UADK_PKEY_KEYEXCH get_default_x448_keyexch(void) -{ - static UADK_PKEY_KEYEXCH s_keyexch; - static int initilazed; - - pthread_mutex_lock(&x448_mutex); - if (!initilazed) { - UADK_PKEY_KEYEXCH *keyexch = - (UADK_PKEY_KEYEXCH *)EVP_KEYEXCH_fetch(NULL, "X448", "provider=default"); - if (keyexch) { - s_keyexch = *keyexch; - EVP_KEYEXCH_free((EVP_KEYEXCH *)keyexch); - initilazed = 1; - } else { - fprintf(stderr, "failed to EVP_KEYEXCH_fetch default X448 provider\n"); - } - } - pthread_mutex_unlock(&x448_mutex); - return s_keyexch; +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_ecx_sw_gen(genctx, cb, cb_params, ECX_KEY_TYPE_X448); + return NULL; } static void *uadk_keyexch_x448_newctx(void *provctx) @@ -1101,12 +1177,15 @@ static int uadk_prov_ecx_derive(PROV_ECX_KEYEXCH_CTX *ecxctx, unsigned char *key goto uninit_iot; ret = uadk_prov_ecc_crypto(ecxctx->sess, &req, (void *)ecxctx); - if (ret == UADK_P_FAIL) + if (ret != UADK_P_SUCCESS) { + ret = UADK_DO_SOFT; goto uninit_iot; - + } wd_ecxdh_get_out_params(req.dst, &s_key); - if (!s_key) + if (!s_key) { + ret = UADK_P_FAIL; goto uninit_iot; + } ret = uadk_prov_reverse_bytes((unsigned char *)s_key->x.data, s_key->x.dsize); if (ret == UADK_P_FAIL) @@ -1155,30 +1234,41 @@ static int uadk_keyexch_x448_derive(void *vecxctx, unsigned char *secret, size_t ret = uadk_prov_keyexch_get_support_state(KEYEXCH_X448); if (ret == UADK_P_FAIL) { fprintf(stderr, "failed to get hardware x448 keyexch support\n"); - return UADK_P_FAIL; + ret = UADK_DO_SOFT; + goto exe_soft; } ret = uadk_prov_ecc_init("x448"); if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to init x448\n"); - return UADK_P_FAIL; + ret = UADK_DO_SOFT; + goto exe_soft; } ecxctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X448); if (ecxctx->sess == (handle_t)0) { fprintf(stderr, "failed to alloc sess\n"); - return UADK_P_FAIL; + ret = UADK_P_FAIL; + goto exe_soft; } ret = uadk_prov_ecx_derive(ecxctx, secret, &ecxctx->keylen); - if (ret == UADK_P_FAIL) + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to do x448 derive\n"); + uadk_prov_ecx_free_sess(ecxctx->sess); + goto exe_soft; + } *secretlen = ecxctx->keylen; uadk_prov_ecx_free_sess(ecxctx->sess); return ret; + +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_ecx_sw_derive(vecxctx, secret, secretlen, outlen, ECX_KEY_TYPE_X448); + return UADK_P_FAIL; } static void *uadk_keyexch_ecx_dupctx(void *vecxctx) @@ -1409,50 +1499,39 @@ static void *uadk_keymgmt_x25519_gen(void *genctx, OSSL_CALLBACK *cb, void *cb_p ret = uadk_prov_keymgmt_get_support_state(KEYMGMT_X25519); if (ret == UADK_P_FAIL) { fprintf(stderr, "failed to get hardware x25519 keygen support\n"); - return NULL; + ret = UADK_DO_SOFT; + goto exe_soft; } ret = uadk_prov_ecc_init("x25519"); if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to init x25519\n"); - return NULL; + ret = UADK_DO_SOFT; + goto exe_soft; } gctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X25519); if (gctx->sess == (handle_t)0) { fprintf(stderr, "failed to alloc x25519 sess\n"); + ret = UADK_P_FAIL; return NULL; } - ecx_key = uadk_prov_ecx_keygen(gctx); - if (ecx_key == NULL) + ret = uadk_prov_ecx_keygen(gctx, &ecx_key); + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to generate x25519 key\n"); + uadk_prov_ecx_free_sess(gctx->sess); + goto exe_soft; + } uadk_prov_ecx_free_sess(gctx->sess); return ecx_key; -} - -static UADK_PKEY_KEYEXCH get_default_x25519_keyexch(void) -{ - static UADK_PKEY_KEYEXCH s_keyexch; - static int initilazed; - - pthread_mutex_lock(&x25519_mutex); - if (!initilazed) { - UADK_PKEY_KEYEXCH *keyexch = - (UADK_PKEY_KEYEXCH *)EVP_KEYEXCH_fetch(NULL, "X25519", "provider=default"); - if (keyexch) { - s_keyexch = *keyexch; - EVP_KEYEXCH_free((EVP_KEYEXCH *)keyexch); - initilazed = 1; - } else { - fprintf(stderr, "failed to EVP_KEYEXCH_fetch default X25519 provider\n"); - } - } - pthread_mutex_unlock(&x25519_mutex); - return s_keyexch; +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_ecx_sw_gen(genctx, cb, cb_params, ECX_KEY_TYPE_X25519); + return NULL; } static void *uadk_keyexch_x25519_newctx(void *provctx) @@ -1559,30 +1638,41 @@ static int uadk_keyexch_x25519_derive(void *vecxctx, unsigned char *secret, size ret = uadk_prov_keyexch_get_support_state(KEYEXCH_X25519); if (ret == UADK_P_FAIL) { fprintf(stderr, "failed to get hardware x25519 keyexch support\n"); - return UADK_P_FAIL; + ret = UADK_DO_SOFT; + goto exe_soft; } ret = uadk_prov_ecc_init("x25519"); if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to init x25519\n"); - return UADK_P_FAIL; + ret = UADK_DO_SOFT; + goto exe_soft; } ecxctx->sess = uadk_prov_ecx_alloc_sess(ECX_KEY_TYPE_X25519); if (ecxctx->sess == (handle_t)0) { fprintf(stderr, "failed to alloc sess\n"); - return UADK_P_FAIL; + ret = UADK_P_FAIL; + goto exe_soft; } ret = uadk_prov_ecx_derive(ecxctx, secret, &ecxctx->keylen); - if (ret == UADK_P_FAIL) + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to do x25519 derive\n"); + uadk_prov_ecx_free_sess(ecxctx->sess); + goto exe_soft; + } *secretlen = ecxctx->keylen; uadk_prov_ecx_free_sess(ecxctx->sess); return ret; + +exe_soft: + if (ret == UADK_DO_SOFT) + return uadk_ecx_sw_derive(vecxctx, secret, secretlen, outlen, ECX_KEY_TYPE_X25519); + return UADK_P_FAIL; } static void *uadk_keyexch_x25519_dupctx(void *vecxctx) -- 2.33.0

From: lizhi <lizhi206@huawei.com> When hardware failures occur, or in cases of insufficient queue resources or unsupported algorithm specifications, the uadk provider can switch ecdh to openssl soft computation. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_ec_kmgmt.c | 27 +++++++++++++---- src/uadk_prov_ecdh_exch.c | 61 +++++++++++++++++++++++++++++++++------ 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/uadk_prov_ec_kmgmt.c b/src/uadk_prov_ec_kmgmt.c index 355d601..01bcc50 100644 --- a/src/uadk_prov_ec_kmgmt.c +++ b/src/uadk_prov_ec_kmgmt.c @@ -54,10 +54,12 @@ static int ec_param_check(struct ec_gen_ctx *gctx, EC_KEY *ec) type = EC_METHOD_get_field_type(EC_GROUP_method_of(group)); if (type != NID_X9_62_prime_field) { fprintf(stderr, "invalid: uadk unsupport Field GF(2m)!\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } - return uadk_prov_ecc_bit_check(group); + ret = uadk_prov_ecc_bit_check(group); + + return ret; } static int ec_set_public_key(EC_KEY *ec, struct wd_ecc_out *ec_out) @@ -235,7 +237,7 @@ static int ec_hw_keygen(EC_KEY *ec, BIGNUM *priv_key) sess = ec_alloc_sess(ec, &ec_out); if (!sess) { fprintf(stderr, "failed to alloc sess!\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } ret = ec_update_private_key(ec, sess, priv_key); @@ -246,8 +248,9 @@ static int ec_hw_keygen(EC_KEY *ec, BIGNUM *priv_key) uadk_prov_ecc_fill_req(&req, WD_ECXDH_GEN_KEY, NULL, ec_out); ret = uadk_prov_ecc_crypto(sess, &req, (void *)sess); - if (!ret) { + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to generate key!\n"); + ret = UADK_DO_SOFT; goto free_sess; } @@ -319,6 +322,15 @@ static int ec_set_check_group_type(EC_KEY *ec, const char *name) return UADK_P_SUCCESS; } +static void *uadk_ec_sw_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) +{ + if (!enable_sw_offload || !get_default_ec_keymgmt().gen) + return NULL; + + fprintf(stderr, "switch to openssl software calculation in ecx generation.\n"); + return get_default_ec_keymgmt().gen(genctx, osslcb, cbarg); +} + static void *uadk_keymgmt_ec_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) { struct ec_gen_ctx *gctx = genctx; @@ -337,7 +349,7 @@ static void *uadk_keymgmt_ec_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbar } ret = ec_param_check(genctx, ec); - if (!ret) { + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to check genctx!\n"); goto free_ec_key; } @@ -345,7 +357,7 @@ static void *uadk_keymgmt_ec_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbar /* Whether you want it or not, you get a keypair, not just one half */ if ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { ret = ec_hw_keygen(ec, gctx->priv_key); - if (!ret) { + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to gen public key!\n"); goto free_ec_key; } @@ -367,6 +379,9 @@ static void *uadk_keymgmt_ec_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbar free_ec_key: EC_KEY_free(ec); + + if (ret == UADK_DO_SOFT) + return uadk_ec_sw_gen(genctx, osslcb, cbarg); return NULL; } diff --git a/src/uadk_prov_ecdh_exch.c b/src/uadk_prov_ecdh_exch.c index 9ac8e58..f3b2e68 100644 --- a/src/uadk_prov_ecdh_exch.c +++ b/src/uadk_prov_ecdh_exch.c @@ -72,6 +72,28 @@ struct ecdh_sess_ctx { }; UADK_PKEY_KEYEXCH_DESCR(ecdh, ECDH); +static pthread_mutex_t ecdh_mutex = PTHREAD_MUTEX_INITIALIZER; +static UADK_PKEY_KEYEXCH get_default_ecdh_keyexch(void) +{ + static UADK_PKEY_KEYEXCH s_keyexch; + static int initilazed; + + pthread_mutex_lock(&ecdh_mutex); + if (!initilazed) { + UADK_PKEY_KEYEXCH *keyexch = + (UADK_PKEY_KEYEXCH *)EVP_KEYEXCH_fetch(NULL, "ecdh", "provider=default"); + if (keyexch) { + s_keyexch = *keyexch; + EVP_KEYEXCH_free((EVP_KEYEXCH *)keyexch); + initilazed = 1; + } else { + fprintf(stderr, "failed to EVP_KEYEXCH_fetch default X448 provider\n"); + } + } + pthread_mutex_unlock(&ecdh_mutex); + + return s_keyexch; +} static size_t ecdh_get_ec_size(const EC_GROUP *group) { @@ -114,12 +136,15 @@ static int ecdh_param_check(struct ecdh_ctx *pecdhctx, struct ecdh_sess_ctx *ses type = EC_METHOD_get_field_type(EC_GROUP_method_of(group)); if (type != NID_X9_62_prime_field) { fprintf(stderr, "invalid: uadk unsupport Field GF(2m)!\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } sess_ctx->group = group; - return uadk_prov_ecc_bit_check(group); + if (uadk_prov_ecc_bit_check(group) != UADK_P_SUCCESS) + return UADK_DO_SOFT; + + return UADK_P_SUCCESS; } static int ecdh_set_privk(struct ecdh_ctx *pecdhctx, @@ -276,7 +301,7 @@ static int ecdh_compute_key(struct ecdh_sess_ctx *sess_ctx, sess = ecdh_alloc_sess(sess_ctx->privk); if (!sess) { fprintf(stderr, "failed to alloc sess to compute key!\n"); - return UADK_P_FAIL; + return UADK_DO_SOFT; } ret = uadk_prov_ecc_set_private_key(sess, sess_ctx->privk); @@ -292,8 +317,9 @@ static int ecdh_compute_key(struct ecdh_sess_ctx *sess_ctx, } ret = uadk_prov_ecc_crypto(sess, &req, (void *)sess); - if (!ret) { + if (ret != UADK_P_SUCCESS) { fprintf(stderr, "failed to calculate shared key!\n"); + ret = UADK_DO_SOFT; goto uninit_req; } @@ -315,7 +341,7 @@ static int ecdh_plain_derive(struct ecdh_ctx *pecdhctx, int ret; ret = ecdh_param_check(pecdhctx, &sess_ctx); - if (!ret) + if (ret != UADK_P_SUCCESS) return ret; ec_size = ecdh_get_ec_size(sess_ctx.group); @@ -389,13 +415,13 @@ static int ecdh_X9_63_kdf_derive(struct ecdh_ctx *pecdhctx, unsigned char *secre } if (outlen < pecdhctx->kdf_outlen) { - fprintf(stderr, "invalid: outlen %lu is less than kdf_outlen %lu!\n", + fprintf(stderr, "invalid: outlen %zu is less than kdf_outlen %zu!\n", outlen, pecdhctx->kdf_outlen); return UADK_P_FAIL; } ret = ecdh_plain_derive(pecdhctx, NULL, &stmplen, 0); - if (!ret) + if (ret != UADK_P_SUCCESS) return ret; stmp = OPENSSL_secure_malloc(stmplen); @@ -419,10 +445,21 @@ static int ecdh_X9_63_kdf_derive(struct ecdh_ctx *pecdhctx, unsigned char *secre return ret; } +static int uadk_ecdh_sw_derive(void *vpecdhctx, unsigned char *secret, + size_t *psecretlen, size_t outlen) +{ + if (!enable_sw_offload || !get_default_ecdh_keyexch().derive) + return UADK_P_FAIL; + + fprintf(stderr, "switch to openssl software calculation in ecdh derivation.\n"); + return get_default_ecdh_keyexch().derive(vpecdhctx, secret, psecretlen, outlen); +} + static int uadk_keyexch_ecdh_derive(void *vpecdhctx, unsigned char *secret, size_t *psecretlen, size_t outlen) { struct ecdh_ctx *pecdhctx = vpecdhctx; + int ret; if (!pecdhctx) { fprintf(stderr, "invalid: vpecdhctx is NULL to derive!\n"); @@ -431,13 +468,19 @@ static int uadk_keyexch_ecdh_derive(void *vpecdhctx, unsigned char *secret, switch (pecdhctx->kdf_type) { case PROV_ECDH_KDF_NONE: - return ecdh_plain_derive(pecdhctx, secret, psecretlen, outlen); + ret = ecdh_plain_derive(pecdhctx, secret, psecretlen, outlen); + break; case PROV_ECDH_KDF_X9_63: - return ecdh_X9_63_kdf_derive(pecdhctx, secret, psecretlen, outlen); + ret = ecdh_X9_63_kdf_derive(pecdhctx, secret, psecretlen, outlen); + break; default: break; } + if (ret == UADK_P_SUCCESS) + return UADK_P_SUCCESS; + if (ret == UADK_DO_SOFT) + return uadk_ecdh_sw_derive(vpecdhctx, secret, psecretlen, outlen); return UADK_P_FAIL; } -- 2.33.0

From: Wenkai Lin <linwenkai6@hisilicon.com> Ensure that soft_ctx is initialized before invoking software calculation and reduces repeated judgments for digest soft init. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_digest.c | 62 ++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/src/uadk_digest.c b/src/uadk_digest.c index cb593a8..c93b0ca 100644 --- a/src/uadk_digest.c +++ b/src/uadk_digest.c @@ -187,72 +187,55 @@ static EVP_MD *uadk_sha256; static EVP_MD *uadk_sha384; static EVP_MD *uadk_sha512; +/* OpenSSL 3.0 has no app_datasize, need set manually, + * check crypto/evp/legacy_md5.c: md5_md as example. + */ +#define SET_APP_DATASIZE(ctx_type) \ +do {\ + app_datasize = EVP_MD_meth_get_app_datasize(priv->soft_md); \ + if (!app_datasize) \ + app_datasize = sizeof(EVP_MD *) + sizeof(ctx_type); \ +} while (0) + static int uadk_e_digests_soft_md(struct digest_priv_ctx *priv) { - int app_datasize = 0; + int app_datasize; - if (priv->soft_md) + if (priv->soft_md && priv->soft_ctx) return 1; switch (priv->e_nid) { case NID_sm3: priv->soft_md = EVP_sm3(); + SET_APP_DATASIZE(SM3_CTX); break; case NID_md5: priv->soft_md = EVP_md5(); + SET_APP_DATASIZE(MD5_CTX); break; case NID_sha1: priv->soft_md = EVP_sha1(); + SET_APP_DATASIZE(SHA_CTX); break; case NID_sha224: priv->soft_md = EVP_sha224(); + SET_APP_DATASIZE(SHA256_CTX); break; case NID_sha256: priv->soft_md = EVP_sha256(); + SET_APP_DATASIZE(SHA256_CTX); break; case NID_sha384: priv->soft_md = EVP_sha384(); + SET_APP_DATASIZE(SHA512_CTX); break; case NID_sha512: priv->soft_md = EVP_sha512(); + SET_APP_DATASIZE(SHA512_CTX); break; default: - break; - } - - if (unlikely(priv->soft_md == NULL)) + fprintf(stderr, "digest nid %d is invalid.\n", priv->e_nid); return 0; - - app_datasize = EVP_MD_meth_get_app_datasize(priv->soft_md); - if (app_datasize == 0) { - /* OpenSSL 3.0 has no app_datasize, need set manually - * check crypto/evp/legacy_md5.c: md5_md as example - */ - switch (priv->e_nid) { - case NID_sm3: - app_datasize = sizeof(EVP_MD *) + sizeof(SM3_CTX); - break; - case NID_md5: - app_datasize = sizeof(EVP_MD *) + sizeof(MD5_CTX); - break; - case NID_sha1: - app_datasize = sizeof(EVP_MD *) + sizeof(SHA_CTX); - break; - case NID_sha224: - app_datasize = sizeof(EVP_MD *) + sizeof(SHA256_CTX); - break; - case NID_sha256: - app_datasize = sizeof(EVP_MD *) + sizeof(SHA256_CTX); - break; - case NID_sha384: - app_datasize = sizeof(EVP_MD *) + sizeof(SHA512_CTX); - break; - case NID_sha512: - app_datasize = sizeof(EVP_MD *) + sizeof(SHA512_CTX); - break; - default: - break; - } } if (priv->soft_ctx == NULL) { @@ -262,8 +245,10 @@ static int uadk_e_digests_soft_md(struct digest_priv_ctx *priv) return 0; ctx->md_data = OPENSSL_malloc(app_datasize); - if (ctx->md_data == NULL) + if (ctx->md_data == NULL) { + EVP_MD_CTX_free(ctx); return 0; + } priv->soft_ctx = ctx; priv->app_datasize = app_datasize; @@ -1033,6 +1018,7 @@ static int uadk_e_digest_copy(EVP_MD_CTX *to, const EVP_MD_CTX *from) } if (t->soft_ctx) { + t->soft_md = NULL; t->soft_ctx = NULL; ret = digest_soft_init(t); if (!ret) -- 2.33.0

When hardware failures, insufficient queue resources, or small packet lengths occur, the service can switch to OpenSSL soft calculation. Signed-off-by: Qi Tao <taoqi10@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_aead.c | 312 ++++++++++++++++++++++++++++++++++------- src/uadk_prov_cipher.c | 83 +++++++---- src/uadk_prov_digest.c | 153 ++++++++++++-------- 3 files changed, 416 insertions(+), 132 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index ce71c15..54e0115 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -40,6 +40,7 @@ #define UADK_OSSL_FAIL 0 #define UADK_AEAD_SUCCESS 1 +#define SWITCH_TO_SOFT 2 #define UADK_AEAD_FAIL (-1) #define UNINITIALISED_SIZET ((size_t)-1) @@ -52,6 +53,8 @@ #define PROV_CIPHER_FLAG_CUSTOM_IV 0x0002 #define AEAD_FLAGS (PROV_CIPHER_FLAG_AEAD | PROV_CIPHER_FLAG_CUSTOM_IV) +#define UADK_DO_SOFT (-0xE0) +#define UADK_DO_HW (-0xF0) #define UADK_AEAD_DEF_CTXS 2 #define UADK_AEAD_OP_NUM 1 @@ -94,6 +97,10 @@ struct aead_priv_ctx { struct wd_aead_req req; enum uadk_aead_mode mode; handle_t sess; + + int stream_switch_flag; /* soft calculation switch flag for stream mode */ + EVP_CIPHER_CTX *sw_ctx; + EVP_CIPHER *sw_aead; }; struct aead_info { @@ -108,6 +115,18 @@ static struct aead_info aead_info_table[] = { { NID_aes_256_gcm, WD_CIPHER_AES, WD_CIPHER_GCM } }; +static EVP_CIPHER_CTX *EVP_CIPHER_CTX_dup(const EVP_CIPHER_CTX *in) +{ + EVP_CIPHER_CTX *out = EVP_CIPHER_CTX_new(); + + if (out != NULL && !EVP_CIPHER_CTX_copy(out, in)) { + EVP_CIPHER_CTX_free(out); + out = NULL; + } + + return out; +} + static int uadk_aead_poll(void *ctx) { __u64 rx_cnt = 0; @@ -134,6 +153,128 @@ static void uadk_aead_mutex_infork(void) pthread_mutex_unlock(&aead_mutex); } +static int uadk_fetch_sw_aead(struct aead_priv_ctx *priv) +{ + if (priv->sw_aead) + return UADK_AEAD_SUCCESS; + + switch (priv->nid) { + case NID_aes_128_gcm: + priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-128-GCM", "provider=default"); + break; + case NID_aes_192_gcm: + priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-192-GCM", "provider=default"); + break; + case NID_aes_256_gcm: + priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-256-GCM", "provider=default"); + break; + default: + break; + } + + if (unlikely(priv->sw_aead == NULL)) { + fprintf(stderr, "aead failed to fetch\n"); + return UADK_AEAD_FAIL; + } + + return UADK_AEAD_SUCCESS; +} + +static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv, const unsigned char *key, + const unsigned char *iv, const OSSL_PARAM *params) +{ + int ret; + + if (!priv->sw_aead || !priv->sw_ctx) + return UADK_AEAD_FAIL; + + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) + ret = EVP_EncryptInit_ex2(priv->sw_ctx, priv->sw_aead, key, iv, params); + else + ret = EVP_DecryptInit_ex2(priv->sw_ctx, priv->sw_aead, key, iv, params); + + if (!ret) { + fprintf(stderr, "aead soft init error!\n"); + return UADK_AEAD_FAIL; + } + + priv->stream_switch_flag = UADK_DO_SOFT; + + return UADK_AEAD_SUCCESS; +} + +static int uadk_aead_soft_update(struct aead_priv_ctx *priv, unsigned char *out, + int *outl, const unsigned char *in, size_t len) +{ + int ret; + + if (!priv->sw_aead || !priv->sw_ctx) + return UADK_AEAD_FAIL; + + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) + ret = EVP_EncryptUpdate(priv->sw_ctx, out, outl, in, len); + else + ret = EVP_DecryptUpdate(priv->sw_ctx, out, outl, in, len); + + if (!ret) { + fprintf(stderr, "aead soft update error.\n"); + return UADK_AEAD_FAIL; + } + + priv->stream_switch_flag = UADK_DO_SOFT; + + return UADK_AEAD_SUCCESS; +} + +static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *digest, size_t *outl) +{ + int ret; + + if (!priv->sw_aead || !priv->sw_ctx) + goto error; + + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) { + ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + if (!ret) + goto error; + + ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_GET_TAG, + priv->taglen, priv->buf); + if (!ret) + goto error; + } else { + ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_SET_TAG, + priv->taglen, priv->buf); + if (!ret) + goto error; + + ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + if (!ret) + goto error; + } + + priv->stream_switch_flag = 0; + + return UADK_AEAD_SUCCESS; + +error: + fprintf(stderr, "aead soft final failed.\n"); + return UADK_AEAD_FAIL; +} + +static void uadk_aead_soft_cleanup(struct aead_priv_ctx *priv) +{ + if (priv->sw_ctx) { + EVP_CIPHER_CTX_free(priv->sw_ctx); + priv->sw_ctx = NULL; + } + + if (priv->sw_aead) { + EVP_CIPHER_free(priv->sw_aead); + priv->sw_aead = NULL; + } +} + static int uadk_prov_aead_dev_init(struct aead_priv_ctx *priv) { struct wd_ctx_nums ctx_set_num; @@ -293,7 +434,8 @@ static void uadk_do_aead_async_prepare(struct aead_priv_ctx *priv, unsigned char } static int uadk_do_aead_sync_inner(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen, enum wd_aead_msg_state state) + const unsigned char *in, size_t inlen, + enum wd_aead_msg_state state) { int ret; @@ -315,21 +457,17 @@ static int uadk_do_aead_sync_inner(struct aead_priv_ctx *priv, unsigned char *ou return UADK_AEAD_FAIL; } - return inlen; + return UADK_AEAD_SUCCESS; } static int uadk_do_aead_sync(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { size_t nbytes, tail, processing_len, max_mid_len; const unsigned char *in_block = in; unsigned char *out_block = out; int ret; - /* Due to a hardware limitation, zero-length aad using block mode. */ - if (!priv->req.assoc_bytes) - return uadk_do_aead_sync_inner(priv, out, in, inlen, AEAD_MSG_BLOCK); - tail = inlen % AES_BLOCK_SIZE; nbytes = inlen - tail; max_mid_len = AEAD_BLOCK_SIZE - priv->req.assoc_bytes; @@ -354,11 +492,11 @@ static int uadk_do_aead_sync(struct aead_priv_ctx *priv, unsigned char *out, return UADK_AEAD_FAIL; } - return inlen; + return UADK_AEAD_SUCCESS; } static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, - unsigned char *out, const unsigned char *in, size_t inlen) + unsigned char *out, const unsigned char *in, size_t inlen) { struct uadk_e_cb_info cb_param; int cnt = 0; @@ -416,11 +554,14 @@ static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, } static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { int ret; if (inlen > MAX_AAD_LEN) { + if (priv->mode != ASYNC_MODE) + return SWITCH_TO_SOFT; + fprintf(stderr, "the aad len is out of range, aad len = %zu.\n", inlen); return UADK_AEAD_FAIL; } @@ -428,20 +569,23 @@ static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char priv->req.assoc_bytes = inlen; /* Asynchronous jobs use the block mode. */ - if (priv->mode == ASYNC_MODE || !priv->req.assoc_bytes) { + if (priv->mode == ASYNC_MODE) { memcpy(priv->data, in, inlen); return UADK_AEAD_SUCCESS; } + if (!priv->req.assoc_bytes) + return SWITCH_TO_SOFT; + ret = uadk_do_aead_sync_inner(priv, out, in, inlen, AEAD_MSG_FIRST); if (unlikely(ret < 0)) - return UADK_AEAD_FAIL; + return SWITCH_TO_SOFT; return UADK_AEAD_SUCCESS; } static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { struct async_op *op; int ret; @@ -464,9 +608,12 @@ static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char } free(op); - return inlen; + return UADK_AEAD_SUCCESS; } + if (priv->stream_switch_flag == UADK_DO_SOFT) + return SWITCH_TO_SOFT; + return uadk_do_aead_sync(priv, out, in, inlen); free_notification: @@ -477,7 +624,7 @@ free_op: } static int uadk_prov_do_aes_gcm_final(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { int ret; @@ -500,8 +647,8 @@ out: } static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, - size_t *outl, size_t outsize, - const unsigned char *in, size_t inlen) + size_t *outl, size_t outsize, + const unsigned char *in, size_t inlen) { int ret; @@ -543,8 +690,8 @@ static OSSL_FUNC_cipher_set_ctx_params_fn uadk_prov_aead_set_ctx_params; static OSSL_FUNC_cipher_settable_ctx_params_fn uadk_prov_aead_settable_ctx_params; static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, - size_t outsize, const unsigned char *in, - size_t inl) + size_t outsize, const unsigned char *in, + size_t inl) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; @@ -566,11 +713,11 @@ static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, } static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, - size_t *outl, size_t outsize, - const unsigned char *in, size_t inl) + size_t *outl, size_t outsize, + const unsigned char *in, size_t inl) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; - int ret; + int ret, outlen; if (!vctx) return UADK_OSSL_FAIL; @@ -580,18 +727,36 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, return UADK_OSSL_FAIL; } + if (priv->stream_switch_flag == UADK_DO_SOFT) + goto do_soft; ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, in, inl); - if (ret < 0) { + if (ret == SWITCH_TO_SOFT) + goto do_soft; + else if (ret < 0) { fprintf(stderr, "stream data update failed.\n"); return UADK_OSSL_FAIL; + } else { + *outl = inl; + return UADK_AEAD_SUCCESS; } - *outl = inl; +do_soft: + if (priv->stream_switch_flag != UADK_DO_SOFT) { + ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); + if (ret <= 0) + return UADK_OSSL_FAIL; + } + + ret = uadk_aead_soft_update(priv, out, &outlen, in, inl); + if (ret <= 0) + return UADK_OSSL_FAIL; + + *outl = outlen; return UADK_AEAD_SUCCESS; } static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, - size_t *outl, size_t outsize) + size_t *outl, size_t outsize) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; @@ -599,6 +764,9 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, if (!vctx || !out || !outl) return UADK_OSSL_FAIL; + if (priv->stream_switch_flag == UADK_DO_SOFT) + goto do_soft; + ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, NULL, 0); if (ret < 0) { fprintf(stderr, "stream data final failed, ret = %d\n", ret); @@ -607,6 +775,15 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, *outl = 0; return UADK_AEAD_SUCCESS; + +do_soft: + ret = uadk_aead_soft_final(priv, out, outl); + if (ret) { + *outl = 0; + return UADK_AEAD_SUCCESS; + } + + return UADK_OSSL_FAIL; } static int uadk_get_aead_info(struct aead_priv_ctx *priv) @@ -630,9 +807,8 @@ static int uadk_get_aead_info(struct aead_priv_ctx *priv) return UADK_AEAD_SUCCESS; } -static int uadk_prov_aead_init(struct aead_priv_ctx *priv, - const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen) +static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char *key, size_t keylen, + const unsigned char *iv, size_t ivlen, const OSSL_PARAM *params) { int ret; @@ -655,16 +831,26 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, priv->key_set = KEY_STATE_SET; } + priv->stream_switch_flag = 0; + + if (uadk_get_sw_offload_state()) + uadk_fetch_sw_aead(priv); + ret = uadk_prov_aead_dev_init(priv); - if (unlikely(ret < 0)) - return UADK_OSSL_FAIL; + if (unlikely(ret < 0)) { + if (ASYNC_get_current_job()) + return UADK_OSSL_FAIL; + + fprintf(stderr, "aead switch to soft init.!\n"); + return uadk_prov_aead_soft_init(priv, key, iv, params); + } return UADK_AEAD_SUCCESS; } static int uadk_prov_aead_einit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) + const unsigned char *iv, size_t ivlen, + const OSSL_PARAM params[]) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; @@ -674,12 +860,12 @@ static int uadk_prov_aead_einit(void *vctx, const unsigned char *key, size_t key priv->req.op_type = WD_CIPHER_ENCRYPTION_DIGEST; priv->enc = 1; - return uadk_prov_aead_init(priv, key, keylen, iv, ivlen); + return uadk_prov_aead_init(priv, key, keylen, iv, ivlen, params); } static int uadk_prov_aead_dinit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) + const unsigned char *iv, size_t ivlen, + const OSSL_PARAM params[]) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; @@ -689,7 +875,7 @@ static int uadk_prov_aead_dinit(void *vctx, const unsigned char *key, size_t key priv->req.op_type = WD_CIPHER_DECRYPTION_DIGEST; priv->enc = 0; - return uadk_prov_aead_init(priv, key, keylen, iv, ivlen); + return uadk_prov_aead_init(priv, key, keylen, iv, ivlen, params); } static const OSSL_PARAM uadk_prov_settable_ctx_params[] = { @@ -869,8 +1055,8 @@ static const OSSL_PARAM *uadk_prov_aead_gettable_params(ossl_unused void *provct } static int uadk_cipher_aead_get_params(OSSL_PARAM params[], unsigned int md, - uint64_t flags, size_t kbits, - size_t blkbits, size_t ivbits) + uint64_t flags, size_t kbits, + size_t blkbits, size_t ivbits) { OSSL_PARAM *p; @@ -910,24 +1096,46 @@ static int uadk_cipher_aead_get_params(OSSL_PARAM params[], unsigned int md, static void *uadk_prov_aead_dupctx(void *ctx) { - struct aead_priv_ctx *priv, *ret; + struct aead_priv_ctx *dst_ctx, *src_ctx; + int ret; - priv = (struct aead_priv_ctx *)ctx; - if (!priv) + src_ctx = (struct aead_priv_ctx *)ctx; + if (!src_ctx) return NULL; - ret = OPENSSL_memdup(priv, sizeof(*priv)); - if (!ret) + dst_ctx = OPENSSL_memdup(src_ctx, sizeof(*src_ctx)); + if (!dst_ctx) return NULL; - ret->sess = 0; - ret->data = OPENSSL_memdup(priv->data, AEAD_BLOCK_SIZE << 1); - if (!ret->data) { - OPENSSL_clear_free(ret, sizeof(*ret)); - return NULL; + dst_ctx->sess = 0; + dst_ctx->data = OPENSSL_memdup(src_ctx->data, AEAD_BLOCK_SIZE << 1); + if (!dst_ctx->data) + goto free_ctx; + + if (dst_ctx->sw_ctx) { + dst_ctx->sw_ctx = EVP_CIPHER_CTX_dup(src_ctx->sw_ctx); + if (!dst_ctx->sw_ctx) { + fprintf(stderr, "EVP_CIPHER_CTX_dup failed in ctx copy.\n"); + goto free_data; + } } - return ret; + if (dst_ctx->sw_aead) { + ret = EVP_CIPHER_up_ref(dst_ctx->sw_aead); + if (!ret) + goto free_dup; + } + + return dst_ctx; + +free_dup: + if (dst_ctx->sw_ctx) + EVP_CIPHER_CTX_free(dst_ctx->sw_ctx); +free_data: + OPENSSL_clear_free(dst_ctx->data, AEAD_BLOCK_SIZE << 1); +free_ctx: + OPENSSL_clear_free(dst_ctx, sizeof(*dst_ctx)); + return NULL; } static void uadk_prov_aead_freectx(void *ctx) @@ -943,6 +1151,9 @@ static void uadk_prov_aead_freectx(void *ctx) if (priv->data) OPENSSL_clear_free(priv->data, AEAD_BLOCK_SIZE << 1); + if (priv->sw_ctx) + uadk_aead_soft_cleanup(priv); + OPENSSL_clear_free(priv, sizeof(*priv)); } @@ -965,6 +1176,9 @@ static void *uadk_##nm##_newctx(void *provctx) \ ctx->ivlen = iv_len; \ ctx->nid = e_nid; \ ctx->taglen = tag_len; \ + ctx->sw_ctx = EVP_CIPHER_CTX_new(); \ + if (ctx->sw_ctx == NULL) \ + fprintf(stderr, "EVP_AEAD_CTX_new failed.\n"); \ strncpy(ctx->alg_name, #algnm, ALG_NAME_SIZE - 1); \ \ return ctx; \ diff --git a/src/uadk_prov_cipher.c b/src/uadk_prov_cipher.c index eb31f2d..db4a957 100644 --- a/src/uadk_prov_cipher.c +++ b/src/uadk_prov_cipher.c @@ -180,7 +180,7 @@ static struct cts_mode_name2id_st cts_modes[] = { { WD_CIPHER_CBC_CS3, OSSL_CIPHER_CTS_MODE_CS3, UADK_CIPHER_CTS_CS3_NAME }, }; -const char *ossl_cipher_cbc_cts_mode_id2name(unsigned int id) +static const char *ossl_cipher_cbc_cts_mode_id2name(unsigned int id) { size_t i; @@ -191,7 +191,7 @@ const char *ossl_cipher_cbc_cts_mode_id2name(unsigned int id) return NULL; } -int ossl_cipher_cbc_cts_mode_name2id(const char *name) +static int ossl_cipher_cbc_cts_mode_name2id(const char *name) { size_t i; @@ -283,12 +283,20 @@ static int uadk_fetch_sw_cipher(struct cipher_priv_ctx *priv) case ID_sm4_ctr: priv->sw_cipher = EVP_CIPHER_fetch(NULL, "SM4-CTR", "provider=default"); break; + case ID_aes_128_xts: + priv->sw_cipher = EVP_CIPHER_fetch(NULL, "AES-128-XTS", "provider=default"); + break; + case ID_aes_256_xts: + priv->sw_cipher = EVP_CIPHER_fetch(NULL, "AES-256-XTS", "provider=default"); + break; default: break; } - if (unlikely(priv->sw_cipher == NULL)) + if (unlikely(priv->sw_cipher == NULL)) { + fprintf(stderr, "cipher failed to fetch\n"); return UADK_P_FAIL; + } return UADK_P_SUCCESS; } @@ -297,28 +305,34 @@ static int uadk_prov_cipher_sw_init(struct cipher_priv_ctx *priv, const unsigned char *key, const unsigned char *iv) { - if (!uadk_fetch_sw_cipher(priv)) + if (!priv->sw_cipher || !priv->sw_ctx) return UADK_P_FAIL; if (!EVP_CipherInit_ex2(priv->sw_ctx, priv->sw_cipher, key, iv, priv->enc, NULL)) { - fprintf(stderr, "SW cipher init error!\n"); + fprintf(stderr, "cipher soft init failed!\n"); return UADK_P_FAIL; } - priv->switch_threshold = SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT; + priv->switch_flag = UADK_DO_SOFT; return UADK_P_SUCCESS; } -static int uadk_prov_cipher_soft_work(struct cipher_priv_ctx *priv, unsigned char *out, - int *outl, const unsigned char *in, size_t len) +static int uadk_prov_cipher_soft_update(struct cipher_priv_ctx *priv, unsigned char *out, + int *outl, const unsigned char *in, size_t len) { - if (priv->sw_cipher == NULL) + if (!priv->sw_cipher || !priv->sw_ctx) return UADK_P_FAIL; + if (!EVP_CipherInit_ex2(priv->sw_ctx, priv->sw_cipher, priv->key, priv->iv, + priv->enc, NULL)) { + fprintf(stderr, "cipher soft init error!\n"); + return UADK_P_FAIL; + } + if (!EVP_CipherUpdate(priv->sw_ctx, out, outl, in, len)) { - fprintf(stderr, "EVP_CipherUpdate sw_ctx failed.\n"); + fprintf(stderr, "cipher soft update error!\n"); return UADK_P_FAIL; } @@ -328,22 +342,25 @@ static int uadk_prov_cipher_soft_work(struct cipher_priv_ctx *priv, unsigned cha } static int uadk_prov_cipher_soft_final(struct cipher_priv_ctx *priv, unsigned char *out, - size_t *outl) + size_t *outl) { int sw_final_len = 0; - if (priv->sw_cipher == NULL) + if (!priv->sw_cipher || !priv->sw_ctx) return UADK_P_FAIL; if (!EVP_CipherFinal_ex(priv->sw_ctx, out, &sw_final_len)) { - fprintf(stderr, "EVP_CipherFinal_ex sw_ctx failed.\n"); + fprintf(stderr, "cipher soft final failed.\n"); return UADK_P_FAIL; } + *outl = sw_final_len; + priv->switch_flag = 0; + return UADK_P_SUCCESS; } -static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv); +static int uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv); static int uadk_cipher_poll(void *ctx) { @@ -407,10 +424,17 @@ static int uadk_prov_cipher_init(struct cipher_priv_ctx *priv, priv->key_set = 1; } - if (enable_sw_offload) - return uadk_prov_cipher_sw_init(priv, key, iv); + priv->switch_flag = 0; + priv->switch_threshold = SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT; - uadk_prov_cipher_dev_init(priv); + if (uadk_get_sw_offload_state()) + uadk_fetch_sw_cipher(priv); + + ret = uadk_prov_cipher_dev_init(priv); + if (unlikely(ret <= 0)) { + fprintf(stderr, "cipher switch to soft init!\n"); + return uadk_prov_cipher_sw_init(priv, key, iv); + } return UADK_P_SUCCESS; } @@ -488,7 +512,7 @@ static void uadk_cipher_mutex_infork(void) pthread_mutex_unlock(&cipher_mutex); } -static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) +static int uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) { int ret; @@ -501,6 +525,7 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) ctx_set_num = calloc(UADK_CIPHER_OP_NUM, sizeof(*ctx_set_num)); if (!ctx_set_num) { fprintf(stderr, "failed to alloc ctx_set_size!\n"); + ret = UADK_P_FAIL; goto init_err; } @@ -510,6 +535,7 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) if (!cparams.bmp) { fprintf(stderr, "failed to create nodemask!\n"); free(ctx_set_num); + ret = UADK_P_FAIL; goto init_err; } @@ -524,6 +550,7 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) if (unlikely(ret)) { fprintf(stderr, "failed to init cipher!\n"); + ret = UADK_P_FAIL; goto init_err; } @@ -531,8 +558,11 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) async_register_poll_fn(ASYNC_TASK_CIPHER, uadk_cipher_poll); } + ret = UADK_P_SUCCESS; + init_err: pthread_mutex_unlock(&cipher_mutex); + return ret; } static int uadk_prov_cipher_ctx_init(struct cipher_priv_ctx *priv) @@ -552,7 +582,9 @@ static int uadk_prov_cipher_ctx_init(struct cipher_priv_ctx *priv) if (priv->switch_flag == UADK_DO_SOFT) return UADK_P_FAIL; - uadk_prov_cipher_dev_init(priv); + ret = uadk_prov_cipher_dev_init(priv); + if (ret <= 0) + return UADK_P_FAIL; /* dec and enc use the same op */ params.type = 0; @@ -802,13 +834,12 @@ do_soft: * 1. small packets * 2. already choose DO_SOFT, can be hw fail case or following sw case */ - ret = uadk_prov_cipher_soft_work(priv, out, &outlint, in, inlen); + ret = uadk_prov_cipher_soft_update(priv, out, &outlint, in, inlen); if (ret) { *outl = outlint; return UADK_P_SUCCESS; } - fprintf(stderr, "do soft ciphers failed.\n"); return UADK_P_FAIL; } @@ -859,8 +890,8 @@ static int uadk_prov_cipher_cipher(void *vctx, unsigned char *output, size_t *ou return UADK_P_SUCCESS; } -static int uadk_prov_cipher_block_encrypto(struct cipher_priv_ctx *priv, - unsigned char *out, size_t *outl, size_t outsize) +static int uadk_prov_cipher_block_encrypto(struct cipher_priv_ctx *priv, unsigned char *out, + size_t *outl, size_t outsize) { size_t blksz = priv->blksize; int ret; @@ -891,8 +922,8 @@ static int uadk_prov_cipher_block_encrypto(struct cipher_priv_ctx *priv, return UADK_P_SUCCESS; } -static int uadk_prov_cipher_block_decrypto(struct cipher_priv_ctx *priv, - unsigned char *out, size_t *outl, size_t outsize) +static int uadk_prov_cipher_block_decrypto(struct cipher_priv_ctx *priv, unsigned char *out, + size_t *outl, size_t outsize) { size_t blksz = priv->blksize; int ret; @@ -1012,7 +1043,7 @@ static int uadk_prov_cipher_stream_update(void *vctx, unsigned char *output, do_soft: /* have isseu if both using hw and soft partly */ - ret = uadk_prov_cipher_soft_work(priv, output, &len, input, inl); + ret = uadk_prov_cipher_soft_update(priv, output, &len, input, inl); if (ret) { *outl = len; return UADK_P_SUCCESS; diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index a2a5f09..7a5bfff 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -70,18 +70,6 @@ struct digest_prov { static struct digest_prov dprov; static pthread_mutex_t digest_mutex = PTHREAD_MUTEX_INITIALIZER; -struct evp_md_ctx_st { - const EVP_MD *digest; - /* Functional reference if 'digest' is ENGINE-provided */ - ENGINE *engine; - unsigned long flags; - void *md_data; - /* Public key context for sign/verify */ - EVP_PKEY_CTX *pctx; - /* Update function: usually copied from EVP_MD */ - int (*update)(EVP_MD_CTX *ctx, const void *data, size_t count); -}; - struct digest_priv_ctx { handle_t sess; struct wd_digest_sess_setup setup; @@ -131,6 +119,17 @@ static struct digest_info digest_info_table[] = { 32, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, }; +static EVP_MD_CTX *EVP_MD_CTX_dup(const EVP_MD_CTX *in) +{ + EVP_MD_CTX *out = EVP_MD_CTX_new(); + + if (out != NULL && !EVP_MD_CTX_copy_ex(out, in)) { + EVP_MD_CTX_free(out); + out = NULL; + } + return out; +} + static int uadk_digests_soft_md(struct digest_priv_ctx *priv) { if (priv->soft_md) @@ -147,53 +146,83 @@ static int uadk_digests_soft_md(struct digest_priv_ctx *priv) priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA1, "provider=default"); break; case NID_sha224: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_224, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_224, "provider=default"); break; case NID_sha256: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_256, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_256, "provider=default"); break; case NID_sha384: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_384, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_384, "provider=default"); break; case NID_sha512: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_512, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_512, "provider=default"); + break; + case NID_sha512_224: + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_512_224, + "provider=default"); + break; + case NID_sha512_256: + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_512_256, + "provider=default"); break; default: break; } - if (unlikely(priv->soft_md == NULL)) + if (unlikely(priv->soft_md == NULL)) { + fprintf(stderr, "digest failed to fetch\n"); return UADK_DIGEST_FAIL; + } return UADK_DIGEST_SUCCESS; } static int uadk_digest_soft_init(struct digest_priv_ctx *priv) { - if (priv->soft_md) - return EVP_DigestInit_ex(priv->soft_ctx, priv->soft_md, NULL); + if (!priv->soft_md || !priv->soft_ctx) + return UADK_DIGEST_FAIL; - return UADK_DIGEST_FAIL; + if (!EVP_DigestInit_ex(priv->soft_ctx, priv->soft_md, NULL)) { + fprintf(stderr, "soft digest init failed.\n"); + return UADK_DIGEST_FAIL; + } + + priv->switch_flag = UADK_DO_SOFT; + + return UADK_DIGEST_SUCCESS; } static int uadk_digest_soft_update(struct digest_priv_ctx *priv, const void *data, size_t len) { - if (priv->soft_md) - return EVP_DigestUpdate(priv->soft_ctx, data, len); + if (priv->soft_md == NULL) + return UADK_DIGEST_FAIL; - return UADK_DIGEST_FAIL; + if (!EVP_DigestUpdate(priv->soft_ctx, data, len)) { + fprintf(stderr, "soft digest update failed.\n"); + return UADK_DIGEST_FAIL; + } + + priv->switch_flag = UADK_DO_SOFT; + + return UADK_DIGEST_SUCCESS; } static int uadk_digest_soft_final(struct digest_priv_ctx *priv, unsigned char *digest) { - if (priv->soft_md) { - unsigned int digest_length; + unsigned int digest_length; - return EVP_DigestFinal_ex(priv->soft_ctx, digest, &digest_length); + if (priv->soft_md == NULL) + return UADK_DIGEST_FAIL; + + if (!EVP_DigestFinal_ex(priv->soft_ctx, digest, &digest_length)) { + fprintf(stderr, "soft digest final failed.\n"); + return UADK_DIGEST_FAIL; } - return UADK_DIGEST_FAIL; + priv->switch_flag = 0; + + return UADK_DIGEST_SUCCESS; } static void digest_soft_cleanup(struct digest_priv_ctx *priv) @@ -201,12 +230,8 @@ static void digest_soft_cleanup(struct digest_priv_ctx *priv) EVP_MD_CTX *ctx = priv->soft_ctx; if (ctx != NULL) { - if (ctx->md_data) { - OPENSSL_free(ctx->md_data); - ctx->md_data = NULL; - } EVP_MD_CTX_free(ctx); - ctx = NULL; + priv->soft_ctx = NULL; } if (priv->soft_md) { @@ -216,7 +241,7 @@ static void digest_soft_cleanup(struct digest_priv_ctx *priv) } static int uadk_digest_soft_work(struct digest_priv_ctx *priv, int len, - unsigned char *digest) + unsigned char *digest) { int ret; @@ -341,7 +366,7 @@ static int uadk_digest_ctx_init(struct digest_priv_ctx *priv) ret = uadk_prov_digest_dev_init(priv); if (unlikely(ret <= 0)) - goto soft_init; + return UADK_DIGEST_FAIL; /* Use the default numa parameters */ params.numa_id = -1; @@ -357,15 +382,7 @@ static int uadk_digest_ctx_init(struct digest_priv_ctx *priv) } } - if (enable_sw_offload) - uadk_digests_soft_md(priv); - return UADK_DIGEST_SUCCESS; - -soft_init: - pthread_mutex_unlock(&digest_mutex); - fprintf(stderr, "uadk failed to initialize digest.\n"); - return uadk_digest_soft_init(priv); } static void uadk_fill_mac_buffer_len(struct digest_priv_ctx *priv, bool is_end) @@ -445,14 +462,15 @@ static int uadk_digest_update_inner(struct digest_priv_ctx *priv, const void *da priv->state = SEC_DIGEST_DOING; priv->req.out = priv->out; - remain_len -= processing_len; - input_data += processing_len; ret = wd_do_digest_sync(priv->sess, &priv->req); if (ret) { fprintf(stderr, "do sec digest update failed, switch to soft digest.\n"); goto do_soft_digest; } + + remain_len -= processing_len; + input_data += processing_len; } while (remain_len > DIGEST_BLOCK_SIZE); priv->last_update_bufflen = remain_len; @@ -462,7 +480,6 @@ static int uadk_digest_update_inner(struct digest_priv_ctx *priv, const void *da do_soft_digest: if (priv->state == SEC_DIGEST_FIRST_UPDATING) { - priv->switch_flag = UADK_DO_SOFT; ret = uadk_digest_soft_init(priv); if (!ret) return ret; @@ -472,6 +489,9 @@ do_soft_digest: ret = uadk_digest_soft_update(priv, priv->data, DIGEST_BLOCK_SIZE); if (!ret) goto out; + + remain_len -= processing_len; + input_data += processing_len; } ret = uadk_digest_soft_update(priv, input_data, remain_len); @@ -548,6 +568,7 @@ static int uadk_do_digest_sync(struct digest_priv_ctx *priv) fprintf(stderr, "do sec digest sync failed, switch to soft digest.\n"); return UADK_DIGEST_FAIL; } + return UADK_DIGEST_SUCCESS; } @@ -558,7 +579,7 @@ static int uadk_do_digest_async(struct digest_priv_ctx *priv, struct async_op *o int cnt = 0; if (unlikely(priv->switch_flag == UADK_DO_SOFT)) { - fprintf(stderr, "async cipher init failed.\n"); + fprintf(stderr, "digest soft switching is not supported in asynchronous mode.\n"); return UADK_DIGEST_FAIL; } @@ -608,9 +629,11 @@ static int uadk_digest_final(struct digest_priv_ctx *priv, unsigned char *digest return UADK_DIGEST_FAIL; } - ret = uadk_digest_ctx_init(priv); - if (ret != UADK_DIGEST_SUCCESS) - return UADK_DIGEST_FAIL; + if (unlikely(priv->switch_flag != UADK_DO_SOFT)) { + ret = uadk_digest_ctx_init(priv); + if (ret != UADK_DIGEST_SUCCESS) + return UADK_DIGEST_FAIL; + } priv->req.in = priv->data; priv->req.out = priv->out; @@ -657,8 +680,8 @@ clear: return ret; } -static int uadk_digest_digest(struct digest_priv_ctx *priv, const void *data, size_t data_len, - unsigned char *digest) +static int uadk_digest_digest(struct digest_priv_ctx *priv, const void *data, + size_t data_len, unsigned char *digest) { struct async_op op; int ret; @@ -706,8 +729,7 @@ static void uadk_digest_cleanup(struct digest_priv_ctx *priv) if (priv->data) OPENSSL_clear_free(priv->data, DIGEST_BLOCK_SIZE); - if (priv->soft_ctx) - OPENSSL_clear_free(priv->soft_ctx, sizeof(EVP_MD_CTX)); + digest_soft_cleanup(priv); } static OSSL_FUNC_digest_freectx_fn uadk_prov_freectx; @@ -780,6 +802,7 @@ static void uadk_prov_freectx(void *dctx) static void *uadk_prov_dupctx(void *dctx) { struct digest_priv_ctx *dst_ctx, *src_ctx; + int ret; src_ctx = (struct digest_priv_ctx *)dctx; if (!dctx) @@ -809,15 +832,24 @@ static void *uadk_prov_dupctx(void *dctx) goto free_ctx; if (dst_ctx->soft_ctx) { - dst_ctx->soft_ctx = EVP_MD_CTX_new(); + dst_ctx->soft_ctx = EVP_MD_CTX_dup(src_ctx->soft_ctx); if (!dst_ctx->soft_ctx) { fprintf(stderr, "EVP_MD_CTX_new failed in ctx copy.\n"); goto free_data; } } + if (dst_ctx->soft_md) { + ret = EVP_MD_up_ref(dst_ctx->soft_md); + if (!ret) + goto free_dup; + } + return dst_ctx; +free_dup: + if (dst_ctx->soft_ctx) + EVP_MD_CTX_free(dst_ctx->soft_ctx); free_data: OPENSSL_clear_free(dst_ctx->data, DIGEST_BLOCK_SIZE); free_ctx: @@ -835,13 +867,20 @@ static int uadk_prov_init(void *dctx, const OSSL_PARAM params[]) return UADK_DIGEST_FAIL; } + if (uadk_get_sw_offload_state()) + uadk_digests_soft_md(priv); + ret = uadk_get_digest_info(priv); if (unlikely(!ret)) return UADK_DIGEST_FAIL; + priv->state = SEC_DIGEST_INIT; + ret = uadk_prov_digest_dev_init(priv); - if (unlikely(ret <= 0)) - return UADK_DIGEST_FAIL; + if (unlikely(ret <= 0)) { + fprintf(stderr, "digest switch to soft init!\n"); + return uadk_digest_soft_init(priv); + } return UADK_DIGEST_SUCCESS; } @@ -879,7 +918,7 @@ static int uadk_prov_final(void *dctx, unsigned char *out, return ret; } - if (unlikely(outl != NULL)) + if (outl) *outl = priv->md_size; return UADK_DIGEST_SUCCESS; -- 2.33.0