When hardware failures, insufficient queue resources, or small packet lengths occur, the service can switch to OpenSSL soft calculation. Signed-off-by: Qi Tao <taoqi10@huawei.com> Signed-off-by: JiangShui Yang <yangjiangshui@h-partners.com> --- src/uadk_prov_aead.c | 312 ++++++++++++++++++++++++++++++++++------- src/uadk_prov_cipher.c | 83 +++++++---- src/uadk_prov_digest.c | 153 ++++++++++++-------- 3 files changed, 416 insertions(+), 132 deletions(-) diff --git a/src/uadk_prov_aead.c b/src/uadk_prov_aead.c index ce71c15..54e0115 100644 --- a/src/uadk_prov_aead.c +++ b/src/uadk_prov_aead.c @@ -40,6 +40,7 @@ #define UADK_OSSL_FAIL 0 #define UADK_AEAD_SUCCESS 1 +#define SWITCH_TO_SOFT 2 #define UADK_AEAD_FAIL (-1) #define UNINITIALISED_SIZET ((size_t)-1) @@ -52,6 +53,8 @@ #define PROV_CIPHER_FLAG_CUSTOM_IV 0x0002 #define AEAD_FLAGS (PROV_CIPHER_FLAG_AEAD | PROV_CIPHER_FLAG_CUSTOM_IV) +#define UADK_DO_SOFT (-0xE0) +#define UADK_DO_HW (-0xF0) #define UADK_AEAD_DEF_CTXS 2 #define UADK_AEAD_OP_NUM 1 @@ -94,6 +97,10 @@ struct aead_priv_ctx { struct wd_aead_req req; enum uadk_aead_mode mode; handle_t sess; + + int stream_switch_flag; /* soft calculation switch flag for stream mode */ + EVP_CIPHER_CTX *sw_ctx; + EVP_CIPHER *sw_aead; }; struct aead_info { @@ -108,6 +115,18 @@ static struct aead_info aead_info_table[] = { { NID_aes_256_gcm, WD_CIPHER_AES, WD_CIPHER_GCM } }; +static EVP_CIPHER_CTX *EVP_CIPHER_CTX_dup(const EVP_CIPHER_CTX *in) +{ + EVP_CIPHER_CTX *out = EVP_CIPHER_CTX_new(); + + if (out != NULL && !EVP_CIPHER_CTX_copy(out, in)) { + EVP_CIPHER_CTX_free(out); + out = NULL; + } + + return out; +} + static int uadk_aead_poll(void *ctx) { __u64 rx_cnt = 0; @@ -134,6 +153,128 @@ static void uadk_aead_mutex_infork(void) pthread_mutex_unlock(&aead_mutex); } +static int uadk_fetch_sw_aead(struct aead_priv_ctx *priv) +{ + if (priv->sw_aead) + return UADK_AEAD_SUCCESS; + + switch (priv->nid) { + case NID_aes_128_gcm: + priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-128-GCM", "provider=default"); + break; + case NID_aes_192_gcm: + priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-192-GCM", "provider=default"); + break; + case NID_aes_256_gcm: + priv->sw_aead = EVP_CIPHER_fetch(NULL, "AES-256-GCM", "provider=default"); + break; + default: + break; + } + + if (unlikely(priv->sw_aead == NULL)) { + fprintf(stderr, "aead failed to fetch\n"); + return UADK_AEAD_FAIL; + } + + return UADK_AEAD_SUCCESS; +} + +static int uadk_prov_aead_soft_init(struct aead_priv_ctx *priv, const unsigned char *key, + const unsigned char *iv, const OSSL_PARAM *params) +{ + int ret; + + if (!priv->sw_aead || !priv->sw_ctx) + return UADK_AEAD_FAIL; + + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) + ret = EVP_EncryptInit_ex2(priv->sw_ctx, priv->sw_aead, key, iv, params); + else + ret = EVP_DecryptInit_ex2(priv->sw_ctx, priv->sw_aead, key, iv, params); + + if (!ret) { + fprintf(stderr, "aead soft init error!\n"); + return UADK_AEAD_FAIL; + } + + priv->stream_switch_flag = UADK_DO_SOFT; + + return UADK_AEAD_SUCCESS; +} + +static int uadk_aead_soft_update(struct aead_priv_ctx *priv, unsigned char *out, + int *outl, const unsigned char *in, size_t len) +{ + int ret; + + if (!priv->sw_aead || !priv->sw_ctx) + return UADK_AEAD_FAIL; + + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) + ret = EVP_EncryptUpdate(priv->sw_ctx, out, outl, in, len); + else + ret = EVP_DecryptUpdate(priv->sw_ctx, out, outl, in, len); + + if (!ret) { + fprintf(stderr, "aead soft update error.\n"); + return UADK_AEAD_FAIL; + } + + priv->stream_switch_flag = UADK_DO_SOFT; + + return UADK_AEAD_SUCCESS; +} + +static int uadk_aead_soft_final(struct aead_priv_ctx *priv, unsigned char *digest, size_t *outl) +{ + int ret; + + if (!priv->sw_aead || !priv->sw_ctx) + goto error; + + if (priv->req.op_type == WD_CIPHER_ENCRYPTION_DIGEST) { + ret = EVP_EncryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + if (!ret) + goto error; + + ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_GET_TAG, + priv->taglen, priv->buf); + if (!ret) + goto error; + } else { + ret = EVP_CIPHER_CTX_ctrl(priv->sw_ctx, EVP_CTRL_GCM_SET_TAG, + priv->taglen, priv->buf); + if (!ret) + goto error; + + ret = EVP_DecryptFinal_ex(priv->sw_ctx, digest, (int *)outl); + if (!ret) + goto error; + } + + priv->stream_switch_flag = 0; + + return UADK_AEAD_SUCCESS; + +error: + fprintf(stderr, "aead soft final failed.\n"); + return UADK_AEAD_FAIL; +} + +static void uadk_aead_soft_cleanup(struct aead_priv_ctx *priv) +{ + if (priv->sw_ctx) { + EVP_CIPHER_CTX_free(priv->sw_ctx); + priv->sw_ctx = NULL; + } + + if (priv->sw_aead) { + EVP_CIPHER_free(priv->sw_aead); + priv->sw_aead = NULL; + } +} + static int uadk_prov_aead_dev_init(struct aead_priv_ctx *priv) { struct wd_ctx_nums ctx_set_num; @@ -293,7 +434,8 @@ static void uadk_do_aead_async_prepare(struct aead_priv_ctx *priv, unsigned char } static int uadk_do_aead_sync_inner(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen, enum wd_aead_msg_state state) + const unsigned char *in, size_t inlen, + enum wd_aead_msg_state state) { int ret; @@ -315,21 +457,17 @@ static int uadk_do_aead_sync_inner(struct aead_priv_ctx *priv, unsigned char *ou return UADK_AEAD_FAIL; } - return inlen; + return UADK_AEAD_SUCCESS; } static int uadk_do_aead_sync(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { size_t nbytes, tail, processing_len, max_mid_len; const unsigned char *in_block = in; unsigned char *out_block = out; int ret; - /* Due to a hardware limitation, zero-length aad using block mode. */ - if (!priv->req.assoc_bytes) - return uadk_do_aead_sync_inner(priv, out, in, inlen, AEAD_MSG_BLOCK); - tail = inlen % AES_BLOCK_SIZE; nbytes = inlen - tail; max_mid_len = AEAD_BLOCK_SIZE - priv->req.assoc_bytes; @@ -354,11 +492,11 @@ static int uadk_do_aead_sync(struct aead_priv_ctx *priv, unsigned char *out, return UADK_AEAD_FAIL; } - return inlen; + return UADK_AEAD_SUCCESS; } static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, - unsigned char *out, const unsigned char *in, size_t inlen) + unsigned char *out, const unsigned char *in, size_t inlen) { struct uadk_e_cb_info cb_param; int cnt = 0; @@ -416,11 +554,14 @@ static int uadk_do_aead_async(struct aead_priv_ctx *priv, struct async_op *op, } static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { int ret; if (inlen > MAX_AAD_LEN) { + if (priv->mode != ASYNC_MODE) + return SWITCH_TO_SOFT; + fprintf(stderr, "the aad len is out of range, aad len = %zu.\n", inlen); return UADK_AEAD_FAIL; } @@ -428,20 +569,23 @@ static int uadk_prov_do_aes_gcm_first(struct aead_priv_ctx *priv, unsigned char priv->req.assoc_bytes = inlen; /* Asynchronous jobs use the block mode. */ - if (priv->mode == ASYNC_MODE || !priv->req.assoc_bytes) { + if (priv->mode == ASYNC_MODE) { memcpy(priv->data, in, inlen); return UADK_AEAD_SUCCESS; } + if (!priv->req.assoc_bytes) + return SWITCH_TO_SOFT; + ret = uadk_do_aead_sync_inner(priv, out, in, inlen, AEAD_MSG_FIRST); if (unlikely(ret < 0)) - return UADK_AEAD_FAIL; + return SWITCH_TO_SOFT; return UADK_AEAD_SUCCESS; } static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { struct async_op *op; int ret; @@ -464,9 +608,12 @@ static int uadk_prov_do_aes_gcm_update(struct aead_priv_ctx *priv, unsigned char } free(op); - return inlen; + return UADK_AEAD_SUCCESS; } + if (priv->stream_switch_flag == UADK_DO_SOFT) + return SWITCH_TO_SOFT; + return uadk_do_aead_sync(priv, out, in, inlen); free_notification: @@ -477,7 +624,7 @@ free_op: } static int uadk_prov_do_aes_gcm_final(struct aead_priv_ctx *priv, unsigned char *out, - const unsigned char *in, size_t inlen) + const unsigned char *in, size_t inlen) { int ret; @@ -500,8 +647,8 @@ out: } static int uadk_prov_do_aes_gcm(struct aead_priv_ctx *priv, unsigned char *out, - size_t *outl, size_t outsize, - const unsigned char *in, size_t inlen) + size_t *outl, size_t outsize, + const unsigned char *in, size_t inlen) { int ret; @@ -543,8 +690,8 @@ static OSSL_FUNC_cipher_set_ctx_params_fn uadk_prov_aead_set_ctx_params; static OSSL_FUNC_cipher_settable_ctx_params_fn uadk_prov_aead_settable_ctx_params; static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, - size_t outsize, const unsigned char *in, - size_t inl) + size_t outsize, const unsigned char *in, + size_t inl) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; @@ -566,11 +713,11 @@ static int uadk_prov_aead_cipher(void *vctx, unsigned char *out, size_t *outl, } static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, - size_t *outl, size_t outsize, - const unsigned char *in, size_t inl) + size_t *outl, size_t outsize, + const unsigned char *in, size_t inl) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; - int ret; + int ret, outlen; if (!vctx) return UADK_OSSL_FAIL; @@ -580,18 +727,36 @@ static int uadk_prov_aead_stream_update(void *vctx, unsigned char *out, return UADK_OSSL_FAIL; } + if (priv->stream_switch_flag == UADK_DO_SOFT) + goto do_soft; ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, in, inl); - if (ret < 0) { + if (ret == SWITCH_TO_SOFT) + goto do_soft; + else if (ret < 0) { fprintf(stderr, "stream data update failed.\n"); return UADK_OSSL_FAIL; + } else { + *outl = inl; + return UADK_AEAD_SUCCESS; } - *outl = inl; +do_soft: + if (priv->stream_switch_flag != UADK_DO_SOFT) { + ret = uadk_prov_aead_soft_init(priv, priv->key, priv->iv, NULL); + if (ret <= 0) + return UADK_OSSL_FAIL; + } + + ret = uadk_aead_soft_update(priv, out, &outlen, in, inl); + if (ret <= 0) + return UADK_OSSL_FAIL; + + *outl = outlen; return UADK_AEAD_SUCCESS; } static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, - size_t *outl, size_t outsize) + size_t *outl, size_t outsize) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; int ret; @@ -599,6 +764,9 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, if (!vctx || !out || !outl) return UADK_OSSL_FAIL; + if (priv->stream_switch_flag == UADK_DO_SOFT) + goto do_soft; + ret = uadk_prov_do_aes_gcm(priv, out, outl, outsize, NULL, 0); if (ret < 0) { fprintf(stderr, "stream data final failed, ret = %d\n", ret); @@ -607,6 +775,15 @@ static int uadk_prov_aead_stream_final(void *vctx, unsigned char *out, *outl = 0; return UADK_AEAD_SUCCESS; + +do_soft: + ret = uadk_aead_soft_final(priv, out, outl); + if (ret) { + *outl = 0; + return UADK_AEAD_SUCCESS; + } + + return UADK_OSSL_FAIL; } static int uadk_get_aead_info(struct aead_priv_ctx *priv) @@ -630,9 +807,8 @@ static int uadk_get_aead_info(struct aead_priv_ctx *priv) return UADK_AEAD_SUCCESS; } -static int uadk_prov_aead_init(struct aead_priv_ctx *priv, - const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen) +static int uadk_prov_aead_init(struct aead_priv_ctx *priv, const unsigned char *key, size_t keylen, + const unsigned char *iv, size_t ivlen, const OSSL_PARAM *params) { int ret; @@ -655,16 +831,26 @@ static int uadk_prov_aead_init(struct aead_priv_ctx *priv, priv->key_set = KEY_STATE_SET; } + priv->stream_switch_flag = 0; + + if (uadk_get_sw_offload_state()) + uadk_fetch_sw_aead(priv); + ret = uadk_prov_aead_dev_init(priv); - if (unlikely(ret < 0)) - return UADK_OSSL_FAIL; + if (unlikely(ret < 0)) { + if (ASYNC_get_current_job()) + return UADK_OSSL_FAIL; + + fprintf(stderr, "aead switch to soft init.!\n"); + return uadk_prov_aead_soft_init(priv, key, iv, params); + } return UADK_AEAD_SUCCESS; } static int uadk_prov_aead_einit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) + const unsigned char *iv, size_t ivlen, + const OSSL_PARAM params[]) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; @@ -674,12 +860,12 @@ static int uadk_prov_aead_einit(void *vctx, const unsigned char *key, size_t key priv->req.op_type = WD_CIPHER_ENCRYPTION_DIGEST; priv->enc = 1; - return uadk_prov_aead_init(priv, key, keylen, iv, ivlen); + return uadk_prov_aead_init(priv, key, keylen, iv, ivlen, params); } static int uadk_prov_aead_dinit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) + const unsigned char *iv, size_t ivlen, + const OSSL_PARAM params[]) { struct aead_priv_ctx *priv = (struct aead_priv_ctx *)vctx; @@ -689,7 +875,7 @@ static int uadk_prov_aead_dinit(void *vctx, const unsigned char *key, size_t key priv->req.op_type = WD_CIPHER_DECRYPTION_DIGEST; priv->enc = 0; - return uadk_prov_aead_init(priv, key, keylen, iv, ivlen); + return uadk_prov_aead_init(priv, key, keylen, iv, ivlen, params); } static const OSSL_PARAM uadk_prov_settable_ctx_params[] = { @@ -869,8 +1055,8 @@ static const OSSL_PARAM *uadk_prov_aead_gettable_params(ossl_unused void *provct } static int uadk_cipher_aead_get_params(OSSL_PARAM params[], unsigned int md, - uint64_t flags, size_t kbits, - size_t blkbits, size_t ivbits) + uint64_t flags, size_t kbits, + size_t blkbits, size_t ivbits) { OSSL_PARAM *p; @@ -910,24 +1096,46 @@ static int uadk_cipher_aead_get_params(OSSL_PARAM params[], unsigned int md, static void *uadk_prov_aead_dupctx(void *ctx) { - struct aead_priv_ctx *priv, *ret; + struct aead_priv_ctx *dst_ctx, *src_ctx; + int ret; - priv = (struct aead_priv_ctx *)ctx; - if (!priv) + src_ctx = (struct aead_priv_ctx *)ctx; + if (!src_ctx) return NULL; - ret = OPENSSL_memdup(priv, sizeof(*priv)); - if (!ret) + dst_ctx = OPENSSL_memdup(src_ctx, sizeof(*src_ctx)); + if (!dst_ctx) return NULL; - ret->sess = 0; - ret->data = OPENSSL_memdup(priv->data, AEAD_BLOCK_SIZE << 1); - if (!ret->data) { - OPENSSL_clear_free(ret, sizeof(*ret)); - return NULL; + dst_ctx->sess = 0; + dst_ctx->data = OPENSSL_memdup(src_ctx->data, AEAD_BLOCK_SIZE << 1); + if (!dst_ctx->data) + goto free_ctx; + + if (dst_ctx->sw_ctx) { + dst_ctx->sw_ctx = EVP_CIPHER_CTX_dup(src_ctx->sw_ctx); + if (!dst_ctx->sw_ctx) { + fprintf(stderr, "EVP_CIPHER_CTX_dup failed in ctx copy.\n"); + goto free_data; + } } - return ret; + if (dst_ctx->sw_aead) { + ret = EVP_CIPHER_up_ref(dst_ctx->sw_aead); + if (!ret) + goto free_dup; + } + + return dst_ctx; + +free_dup: + if (dst_ctx->sw_ctx) + EVP_CIPHER_CTX_free(dst_ctx->sw_ctx); +free_data: + OPENSSL_clear_free(dst_ctx->data, AEAD_BLOCK_SIZE << 1); +free_ctx: + OPENSSL_clear_free(dst_ctx, sizeof(*dst_ctx)); + return NULL; } static void uadk_prov_aead_freectx(void *ctx) @@ -943,6 +1151,9 @@ static void uadk_prov_aead_freectx(void *ctx) if (priv->data) OPENSSL_clear_free(priv->data, AEAD_BLOCK_SIZE << 1); + if (priv->sw_ctx) + uadk_aead_soft_cleanup(priv); + OPENSSL_clear_free(priv, sizeof(*priv)); } @@ -965,6 +1176,9 @@ static void *uadk_##nm##_newctx(void *provctx) \ ctx->ivlen = iv_len; \ ctx->nid = e_nid; \ ctx->taglen = tag_len; \ + ctx->sw_ctx = EVP_CIPHER_CTX_new(); \ + if (ctx->sw_ctx == NULL) \ + fprintf(stderr, "EVP_AEAD_CTX_new failed.\n"); \ strncpy(ctx->alg_name, #algnm, ALG_NAME_SIZE - 1); \ \ return ctx; \ diff --git a/src/uadk_prov_cipher.c b/src/uadk_prov_cipher.c index eb31f2d..db4a957 100644 --- a/src/uadk_prov_cipher.c +++ b/src/uadk_prov_cipher.c @@ -180,7 +180,7 @@ static struct cts_mode_name2id_st cts_modes[] = { { WD_CIPHER_CBC_CS3, OSSL_CIPHER_CTS_MODE_CS3, UADK_CIPHER_CTS_CS3_NAME }, }; -const char *ossl_cipher_cbc_cts_mode_id2name(unsigned int id) +static const char *ossl_cipher_cbc_cts_mode_id2name(unsigned int id) { size_t i; @@ -191,7 +191,7 @@ const char *ossl_cipher_cbc_cts_mode_id2name(unsigned int id) return NULL; } -int ossl_cipher_cbc_cts_mode_name2id(const char *name) +static int ossl_cipher_cbc_cts_mode_name2id(const char *name) { size_t i; @@ -283,12 +283,20 @@ static int uadk_fetch_sw_cipher(struct cipher_priv_ctx *priv) case ID_sm4_ctr: priv->sw_cipher = EVP_CIPHER_fetch(NULL, "SM4-CTR", "provider=default"); break; + case ID_aes_128_xts: + priv->sw_cipher = EVP_CIPHER_fetch(NULL, "AES-128-XTS", "provider=default"); + break; + case ID_aes_256_xts: + priv->sw_cipher = EVP_CIPHER_fetch(NULL, "AES-256-XTS", "provider=default"); + break; default: break; } - if (unlikely(priv->sw_cipher == NULL)) + if (unlikely(priv->sw_cipher == NULL)) { + fprintf(stderr, "cipher failed to fetch\n"); return UADK_P_FAIL; + } return UADK_P_SUCCESS; } @@ -297,28 +305,34 @@ static int uadk_prov_cipher_sw_init(struct cipher_priv_ctx *priv, const unsigned char *key, const unsigned char *iv) { - if (!uadk_fetch_sw_cipher(priv)) + if (!priv->sw_cipher || !priv->sw_ctx) return UADK_P_FAIL; if (!EVP_CipherInit_ex2(priv->sw_ctx, priv->sw_cipher, key, iv, priv->enc, NULL)) { - fprintf(stderr, "SW cipher init error!\n"); + fprintf(stderr, "cipher soft init failed!\n"); return UADK_P_FAIL; } - priv->switch_threshold = SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT; + priv->switch_flag = UADK_DO_SOFT; return UADK_P_SUCCESS; } -static int uadk_prov_cipher_soft_work(struct cipher_priv_ctx *priv, unsigned char *out, - int *outl, const unsigned char *in, size_t len) +static int uadk_prov_cipher_soft_update(struct cipher_priv_ctx *priv, unsigned char *out, + int *outl, const unsigned char *in, size_t len) { - if (priv->sw_cipher == NULL) + if (!priv->sw_cipher || !priv->sw_ctx) return UADK_P_FAIL; + if (!EVP_CipherInit_ex2(priv->sw_ctx, priv->sw_cipher, priv->key, priv->iv, + priv->enc, NULL)) { + fprintf(stderr, "cipher soft init error!\n"); + return UADK_P_FAIL; + } + if (!EVP_CipherUpdate(priv->sw_ctx, out, outl, in, len)) { - fprintf(stderr, "EVP_CipherUpdate sw_ctx failed.\n"); + fprintf(stderr, "cipher soft update error!\n"); return UADK_P_FAIL; } @@ -328,22 +342,25 @@ static int uadk_prov_cipher_soft_work(struct cipher_priv_ctx *priv, unsigned cha } static int uadk_prov_cipher_soft_final(struct cipher_priv_ctx *priv, unsigned char *out, - size_t *outl) + size_t *outl) { int sw_final_len = 0; - if (priv->sw_cipher == NULL) + if (!priv->sw_cipher || !priv->sw_ctx) return UADK_P_FAIL; if (!EVP_CipherFinal_ex(priv->sw_ctx, out, &sw_final_len)) { - fprintf(stderr, "EVP_CipherFinal_ex sw_ctx failed.\n"); + fprintf(stderr, "cipher soft final failed.\n"); return UADK_P_FAIL; } + *outl = sw_final_len; + priv->switch_flag = 0; + return UADK_P_SUCCESS; } -static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv); +static int uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv); static int uadk_cipher_poll(void *ctx) { @@ -407,10 +424,17 @@ static int uadk_prov_cipher_init(struct cipher_priv_ctx *priv, priv->key_set = 1; } - if (enable_sw_offload) - return uadk_prov_cipher_sw_init(priv, key, iv); + priv->switch_flag = 0; + priv->switch_threshold = SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT; - uadk_prov_cipher_dev_init(priv); + if (uadk_get_sw_offload_state()) + uadk_fetch_sw_cipher(priv); + + ret = uadk_prov_cipher_dev_init(priv); + if (unlikely(ret <= 0)) { + fprintf(stderr, "cipher switch to soft init!\n"); + return uadk_prov_cipher_sw_init(priv, key, iv); + } return UADK_P_SUCCESS; } @@ -488,7 +512,7 @@ static void uadk_cipher_mutex_infork(void) pthread_mutex_unlock(&cipher_mutex); } -static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) +static int uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) { int ret; @@ -501,6 +525,7 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) ctx_set_num = calloc(UADK_CIPHER_OP_NUM, sizeof(*ctx_set_num)); if (!ctx_set_num) { fprintf(stderr, "failed to alloc ctx_set_size!\n"); + ret = UADK_P_FAIL; goto init_err; } @@ -510,6 +535,7 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) if (!cparams.bmp) { fprintf(stderr, "failed to create nodemask!\n"); free(ctx_set_num); + ret = UADK_P_FAIL; goto init_err; } @@ -524,6 +550,7 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) if (unlikely(ret)) { fprintf(stderr, "failed to init cipher!\n"); + ret = UADK_P_FAIL; goto init_err; } @@ -531,8 +558,11 @@ static void uadk_prov_cipher_dev_init(struct cipher_priv_ctx *priv) async_register_poll_fn(ASYNC_TASK_CIPHER, uadk_cipher_poll); } + ret = UADK_P_SUCCESS; + init_err: pthread_mutex_unlock(&cipher_mutex); + return ret; } static int uadk_prov_cipher_ctx_init(struct cipher_priv_ctx *priv) @@ -552,7 +582,9 @@ static int uadk_prov_cipher_ctx_init(struct cipher_priv_ctx *priv) if (priv->switch_flag == UADK_DO_SOFT) return UADK_P_FAIL; - uadk_prov_cipher_dev_init(priv); + ret = uadk_prov_cipher_dev_init(priv); + if (ret <= 0) + return UADK_P_FAIL; /* dec and enc use the same op */ params.type = 0; @@ -802,13 +834,12 @@ do_soft: * 1. small packets * 2. already choose DO_SOFT, can be hw fail case or following sw case */ - ret = uadk_prov_cipher_soft_work(priv, out, &outlint, in, inlen); + ret = uadk_prov_cipher_soft_update(priv, out, &outlint, in, inlen); if (ret) { *outl = outlint; return UADK_P_SUCCESS; } - fprintf(stderr, "do soft ciphers failed.\n"); return UADK_P_FAIL; } @@ -859,8 +890,8 @@ static int uadk_prov_cipher_cipher(void *vctx, unsigned char *output, size_t *ou return UADK_P_SUCCESS; } -static int uadk_prov_cipher_block_encrypto(struct cipher_priv_ctx *priv, - unsigned char *out, size_t *outl, size_t outsize) +static int uadk_prov_cipher_block_encrypto(struct cipher_priv_ctx *priv, unsigned char *out, + size_t *outl, size_t outsize) { size_t blksz = priv->blksize; int ret; @@ -891,8 +922,8 @@ static int uadk_prov_cipher_block_encrypto(struct cipher_priv_ctx *priv, return UADK_P_SUCCESS; } -static int uadk_prov_cipher_block_decrypto(struct cipher_priv_ctx *priv, - unsigned char *out, size_t *outl, size_t outsize) +static int uadk_prov_cipher_block_decrypto(struct cipher_priv_ctx *priv, unsigned char *out, + size_t *outl, size_t outsize) { size_t blksz = priv->blksize; int ret; @@ -1012,7 +1043,7 @@ static int uadk_prov_cipher_stream_update(void *vctx, unsigned char *output, do_soft: /* have isseu if both using hw and soft partly */ - ret = uadk_prov_cipher_soft_work(priv, output, &len, input, inl); + ret = uadk_prov_cipher_soft_update(priv, output, &len, input, inl); if (ret) { *outl = len; return UADK_P_SUCCESS; diff --git a/src/uadk_prov_digest.c b/src/uadk_prov_digest.c index a2a5f09..7a5bfff 100644 --- a/src/uadk_prov_digest.c +++ b/src/uadk_prov_digest.c @@ -70,18 +70,6 @@ struct digest_prov { static struct digest_prov dprov; static pthread_mutex_t digest_mutex = PTHREAD_MUTEX_INITIALIZER; -struct evp_md_ctx_st { - const EVP_MD *digest; - /* Functional reference if 'digest' is ENGINE-provided */ - ENGINE *engine; - unsigned long flags; - void *md_data; - /* Public key context for sign/verify */ - EVP_PKEY_CTX *pctx; - /* Update function: usually copied from EVP_MD */ - int (*update)(EVP_MD_CTX *ctx, const void *data, size_t count); -}; - struct digest_priv_ctx { handle_t sess; struct wd_digest_sess_setup setup; @@ -131,6 +119,17 @@ static struct digest_info digest_info_table[] = { 32, SHA_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, }; +static EVP_MD_CTX *EVP_MD_CTX_dup(const EVP_MD_CTX *in) +{ + EVP_MD_CTX *out = EVP_MD_CTX_new(); + + if (out != NULL && !EVP_MD_CTX_copy_ex(out, in)) { + EVP_MD_CTX_free(out); + out = NULL; + } + return out; +} + static int uadk_digests_soft_md(struct digest_priv_ctx *priv) { if (priv->soft_md) @@ -147,53 +146,83 @@ static int uadk_digests_soft_md(struct digest_priv_ctx *priv) priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA1, "provider=default"); break; case NID_sha224: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_224, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_224, "provider=default"); break; case NID_sha256: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_256, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_256, "provider=default"); break; case NID_sha384: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_384, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_384, "provider=default"); break; case NID_sha512: - priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA3_512, "provider=default"); + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_512, "provider=default"); + break; + case NID_sha512_224: + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_512_224, + "provider=default"); + break; + case NID_sha512_256: + priv->soft_md = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_SHA2_512_256, + "provider=default"); break; default: break; } - if (unlikely(priv->soft_md == NULL)) + if (unlikely(priv->soft_md == NULL)) { + fprintf(stderr, "digest failed to fetch\n"); return UADK_DIGEST_FAIL; + } return UADK_DIGEST_SUCCESS; } static int uadk_digest_soft_init(struct digest_priv_ctx *priv) { - if (priv->soft_md) - return EVP_DigestInit_ex(priv->soft_ctx, priv->soft_md, NULL); + if (!priv->soft_md || !priv->soft_ctx) + return UADK_DIGEST_FAIL; - return UADK_DIGEST_FAIL; + if (!EVP_DigestInit_ex(priv->soft_ctx, priv->soft_md, NULL)) { + fprintf(stderr, "soft digest init failed.\n"); + return UADK_DIGEST_FAIL; + } + + priv->switch_flag = UADK_DO_SOFT; + + return UADK_DIGEST_SUCCESS; } static int uadk_digest_soft_update(struct digest_priv_ctx *priv, const void *data, size_t len) { - if (priv->soft_md) - return EVP_DigestUpdate(priv->soft_ctx, data, len); + if (priv->soft_md == NULL) + return UADK_DIGEST_FAIL; - return UADK_DIGEST_FAIL; + if (!EVP_DigestUpdate(priv->soft_ctx, data, len)) { + fprintf(stderr, "soft digest update failed.\n"); + return UADK_DIGEST_FAIL; + } + + priv->switch_flag = UADK_DO_SOFT; + + return UADK_DIGEST_SUCCESS; } static int uadk_digest_soft_final(struct digest_priv_ctx *priv, unsigned char *digest) { - if (priv->soft_md) { - unsigned int digest_length; + unsigned int digest_length; - return EVP_DigestFinal_ex(priv->soft_ctx, digest, &digest_length); + if (priv->soft_md == NULL) + return UADK_DIGEST_FAIL; + + if (!EVP_DigestFinal_ex(priv->soft_ctx, digest, &digest_length)) { + fprintf(stderr, "soft digest final failed.\n"); + return UADK_DIGEST_FAIL; } - return UADK_DIGEST_FAIL; + priv->switch_flag = 0; + + return UADK_DIGEST_SUCCESS; } static void digest_soft_cleanup(struct digest_priv_ctx *priv) @@ -201,12 +230,8 @@ static void digest_soft_cleanup(struct digest_priv_ctx *priv) EVP_MD_CTX *ctx = priv->soft_ctx; if (ctx != NULL) { - if (ctx->md_data) { - OPENSSL_free(ctx->md_data); - ctx->md_data = NULL; - } EVP_MD_CTX_free(ctx); - ctx = NULL; + priv->soft_ctx = NULL; } if (priv->soft_md) { @@ -216,7 +241,7 @@ static void digest_soft_cleanup(struct digest_priv_ctx *priv) } static int uadk_digest_soft_work(struct digest_priv_ctx *priv, int len, - unsigned char *digest) + unsigned char *digest) { int ret; @@ -341,7 +366,7 @@ static int uadk_digest_ctx_init(struct digest_priv_ctx *priv) ret = uadk_prov_digest_dev_init(priv); if (unlikely(ret <= 0)) - goto soft_init; + return UADK_DIGEST_FAIL; /* Use the default numa parameters */ params.numa_id = -1; @@ -357,15 +382,7 @@ static int uadk_digest_ctx_init(struct digest_priv_ctx *priv) } } - if (enable_sw_offload) - uadk_digests_soft_md(priv); - return UADK_DIGEST_SUCCESS; - -soft_init: - pthread_mutex_unlock(&digest_mutex); - fprintf(stderr, "uadk failed to initialize digest.\n"); - return uadk_digest_soft_init(priv); } static void uadk_fill_mac_buffer_len(struct digest_priv_ctx *priv, bool is_end) @@ -445,14 +462,15 @@ static int uadk_digest_update_inner(struct digest_priv_ctx *priv, const void *da priv->state = SEC_DIGEST_DOING; priv->req.out = priv->out; - remain_len -= processing_len; - input_data += processing_len; ret = wd_do_digest_sync(priv->sess, &priv->req); if (ret) { fprintf(stderr, "do sec digest update failed, switch to soft digest.\n"); goto do_soft_digest; } + + remain_len -= processing_len; + input_data += processing_len; } while (remain_len > DIGEST_BLOCK_SIZE); priv->last_update_bufflen = remain_len; @@ -462,7 +480,6 @@ static int uadk_digest_update_inner(struct digest_priv_ctx *priv, const void *da do_soft_digest: if (priv->state == SEC_DIGEST_FIRST_UPDATING) { - priv->switch_flag = UADK_DO_SOFT; ret = uadk_digest_soft_init(priv); if (!ret) return ret; @@ -472,6 +489,9 @@ do_soft_digest: ret = uadk_digest_soft_update(priv, priv->data, DIGEST_BLOCK_SIZE); if (!ret) goto out; + + remain_len -= processing_len; + input_data += processing_len; } ret = uadk_digest_soft_update(priv, input_data, remain_len); @@ -548,6 +568,7 @@ static int uadk_do_digest_sync(struct digest_priv_ctx *priv) fprintf(stderr, "do sec digest sync failed, switch to soft digest.\n"); return UADK_DIGEST_FAIL; } + return UADK_DIGEST_SUCCESS; } @@ -558,7 +579,7 @@ static int uadk_do_digest_async(struct digest_priv_ctx *priv, struct async_op *o int cnt = 0; if (unlikely(priv->switch_flag == UADK_DO_SOFT)) { - fprintf(stderr, "async cipher init failed.\n"); + fprintf(stderr, "digest soft switching is not supported in asynchronous mode.\n"); return UADK_DIGEST_FAIL; } @@ -608,9 +629,11 @@ static int uadk_digest_final(struct digest_priv_ctx *priv, unsigned char *digest return UADK_DIGEST_FAIL; } - ret = uadk_digest_ctx_init(priv); - if (ret != UADK_DIGEST_SUCCESS) - return UADK_DIGEST_FAIL; + if (unlikely(priv->switch_flag != UADK_DO_SOFT)) { + ret = uadk_digest_ctx_init(priv); + if (ret != UADK_DIGEST_SUCCESS) + return UADK_DIGEST_FAIL; + } priv->req.in = priv->data; priv->req.out = priv->out; @@ -657,8 +680,8 @@ clear: return ret; } -static int uadk_digest_digest(struct digest_priv_ctx *priv, const void *data, size_t data_len, - unsigned char *digest) +static int uadk_digest_digest(struct digest_priv_ctx *priv, const void *data, + size_t data_len, unsigned char *digest) { struct async_op op; int ret; @@ -706,8 +729,7 @@ static void uadk_digest_cleanup(struct digest_priv_ctx *priv) if (priv->data) OPENSSL_clear_free(priv->data, DIGEST_BLOCK_SIZE); - if (priv->soft_ctx) - OPENSSL_clear_free(priv->soft_ctx, sizeof(EVP_MD_CTX)); + digest_soft_cleanup(priv); } static OSSL_FUNC_digest_freectx_fn uadk_prov_freectx; @@ -780,6 +802,7 @@ static void uadk_prov_freectx(void *dctx) static void *uadk_prov_dupctx(void *dctx) { struct digest_priv_ctx *dst_ctx, *src_ctx; + int ret; src_ctx = (struct digest_priv_ctx *)dctx; if (!dctx) @@ -809,15 +832,24 @@ static void *uadk_prov_dupctx(void *dctx) goto free_ctx; if (dst_ctx->soft_ctx) { - dst_ctx->soft_ctx = EVP_MD_CTX_new(); + dst_ctx->soft_ctx = EVP_MD_CTX_dup(src_ctx->soft_ctx); if (!dst_ctx->soft_ctx) { fprintf(stderr, "EVP_MD_CTX_new failed in ctx copy.\n"); goto free_data; } } + if (dst_ctx->soft_md) { + ret = EVP_MD_up_ref(dst_ctx->soft_md); + if (!ret) + goto free_dup; + } + return dst_ctx; +free_dup: + if (dst_ctx->soft_ctx) + EVP_MD_CTX_free(dst_ctx->soft_ctx); free_data: OPENSSL_clear_free(dst_ctx->data, DIGEST_BLOCK_SIZE); free_ctx: @@ -835,13 +867,20 @@ static int uadk_prov_init(void *dctx, const OSSL_PARAM params[]) return UADK_DIGEST_FAIL; } + if (uadk_get_sw_offload_state()) + uadk_digests_soft_md(priv); + ret = uadk_get_digest_info(priv); if (unlikely(!ret)) return UADK_DIGEST_FAIL; + priv->state = SEC_DIGEST_INIT; + ret = uadk_prov_digest_dev_init(priv); - if (unlikely(ret <= 0)) - return UADK_DIGEST_FAIL; + if (unlikely(ret <= 0)) { + fprintf(stderr, "digest switch to soft init!\n"); + return uadk_digest_soft_init(priv); + } return UADK_DIGEST_SUCCESS; } @@ -879,7 +918,7 @@ static int uadk_prov_final(void *dctx, unsigned char *out, return ret; } - if (unlikely(outl != NULL)) + if (outl) *outl = priv->md_size; return UADK_DIGEST_SUCCESS; -- 2.33.0