From: JiangShui Yang yangjiangshui@h-partners.com
Wenkai Lin (6): uadk: fix for wd_memset_zero occupies high cpu usage uadk: optimize session update for long hash mode uadk: support digest message carrying long data information uadk: add out_bytes check for long hash scene uadk: bugfix for aead set decrypt mac for stream mode uadk: fix for digest mac full length check
drv/hisi_sec.c | 4 +- include/drv/wd_aead_drv.h | 2 +- include/wd_digest.h | 22 +++++++++- v1/drv/hisi_sec_udrv.c | 4 ++ v1/wd_digest.c | 34 +++++++++++---- v1/wd_digest.h | 12 ++++++ wd_aead.c | 18 +++++--- wd_cipher.c | 2 +- wd_digest.c | 89 ++++++++++++++++++++++++--------------- 9 files changed, 131 insertions(+), 56 deletions(-)
From: Wenkai Lin linwenkai6@hisilicon.com
When key bytes is 0, it should not clean session key, CPU usage is occupied by wd_memset_zero now, which will causing performance deterioration.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd_aead.c | 4 ++-- wd_cipher.c | 2 +- wd_digest.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/wd_aead.c b/wd_aead.c index ed7b987..e5e1b19 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -342,8 +342,8 @@ void wd_aead_free_sess(handle_t h_sess) return; }
- wd_memset_zero(sess->ckey, MAX_CIPHER_KEY_SIZE); - wd_memset_zero(sess->akey, MAX_HMAC_KEY_SIZE); + wd_memset_zero(sess->ckey, sess->ckey_bytes); + wd_memset_zero(sess->akey, sess->akey_bytes);
if (sess->sched_key) free(sess->sched_key); diff --git a/wd_cipher.c b/wd_cipher.c index 6f57e17..2eaa77b 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -284,7 +284,7 @@ void wd_cipher_free_sess(handle_t h_sess) return; }
- wd_memset_zero(sess->key, MAX_CIPHER_KEY_SIZE); + wd_memset_zero(sess->key, sess->key_bytes);
if (sess->sched_key) free(sess->sched_key); diff --git a/wd_digest.c b/wd_digest.c index c8ccc8d..0847fe3 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -211,7 +211,7 @@ void wd_digest_free_sess(handle_t h_sess) return; }
- wd_memset_zero(sess->key, MAX_HMAC_KEY_SIZE); + wd_memset_zero(sess->key, sess->key_bytes); if (sess->sched_key) free(sess->sched_key); free(sess);
From: Wenkai Lin linwenkai6@hisilicon.com
The session should be updated after job is done successfully.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/wd_digest.h | 2 +- wd_digest.c | 44 +++++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/include/wd_digest.h b/include/wd_digest.h index 874e9c1..9abbe16 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -180,7 +180,7 @@ void wd_digest_uninit2(void); handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup);
/** - * wd_alg_digest_free_sess() - Free digest session. + * wd_digest_free_sess() - Free digest session. * @h_sess: session handler which will be free */ void wd_digest_free_sess(handle_t h_sess); diff --git a/wd_digest.c b/wd_digest.c index 0847fe3..fd26cc8 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -63,12 +63,13 @@ struct wd_digest_sess { __u32 key_bytes; void *sched_key; /* - * Notify the BD state, zero is frist BD, non-zero - * is middle or final BD. + * Notify the stream message state, zero is frist message, + * non-zero is middle or final message. */ - int bd_state; - /* Total of data for stream mode */ - __u64 long_data_len; + int msg_state; + + /* Total data length for stream mode */ + __u64 long_data_len; };
struct wd_env_config wd_digest_env_config; @@ -542,15 +543,10 @@ static void fill_request_msg(struct wd_digest_msg *msg, msg->out_bytes = req->out_bytes; msg->data_fmt = req->data_fmt; msg->has_next = req->has_next; - sess->long_data_len += req->in_bytes; - msg->long_data_len = sess->long_data_len; + msg->long_data_len = sess->long_data_len + req->in_bytes;
- /* To store the stream BD state, iv_bytes also means BD state */ - msg->iv_bytes = sess->bd_state; - if (req->has_next == 0) { - sess->long_data_len = 0; - sess->bd_state = 0; - } + /* Use iv_bytes to store the stream message state */ + msg->iv_bytes = sess->msg_state; }
static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *dsess, @@ -565,17 +561,23 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *ds pthread_spin_lock(&ctx->lock); ret = wd_handle_msg_sync(wd_digest_setting.driver, &msg_handle, ctx->ctx, msg, NULL, wd_digest_setting.config.epoll_en); + pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) - goto out; + return ret;
- /* - * non-zero is final BD or middle BD as stream mode. - */ - dsess->bd_state = msg->has_next; + /* After a stream mode job was done, update session long_data_len */ + if (msg->has_next) { + /* Long hash(first and middle message) */ + dsess->long_data_len += msg->in_bytes; + } else if (msg->iv_bytes) { + /* Long hash(final message) */ + dsess->long_data_len = 0; + }
-out: - pthread_spin_unlock(&ctx->lock); - return ret; + /* Update session message state */ + dsess->msg_state = msg->has_next; + + return 0; }
int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req)
From: Wenkai Lin linwenkai6@hisilicon.com
Currently, we use 0 and 1 to fill the has_next filed of digest request, 0 indicates that there is no next packet, 1 indicates there are next packets, we use WD_DIGEST_END and WD_DIGEST_DOING to mark those messages.
In another scenario, when user calculates on data flow A, after partial data is calculated, the result may be copied to data flow B so that flow B can calculate different data. However, flow B uses a new session, there's no information required for continuing calculation. Therefore, messages with this information need to be delivered to uadk, so we use WD_DIGEST_STREAM_END and WD_DIGEST_STREAM_DOING to mark those messages, long_data_len is also added to messages to store the length of the processed data.
After uadk set long data length and msg_state for the session, it will also change the status of has_next, change WD_DIGEST_STREAM_END to WD_DIGEST_END, WD_DIGEST_STREAM_DOING to WD_DIGEST_DOING.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/wd_digest.h | 19 ++++++++++++++++++- wd_digest.c | 10 ++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/include/wd_digest.h b/include/wd_digest.h index 9abbe16..7539866 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -78,6 +78,20 @@ enum wd_digest_mode { WD_DIGEST_MODE_MAX, };
+/** + * wd_digest_msg_state - Message state of digest + * @WD_DIGEST_END: Final message or single block message + * @WD_DIGEST_DOING: First message or middle message + * @WD_DIGEST_STREAM_END: Final message carrying long data information + * @WD_DIGEST_STREAM_DOING: Middle message carrying long data information + */ +enum wd_digest_msg_state { + WD_DIGEST_END = 0x0, + WD_DIGEST_DOING, + WD_DIGEST_STREAM_END, + WD_DIGEST_STREAM_DOING, +}; + /** * wd_digest_sess_setup - Parameters which is used to allocate a digest session * @alg: digest algorithm type, denoted by enum wd_digest_type @@ -100,9 +114,11 @@ typedef void *wd_digest_cb_t(void *cb_param); * @out_buf_bytes: actual output buffer size * @iv: input iv data addrss for AES_GMAC * @iv_bytes: input iv data size - * @has_next: is there next data block + * @has_next: message state, all types are defined in enum wd_digest_msg_state. * @cb: callback function for async mode * @cb_param: pointer of callback parameter + * @long_data_len: total length of data has been processed, it is only needed by + * the data flow switched to another session for processing. * * Note: If there is a alg selected in session, alg below will be ignore * otherwise, alg here will be used. Same as mode below. @@ -125,6 +141,7 @@ struct wd_digest_req { __u8 data_fmt; wd_digest_cb_t *cb; void *cb_param; + __u64 long_data_len; };
struct wd_cb_tag { diff --git a/wd_digest.c b/wd_digest.c index fd26cc8..69c4b28 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -531,6 +531,16 @@ static void fill_request_msg(struct wd_digest_msg *msg, { memcpy(&msg->req, req, sizeof(struct wd_digest_req));
+ if (unlikely(req->has_next == WD_DIGEST_STREAM_END)) { + sess->long_data_len = req->long_data_len; + sess->msg_state = WD_DIGEST_DOING; + req->has_next = WD_DIGEST_END; + } else if (unlikely(req->has_next == WD_DIGEST_STREAM_DOING)) { + sess->long_data_len = req->long_data_len; + sess->msg_state = WD_DIGEST_DOING; + req->has_next = WD_DIGEST_DOING; + } + msg->alg_type = WD_DIGEST; msg->alg = sess->alg; msg->mode = sess->mode;
From: Wenkai Lin linwenkai6@hisilicon.com
In long hash scene, the first and middle hash task should ensure full mac len out buffer, so we add out_bytes check here. But out_bytes of the last hash task should be actual length.
mac length should also be set for hardware mac check.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- v1/drv/hisi_sec_udrv.c | 4 ++++ v1/wd_digest.c | 34 ++++++++++++++++++++++++++-------- v1/wd_digest.h | 12 ++++++++++++ 3 files changed, 42 insertions(+), 8 deletions(-)
diff --git a/v1/drv/hisi_sec_udrv.c b/v1/drv/hisi_sec_udrv.c index ecb2ac2..fb6c933 100644 --- a/v1/drv/hisi_sec_udrv.c +++ b/v1/drv/hisi_sec_udrv.c @@ -1178,11 +1178,13 @@ static void qm_fill_digest_long_bd(struct wcrypto_digest_msg *msg,
if (msg->has_next && (msg->iv_bytes == 0)) { /* LONG BD FIRST */ + sqe->type2.mac_len = 0x1; sqe->type2.ai_gen = AI_GEN_INNER; sqe->type2.a_pad = AUTHPAD_NOPAD; msg->iv_bytes = msg->out_bytes; } else if (msg->has_next && (msg->iv_bytes != 0)) { /* LONG BD MIDDLE */ + sqe->type2.mac_len = 0x1; sqe->type2.ai_gen = AI_GEN_IVIN_ADDR; sqe->type2.a_pad = AUTHPAD_NOPAD; sqe->type2.a_ivin_addr_h = sqe->type2.mac_addr_h; @@ -1508,11 +1510,13 @@ static void qm_fill_digest_long_bd3(struct wcrypto_digest_msg *msg, /* iv_bytes is multiplexed as a flag bit to determine whether it is LOGN BD FIRST */ if (msg->has_next && msg->iv_bytes == 0) { /* LONG BD FIRST */ + sqe->mac_len = 0x1; sqe->ai_gen = AI_GEN_INNER; sqe->stream_scene.auth_pad = AUTHPAD_NOPAD; msg->iv_bytes = msg->out_bytes; } else if (msg->has_next && msg->iv_bytes != 0) { /* LONG BD MIDDLE */ + sqe->mac_len = 0x1; sqe->ai_gen = AI_GEN_IVIN_ADDR; sqe->stream_scene.auth_pad = AUTHPAD_NOPAD; sqe->auth_key_iv.a_ivin_addr_h = sqe->mac_addr_h; diff --git a/v1/wd_digest.c b/v1/wd_digest.c index cb0d4d8..5057f49 100644 --- a/v1/wd_digest.c +++ b/v1/wd_digest.c @@ -54,6 +54,14 @@ static __u32 g_digest_mac_len[WCRYPTO_MAX_DIGEST_TYPE] = { WCRYPTO_DIGEST_SHA512_224_LEN, WCRYPTO_DIGEST_SHA512_256_LEN };
+static __u32 g_digest_mac_full_len[WCRYPTO_MAX_DIGEST_TYPE] = { + WCRYPTO_DIGEST_SM3_FULL_LEN, WCRYPTO_DIGEST_MD5_FULL_LEN, + WCRYPTO_DIGEST_SHA1_FULL_LEN, WCRYPTO_DIGEST_SHA256_FULL_LEN, + WCRYPTO_DIGEST_SHA224_FULL_LEN, WCRYPTO_DIGEST_SHA384_FULL_LEN, + WCRYPTO_DIGEST_SHA512_FULL_LEN, WCRYPTO_DIGEST_SHA512_224_FULL_LEN, + WCRYPTO_DIGEST_SHA512_256_FULL_LEN +}; + static void del_ctx_key(struct wcrypto_digest_ctx *ctx) { struct wd_mm_br *br = &(ctx->setup.br); @@ -330,20 +338,30 @@ static int param_check(struct wcrypto_digest_ctx *d_ctx, return -WD_EINVAL; }
- if (unlikely(num != 1 && d_opdata[i]->has_next)) { - WD_ERR("num > 1, wcrypto_burst_digest does not support stream mode!\n"); + if (unlikely(!d_opdata[i]->out_bytes)) { + WD_ERR("invalid: digest mac length is 0.\n"); return -WD_EINVAL; }
- if (unlikely(d_opdata[0]->has_next && d_opdata[0]->in_bytes % d_ctx->align_sz)) { - WD_ERR("digest stream mode must be %u-byte aligned!\n", d_ctx->align_sz); - return -WD_EINVAL; - } - if (d_opdata[i]->out_bytes == 0 || - d_opdata[i]->out_bytes > g_digest_mac_len[alg]) { + if (d_opdata[i]->has_next) { + if (unlikely(num != 1)) { + WD_ERR("num > 1, wcrypto_burst_digest does not support stream mode!\n"); + return -WD_EINVAL; + } + if (unlikely(d_opdata[i]->in_bytes % d_ctx->align_sz)) { + WD_ERR("digest stream mode must be %u-byte aligned!\n", d_ctx->align_sz); + return -WD_EINVAL; + } + if (unlikely(d_opdata[i]->out_bytes < g_digest_mac_full_len[alg])) { + WD_ERR("digest stream mode out buffer space is not enough!\n"); + return -WD_EINVAL; + } + } else { + if (unlikely(d_opdata[i]->out_bytes > g_digest_mac_len[alg])) { WD_ERR("failed to check digest mac length!\n"); return -WD_EINVAL; } + }
if (unlikely(tag && !tag[i])) { WD_ERR("tag[%u] is NULL!\n", i); diff --git a/v1/wd_digest.h b/v1/wd_digest.h index 6ad4c85..8f7ac2e 100644 --- a/v1/wd_digest.h +++ b/v1/wd_digest.h @@ -51,6 +51,18 @@ enum wd_digest_mac_len { WCRYPTO_DIGEST_SHA512_256_LEN = 32 };
+enum wcrypto_digest_mac_full_len { + WCRYPTO_DIGEST_SM3_FULL_LEN = 32, + WCRYPTO_DIGEST_MD5_FULL_LEN = 16, + WCRYPTO_DIGEST_SHA1_FULL_LEN = 20, + WCRYPTO_DIGEST_SHA256_FULL_LEN = 32, + WCRYPTO_DIGEST_SHA224_FULL_LEN = 32, + WCRYPTO_DIGEST_SHA384_FULL_LEN = 64, + WCRYPTO_DIGEST_SHA512_FULL_LEN = 64, + WCRYPTO_DIGEST_SHA512_224_FULL_LEN = 64, + WCRYPTO_DIGEST_SHA512_256_FULL_LEN = 64, +}; + enum wcrypto_digest_mode { WCRYPTO_DIGEST_NORMAL, WCRYPTO_DIGEST_HMAC,
From: Wenkai Lin linwenkai6@hisilicon.com
For the aead stream mode, packets are sent in three phases: first, middle and final, user may deliver the decrypt mac through the first or final message.
Currently uadk stores the first message's mac to the session, then restore it when process the final message, it can't work when the first message does not carry a valid decrypt mac.
Now uadk can cache the mac calculated based on the first message and the middle message to the session and use the decrypt mac until the final message is processed.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- drv/hisi_sec.c | 4 ++-- include/drv/wd_aead_drv.h | 2 +- wd_aead.c | 14 +++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 3341b09..b0dcbba 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -2264,7 +2264,7 @@ static void gcm_auth_ivin(struct wd_aead_msg *msg)
/* Use the user's origin mac for decrypt icv check */ if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) - memcpy(msg->mac, msg->mac_bak, msg->auth_bytes); + memcpy(msg->mac, msg->dec_mac, msg->auth_bytes); }
static void fill_gcm_first_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) @@ -2357,7 +2357,7 @@ static int gcm_do_soft_mac(struct wd_aead_msg *msg) msg->mac[i] = g[i] ^ ctr_r[i];
if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) { - ret = memcmp(msg->mac, msg->mac_bak, msg->auth_bytes); + ret = memcmp(msg->mac, msg->dec_mac, msg->auth_bytes); if (ret) { msg->result = WD_IN_EPARA; WD_ERR("failed to do the gcm authentication!\n"); diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 5e4f73a..a9c0e7c 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -64,7 +64,7 @@ struct wd_aead_msg { /* mac */ __u8 *mac; /* mac data pointer for decrypto as stream mode */ - __u8 mac_bak[AES_BLOCK_SIZE]; + __u8 *dec_mac; /* total of data for stream mode */ __u64 long_data_len; enum wd_aead_msg_state msg_state; diff --git a/wd_aead.c b/wd_aead.c index e5e1b19..87d61c3 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -635,20 +635,24 @@ static void fill_stream_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, memset(sess->iv, 0, MAX_IV_SIZE); memcpy(sess->iv, req->iv, GCM_IV_SIZE);
- /* Store the original mac of first message to session */ if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) - memcpy(sess->mac_bak, req->mac, sess->auth_bytes); + msg->mac = sess->mac_bak; break; case AEAD_MSG_MIDDLE: /* Middle messages need to store the stream's total length to session */ sess->long_data_len += req->in_bytes;
msg->long_data_len = sess->long_data_len; + + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + msg->mac = sess->mac_bak; break; case AEAD_MSG_END: - /* Sets the original mac for final message */ - if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) - memcpy(msg->mac_bak, sess->mac_bak, sess->auth_bytes); + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) { + /* Sets the original mac for final message */ + msg->dec_mac = req->mac; + msg->mac = sess->mac_bak; + }
msg->long_data_len = sess->long_data_len + req->in_bytes; /* Reset the session's long_data_len */
From: Wenkai Lin linwenkai6@hisilicon.com
An uadk check error caused because digest mac full length check does not consider the new added request message type, so mac full length needs to be checked based on all message types.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/wd_digest.h | 1 + wd_digest.c | 33 ++++++++++++++++++++------------- 2 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/include/wd_digest.h b/include/wd_digest.h index 7539866..ad4d579 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -90,6 +90,7 @@ enum wd_digest_msg_state { WD_DIGEST_DOING, WD_DIGEST_STREAM_END, WD_DIGEST_STREAM_DOING, + WD_DIGEST_MSG_STATE_MAX, };
/** diff --git a/wd_digest.c b/wd_digest.c index 69c4b28..2307bf1 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -463,19 +463,26 @@ static int wd_mac_length_check(struct wd_digest_sess *sess, return -WD_EINVAL; }
- if (unlikely(!req->has_next && - req->out_bytes > g_digest_mac_len[sess->alg])) { - WD_ERR("invalid: digest mac length, alg = %d, out_bytes = %u\n", - sess->alg, req->out_bytes); - return -WD_EINVAL; - } - - /* User need to input full mac buffer in first and middle hash */ - if (unlikely(req->has_next && - req->out_bytes != g_digest_mac_full_len[sess->alg])) { - WD_ERR("invalid: digest mac full length is error, alg = %d, out_bytes = %u\n", - sess->alg, req->out_bytes); - return -WD_EINVAL; + switch (req->has_next) { + case WD_DIGEST_END: + case WD_DIGEST_STREAM_END: + if (unlikely(req->out_bytes > g_digest_mac_len[sess->alg])) { + WD_ERR("invalid: digest mac length, alg = %d, out_bytes = %u\n", + sess->alg, req->out_bytes); + return -WD_EINVAL; + } + break; + case WD_DIGEST_DOING: + case WD_DIGEST_STREAM_DOING: + /* User need to input full mac buffer in first and middle hash */ + if (unlikely(req->out_bytes != g_digest_mac_full_len[sess->alg])) { + WD_ERR("invalid: digest mac full length, alg = %d, out_bytes = %u\n", + sess->alg, req->out_bytes); + return -WD_EINVAL; + } + break; + default: + break; }
return 0;
-
Qi Tao