In stream processing encryption mode, a long file needs to be encrypted. When the accelerator is invoked, the encryption result of each block is assembled. The assembled result is the same as the result of encrypting the entire file at a time. For hisi_sec, AAD data is completed by the first BD, plaintext data are done with middle BDs and end BD. In an encrypted stream, the first BD and end BD are unique and must be delivered to hardware.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- Makefile.am | 3 +- drv/hisi_sec.c | 392 ++++++++++++++++++++++++++++++++++-- include/crypto/aes.h | 32 +++ include/crypto/galois.h | 19 ++ include/drv/wd_aead_drv.h | 7 + include/wd.h | 1 + include/wd_aead.h | 15 ++ lib/crypto/aes.c | 405 ++++++++++++++++++++++++++++++++++++++ lib/crypto/galois.c | 94 +++++++++ wd_aead.c | 44 +++++ 10 files changed, 992 insertions(+), 20 deletions(-) create mode 100644 include/crypto/aes.h create mode 100644 include/crypto/galois.h create mode 100644 lib/crypto/aes.c create mode 100644 lib/crypto/galois.c
diff --git a/Makefile.am b/Makefile.am index f6b73c1..37ac9b0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -75,7 +75,8 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_sched.c wd_sched.h
libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h + lib/crypto/aes.c lib/crypto/galois.c \ + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h galois.h
libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h wd_hpre_drv.h diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 41285ba..ff78419 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1,14 +1,17 @@ /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */
-#include "../include/drv/wd_cipher_drv.h" -#include "../include/drv/wd_digest_drv.h" -#include "../include/drv/wd_aead_drv.h" +#include "drv/wd_cipher_drv.h" +#include "drv/wd_digest_drv.h" +#include "drv/wd_aead_drv.h" +#include "crypto/aes.h" +#include "crypto/galois.h" #include "hisi_qm_udrv.h" #include "wd_cipher.h" #include "wd_digest.h" #include "wd_aead.h"
+#define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 #define WORD_ALIGNMENT_MASK 0x3 #define CTR_MODE_LEN_SHIFT 4 @@ -68,6 +71,15 @@ #define DES3_BLOCK_SIZE 8 #define AES_BLOCK_SIZE 16 #define CTR_128BIT_COUNTER 16 +#define GCM_FINAL_COUNTER 0x1000000 +#define GCM_FINAL_COUNTER_LEN 4 +#define GCM_STREAM_MAC_OFFSET 32 +#define GCM_FULL_MAC_LEN 16 +#define GCM_AUTH_MAC_OFFSET 47 +#define GCM_BLOCK_SIZE AES_BLOCK_SIZE +#define AKEY_LEN(c_key_len) (2 * (c_key_len) + 0x4) +#define MAC_LEN 4 +#define LONG_AUTH_DATA_OFFSET 24
/* The max BD data length is 16M-512B */ #define MAX_INPUT_DATA_LEN 0xFFFE00 @@ -175,8 +187,10 @@ enum { };
enum sec_cipher_dir { + SEC_NO_CIPHER = 0x0, SEC_CIPHER_ENC = 0x1, SEC_CIPHER_DEC = 0x2, + SEC_CIPHER_COPY = 0x3, };
enum sec_bd_type { @@ -369,7 +383,8 @@ struct bd3_stream_scene { /* * auth_pad: 0~1 bits * stream_protocol: 2~4 bits - * reserved: 5~7 bits + * mac_sel: 5 bits + * reserved: 6~7 bits */ __u8 stream_auth_pad; __u8 plaintext_type; @@ -591,6 +606,7 @@ static void dump_sec_msg(void *msg, const char *alg) dmsg->key_bytes, dmsg->iv_bytes, dmsg->in_bytes, dmsg->out_bytes); } else if (!strcmp(alg, "aead")) { amsg = (struct wd_aead_msg *)msg; + WD_ERR("MSG_STATE:%u\n", amsg->msg_state); WD_ERR("type:%u calg:%u op_type:%u cmode:%u\n", amsg->alg_type, amsg->calg, amsg->op_type, amsg->cmode); WD_ERR("data_fmt:%u ckey_bytes:%u auth_bytes:%u\n", @@ -2095,6 +2111,10 @@ static int fill_aead_bd2_mode(struct wd_aead_msg *msg, case WD_CIPHER_GCM: c_mode = C_MODE_GCM; sqe->type_auth_cipher &= SEC_AUTH_MASK; + if ((msg->msg_state == AEAD_MSG_FIRST) && !msg->assoc_bytes) { + WD_ERR("invalid: first bd assoc bytes is 0!\n"); + return -WD_EINVAL; + } sqe->type2.alen_ivllen = msg->assoc_bytes; sqe->type2.icvw_kmode |= msg->auth_bytes; break; @@ -2170,8 +2190,22 @@ static void fill_aead_bd2_addr(struct wd_aead_msg *msg, sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; }
-static int aead_len_check(struct wd_aead_msg *msg) +static int aead_len_check(struct wd_aead_msg *msg, enum sec_bd_type type) { + if (msg->msg_state == AEAD_MSG_MIDDLE) { + if ((!msg->in_bytes || (msg->in_bytes & (AES_BLOCK_SIZE - 1)))) { + WD_ERR("invalid: middle bd input size is 0 or not 16 bytes aligned!\n"); + return -WD_EINVAL; + } + } + + if (msg->cmode == WD_CIPHER_GCM || msg->cmode == WD_CIPHER_CCM) { + if (msg->msg_state == AEAD_MSG_BLOCK && type == BD_TYPE2 && !msg->in_bytes) { + WD_ERR("invalid: ccm/gcm block mode input size is 0 for hw_v2!\n"); + return -WD_EINVAL; + } + } + if (unlikely(msg->in_bytes + msg->assoc_bytes > MAX_INPUT_DATA_LEN)) { WD_ERR("aead input data length is too long, size = %u\n", msg->in_bytes + msg->assoc_bytes); @@ -2188,16 +2222,184 @@ static int aead_len_check(struct wd_aead_msg *msg) return 0; }
+static void fill_gcm_akey_len(struct wd_aead_msg *msg, void *sqe, enum sec_bd_type type) +{ + struct hisi_sec_sqe3 *sqe3; + struct hisi_sec_sqe *sqe2; + __u8 c_key_len = 0; + + aead_get_aes_key_len(msg, &c_key_len); + + if (type == BD_TYPE2) { + sqe2 = (struct hisi_sec_sqe *)sqe; + sqe2->type2.mac_key_alg |= (__u32)AKEY_LEN(c_key_len) << MAC_LEN_OFFSET; + } else if (type == BD_TYPE3) { + sqe3 = (struct hisi_sec_sqe3 *)sqe; + sqe3->auth_mac_key |= (__u32)AKEY_LEN(c_key_len) << SEC_AKEY_OFFSET_V3; + } +} + +static void gcm_auth_ivin(struct wd_aead_msg *msg) +{ + __u32 final_counter = GCM_FINAL_COUNTER; + + /* auth_ivin = {cipher_ivin(16B), null(16B), auth_mac(16B), null(16B)} */ + memset(msg->aiv_stream, 0, AIV_STREAM_LEN); + + memcpy(msg->aiv_stream, msg->iv, GCM_IV_SIZE); + /* The last 4 bytes of c_ivin are counters */ + memcpy(msg->aiv_stream + GCM_IV_SIZE, &final_counter, GCM_FINAL_COUNTER_LEN); + + /* Fill auth_ivin with the mac of last MIDDLE BD */ + memcpy(msg->aiv_stream + GCM_STREAM_MAC_OFFSET, msg->mac, GCM_FULL_MAC_LEN); + + /* Use the user's origin mac for decrypt icv check */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac, msg->mac_bak, msg->auth_bytes); +} + +static void fill_gcm_first_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_INNER; + sqe->ai_apd_cs |= AUTHPAD_NOPAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher &= ~(SEC_CIPHER_COPY << SEC_CIPHER_OFFSET); + sqe->type_auth_cipher |= AUTH_HMAC_CALCULATE << AUTHTYPE_OFFSET; + sqe->type2.mac_key_alg = MAC_LEN; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.mac_key_alg |= A_ALG_AES_GMAC << AUTH_ALG_OFFSET; + sqe->type2.clen_ivhlen = 0; + sqe->type2.icvw_kmode = 0; + sqe->type2.a_ivin_addr = 0; + sqe->type2.c_key_addr = 0; + sqe->type2.c_alg = 0; + sqe->type2.auth_src_offset = 0; + sqe->type2.alen_ivllen = msg->assoc_bytes; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_middle_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_IVIN_ADDR; + sqe->ai_apd_cs |= AUTHPAD_NOPAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher |= NO_AUTH << AUTHTYPE_OFFSET; + sqe->type2.cipher_src_offset = 0; + sqe->type2.auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.alen_ivllen = 0; + sqe->type2.a_ivin_addr = sqe->type2.mac_addr; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_final_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_IVIN_ADDR; + sqe->ai_apd_cs |= AUTHPAD_PAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher |= NO_AUTH << AUTHTYPE_OFFSET; + sqe->type2.cipher_src_offset = 0; + sqe->type2.auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.alen_ivllen = 0; + sqe->type2.long_a_data_len = msg->assoc_bytes; + sqe->type2.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; +} + +static void get_galois_vector_s(struct wd_aead_msg *msg, __u8 *s) +{ + unsigned int aad_len, cipher_len; + __u8 a_c[GCM_BLOCK_SIZE] = {0}; + int i; + + aad_len = msg->assoc_bytes * BYTE_BITS; + memcpy(&a_c[BYTE_BITS], &aad_len, sizeof(unsigned int)); + + cipher_len = msg->long_data_len * BYTE_BITS; + memcpy(&a_c[0], &cipher_len, sizeof(unsigned int)); + + /* Based the little-endian operation */ + for (i = 0; i < GCM_BLOCK_SIZE; i++) + s[i] = a_c[i] ^ msg->aiv_stream[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; +} + +static int gcm_do_soft_mac(struct wd_aead_msg *msg) +{ + __u8 ctr_r[GCM_BLOCK_SIZE] = {0}; + __u8 data[GCM_BLOCK_SIZE] = {0}; + __u8 H[GCM_BLOCK_SIZE] = {0}; + __u8 S[GCM_BLOCK_SIZE] = {0}; + __u8 g[GCM_BLOCK_SIZE] = {0}; + int i, ret; + + aes_encrypt(msg->ckey, msg->ckey_bytes, data, H); + + get_galois_vector_s(msg, S); + + galois_compute(S, H, g, GCM_BLOCK_SIZE); + + /* Encrypt ctr0 based on AES_ECB */ + aes_encrypt(msg->ckey, msg->ckey_bytes, msg->aiv_stream, ctr_r); + + /* Get the GMAC tag final */ + for (i = 0; i < GCM_BLOCK_SIZE; i++) + msg->mac[i] = g[i] ^ ctr_r[i]; + + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) { + ret = memcmp(msg->mac, msg->mac_bak, msg->auth_bytes); + if (ret) { + msg->result = WD_IN_EPARA; + WD_ERR("failed to do the gcm authentication!\n"); + return -WD_EINVAL; + } + } + + msg->result = WD_SUCCESS; + + return 0; +} + +static int fill_stream_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + int ret = 0; + + switch (msg->msg_state) { + case AEAD_MSG_FIRST: + fill_gcm_first_bd2(msg, sqe); + break; + case AEAD_MSG_MIDDLE: + fill_gcm_middle_bd2(msg, sqe); + break; + case AEAD_MSG_END: + gcm_auth_ivin(msg); + if (msg->in_bytes) { + fill_gcm_final_bd2(msg, sqe); + } else { + ret = gcm_do_soft_mac(msg); + if (!ret) + ret = WD_SOFT_COMPUTING; + } + break; + default: + break; + } + + return ret; +} + static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) { __u8 scene, cipher, de; int ret;
- /* config BD type */ sqe->type_auth_cipher = BD_TYPE2; - /* config scene */ - scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET; - de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET; + + if (msg->msg_state == AEAD_MSG_BLOCK) + scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET; + else + scene = SEC_STREAM_SCENE << SEC_SCENE_OFFSET;
if (msg->op_type == WD_CIPHER_ENCRYPTION_DIGEST) { cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; @@ -2213,6 +2415,8 @@ static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) WD_ERR("failed to check aead op type, op = %u\n", msg->op_type); return -WD_EINVAL; } + + de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET; sqe->sds_sa_type |= (__u8)(de | scene); sqe->type_auth_cipher |= cipher;
@@ -2235,6 +2439,23 @@ static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) return 0; }
+int aead_msg_state_check(struct wd_aead_msg *msg) +{ + if (msg->cmode == WD_CIPHER_GCM) { + if (unlikely(msg->msg_state >= AEAD_MSG_INVALID)) { + WD_ERR("invalid: GCM input msg state is wrong!\n"); + return -WD_EINVAL; + } + } else { + if (unlikely(msg->msg_state != AEAD_MSG_BLOCK)) { + WD_ERR("invalid: cmode not support stream msg state!\n"); + return -WD_EINVAL; + } + } + + return 0; +} + int hisi_sec_aead_send(handle_t ctx, void *aead_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); @@ -2248,12 +2469,11 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) return -WD_EINVAL; }
- if (unlikely(msg->cmode != WD_CIPHER_CBC && !msg->in_bytes)) { - WD_ERR("ccm or gcm not supports 0 packet size at hw_v2!\n"); - return -WD_EINVAL; - } + ret = aead_msg_state_check(msg); + if (unlikely(ret)) + return ret;
- ret = aead_len_check(msg); + ret = aead_len_check(msg, BD_TYPE2); if (unlikely(ret)) return ret;
@@ -2270,6 +2490,13 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) }
fill_aead_bd2_addr(msg, &sqe); + + ret = fill_stream_bd2(msg, &sqe); + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret; + hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.tag = (__u16)msg->tag;
@@ -2288,6 +2515,20 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) return 0; }
+static void update_stream_counter(struct wd_aead_msg *recv_msg) +{ + /* + * The counter of the first middle BD is set to 1. + * Other middle BDs and tail BD are set based on + * cipher_len and the counter of the previous BD. + */ + if (recv_msg->msg_state == AEAD_MSG_FIRST) { + recv_msg->iv[MAX_IV_SIZE - 1] = 0x1; + } else if (recv_msg->msg_state == AEAD_MSG_MIDDLE) { + ctr_iv_inc(recv_msg->iv, recv_msg->in_bytes >> CTR_MODE_LEN_SHIFT); + } +} + static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { @@ -2324,10 +2565,19 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, temp_msg = recv_msg; }
+ update_stream_counter(recv_msg); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); }
+static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) +{ + /* Asynchronous mode does not use the sent message, so ignores it */ + return (msg->msg_state == AEAD_MSG_END) && !msg->in_bytes && + qp->q_info.qp_mode == CTX_MODE_SYNC; +} + int hisi_sec_aead_recv(handle_t ctx, void *aead_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); @@ -2336,6 +2586,9 @@ int hisi_sec_aead_recv(handle_t ctx, void *aead_msg) __u16 count = 0; int ret;
+ if (soft_compute_check((struct hisi_qp *)h_qp, recv_msg)) + return 0; + ret = hisi_qm_recv(h_qp, &sqe, 1, &count); if (ret < 0) return ret; @@ -2436,6 +2689,10 @@ static int fill_aead_bd3_mode(struct wd_aead_msg *msg, case WD_CIPHER_GCM: sqe->c_mode_alg |= C_MODE_GCM; sqe->auth_mac_key &= SEC_AUTH_MASK_V3; + if ((msg->msg_state == AEAD_MSG_FIRST) && !msg->assoc_bytes) { + WD_ERR("invalid: first bd assoc bytes is 0!\n"); + return -WD_EINVAL; + } sqe->a_len_key = msg->assoc_bytes; sqe->c_icv_key |= msg->auth_bytes << SEC_MAC_OFFSET_V3; break; @@ -2465,17 +2722,97 @@ static void fill_aead_bd3_addr(struct wd_aead_msg *msg, sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; }
+static void fill_gcm_first_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_INNER << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_NOPAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->c_icv_key &= ~0x3; + sqe->auth_mac_key = AUTH_HMAC_CALCULATE; + sqe->auth_mac_key |= MAC_LEN << SEC_MAC_OFFSET_V3; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->auth_mac_key |= A_ALG_AES_GMAC << SEC_AUTH_ALG_OFFSET_V3; + sqe->c_len_ivin = 0; + sqe->auth_ivin.a_ivin_addr = 0; + sqe->c_key_addr = 0; + sqe->c_mode_alg &= ~(0x7 << SEC_CALG_OFFSET_V3); + sqe->auth_src_offset = 0; + sqe->a_len_key = msg->assoc_bytes; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_middle_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_NOPAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->auth_mac_key |= NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->a_len_key = 0; + sqe->auth_ivin.a_ivin_addr = sqe->mac_addr; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_final_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_PAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->auth_mac_key |= NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->a_len_key = 0; + sqe->stream_scene.long_a_data_len = msg->assoc_bytes; + sqe->stream_scene.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; +} + +static int fill_stream_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + int ret = 0; + + switch (msg->msg_state) { + case AEAD_MSG_FIRST: + fill_gcm_first_bd3(msg, sqe); + break; + case AEAD_MSG_MIDDLE: + fill_gcm_middle_bd3(msg, sqe); + break; + case AEAD_MSG_END: + gcm_auth_ivin(msg); + if (msg->in_bytes) { + fill_gcm_final_bd3(msg, sqe); + } else { + ret = gcm_do_soft_mac(msg); + if (!ret) + ret = WD_SOFT_COMPUTING; + } + break; + default: + break; + } + + return ret; +}
static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) { __u16 scene, de; int ret;
- /* config BD type */ sqe->bd_param = BD_TYPE3; - /* config scene */ - scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET_V3; - de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET_V3; + + if (msg->msg_state == AEAD_MSG_BLOCK) + scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET_V3; + else + scene = SEC_STREAM_SCENE << SEC_SCENE_OFFSET_V3;
if (msg->op_type == WD_CIPHER_ENCRYPTION_DIGEST) { sqe->c_icv_key = SEC_CIPHER_ENC; @@ -2489,6 +2826,8 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) WD_ERR("failed to check aead op type, op = %u\n", msg->op_type); return -WD_EINVAL; } + + de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET_V3; sqe->bd_param |= (__u16)(de | scene);
sqe->c_len_ivin = msg->in_bytes; @@ -2526,7 +2865,11 @@ int hisi_sec_aead_send_v3(handle_t ctx, void *aead_msg) return -WD_EINVAL; }
- ret = aead_len_check(msg); + ret = aead_msg_state_check(msg); + if (unlikely(ret)) + return ret; + + ret = aead_len_check(msg, BD_TYPE3); if (unlikely(ret)) return ret;
@@ -2544,6 +2887,12 @@ int hisi_sec_aead_send_v3(handle_t ctx, void *aead_msg)
fill_aead_bd3_addr(msg, &sqe);
+ ret = fill_stream_bd3(msg, &sqe); + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret; + hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = msg->tag; ret = hisi_qm_send(h_qp, &sqe, 1, &count); @@ -2598,6 +2947,8 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, temp_msg = recv_msg; }
+ update_stream_counter(recv_msg); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); } @@ -2610,6 +2961,9 @@ int hisi_sec_aead_recv_v3(handle_t ctx, void *aead_msg) __u16 count = 0; int ret;
+ if (soft_compute_check((struct hisi_qp *)h_qp, recv_msg)) + return 0; + ret = hisi_qm_recv(h_qp, &sqe, 1, &count); if (ret < 0) return ret; diff --git a/include/crypto/aes.h b/include/crypto/aes.h new file mode 100644 index 0000000..eca58ba --- /dev/null +++ b/include/crypto/aes.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_AES_H__ +#define __WD_AES_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define UINT_B_CNT 8 +#define AES_MAXNR 14 + +struct aes_key { + unsigned int rd_key[4 * (AES_MAXNR + 1)]; + __u8 rounds; +}; + +union uni { + unsigned char b[UINT_B_CNT]; + __u32 w[2]; + __u64 d; +}; + +void aes_encrypt(__u8 *key, __u32 key_len, __u8 *src, __u8 *dst); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/crypto/galois.h b/include/crypto/galois.h new file mode 100644 index 0000000..07723be --- /dev/null +++ b/include/crypto/galois.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_GALOIS_H__ +#define __WD_GALOIS_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +void galois_compute(__u8 *S, __u8 *H, __u8 *g, __u32 len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 195205c..b43bb36 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -57,12 +57,19 @@ struct wd_aead_msg { __u8 *iv; /* input auth iv pointer */ __u8 aiv[MAX_IV_SIZE]; + /* input auth iv pointer for stream mode */ + __u8 aiv_stream[AIV_STREAM_LEN]; /* input data pointer */ __u8 *in; /* output data pointer */ __u8 *out; /* mac */ __u8 *mac; + /* mac data pointer for decrypto as stream mode */ + __u8 mac_bak[AES_BLOCK_SIZE]; + /* total of data for stream mode */ + __u64 long_data_len; + enum wd_aead_msg_state msg_state; };
struct wd_aead_driver { diff --git a/include/wd.h b/include/wd.h index da3cfc6..97956e2 100644 --- a/include/wd.h +++ b/include/wd.h @@ -71,6 +71,7 @@ typedef void (*wd_log)(const char *format, ...); #define WD_SUCCESS 0 #define WD_STREAM_END 1 #define WD_STREAM_START 2 +#define WD_SOFT_COMPUTING 3 #define WD_EIO EIO #define WD_EAGAIN EAGAIN #define WD_ENOMEM ENOMEM diff --git a/include/wd_aead.h b/include/wd_aead.h index 9f6f4e8..eefbed3 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -17,6 +17,7 @@ extern "C" { #endif
+#define AIV_STREAM_LEN 64 /** * wd_aead_op_type - Algorithm type of option */ @@ -43,6 +44,18 @@ struct wd_aead_sess_setup { void *sched_param; };
+/** + * wd_aead_msg_state - Notify the message state + * zero is message for block mode, non-zero is message for stream mode. + */ +enum wd_aead_msg_state { + AEAD_MSG_BLOCK = 0x0, + AEAD_MSG_FIRST, + AEAD_MSG_MIDDLE, + AEAD_MSG_END, + AEAD_MSG_INVALID, +}; + struct wd_aead_req; typedef void *wd_alg_aead_cb_t(struct wd_aead_req *req, void *cb_param);
@@ -84,6 +97,8 @@ struct wd_aead_req { __u8 data_fmt; wd_alg_aead_cb_t *cb; void *cb_param; + + enum wd_aead_msg_state msg_state; };
/** diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c new file mode 100644 index 0000000..2145d24 --- /dev/null +++ b/lib/crypto/aes.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include "crypto/aes.h" + +#define WORD(n) (0x##n##n##n##n) +#define LONG(n) (0x##n##n##n##n##n##n##n##n) + +#define STATE_CNT 2 + +static void xtimeword(__u32 *w) +{ + __u32 a, b; + + a = *w; + b = a & WORD(80); + a ^= b; + b -= b >> 0x7; + b &= WORD(1B); + b ^= a << 0x1; + *w = b; +} + +static void xtimelong(__u64 *w) +{ + __u64 a, b; + + a = *w; + b = a & LONG(80); + a ^= b; + b -= b >> 0x7; + b &= LONG(1B); + b ^= a << 0x1; + *w = b; +} + +static __u32 caculate_x_final(__u32 x_1) +{ + __u32 x, y; + + x = x_1; + y = ((x & WORD(FE)) >> 0x1) | ((x & WORD(01)) << 0x7); + x &= WORD(39); + x ^= y & WORD(3F); + y = ((y & WORD(FC)) >> 0x2) | ((y & WORD(03)) << 0x6); + x ^= y & WORD(97); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(9B); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(3C); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(DD); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(72); + x ^= WORD(63); + + return x; +} + +static __u32 caculate_x(__u32 *w) +{ + __u32 x, y; + + x = *w; + y = ((x & WORD(FE)) >> 0x1) | ((x & WORD(01)) << 0x7); + x &= WORD(DD); + x ^= y & WORD(57); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(1C); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(4A); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(42); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(64); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(E0); + return x; +} + +static void subword(__u32 *w) +{ +#define SPECIAL_WORD 0x0A0A0A0A0Au + __u32 x, a1, a2, a3, a4, a5, a6; + + x = caculate_x(w); + a1 = x ^ (x & WORD(F0)) >> 0x4; + a2 = ((x & WORD(CC)) >> 0x2) | ((x & WORD(33)) << 0x2); + a3 = (x & a1) ^ ((x & a1) & WORD(AA)) >> 0x1; + a3 ^= (((x << 0x1) & a1) ^ ((a1 << 0x1) & x)) & WORD(AA); + a4 = (a2 & a1) ^ ((a2 & a1) & WORD(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a1) ^ ((a1 << 0x1) & a2)) & WORD(AA); + a5 = (a3 & WORD(CC)) >> 0x2; + a3 ^= ((a4 << 0x2) ^ a4) & WORD(CC); + a4 = (a5 & WORD(22)) | (a5 & WORD(22)) >> 0x1; + a4 ^= (a5 << 0x1) & WORD(22); + a3 ^= a4; + a5 = (a3 & WORD(A0)) | (a3 & WORD(A0)) >> 0x1; + a5 ^= (a3 << 0x1) & WORD(A0); + a4 = a5 & WORD(C0); + a6 = a4 >> 0x2; + a4 ^= (a5 << 0x2) & WORD(C0); + a5 = (a6 & WORD(20)) | (a6 & WORD(20)) >> 0x1; + a5 ^= (a6 << 0x1) & WORD(20); + a4 |= a5; + a3 ^= a4 >> 0x4; + a3 &= WORD(0F); + a2 = a3 ^ (a3 & WORD(0C)) >> 0x2; + a4 = a3 & a2; + a4 ^= (a4 & SPECIAL_WORD) >> 0x1; + a4 ^= (((a3 << 0x1) & a2) ^ ((a2 << 0x1) & a3)) & WORD(0A); + a5 = (a4 & WORD(08)) | (a4 & WORD(08)) >> 0x1; + a5 ^= (a4 << 0x1) & WORD(08); + a4 ^= a5 >> 0x2; + a4 &= WORD(03); + a4 ^= (a4 & WORD(02)) >> 0x1; + a4 |= a4 << 0x2; + a3 = (a2 & a4) ^ ((a2 & a4) & WORD(0A)) >> 0x1; + a3 ^= (((a2 << 0x1) & a4) ^ ((a4 << 0x1) & a2)) & WORD(0A); + a3 |= a3 << 0x4; + a2 = ((a1 & WORD(CC)) >> 0x2) | ((a1 & WORD(33)) << 0x2); + x = (a1 & a3) ^ ((a1 & a3) & WORD(AA)) >> 0x1; + x ^= (((a1 << 0x1) & a3) ^ ((a3 << 0x1) & a1)) & WORD(AA); + a4 = (a2 & a3) ^ ((a2 & a3) & WORD(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a3) ^ ((a3 << 0x1) & a2)) & WORD(AA); + a5 = (x & WORD(CC)) >> 0x2; + x ^= ((a4 << 0x2) ^ a4) & WORD(CC); + a4 = (a5 & WORD(22)) | (a5 & WORD(22)) >> 0x1; + a4 ^= (a5 << 0x1) & WORD(22); + *w = caculate_x_final(x ^ a4); +} + +static __u64 caculate_long_x_final(__u64 x_1) +{ + __u64 x, y; + + x = x_1; + y = ((x & LONG(FE)) >> 0x1) | ((x & LONG(01)) << 0x7); + x &= LONG(39); + x ^= y & LONG(3F); + y = ((y & LONG(FC)) >> 0x2) | ((y & LONG(03)) << 0x6); + x ^= y & LONG(97); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(9B); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(3C); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(DD); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(72); + x ^= LONG(63); + + return x; +} + +static __u64 caculate_long_x(__u64 *w) +{ + __u64 x, y; + + x = *w; + y = ((x & LONG(FE)) >> 0x1) | ((x & LONG(01)) << 0x7); + x &= LONG(DD); + x ^= y & LONG(57); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(1C); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(4A); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(42); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(64); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(E0); + + return x; +} + +static void sublong(__u64 *w) +{ + __u64 x, a1, a2, a3, a4, a5, a6; + + x = caculate_long_x(w); + a1 = x ^ (x & LONG(F0)) >> 0x4; + a2 = ((x & LONG(CC)) >> 0x2) | ((x & LONG(33)) << 0x2); + a3 = (x & a1) ^ ((x & a1) & LONG(AA)) >> 0x1; + a3 ^= (((x << 0x1) & a1) ^ ((a1 << 0x1) & x)) & LONG(AA); + a4 = (a2 & a1) ^ ((a2 & a1) & LONG(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a1) ^ ((a1 << 0x1) & a2)) & LONG(AA); + a5 = (a3 & LONG(CC)) >> 0x2; + a3 ^= ((a4 << 0x2) ^ a4) & LONG(CC); + a4 = (a5 & LONG(22)) | (a5 & LONG(22)) >> 0x1; + a4 ^= (a5 << 0x1) & LONG(22); + a3 ^= a4; + a5 = (a3 & LONG(A0)) | (a3 & LONG(A0)) >> 0x1; + a5 ^= (a3 << 0x1) & LONG(A0); + a4 = a5 & LONG(C0); + a6 = a4 >> 0x2; + a4 ^= (a5 << 0x2) & LONG(C0); + a5 = (a6 & LONG(20)) | (a6 & LONG(20)) >> 0x1; + a5 ^= (a6 << 0x1) & LONG(20); + a4 |= a5; + a3 ^= a4 >> 0x4; + a3 &= LONG(0F); + a2 = a3 ^ (a3 & LONG(0C)) >> 0x2; + a4 = a3 & a2; + a4 ^= (a4 & LONG(0A)) >> 0x1; + a4 ^= (((a3 << 0x1) & a2) ^ ((a2 << 0x1) & a3)) & LONG(0A); + a5 = (a4 & LONG(08)) | (a4 & LONG(08)) >> 0x1; + a5 ^= (a4 << 0x1) & LONG(08); + a4 ^= a5 >> 0x2; + a4 &= LONG(03); + a4 ^= (a4 & LONG(02)) >> 0x1; + a4 |= a4 << 0x2; + a3 = (a2 & a4) ^ ((a2 & a4) & LONG(0A)) >> 0x1; + a3 ^= (((a2 << 0x1) & a4) ^ ((a4 << 0x1) & a2)) & LONG(0A); + a3 |= a3 << 0x4; + a2 = ((a1 & LONG(CC)) >> 0x2) | ((a1 & LONG(33)) << 0x2); + x = (a1 & a3) ^ ((a1 & a3) & LONG(AA)) >> 0x1; + x ^= (((a1 << 0x1) & a3) ^ ((a3 << 0x1) & a1)) & LONG(AA); + a4 = (a2 & a3) ^ ((a2 & a3) & LONG(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a3) ^ ((a3 << 0x1) & a2)) & LONG(AA); + a5 = (x & LONG(CC)) >> 0x2; + x ^= ((a4 << 0x2) ^ a4) & LONG(CC); + a4 = (a5 & LONG(22)) | (a5 & LONG(22)) >> 0x1; + a4 ^= (a5 << 0x1) & LONG(22); + *w = caculate_long_x_final(x ^ a4); +} + +static void shift_rows(__u64 *state) +{ +#define S_CNT 4 + unsigned char s[S_CNT]; + unsigned char *s0; + __u8 r, i; + + s0 = (unsigned char *)state; + for (r = 0; r < S_CNT; r++) { + for (i = 0; i < S_CNT; i++) + s[i] = s0[i * S_CNT + r]; + + for (i = 0; i < S_CNT; i++) + s0[i * S_CNT + r] = s[(r + i) % S_CNT]; + } +} + +static void mix_columns(__u64 *state) +{ +#define REVERT_OFFSET 0x3 +#define A 0xFFFF0000FFFF0000uLL +#define B 0xFF00FF00FF00FF00uLL + union uni s1; + union uni s; + __u8 c, i; + + for (c = 0; c < STATE_CNT; c++) { + s1.d = state[c]; + s.d = s1.d; + s.d ^= ((s.d & A) >> 0x10) + | ((s.d & ~A) << 0x10); + s.d ^= ((s.d & B) >> 0x8) + | ((s.d & ~B) << 0x8); + s.d ^= s1.d; + xtimelong(&s1.d); + s.d ^= s1.d; + for (i = 0; i < UINT_B_CNT; i++) { + if (i == UINT_B_CNT - 1 || i == (UINT_B_CNT >> 0x1) - 1) + s.b[i] ^= s1.b[i - REVERT_OFFSET]; + else + s.b[i] ^= s1.b[i + 1]; + } + state[c] = s.d; + } +} + +static void add_round_key(__u64 *state, const __u64 *w) +{ + state[0] ^= w[0]; + state[1] ^= w[1]; +} + +static void cipher(const unsigned char *in, unsigned char *out, + const __u64 *w, __u8 nr) +{ +#define STATE_BYTE 16 + __u64 state[STATE_CNT]; + __u8 i; + + memcpy(state, in, STATE_BYTE); + + add_round_key(state, w); + + for (i = 1; i < nr; i++) { + sublong(&state[0]); + sublong(&state[1]); + shift_rows(state); + mix_columns(state); + add_round_key(state, w + i * STATE_CNT); + } + + sublong(&state[0]); + sublong(&state[1]); + shift_rows(state); + add_round_key(state, w + nr * STATE_CNT); + + memcpy(out, state, STATE_BYTE); +} + +static void rotword(__u32 *x) +{ +#define WORDBYTE 4 + unsigned char *w0; + unsigned char tmp; + __u8 i; + + w0 = (unsigned char *)x; + tmp = w0[0]; + for (i = 0; i < WORDBYTE - 1; i++) + w0[i] = w0[i + 1]; + + w0[WORDBYTE - 1] = tmp; +} + +static void key_expansion(const unsigned char *key, __u64 *w, __u8 nr, __u8 nk) +{ +#define RCON_LEN 4 +#define N 2 +#define NK 6 + __u32 rcon; + union uni prev; + __u32 temp; + __u8 i, n, m; + + memcpy(w, key, nk * RCON_LEN); + memcpy(&rcon, "\1\0\0\0", RCON_LEN); + n = nk / N; + m = (nr + 1) * N; + prev.d = w[n - 1]; + for (i = n; i < m; i++) { + temp = prev.w[1]; + if (i % n == 0) { + rotword(&temp); + subword(&temp); + temp ^= rcon; + xtimeword(&rcon); + } else if (nk > NK && i % n == N) { + subword(&temp); + } + prev.d = w[i - n]; + prev.w[0] ^= temp; + prev.w[1] ^= prev.w[0]; + w[i] = prev.d; + } +} + +static int aes_set_encrypt_key(const unsigned char *userkey, const int bits, + struct aes_key *key) +{ +#define AES_128_BIT 128 +#define AES_192_BIT 192 +#define AES_256_BIT 256 +#define AES_128_ROUNDS 10 +#define AES_192_ROUNDS 12 +#define AES_256_ROUNDS 14 +#define KEY_NK 32 + __u64 *rk; + + if (!userkey || !key) + return -1; + if (bits != AES_128_BIT && bits != AES_192_BIT && bits != AES_256_BIT) + return -1; + + rk = (__u64 *)key->rd_key; + + if (bits == AES_128_BIT) + key->rounds = AES_128_ROUNDS; + else if (bits == AES_192_BIT) + key->rounds = AES_192_ROUNDS; + else + key->rounds = AES_256_ROUNDS; + + key_expansion(userkey, rk, key->rounds, bits / KEY_NK); + return 0; +} + +static void aes_encrypt_(const __u8 *in, __u8 *out, const struct aes_key *key) +{ + const __u64 *rk; + + rk = (__u64 *)key->rd_key; + + cipher(in, out, rk, key->rounds); +} + +void aes_encrypt(__u8 *key, __u32 key_len, __u8 *src, __u8 *dst) +{ + struct aes_key local_key; + int ret; + + ret = aes_set_encrypt_key(key, key_len << 0x3, &local_key); + if (ret) + return; + + aes_encrypt_(src, dst, &local_key); +} diff --git a/lib/crypto/galois.c b/lib/crypto/galois.c new file mode 100644 index 0000000..94f97fe --- /dev/null +++ b/lib/crypto/galois.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include "crypto/galois.h" + +#define GF_LSB1_MASK 0x1 +#define GF_R 0xe1000000 +#define GALOIS_BITS 128 +#define UINT_BITS (sizeof(unsigned int) * 8) + +#define GALOIS_UL_COUNT 4 +#define GALOIS_UL_COUNT_H 12 + +#define GALOIS_PARA_TRANS_S(S, i) (((unsigned int)(S)[(i) + 3] << 24) | \ + ((unsigned int)(S)[(i) + 2] << 16) | \ + ((unsigned int)(S)[(i) + 1] << 8) | \ + ((unsigned int)(S)[(i)])) +#define GALOIS_PARA_TRANS_H(H, i) (((unsigned int)(H)[(i)] << 24) | \ + ((unsigned int)(H)[(i) + 1] << 16) | \ + ((unsigned int)(H)[(i) + 2] << 8) | \ + ((unsigned int)(H)[(i) + 3])) + +#define GALOIS_UINT_TRANS_CHAR(src_addr) \ + do { \ + unsigned int temp = *(src_addr); \ + *(src_addr) = ((temp & 0xff) << 24) | ((temp & 0xff00) << 8) \ + | ((temp & 0xff0000) >> 8) | ((temp & 0xff000000) >> 24); \ + } while (0) + +static void galois_xtime_128(unsigned int *src, unsigned int *dst) +{ + unsigned int tmp; + __u8 i; + + /* Based the NIST Special Publication 800-38D */ + for (i = 0; i < GALOIS_UL_COUNT; i++) { + tmp = src[i] >> 0x1; + dst[i] = tmp | (src[i + 1] << (UINT_BITS - 1)); + if (i == GALOIS_UL_COUNT - 1) { + if (src[0] & GF_LSB1_MASK) + dst[i] = tmp ^ GF_R; + else + dst[i] = tmp; + } + } +} + +static void galois_multi(unsigned int *input_a, unsigned int *input_b, + unsigned int *mul, unsigned int array_size) +{ + /* 4 * (unsigned int), 4 * 32bit = 128bit */ + unsigned int gf_V[GALOIS_BITS][GALOIS_UL_COUNT] = {0}; + unsigned int *gf_Z = mul; + __u8 i, j, k; + + memcpy(gf_V[0], input_a, sizeof(unsigned int) * array_size); + + for (i = 1; i < GALOIS_BITS; i++) + galois_xtime_128(gf_V[i - 1], gf_V[i]); + + for (i = 0; i < GALOIS_BITS; i++) { + k = GALOIS_BITS - i - 1; + if ((input_b[(k / UINT_BITS)] >> (k % UINT_BITS)) & 0x1) { + for (j = 0; j < array_size; j++) + gf_Z[j] ^= gf_V[i][j]; + } + } +} + +void galois_compute(__u8 *S, __u8 *H, __u8 *g, __u32 len) +{ + unsigned int SL[GALOIS_UL_COUNT] = {0}; + unsigned int HL[GALOIS_UL_COUNT] = {0}; + unsigned int G[GALOIS_UL_COUNT] = {0}; + __u8 i, j; + + /* Do galois multiplication operation on blocks: G = S x H */ + for (i = 0; i < GALOIS_UL_COUNT; i++) { + j = i * GALOIS_UL_COUNT; + SL[i] = GALOIS_PARA_TRANS_S(S, j); + j = GALOIS_UL_COUNT_H - j; + HL[i] = GALOIS_PARA_TRANS_H(H, j); + } + + galois_multi(SL, HL, G, GALOIS_UL_COUNT); + + j = len - GALOIS_UL_COUNT; + for (i = 0; i < GALOIS_UL_COUNT; i++) { + GALOIS_UINT_TRANS_CHAR(&G[i]); + memcpy(&g[j], &G[i], sizeof(unsigned int)); + j -= GALOIS_UL_COUNT; + } +} diff --git a/wd_aead.c b/wd_aead.c index 38d3f8f..c1fc354 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -48,11 +48,17 @@ struct wd_aead_sess { enum wd_digest_mode dmode; unsigned char ckey[MAX_CIPHER_KEY_SIZE]; unsigned char akey[MAX_HMAC_KEY_SIZE]; + /* Mac data pointer for decrypto as stream mode */ + unsigned char mac_bak[WD_AEAD_CCM_GCM_MAX]; __u16 ckey_bytes; __u16 akey_bytes; __u16 auth_bytes; void *priv; void *sched_key; + /* Stored the counter for gcm stream mode */ + __u8 iv[MAX_IV_SIZE]; + /* Total of data for stream mode */ + __u64 long_data_len; };
struct wd_env_config wd_aead_env_config; @@ -485,6 +491,41 @@ void wd_aead_uninit(void) wd_alg_clear_init(&wd_aead_setting.status); }
+static void fill_stream_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, + struct wd_aead_sess *sess) +{ + switch (req->msg_state) { + case AEAD_MSG_FIRST: + /* Stream iv is extended to 16 bytes and last 4 bytes must be zero */ + memset(sess->iv, 0, MAX_IV_SIZE); + memcpy(sess->iv, req->iv, GCM_IV_SIZE); + + /* Store the original mac of first message to session */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(sess->mac_bak, req->mac, sess->auth_bytes); + break; + case AEAD_MSG_MIDDLE: + /* Middle messages need to store the stream's total length to session */ + sess->long_data_len += req->in_bytes; + + msg->long_data_len = sess->long_data_len; + break; + case AEAD_MSG_END: + /* Sets the original mac for final message */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac_bak, sess->mac_bak, sess->auth_bytes); + + msg->long_data_len = sess->long_data_len + req->in_bytes; + /* Reset the session's long_data_len */ + sess->long_data_len = 0; + break; + default: + return; + } + + msg->iv = sess->iv; +} + static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, struct wd_aead_sess *sess) { @@ -510,6 +551,9 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, msg->mac = req->mac; msg->auth_bytes = sess->auth_bytes; msg->data_fmt = req->data_fmt; + + msg->msg_state = req->msg_state; + fill_stream_msg(msg, req, sess); }
static int send_recv_sync(struct wd_ctx_internal *ctx,
Include file '../wd_alg_common.h' is redundant. No declarations or interfaces is used.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/drv/wd_aead_drv.h | 1 - include/drv/wd_digest_drv.h | 1 - 2 files changed, 2 deletions(-)
diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index b43bb36..5aefbed 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -6,7 +6,6 @@
#include <asm/types.h>
-#include "../wd_alg_common.h" #include "../wd_aead.h"
#ifdef __cplusplus diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index 17e4630..fa60bfa 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -5,7 +5,6 @@
#include <asm/types.h>
-#include "../wd_alg_common.h" #include "../wd_digest.h"
#ifdef __cplusplus
Header files are not self contained, fix it.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/wd.h | 1 + include/wd_alg_common.h | 1 + include/wd_cipher.h | 1 + include/wd_sched.h | 1 + include/wd_util.h | 2 ++ 5 files changed, 6 insertions(+)
diff --git a/include/wd.h b/include/wd.h index 97956e2..9b33dfc 100644 --- a/include/wd.h +++ b/include/wd.h @@ -7,6 +7,7 @@ #ifndef __WD_H #define __WD_H #include <errno.h> +#include <numa.h> #include <fcntl.h> #include <stdbool.h> #include <stdint.h> diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index 4c12e78..d077b87 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -9,6 +9,7 @@
#include <pthread.h> #include <stdbool.h> +#include <numa.h> #include <asm/types.h>
#include "wd.h" diff --git a/include/wd_cipher.h b/include/wd_cipher.h index 8f043fd..26fb288 100644 --- a/include/wd_cipher.h +++ b/include/wd_cipher.h @@ -8,6 +8,7 @@ #define __WD_CIPHER_H
#include <dlfcn.h> +#include <asm/types.h> #include "wd_alg_common.h"
#ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index a492d70..b145172 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -6,6 +6,7 @@
#ifndef SCHED_SAMPLE_h #define SCHED_SAMPLE_h +#include <asm/types.h> #include "wd_alg_common.h"
#ifdef __cplusplus diff --git a/include/wd_util.h b/include/wd_util.h index 036fd51..037e18a 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -13,7 +13,9 @@ #include <sys/shm.h> #include <asm/types.h>
+#include "wd.h" #include "wd_sched.h" +#include "wd_alg.h" #include "wd_alg_common.h"
#ifdef __cplusplus
When NUMA is not configured, the node id is set to 0.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd.c | 6 ++++++ wd_util.c | 4 +--- 2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/wd.c b/wd.c index b614ea2..d1fc207 100644 --- a/wd.c +++ b/wd.c @@ -246,6 +246,12 @@ static int get_dev_info(struct uacce_dev *dev) if (ret < 0) return ret;
+ /* Special processing is performed when NUMA is not configured */ + if (dev->numa_id < 0) { + WD_INFO("numa node of the device is not configured, set it to 0!\n"); + dev->numa_id = 0; + } + ret = get_str_attr(dev, "api", dev->api, WD_NAME_SIZE); if (ret < 0) return ret; diff --git a/wd_util.c b/wd_util.c index 67b1afc..330d53a 100644 --- a/wd_util.c +++ b/wd_util.c @@ -550,9 +550,7 @@ static __u16 wd_get_dev_numa(struct uacce_dev_list *head, __u16 numa_num = 0;
while (list) { - if (list->dev->numa_id < 0) { - list->dev->numa_id = 0; - } else if (list->dev->numa_id >= size) { + if (list->dev->numa_id >= size) { WD_ERR("invalid: numa id is %d!\n", list->dev->numa_id); return 0; }
Add RTLD_NODELETE, do not unload shared objects during dlclose(). Therefore, if the object is reloaded later using dlopen(), the object's static variables are not reinitialized.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/wd_util.c b/wd_util.c index 330d53a..5b3aa8d 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2264,7 +2264,7 @@ void *wd_dlopen_drv(const char *cust_lib_dir) if (ret < 0) goto free_node;
- node->dlhandle = dlopen(lib_path, RTLD_NOW); + node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); if (!node->dlhandle) { free(node); /* there are many other files need to skip */
UADK_ALG_HW is calc_type of driver property, fix it.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/wd_util.c b/wd_util.c index 5b3aa8d..9cf95bf 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2008,7 +2008,7 @@ void wd_alg_uninit_driver(struct wd_ctx_config_internal *config,
driver->exit(priv); /* Ctx config just need clear once */ - if (driver->priority == UADK_ALG_HW) + if (driver->calc_type == UADK_ALG_HW) wd_clear_ctx_config(config);
if (driver->fallback)
ctx params uninit should be after ctx params init.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd_rsa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/wd_rsa.c b/wd_rsa.c index 8ead706..21f80e7 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -296,7 +296,6 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para if (ret == -WD_EAGAIN) { wd_disable_drv(wd_rsa_setting.driver); wd_alg_drv_unbind(wd_rsa_setting.driver); - wd_ctx_param_uninit(&rsa_ctx_params); continue; } goto out_driver; @@ -313,6 +312,7 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para if (ret == -WD_ENODEV) { wd_disable_drv(wd_rsa_setting.driver); wd_alg_drv_unbind(wd_rsa_setting.driver); + wd_ctx_param_uninit(&rsa_ctx_params); continue; } WD_ERR("failed to init alg attrs!\n");
Include file 'xxx.h' is redundant, no declarations or interfaces is used, remove them.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/drv/wd_dh_drv.h | 1 - include/drv/wd_ecc_drv.h | 1 - include/wd_rsa.h | 1 - 3 files changed, 3 deletions(-)
diff --git a/include/drv/wd_dh_drv.h b/include/drv/wd_dh_drv.h index 61a7fcc..e2df07f 100644 --- a/include/drv/wd_dh_drv.h +++ b/include/drv/wd_dh_drv.h @@ -5,7 +5,6 @@ #define __WD_DH_DRV_H
#include <asm/types.h> -#include "../wd_alg_common.h" #include "../wd_dh.h"
#ifdef __cplusplus diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index 7dcb56b..cb2f158 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -8,7 +8,6 @@ #include <asm/types.h>
#include "../wd.h" -#include "../wd_alg_common.h" #include "../wd_ecc.h"
#ifdef __cplusplus diff --git a/include/wd_rsa.h b/include/wd_rsa.h index 36e8c94..200ff14 100644 --- a/include/wd_rsa.h +++ b/include/wd_rsa.h @@ -9,7 +9,6 @@
#include <stdlib.h> #include <stdbool.h> -#include <asm/types.h>
#include "wd.h" #include "wd_alg_common.h"
Library real path maybe not exist, use realpath to check it.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd_util.c | 6 ++++++ 1 file changed, 6 insertions(+)
diff --git a/wd_util.c b/wd_util.c index 9cf95bf..d8d6482 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2185,6 +2185,7 @@ static void dladdr_empty(void) {} int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir) { char file_path[PATH_STR_SIZE] = {0}; + char path[PATH_MAX]; Dl_info file_info; int len, rc, i;
@@ -2214,6 +2215,11 @@ int wd_get_lib_file_path(char *lib_file, char *lib_path, bool is_dir) if (len < 0) return -WD_EINVAL;
+ if (realpath(lib_path, path) == NULL) { + WD_ERR("%s: no such file or directory!\n", path); + return -WD_EINVAL; + } + return 0; }
The length of the MAC written back by the hardware is equal to the auth_size input by the software. The auth_size has been verified first, so it only need to ensure that the MAC length is greater than the auth_size.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- wd_aead.c | 34 +++------------------------------- 1 file changed, 3 insertions(+), 31 deletions(-)
diff --git a/wd_aead.c b/wd_aead.c index 83c743c..181dab1 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -352,35 +352,6 @@ void wd_aead_free_sess(handle_t h_sess) free(sess); }
-static int aead_mac_param_check(struct wd_aead_sess *sess, - struct wd_aead_req *req) -{ - int ret = 0; - - switch (sess->cmode) { - case WD_CIPHER_CBC: - if (req->mac_bytes < g_aead_mac_len[sess->dalg]) { - WD_ERR("failed to check cbc-hmac mac buffer length, size = %u\n", - req->mac_bytes); - ret = -WD_EINVAL; - } - break; - case WD_CIPHER_CCM: - case WD_CIPHER_GCM: - if (req->mac_bytes < WD_AEAD_CCM_GCM_MAX) { - WD_ERR("failed to check CCM or GCM mac buffer length, size = %u\n", - req->mac_bytes); - ret = -WD_EINVAL; - } - break; - default: - ret = -WD_EINVAL; - WD_ERR("set the aead cmode is error, cmode = %d\n", sess->cmode); - } - - return ret; -} - static int wd_aead_param_check(struct wd_aead_sess *sess, struct wd_aead_req *req) { @@ -410,9 +381,10 @@ static int wd_aead_param_check(struct wd_aead_sess *sess, return -WD_EINVAL; }
- ret = aead_mac_param_check(sess, req); - if (unlikely(ret)) + if (unlikely(req->mac_bytes < sess->auth_bytes)) { + WD_ERR("failed to check aead mac length, size = %u\n", req->mac_bytes); return -WD_EINVAL; + }
if (req->data_fmt == WD_SGL_BUF) { len = req->in_bytes + req->assoc_bytes;
Because the unsigne long exists, the structure members are 8-byte aligned. If the int type is placed in the head and tail, 8 bytes are added. If the structure alignment is not considered, unpredictable memory exceptions, such as segment errors and malloc failures, may occur.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- include/wd.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/include/wd.h b/include/wd.h index 9b33dfc..5384afe 100644 --- a/include/wd.h +++ b/include/wd.h @@ -109,22 +109,23 @@ struct wd_dtb { __u32 bsize; };
+/* Members should be exactly 8-byte aligned */ struct uacce_dev { /* sysfs node content */ /* flag: SVA */ int flags; + int numa_id; /* HW context type */ char api[WD_NAME_SIZE]; /* dev supported algorithms */ char algs[MAX_ATTR_STR_SIZE]; - unsigned long qfrs_offs[UACCE_QFRT_MAX]; /* sysfs path with dev name */ char dev_root[PATH_STR_SIZE];
/* dev path in devfs */ char char_dev_path[MAX_DEV_NAME_LEN];
- int numa_id; + unsigned long qfrs_offs[UACCE_QFRT_MAX]; };
struct uacce_dev_list {
-
Wenkai Lin