In stream processing encryption mode, a long file needs to be encrypted. When the accelerator is invoked, the encryption result of each block is assembled. The assembled result is the same as the result of encrypting the entire file at a time. For hisi_sec, AAD data is completed by the first BD, plaintext data are done with middle BDs and end BD. In an encrypted stream, the first BD and end BD are unique and must be delivered to hardware.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- Makefile.am | 3 +- drv/hisi_sec.c | 392 ++++++++++++++++++++++++++++++++++-- include/crypto/aes.h | 32 +++ include/crypto/galois.h | 19 ++ include/drv/wd_aead_drv.h | 7 + include/wd.h | 1 + include/wd_aead.h | 15 ++ lib/crypto/aes.c | 405 ++++++++++++++++++++++++++++++++++++++ lib/crypto/galois.c | 94 +++++++++ wd_aead.c | 44 +++++ 10 files changed, 992 insertions(+), 20 deletions(-) create mode 100644 include/crypto/aes.h create mode 100644 include/crypto/galois.h create mode 100644 lib/crypto/aes.c create mode 100644 lib/crypto/galois.c
diff --git a/Makefile.am b/Makefile.am index f6b73c1..37ac9b0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -75,7 +75,8 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_sched.c wd_sched.h
libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h + lib/crypto/aes.c lib/crypto/galois.c \ + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h galois.h
libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h wd_hpre_drv.h diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 41285ba..ff78419 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1,14 +1,17 @@ /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */
-#include "../include/drv/wd_cipher_drv.h" -#include "../include/drv/wd_digest_drv.h" -#include "../include/drv/wd_aead_drv.h" +#include "drv/wd_cipher_drv.h" +#include "drv/wd_digest_drv.h" +#include "drv/wd_aead_drv.h" +#include "crypto/aes.h" +#include "crypto/galois.h" #include "hisi_qm_udrv.h" #include "wd_cipher.h" #include "wd_digest.h" #include "wd_aead.h"
+#define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 #define WORD_ALIGNMENT_MASK 0x3 #define CTR_MODE_LEN_SHIFT 4 @@ -68,6 +71,15 @@ #define DES3_BLOCK_SIZE 8 #define AES_BLOCK_SIZE 16 #define CTR_128BIT_COUNTER 16 +#define GCM_FINAL_COUNTER 0x1000000 +#define GCM_FINAL_COUNTER_LEN 4 +#define GCM_STREAM_MAC_OFFSET 32 +#define GCM_FULL_MAC_LEN 16 +#define GCM_AUTH_MAC_OFFSET 47 +#define GCM_BLOCK_SIZE AES_BLOCK_SIZE +#define AKEY_LEN(c_key_len) (2 * (c_key_len) + 0x4) +#define MAC_LEN 4 +#define LONG_AUTH_DATA_OFFSET 24
/* The max BD data length is 16M-512B */ #define MAX_INPUT_DATA_LEN 0xFFFE00 @@ -175,8 +187,10 @@ enum { };
enum sec_cipher_dir { + SEC_NO_CIPHER = 0x0, SEC_CIPHER_ENC = 0x1, SEC_CIPHER_DEC = 0x2, + SEC_CIPHER_COPY = 0x3, };
enum sec_bd_type { @@ -369,7 +383,8 @@ struct bd3_stream_scene { /* * auth_pad: 0~1 bits * stream_protocol: 2~4 bits - * reserved: 5~7 bits + * mac_sel: 5 bits + * reserved: 6~7 bits */ __u8 stream_auth_pad; __u8 plaintext_type; @@ -591,6 +606,7 @@ static void dump_sec_msg(void *msg, const char *alg) dmsg->key_bytes, dmsg->iv_bytes, dmsg->in_bytes, dmsg->out_bytes); } else if (!strcmp(alg, "aead")) { amsg = (struct wd_aead_msg *)msg; + WD_ERR("MSG_STATE:%u\n", amsg->msg_state); WD_ERR("type:%u calg:%u op_type:%u cmode:%u\n", amsg->alg_type, amsg->calg, amsg->op_type, amsg->cmode); WD_ERR("data_fmt:%u ckey_bytes:%u auth_bytes:%u\n", @@ -2095,6 +2111,10 @@ static int fill_aead_bd2_mode(struct wd_aead_msg *msg, case WD_CIPHER_GCM: c_mode = C_MODE_GCM; sqe->type_auth_cipher &= SEC_AUTH_MASK; + if ((msg->msg_state == AEAD_MSG_FIRST) && !msg->assoc_bytes) { + WD_ERR("invalid: first bd assoc bytes is 0!\n"); + return -WD_EINVAL; + } sqe->type2.alen_ivllen = msg->assoc_bytes; sqe->type2.icvw_kmode |= msg->auth_bytes; break; @@ -2170,8 +2190,22 @@ static void fill_aead_bd2_addr(struct wd_aead_msg *msg, sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; }
-static int aead_len_check(struct wd_aead_msg *msg) +static int aead_len_check(struct wd_aead_msg *msg, enum sec_bd_type type) { + if (msg->msg_state == AEAD_MSG_MIDDLE) { + if ((!msg->in_bytes || (msg->in_bytes & (AES_BLOCK_SIZE - 1)))) { + WD_ERR("invalid: middle bd input size is 0 or not 16 bytes aligned!\n"); + return -WD_EINVAL; + } + } + + if (msg->cmode == WD_CIPHER_GCM || msg->cmode == WD_CIPHER_CCM) { + if (msg->msg_state == AEAD_MSG_BLOCK && type == BD_TYPE2 && !msg->in_bytes) { + WD_ERR("invalid: ccm/gcm block mode input size is 0 for hw_v2!\n"); + return -WD_EINVAL; + } + } + if (unlikely(msg->in_bytes + msg->assoc_bytes > MAX_INPUT_DATA_LEN)) { WD_ERR("aead input data length is too long, size = %u\n", msg->in_bytes + msg->assoc_bytes); @@ -2188,16 +2222,184 @@ static int aead_len_check(struct wd_aead_msg *msg) return 0; }
+static void fill_gcm_akey_len(struct wd_aead_msg *msg, void *sqe, enum sec_bd_type type) +{ + struct hisi_sec_sqe3 *sqe3; + struct hisi_sec_sqe *sqe2; + __u8 c_key_len = 0; + + aead_get_aes_key_len(msg, &c_key_len); + + if (type == BD_TYPE2) { + sqe2 = (struct hisi_sec_sqe *)sqe; + sqe2->type2.mac_key_alg |= (__u32)AKEY_LEN(c_key_len) << MAC_LEN_OFFSET; + } else if (type == BD_TYPE3) { + sqe3 = (struct hisi_sec_sqe3 *)sqe; + sqe3->auth_mac_key |= (__u32)AKEY_LEN(c_key_len) << SEC_AKEY_OFFSET_V3; + } +} + +static void gcm_auth_ivin(struct wd_aead_msg *msg) +{ + __u32 final_counter = GCM_FINAL_COUNTER; + + /* auth_ivin = {cipher_ivin(16B), null(16B), auth_mac(16B), null(16B)} */ + memset(msg->aiv_stream, 0, AIV_STREAM_LEN); + + memcpy(msg->aiv_stream, msg->iv, GCM_IV_SIZE); + /* The last 4 bytes of c_ivin are counters */ + memcpy(msg->aiv_stream + GCM_IV_SIZE, &final_counter, GCM_FINAL_COUNTER_LEN); + + /* Fill auth_ivin with the mac of last MIDDLE BD */ + memcpy(msg->aiv_stream + GCM_STREAM_MAC_OFFSET, msg->mac, GCM_FULL_MAC_LEN); + + /* Use the user's origin mac for decrypt icv check */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac, msg->mac_bak, msg->auth_bytes); +} + +static void fill_gcm_first_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_INNER; + sqe->ai_apd_cs |= AUTHPAD_NOPAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher &= ~(SEC_CIPHER_COPY << SEC_CIPHER_OFFSET); + sqe->type_auth_cipher |= AUTH_HMAC_CALCULATE << AUTHTYPE_OFFSET; + sqe->type2.mac_key_alg = MAC_LEN; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.mac_key_alg |= A_ALG_AES_GMAC << AUTH_ALG_OFFSET; + sqe->type2.clen_ivhlen = 0; + sqe->type2.icvw_kmode = 0; + sqe->type2.a_ivin_addr = 0; + sqe->type2.c_key_addr = 0; + sqe->type2.c_alg = 0; + sqe->type2.auth_src_offset = 0; + sqe->type2.alen_ivllen = msg->assoc_bytes; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_middle_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_IVIN_ADDR; + sqe->ai_apd_cs |= AUTHPAD_NOPAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher |= NO_AUTH << AUTHTYPE_OFFSET; + sqe->type2.cipher_src_offset = 0; + sqe->type2.auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.alen_ivllen = 0; + sqe->type2.a_ivin_addr = sqe->type2.mac_addr; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_final_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_IVIN_ADDR; + sqe->ai_apd_cs |= AUTHPAD_PAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher |= NO_AUTH << AUTHTYPE_OFFSET; + sqe->type2.cipher_src_offset = 0; + sqe->type2.auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.alen_ivllen = 0; + sqe->type2.long_a_data_len = msg->assoc_bytes; + sqe->type2.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; +} + +static void get_galois_vector_s(struct wd_aead_msg *msg, __u8 *s) +{ + unsigned int aad_len, cipher_len; + __u8 a_c[GCM_BLOCK_SIZE] = {0}; + int i; + + aad_len = msg->assoc_bytes * BYTE_BITS; + memcpy(&a_c[BYTE_BITS], &aad_len, sizeof(unsigned int)); + + cipher_len = msg->long_data_len * BYTE_BITS; + memcpy(&a_c[0], &cipher_len, sizeof(unsigned int)); + + /* Based the little-endian operation */ + for (i = 0; i < GCM_BLOCK_SIZE; i++) + s[i] = a_c[i] ^ msg->aiv_stream[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; +} + +static int gcm_do_soft_mac(struct wd_aead_msg *msg) +{ + __u8 ctr_r[GCM_BLOCK_SIZE] = {0}; + __u8 data[GCM_BLOCK_SIZE] = {0}; + __u8 H[GCM_BLOCK_SIZE] = {0}; + __u8 S[GCM_BLOCK_SIZE] = {0}; + __u8 g[GCM_BLOCK_SIZE] = {0}; + int i, ret; + + aes_encrypt(msg->ckey, msg->ckey_bytes, data, H); + + get_galois_vector_s(msg, S); + + galois_compute(S, H, g, GCM_BLOCK_SIZE); + + /* Encrypt ctr0 based on AES_ECB */ + aes_encrypt(msg->ckey, msg->ckey_bytes, msg->aiv_stream, ctr_r); + + /* Get the GMAC tag final */ + for (i = 0; i < GCM_BLOCK_SIZE; i++) + msg->mac[i] = g[i] ^ ctr_r[i]; + + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) { + ret = memcmp(msg->mac, msg->mac_bak, msg->auth_bytes); + if (ret) { + msg->result = WD_IN_EPARA; + WD_ERR("failed to do the gcm authentication!\n"); + return -WD_EINVAL; + } + } + + msg->result = WD_SUCCESS; + + return 0; +} + +static int fill_stream_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + int ret = 0; + + switch (msg->msg_state) { + case AEAD_MSG_FIRST: + fill_gcm_first_bd2(msg, sqe); + break; + case AEAD_MSG_MIDDLE: + fill_gcm_middle_bd2(msg, sqe); + break; + case AEAD_MSG_END: + gcm_auth_ivin(msg); + if (msg->in_bytes) { + fill_gcm_final_bd2(msg, sqe); + } else { + ret = gcm_do_soft_mac(msg); + if (!ret) + ret = WD_SOFT_COMPUTING; + } + break; + default: + break; + } + + return ret; +} + static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) { __u8 scene, cipher, de; int ret;
- /* config BD type */ sqe->type_auth_cipher = BD_TYPE2; - /* config scene */ - scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET; - de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET; + + if (msg->msg_state == AEAD_MSG_BLOCK) + scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET; + else + scene = SEC_STREAM_SCENE << SEC_SCENE_OFFSET;
if (msg->op_type == WD_CIPHER_ENCRYPTION_DIGEST) { cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; @@ -2213,6 +2415,8 @@ static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) WD_ERR("failed to check aead op type, op = %u\n", msg->op_type); return -WD_EINVAL; } + + de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET; sqe->sds_sa_type |= (__u8)(de | scene); sqe->type_auth_cipher |= cipher;
@@ -2235,6 +2439,23 @@ static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) return 0; }
+int aead_msg_state_check(struct wd_aead_msg *msg) +{ + if (msg->cmode == WD_CIPHER_GCM) { + if (unlikely(msg->msg_state >= AEAD_MSG_INVALID)) { + WD_ERR("invalid: GCM input msg state is wrong!\n"); + return -WD_EINVAL; + } + } else { + if (unlikely(msg->msg_state != AEAD_MSG_BLOCK)) { + WD_ERR("invalid: cmode not support stream msg state!\n"); + return -WD_EINVAL; + } + } + + return 0; +} + int hisi_sec_aead_send(handle_t ctx, void *aead_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); @@ -2248,12 +2469,11 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) return -WD_EINVAL; }
- if (unlikely(msg->cmode != WD_CIPHER_CBC && !msg->in_bytes)) { - WD_ERR("ccm or gcm not supports 0 packet size at hw_v2!\n"); - return -WD_EINVAL; - } + ret = aead_msg_state_check(msg); + if (unlikely(ret)) + return ret;
- ret = aead_len_check(msg); + ret = aead_len_check(msg, BD_TYPE2); if (unlikely(ret)) return ret;
@@ -2270,6 +2490,13 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) }
fill_aead_bd2_addr(msg, &sqe); + + ret = fill_stream_bd2(msg, &sqe); + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret; + hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.tag = (__u16)msg->tag;
@@ -2288,6 +2515,20 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) return 0; }
+static void update_stream_counter(struct wd_aead_msg *recv_msg) +{ + /* + * The counter of the first middle BD is set to 1. + * Other middle BDs and tail BD are set based on + * cipher_len and the counter of the previous BD. + */ + if (recv_msg->msg_state == AEAD_MSG_FIRST) { + recv_msg->iv[MAX_IV_SIZE - 1] = 0x1; + } else if (recv_msg->msg_state == AEAD_MSG_MIDDLE) { + ctr_iv_inc(recv_msg->iv, recv_msg->in_bytes >> CTR_MODE_LEN_SHIFT); + } +} + static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { @@ -2324,10 +2565,19 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, temp_msg = recv_msg; }
+ update_stream_counter(recv_msg); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); }
+static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) +{ + /* Asynchronous mode does not use the sent message, so ignores it */ + return (msg->msg_state == AEAD_MSG_END) && !msg->in_bytes && + qp->q_info.qp_mode == CTX_MODE_SYNC; +} + int hisi_sec_aead_recv(handle_t ctx, void *aead_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); @@ -2336,6 +2586,9 @@ int hisi_sec_aead_recv(handle_t ctx, void *aead_msg) __u16 count = 0; int ret;
+ if (soft_compute_check((struct hisi_qp *)h_qp, recv_msg)) + return 0; + ret = hisi_qm_recv(h_qp, &sqe, 1, &count); if (ret < 0) return ret; @@ -2436,6 +2689,10 @@ static int fill_aead_bd3_mode(struct wd_aead_msg *msg, case WD_CIPHER_GCM: sqe->c_mode_alg |= C_MODE_GCM; sqe->auth_mac_key &= SEC_AUTH_MASK_V3; + if ((msg->msg_state == AEAD_MSG_FIRST) && !msg->assoc_bytes) { + WD_ERR("invalid: first bd assoc bytes is 0!\n"); + return -WD_EINVAL; + } sqe->a_len_key = msg->assoc_bytes; sqe->c_icv_key |= msg->auth_bytes << SEC_MAC_OFFSET_V3; break; @@ -2465,17 +2722,97 @@ static void fill_aead_bd3_addr(struct wd_aead_msg *msg, sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; }
+static void fill_gcm_first_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_INNER << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_NOPAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->c_icv_key &= ~0x3; + sqe->auth_mac_key = AUTH_HMAC_CALCULATE; + sqe->auth_mac_key |= MAC_LEN << SEC_MAC_OFFSET_V3; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->auth_mac_key |= A_ALG_AES_GMAC << SEC_AUTH_ALG_OFFSET_V3; + sqe->c_len_ivin = 0; + sqe->auth_ivin.a_ivin_addr = 0; + sqe->c_key_addr = 0; + sqe->c_mode_alg &= ~(0x7 << SEC_CALG_OFFSET_V3); + sqe->auth_src_offset = 0; + sqe->a_len_key = msg->assoc_bytes; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_middle_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_NOPAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->auth_mac_key |= NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->a_len_key = 0; + sqe->auth_ivin.a_ivin_addr = sqe->mac_addr; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_final_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_PAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->auth_mac_key |= NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->a_len_key = 0; + sqe->stream_scene.long_a_data_len = msg->assoc_bytes; + sqe->stream_scene.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; +} + +static int fill_stream_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + int ret = 0; + + switch (msg->msg_state) { + case AEAD_MSG_FIRST: + fill_gcm_first_bd3(msg, sqe); + break; + case AEAD_MSG_MIDDLE: + fill_gcm_middle_bd3(msg, sqe); + break; + case AEAD_MSG_END: + gcm_auth_ivin(msg); + if (msg->in_bytes) { + fill_gcm_final_bd3(msg, sqe); + } else { + ret = gcm_do_soft_mac(msg); + if (!ret) + ret = WD_SOFT_COMPUTING; + } + break; + default: + break; + } + + return ret; +}
static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) { __u16 scene, de; int ret;
- /* config BD type */ sqe->bd_param = BD_TYPE3; - /* config scene */ - scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET_V3; - de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET_V3; + + if (msg->msg_state == AEAD_MSG_BLOCK) + scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET_V3; + else + scene = SEC_STREAM_SCENE << SEC_SCENE_OFFSET_V3;
if (msg->op_type == WD_CIPHER_ENCRYPTION_DIGEST) { sqe->c_icv_key = SEC_CIPHER_ENC; @@ -2489,6 +2826,8 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) WD_ERR("failed to check aead op type, op = %u\n", msg->op_type); return -WD_EINVAL; } + + de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET_V3; sqe->bd_param |= (__u16)(de | scene);
sqe->c_len_ivin = msg->in_bytes; @@ -2526,7 +2865,11 @@ int hisi_sec_aead_send_v3(handle_t ctx, void *aead_msg) return -WD_EINVAL; }
- ret = aead_len_check(msg); + ret = aead_msg_state_check(msg); + if (unlikely(ret)) + return ret; + + ret = aead_len_check(msg, BD_TYPE3); if (unlikely(ret)) return ret;
@@ -2544,6 +2887,12 @@ int hisi_sec_aead_send_v3(handle_t ctx, void *aead_msg)
fill_aead_bd3_addr(msg, &sqe);
+ ret = fill_stream_bd3(msg, &sqe); + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret; + hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = msg->tag; ret = hisi_qm_send(h_qp, &sqe, 1, &count); @@ -2598,6 +2947,8 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, temp_msg = recv_msg; }
+ update_stream_counter(recv_msg); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); } @@ -2610,6 +2961,9 @@ int hisi_sec_aead_recv_v3(handle_t ctx, void *aead_msg) __u16 count = 0; int ret;
+ if (soft_compute_check((struct hisi_qp *)h_qp, recv_msg)) + return 0; + ret = hisi_qm_recv(h_qp, &sqe, 1, &count); if (ret < 0) return ret; diff --git a/include/crypto/aes.h b/include/crypto/aes.h new file mode 100644 index 0000000..eca58ba --- /dev/null +++ b/include/crypto/aes.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_AES_H__ +#define __WD_AES_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define UINT_B_CNT 8 +#define AES_MAXNR 14 + +struct aes_key { + unsigned int rd_key[4 * (AES_MAXNR + 1)]; + __u8 rounds; +}; + +union uni { + unsigned char b[UINT_B_CNT]; + __u32 w[2]; + __u64 d; +}; + +void aes_encrypt(__u8 *key, __u32 key_len, __u8 *src, __u8 *dst); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/crypto/galois.h b/include/crypto/galois.h new file mode 100644 index 0000000..07723be --- /dev/null +++ b/include/crypto/galois.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_GALOIS_H__ +#define __WD_GALOIS_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +void galois_compute(__u8 *S, __u8 *H, __u8 *g, __u32 len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 195205c..b43bb36 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -57,12 +57,19 @@ struct wd_aead_msg { __u8 *iv; /* input auth iv pointer */ __u8 aiv[MAX_IV_SIZE]; + /* input auth iv pointer for stream mode */ + __u8 aiv_stream[AIV_STREAM_LEN]; /* input data pointer */ __u8 *in; /* output data pointer */ __u8 *out; /* mac */ __u8 *mac; + /* mac data pointer for decrypto as stream mode */ + __u8 mac_bak[AES_BLOCK_SIZE]; + /* total of data for stream mode */ + __u64 long_data_len; + enum wd_aead_msg_state msg_state; };
struct wd_aead_driver { diff --git a/include/wd.h b/include/wd.h index da3cfc6..97956e2 100644 --- a/include/wd.h +++ b/include/wd.h @@ -71,6 +71,7 @@ typedef void (*wd_log)(const char *format, ...); #define WD_SUCCESS 0 #define WD_STREAM_END 1 #define WD_STREAM_START 2 +#define WD_SOFT_COMPUTING 3 #define WD_EIO EIO #define WD_EAGAIN EAGAIN #define WD_ENOMEM ENOMEM diff --git a/include/wd_aead.h b/include/wd_aead.h index 9f6f4e8..eefbed3 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -17,6 +17,7 @@ extern "C" { #endif
+#define AIV_STREAM_LEN 64 /** * wd_aead_op_type - Algorithm type of option */ @@ -43,6 +44,18 @@ struct wd_aead_sess_setup { void *sched_param; };
+/** + * wd_aead_msg_state - Notify the message state + * zero is message for block mode, non-zero is message for stream mode. + */ +enum wd_aead_msg_state { + AEAD_MSG_BLOCK = 0x0, + AEAD_MSG_FIRST, + AEAD_MSG_MIDDLE, + AEAD_MSG_END, + AEAD_MSG_INVALID, +}; + struct wd_aead_req; typedef void *wd_alg_aead_cb_t(struct wd_aead_req *req, void *cb_param);
@@ -84,6 +97,8 @@ struct wd_aead_req { __u8 data_fmt; wd_alg_aead_cb_t *cb; void *cb_param; + + enum wd_aead_msg_state msg_state; };
/** diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c new file mode 100644 index 0000000..2145d24 --- /dev/null +++ b/lib/crypto/aes.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include "crypto/aes.h" + +#define WORD(n) (0x##n##n##n##n) +#define LONG(n) (0x##n##n##n##n##n##n##n##n) + +#define STATE_CNT 2 + +static void xtimeword(__u32 *w) +{ + __u32 a, b; + + a = *w; + b = a & WORD(80); + a ^= b; + b -= b >> 0x7; + b &= WORD(1B); + b ^= a << 0x1; + *w = b; +} + +static void xtimelong(__u64 *w) +{ + __u64 a, b; + + a = *w; + b = a & LONG(80); + a ^= b; + b -= b >> 0x7; + b &= LONG(1B); + b ^= a << 0x1; + *w = b; +} + +static __u32 caculate_x_final(__u32 x_1) +{ + __u32 x, y; + + x = x_1; + y = ((x & WORD(FE)) >> 0x1) | ((x & WORD(01)) << 0x7); + x &= WORD(39); + x ^= y & WORD(3F); + y = ((y & WORD(FC)) >> 0x2) | ((y & WORD(03)) << 0x6); + x ^= y & WORD(97); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(9B); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(3C); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(DD); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(72); + x ^= WORD(63); + + return x; +} + +static __u32 caculate_x(__u32 *w) +{ + __u32 x, y; + + x = *w; + y = ((x & WORD(FE)) >> 0x1) | ((x & WORD(01)) << 0x7); + x &= WORD(DD); + x ^= y & WORD(57); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(1C); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(4A); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(42); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(64); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(E0); + return x; +} + +static void subword(__u32 *w) +{ +#define SPECIAL_WORD 0x0A0A0A0A0Au + __u32 x, a1, a2, a3, a4, a5, a6; + + x = caculate_x(w); + a1 = x ^ (x & WORD(F0)) >> 0x4; + a2 = ((x & WORD(CC)) >> 0x2) | ((x & WORD(33)) << 0x2); + a3 = (x & a1) ^ ((x & a1) & WORD(AA)) >> 0x1; + a3 ^= (((x << 0x1) & a1) ^ ((a1 << 0x1) & x)) & WORD(AA); + a4 = (a2 & a1) ^ ((a2 & a1) & WORD(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a1) ^ ((a1 << 0x1) & a2)) & WORD(AA); + a5 = (a3 & WORD(CC)) >> 0x2; + a3 ^= ((a4 << 0x2) ^ a4) & WORD(CC); + a4 = (a5 & WORD(22)) | (a5 & WORD(22)) >> 0x1; + a4 ^= (a5 << 0x1) & WORD(22); + a3 ^= a4; + a5 = (a3 & WORD(A0)) | (a3 & WORD(A0)) >> 0x1; + a5 ^= (a3 << 0x1) & WORD(A0); + a4 = a5 & WORD(C0); + a6 = a4 >> 0x2; + a4 ^= (a5 << 0x2) & WORD(C0); + a5 = (a6 & WORD(20)) | (a6 & WORD(20)) >> 0x1; + a5 ^= (a6 << 0x1) & WORD(20); + a4 |= a5; + a3 ^= a4 >> 0x4; + a3 &= WORD(0F); + a2 = a3 ^ (a3 & WORD(0C)) >> 0x2; + a4 = a3 & a2; + a4 ^= (a4 & SPECIAL_WORD) >> 0x1; + a4 ^= (((a3 << 0x1) & a2) ^ ((a2 << 0x1) & a3)) & WORD(0A); + a5 = (a4 & WORD(08)) | (a4 & WORD(08)) >> 0x1; + a5 ^= (a4 << 0x1) & WORD(08); + a4 ^= a5 >> 0x2; + a4 &= WORD(03); + a4 ^= (a4 & WORD(02)) >> 0x1; + a4 |= a4 << 0x2; + a3 = (a2 & a4) ^ ((a2 & a4) & WORD(0A)) >> 0x1; + a3 ^= (((a2 << 0x1) & a4) ^ ((a4 << 0x1) & a2)) & WORD(0A); + a3 |= a3 << 0x4; + a2 = ((a1 & WORD(CC)) >> 0x2) | ((a1 & WORD(33)) << 0x2); + x = (a1 & a3) ^ ((a1 & a3) & WORD(AA)) >> 0x1; + x ^= (((a1 << 0x1) & a3) ^ ((a3 << 0x1) & a1)) & WORD(AA); + a4 = (a2 & a3) ^ ((a2 & a3) & WORD(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a3) ^ ((a3 << 0x1) & a2)) & WORD(AA); + a5 = (x & WORD(CC)) >> 0x2; + x ^= ((a4 << 0x2) ^ a4) & WORD(CC); + a4 = (a5 & WORD(22)) | (a5 & WORD(22)) >> 0x1; + a4 ^= (a5 << 0x1) & WORD(22); + *w = caculate_x_final(x ^ a4); +} + +static __u64 caculate_long_x_final(__u64 x_1) +{ + __u64 x, y; + + x = x_1; + y = ((x & LONG(FE)) >> 0x1) | ((x & LONG(01)) << 0x7); + x &= LONG(39); + x ^= y & LONG(3F); + y = ((y & LONG(FC)) >> 0x2) | ((y & LONG(03)) << 0x6); + x ^= y & LONG(97); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(9B); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(3C); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(DD); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(72); + x ^= LONG(63); + + return x; +} + +static __u64 caculate_long_x(__u64 *w) +{ + __u64 x, y; + + x = *w; + y = ((x & LONG(FE)) >> 0x1) | ((x & LONG(01)) << 0x7); + x &= LONG(DD); + x ^= y & LONG(57); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(1C); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(4A); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(42); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(64); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(E0); + + return x; +} + +static void sublong(__u64 *w) +{ + __u64 x, a1, a2, a3, a4, a5, a6; + + x = caculate_long_x(w); + a1 = x ^ (x & LONG(F0)) >> 0x4; + a2 = ((x & LONG(CC)) >> 0x2) | ((x & LONG(33)) << 0x2); + a3 = (x & a1) ^ ((x & a1) & LONG(AA)) >> 0x1; + a3 ^= (((x << 0x1) & a1) ^ ((a1 << 0x1) & x)) & LONG(AA); + a4 = (a2 & a1) ^ ((a2 & a1) & LONG(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a1) ^ ((a1 << 0x1) & a2)) & LONG(AA); + a5 = (a3 & LONG(CC)) >> 0x2; + a3 ^= ((a4 << 0x2) ^ a4) & LONG(CC); + a4 = (a5 & LONG(22)) | (a5 & LONG(22)) >> 0x1; + a4 ^= (a5 << 0x1) & LONG(22); + a3 ^= a4; + a5 = (a3 & LONG(A0)) | (a3 & LONG(A0)) >> 0x1; + a5 ^= (a3 << 0x1) & LONG(A0); + a4 = a5 & LONG(C0); + a6 = a4 >> 0x2; + a4 ^= (a5 << 0x2) & LONG(C0); + a5 = (a6 & LONG(20)) | (a6 & LONG(20)) >> 0x1; + a5 ^= (a6 << 0x1) & LONG(20); + a4 |= a5; + a3 ^= a4 >> 0x4; + a3 &= LONG(0F); + a2 = a3 ^ (a3 & LONG(0C)) >> 0x2; + a4 = a3 & a2; + a4 ^= (a4 & LONG(0A)) >> 0x1; + a4 ^= (((a3 << 0x1) & a2) ^ ((a2 << 0x1) & a3)) & LONG(0A); + a5 = (a4 & LONG(08)) | (a4 & LONG(08)) >> 0x1; + a5 ^= (a4 << 0x1) & LONG(08); + a4 ^= a5 >> 0x2; + a4 &= LONG(03); + a4 ^= (a4 & LONG(02)) >> 0x1; + a4 |= a4 << 0x2; + a3 = (a2 & a4) ^ ((a2 & a4) & LONG(0A)) >> 0x1; + a3 ^= (((a2 << 0x1) & a4) ^ ((a4 << 0x1) & a2)) & LONG(0A); + a3 |= a3 << 0x4; + a2 = ((a1 & LONG(CC)) >> 0x2) | ((a1 & LONG(33)) << 0x2); + x = (a1 & a3) ^ ((a1 & a3) & LONG(AA)) >> 0x1; + x ^= (((a1 << 0x1) & a3) ^ ((a3 << 0x1) & a1)) & LONG(AA); + a4 = (a2 & a3) ^ ((a2 & a3) & LONG(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a3) ^ ((a3 << 0x1) & a2)) & LONG(AA); + a5 = (x & LONG(CC)) >> 0x2; + x ^= ((a4 << 0x2) ^ a4) & LONG(CC); + a4 = (a5 & LONG(22)) | (a5 & LONG(22)) >> 0x1; + a4 ^= (a5 << 0x1) & LONG(22); + *w = caculate_long_x_final(x ^ a4); +} + +static void shift_rows(__u64 *state) +{ +#define S_CNT 4 + unsigned char s[S_CNT]; + unsigned char *s0; + __u8 r, i; + + s0 = (unsigned char *)state; + for (r = 0; r < S_CNT; r++) { + for (i = 0; i < S_CNT; i++) + s[i] = s0[i * S_CNT + r]; + + for (i = 0; i < S_CNT; i++) + s0[i * S_CNT + r] = s[(r + i) % S_CNT]; + } +} + +static void mix_columns(__u64 *state) +{ +#define REVERT_OFFSET 0x3 +#define A 0xFFFF0000FFFF0000uLL +#define B 0xFF00FF00FF00FF00uLL + union uni s1; + union uni s; + __u8 c, i; + + for (c = 0; c < STATE_CNT; c++) { + s1.d = state[c]; + s.d = s1.d; + s.d ^= ((s.d & A) >> 0x10) + | ((s.d & ~A) << 0x10); + s.d ^= ((s.d & B) >> 0x8) + | ((s.d & ~B) << 0x8); + s.d ^= s1.d; + xtimelong(&s1.d); + s.d ^= s1.d; + for (i = 0; i < UINT_B_CNT; i++) { + if (i == UINT_B_CNT - 1 || i == (UINT_B_CNT >> 0x1) - 1) + s.b[i] ^= s1.b[i - REVERT_OFFSET]; + else + s.b[i] ^= s1.b[i + 1]; + } + state[c] = s.d; + } +} + +static void add_round_key(__u64 *state, const __u64 *w) +{ + state[0] ^= w[0]; + state[1] ^= w[1]; +} + +static void cipher(const unsigned char *in, unsigned char *out, + const __u64 *w, __u8 nr) +{ +#define STATE_BYTE 16 + __u64 state[STATE_CNT]; + __u8 i; + + memcpy(state, in, STATE_BYTE); + + add_round_key(state, w); + + for (i = 1; i < nr; i++) { + sublong(&state[0]); + sublong(&state[1]); + shift_rows(state); + mix_columns(state); + add_round_key(state, w + i * STATE_CNT); + } + + sublong(&state[0]); + sublong(&state[1]); + shift_rows(state); + add_round_key(state, w + nr * STATE_CNT); + + memcpy(out, state, STATE_BYTE); +} + +static void rotword(__u32 *x) +{ +#define WORDBYTE 4 + unsigned char *w0; + unsigned char tmp; + __u8 i; + + w0 = (unsigned char *)x; + tmp = w0[0]; + for (i = 0; i < WORDBYTE - 1; i++) + w0[i] = w0[i + 1]; + + w0[WORDBYTE - 1] = tmp; +} + +static void key_expansion(const unsigned char *key, __u64 *w, __u8 nr, __u8 nk) +{ +#define RCON_LEN 4 +#define N 2 +#define NK 6 + __u32 rcon; + union uni prev; + __u32 temp; + __u8 i, n, m; + + memcpy(w, key, nk * RCON_LEN); + memcpy(&rcon, "\1\0\0\0", RCON_LEN); + n = nk / N; + m = (nr + 1) * N; + prev.d = w[n - 1]; + for (i = n; i < m; i++) { + temp = prev.w[1]; + if (i % n == 0) { + rotword(&temp); + subword(&temp); + temp ^= rcon; + xtimeword(&rcon); + } else if (nk > NK && i % n == N) { + subword(&temp); + } + prev.d = w[i - n]; + prev.w[0] ^= temp; + prev.w[1] ^= prev.w[0]; + w[i] = prev.d; + } +} + +static int aes_set_encrypt_key(const unsigned char *userkey, const int bits, + struct aes_key *key) +{ +#define AES_128_BIT 128 +#define AES_192_BIT 192 +#define AES_256_BIT 256 +#define AES_128_ROUNDS 10 +#define AES_192_ROUNDS 12 +#define AES_256_ROUNDS 14 +#define KEY_NK 32 + __u64 *rk; + + if (!userkey || !key) + return -1; + if (bits != AES_128_BIT && bits != AES_192_BIT && bits != AES_256_BIT) + return -1; + + rk = (__u64 *)key->rd_key; + + if (bits == AES_128_BIT) + key->rounds = AES_128_ROUNDS; + else if (bits == AES_192_BIT) + key->rounds = AES_192_ROUNDS; + else + key->rounds = AES_256_ROUNDS; + + key_expansion(userkey, rk, key->rounds, bits / KEY_NK); + return 0; +} + +static void aes_encrypt_(const __u8 *in, __u8 *out, const struct aes_key *key) +{ + const __u64 *rk; + + rk = (__u64 *)key->rd_key; + + cipher(in, out, rk, key->rounds); +} + +void aes_encrypt(__u8 *key, __u32 key_len, __u8 *src, __u8 *dst) +{ + struct aes_key local_key; + int ret; + + ret = aes_set_encrypt_key(key, key_len << 0x3, &local_key); + if (ret) + return; + + aes_encrypt_(src, dst, &local_key); +} diff --git a/lib/crypto/galois.c b/lib/crypto/galois.c new file mode 100644 index 0000000..94f97fe --- /dev/null +++ b/lib/crypto/galois.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include "crypto/galois.h" + +#define GF_LSB1_MASK 0x1 +#define GF_R 0xe1000000 +#define GALOIS_BITS 128 +#define UINT_BITS (sizeof(unsigned int) * 8) + +#define GALOIS_UL_COUNT 4 +#define GALOIS_UL_COUNT_H 12 + +#define GALOIS_PARA_TRANS_S(S, i) (((unsigned int)(S)[(i) + 3] << 24) | \ + ((unsigned int)(S)[(i) + 2] << 16) | \ + ((unsigned int)(S)[(i) + 1] << 8) | \ + ((unsigned int)(S)[(i)])) +#define GALOIS_PARA_TRANS_H(H, i) (((unsigned int)(H)[(i)] << 24) | \ + ((unsigned int)(H)[(i) + 1] << 16) | \ + ((unsigned int)(H)[(i) + 2] << 8) | \ + ((unsigned int)(H)[(i) + 3])) + +#define GALOIS_UINT_TRANS_CHAR(src_addr) \ + do { \ + unsigned int temp = *(src_addr); \ + *(src_addr) = ((temp & 0xff) << 24) | ((temp & 0xff00) << 8) \ + | ((temp & 0xff0000) >> 8) | ((temp & 0xff000000) >> 24); \ + } while (0) + +static void galois_xtime_128(unsigned int *src, unsigned int *dst) +{ + unsigned int tmp; + __u8 i; + + /* Based the NIST Special Publication 800-38D */ + for (i = 0; i < GALOIS_UL_COUNT; i++) { + tmp = src[i] >> 0x1; + dst[i] = tmp | (src[i + 1] << (UINT_BITS - 1)); + if (i == GALOIS_UL_COUNT - 1) { + if (src[0] & GF_LSB1_MASK) + dst[i] = tmp ^ GF_R; + else + dst[i] = tmp; + } + } +} + +static void galois_multi(unsigned int *input_a, unsigned int *input_b, + unsigned int *mul, unsigned int array_size) +{ + /* 4 * (unsigned int), 4 * 32bit = 128bit */ + unsigned int gf_V[GALOIS_BITS][GALOIS_UL_COUNT] = {0}; + unsigned int *gf_Z = mul; + __u8 i, j, k; + + memcpy(gf_V[0], input_a, sizeof(unsigned int) * array_size); + + for (i = 1; i < GALOIS_BITS; i++) + galois_xtime_128(gf_V[i - 1], gf_V[i]); + + for (i = 0; i < GALOIS_BITS; i++) { + k = GALOIS_BITS - i - 1; + if ((input_b[(k / UINT_BITS)] >> (k % UINT_BITS)) & 0x1) { + for (j = 0; j < array_size; j++) + gf_Z[j] ^= gf_V[i][j]; + } + } +} + +void galois_compute(__u8 *S, __u8 *H, __u8 *g, __u32 len) +{ + unsigned int SL[GALOIS_UL_COUNT] = {0}; + unsigned int HL[GALOIS_UL_COUNT] = {0}; + unsigned int G[GALOIS_UL_COUNT] = {0}; + __u8 i, j; + + /* Do galois multiplication operation on blocks: G = S x H */ + for (i = 0; i < GALOIS_UL_COUNT; i++) { + j = i * GALOIS_UL_COUNT; + SL[i] = GALOIS_PARA_TRANS_S(S, j); + j = GALOIS_UL_COUNT_H - j; + HL[i] = GALOIS_PARA_TRANS_H(H, j); + } + + galois_multi(SL, HL, G, GALOIS_UL_COUNT); + + j = len - GALOIS_UL_COUNT; + for (i = 0; i < GALOIS_UL_COUNT; i++) { + GALOIS_UINT_TRANS_CHAR(&G[i]); + memcpy(&g[j], &G[i], sizeof(unsigned int)); + j -= GALOIS_UL_COUNT; + } +} diff --git a/wd_aead.c b/wd_aead.c index 38d3f8f..c1fc354 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -48,11 +48,17 @@ struct wd_aead_sess { enum wd_digest_mode dmode; unsigned char ckey[MAX_CIPHER_KEY_SIZE]; unsigned char akey[MAX_HMAC_KEY_SIZE]; + /* Mac data pointer for decrypto as stream mode */ + unsigned char mac_bak[WD_AEAD_CCM_GCM_MAX]; __u16 ckey_bytes; __u16 akey_bytes; __u16 auth_bytes; void *priv; void *sched_key; + /* Stored the counter for gcm stream mode */ + __u8 iv[MAX_IV_SIZE]; + /* Total of data for stream mode */ + __u64 long_data_len; };
struct wd_env_config wd_aead_env_config; @@ -485,6 +491,41 @@ void wd_aead_uninit(void) wd_alg_clear_init(&wd_aead_setting.status); }
+static void fill_stream_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, + struct wd_aead_sess *sess) +{ + switch (req->msg_state) { + case AEAD_MSG_FIRST: + /* Stream iv is extended to 16 bytes and last 4 bytes must be zero */ + memset(sess->iv, 0, MAX_IV_SIZE); + memcpy(sess->iv, req->iv, GCM_IV_SIZE); + + /* Store the original mac of first message to session */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(sess->mac_bak, req->mac, sess->auth_bytes); + break; + case AEAD_MSG_MIDDLE: + /* Middle messages need to store the stream's total length to session */ + sess->long_data_len += req->in_bytes; + + msg->long_data_len = sess->long_data_len; + break; + case AEAD_MSG_END: + /* Sets the original mac for final message */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac_bak, sess->mac_bak, sess->auth_bytes); + + msg->long_data_len = sess->long_data_len + req->in_bytes; + /* Reset the session's long_data_len */ + sess->long_data_len = 0; + break; + default: + return; + } + + msg->iv = sess->iv; +} + static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, struct wd_aead_sess *sess) { @@ -510,6 +551,9 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, msg->mac = req->mac; msg->auth_bytes = sess->auth_bytes; msg->data_fmt = req->data_fmt; + + msg->msg_state = req->msg_state; + fill_stream_msg(msg, req, sess); }
static int send_recv_sync(struct wd_ctx_internal *ctx,