From: Wenkai Lin linwenkai6@hisilicon.com
In stream processing encryption mode, a long file needs to be encrypted. When the accelerator is invoked, the encryption result of each block is assembled. The assembled result is the same as the result of encrypting the entire file at a time. For hisi_sec, the AAD is filled to the first message, plaintext are done with the middle and the end message. In an encrypted stream, the first and the end message are unique and must be delivered to hardware.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com --- Makefile.am | 6 +- include/crypto/sm4.h | 18 +++ lib/crypto/sm4.c | 176 ++++++++++++++++++++++++++++ v1/drv/hisi_sec_udrv.c | 259 +++++++++++++++++++++++++++++++++++++++++ v1/drv/hisi_sec_udrv.h | 9 +- v1/wd_aead.c | 67 ++++++++++- v1/wd_aead.h | 12 ++ 7 files changed, 539 insertions(+), 8 deletions(-) create mode 100644 include/crypto/sm4.h create mode 100644 lib/crypto/sm4.c
diff --git a/Makefile.am b/Makefile.am index 77b369a9..6b4c6cbc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -69,6 +69,8 @@ libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ v1/wd_bmm.c v1/wd_bmm.h \ v1/wd_ecc.c v1/wd_ecc.h \ v1/wd_sgl.c v1/wd_sgl.h \ + aes.h sm4.h galois.h \ + lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ v1/drv/hisi_qm_udrv.c v1/drv/hisi_qm_udrv.h \ v1/drv/hisi_zip_udrv.c v1/drv/hisi_zip_udrv.h \ v1/drv/hisi_hpre_udrv.c v1/drv/hisi_hpre_udrv.h \ @@ -95,8 +97,8 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd.c wd.h
libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ - lib/crypto/aes.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h galois.h + lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h
libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h diff --git a/include/crypto/sm4.h b/include/crypto/sm4.h new file mode 100644 index 00000000..e242b3a9 --- /dev/null +++ b/include/crypto/sm4.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_SM4_H__ +#define __WD_SM4_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +void sm4_encrypt(__u8 *key, __u32 key_len, __u8 *input, __u8 *output); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/crypto/sm4.c b/lib/crypto/sm4.c new file mode 100644 index 00000000..97c0fb2b --- /dev/null +++ b/lib/crypto/sm4.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include <stdint.h> +#include "crypto/sm4.h" + +#define AES_BLOCK_SIZE 16 +#define U32_BYTES 4 +#define BYTE_TO_BIT 8 +#define U32_BITS 32 +#define FIRST_KEY_STEP 13 +#define SECOND_KEY_STEP 23 + +#define FIRST_DATA_STEP 2 +#define SECOND_DATA_STEP 10 +#define THIRD_DATA_STEP 18 +#define FOURTH_DATA_STEP 24 +#define TOTAL_ROUND 36 +#define F_ROUND 32 +#define BUF_LEN 128 + +const __u32 TBL_SYS_PARAMS[4] = { + 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +}; + +const __u32 TBL_FIX_PARAMS[32] = { + 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, + 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, + 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, + 0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9, + 0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229, + 0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299, + 0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209, + 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 +}; + +const __u8 TBL_SBOX[256] = { + 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, + 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, + 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, + 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99, + 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, + 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62, + 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, + 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6, + 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, + 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8, + 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, + 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35, + 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, + 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87, + 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, + 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e, + 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, + 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1, + 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, + 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3, + 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, + 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f, + 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, + 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51, + 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, + 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8, + 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, + 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0, + 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, + 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84, + 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, + 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48 +}; + +static void get_u32(__u8 *in, __u32 *out) +{ + __u32 i = 0; + + *out = 0; + for (i = 0; i < U32_BYTES; i++) + *out = ((__u32)in[i] << ((U32_BYTES - 1 - i) * BYTE_TO_BIT)) ^ *out; +} + +static void put_u32(__u32 in, __u8 *out) +{ + __u32 i = 0; + + for (i = 0; i < U32_BYTES; i++) + *(out + i) = (__u32)(in >> ((U32_BYTES - 1 - i) * BYTE_TO_BIT)); +} + +static __u32 move(__u32 data, __u32 length) +{ + __u32 result = 0; + + result = (data << length) ^ (data >> (U32_BITS - length)); + + return result; +} + +static __u32 func_key(__u32 input) +{ + __u8 ucSboxValueList[U32_BYTES] = { 0 }; + __u8 ucIndexList[U32_BYTES] = { 0 }; + __u32 i, ulTmp; + + put_u32(input, ucIndexList); + for (i = 0; i < U32_BYTES; i++) + ucSboxValueList[i] = TBL_SBOX[ucIndexList[i]]; + + get_u32(ucSboxValueList, &ulTmp); + ulTmp = ulTmp ^ move(ulTmp, FIRST_KEY_STEP) ^ + move(ulTmp, SECOND_KEY_STEP); + + return ulTmp; +} + +static __u32 func_data(__u32 input) +{ + __u8 ucSboxValueList[4] = { 0 }; + __u8 ucIndexList[4] = { 0 }; + __u32 i, ulTmp; + + put_u32(input, ucIndexList); + for (i = 0; i < U32_BYTES; i++) + ucSboxValueList[i] = TBL_SBOX[ucIndexList[i]]; + + get_u32(ucSboxValueList, &ulTmp); + ulTmp = ulTmp ^ move(ulTmp, FIRST_DATA_STEP) ^ + move(ulTmp, SECOND_DATA_STEP) ^ + move(ulTmp, THIRD_DATA_STEP) ^ + move(ulTmp, FOURTH_DATA_STEP); + + return ulTmp; +} + +void sm4_encrypt(__u8 *key, __u32 key_len, __u8 *input, __u8 *output) +{ + __u32 tmp_keys[U32_BYTES] = { 0 }; + __u32 keys[TOTAL_ROUND] = { 0 }; + __u32 data[TOTAL_ROUND] = { 0 }; + __u32 i, j, k, in_len; + __u8 p[BUF_LEN]; + + if (key_len < AES_BLOCK_SIZE) + return; + + for (i = 0; i < U32_BYTES; i++) { + get_u32(key + U32_BYTES * i, &tmp_keys[i]); + keys[i] = tmp_keys[i] ^ TBL_SYS_PARAMS[i]; + } + + for (i = 0; i < F_ROUND; i++) + keys[i + 0x4] = keys[i] ^ func_key(keys[i + 0x1] ^ keys[i + 0x2] ^ + keys[i + 0x3] ^ TBL_FIX_PARAMS[i]); + + in_len = AES_BLOCK_SIZE; + for (i = 0; i < in_len; i++) + *(p + i) = *(input + i); + + k = AES_BLOCK_SIZE - (in_len % AES_BLOCK_SIZE); + for (i = 0; i < k; i++) + *(p + in_len + i) = 0; + + k = in_len / AES_BLOCK_SIZE + ((in_len % AES_BLOCK_SIZE) ? 1 : 0); + for (j = 0; j < k; j++) { + for (i = 0; i < TOTAL_ROUND - F_ROUND; i++) + get_u32(p + AES_BLOCK_SIZE * j + U32_BYTES * i, &data[i]); + + for (i = 0; i < F_ROUND; i++) + data[i + 0x4] = data[i] ^ func_data(data[i + 0x1] ^ data[i + 0x2] ^ + data[i + 0x3] ^ keys[i + 0x4]); + + for (i = 0; i < TOTAL_ROUND - F_ROUND; i++) + put_u32(data[TOTAL_ROUND - i - 1], + output + AES_BLOCK_SIZE * j + U32_BYTES * i); + } +} diff --git a/v1/drv/hisi_sec_udrv.c b/v1/drv/hisi_sec_udrv.c index c7919e3c..5dd2a696 100644 --- a/v1/drv/hisi_sec_udrv.c +++ b/v1/drv/hisi_sec_udrv.c @@ -24,6 +24,9 @@ #include <sys/eventfd.h> #include <sys/types.h>
+#include "crypto/aes.h" +#include "crypto/sm4.h" +#include "crypto/galois.h" #include "hisi_sec_udrv.h"
#define SEC_HW_TASK_DONE 1 @@ -43,6 +46,18 @@ #define AEAD_IV_MAX_BYTES 64 #define MAX_CCM_AAD_LEN 65279 #define SEC_GMAC_IV_LEN 16 +#define MAC_LEN 4 +#define GCM_FINAL_COUNTER 0x1000000 +#define GCM_FINAL_COUNTER_LEN 4 +#define GCM_STREAM_MAC_OFFSET 32 +#define GCM_FULL_MAC_LEN 16 +#define LONG_AUTH_DATA_OFFSET 24 +#define GCM_IV_SIZE 12 +#define AIV_STREAM_LEN 64 +#define AKEY_LEN(c_key_len) (2 * (c_key_len) + 0x4) +#define SM4_AKEY_LEN 4 +#define GCM_AUTH_MAC_OFFSET 47 +#define GCM_BLOCK_OFFSET (AES_BLOCK_SIZE - 1)
static int g_digest_a_alg[WCRYPTO_MAX_DIGEST_TYPE] = { A_ALG_SM3, A_ALG_MD5, A_ALG_SHA1, A_ALG_SHA256, A_ALG_SHA224, @@ -2259,6 +2274,9 @@ static int fill_aead_bd3_addr_src(struct wd_queue *q, sqe->data_src_addr_l = (__u32)(phy1 & QM_L32BITS_MASK); sqe->data_src_addr_h = HI_U32(phy1);
+ if (msg->msg_state != WCRYPTO_AEAD_MSG_BLOCK) + return WD_SUCCESS; + if (msg->op_type == WCRYPTO_CIPHER_DECRYPTION_DIGEST && msg->data_fmt == WD_FLAT_BUF) { phy2 = phy1 + msg->assoc_bytes + msg->in_bytes; @@ -2282,6 +2300,9 @@ static int fill_aead_bd3_addr_dst(struct wd_queue *q, sqe->data_dst_addr_l = (__u32)(phy1 & QM_L32BITS_MASK); sqe->data_dst_addr_h = HI_U32(phy1);
+ if (msg->msg_state != WCRYPTO_AEAD_MSG_BLOCK) + return WD_SUCCESS; + if (msg->op_type == WCRYPTO_CIPHER_ENCRYPTION_DIGEST && msg->data_fmt == WD_FLAT_BUF) { phy2 = phy1 + msg->out_bytes - msg->auth_bytes; @@ -2394,6 +2415,9 @@ static int fill_aead_bd3_addr(struct wd_queue *q, if (unlikely(ret)) goto map_civ_error;
+ if (msg->msg_state != WCRYPTO_AEAD_MSG_BLOCK) + return WD_SUCCESS; + /* CCM/GCM should init a_iv */ set_aead_auth_iv(msg); ret = map_addr(q, msg->aiv, msg->iv_bytes, &sqe->auth_key_iv.a_ivin_addr_l, @@ -2425,6 +2449,225 @@ map_out_error: return -WD_ENOMEM; }
+static void fill_gcm_akey_len(struct wcrypto_aead_msg *msg, struct hisi_sec_bd3_sqe *sqe) +{ + __u8 c_key_len = 0; + + if (msg->calg == WCRYPTO_CIPHER_AES) { + sqe->a_alg = A_ALG_AES_GMAC; + get_aes_c_key_len(msg->cmode, msg->ckey_bytes, &c_key_len); + sqe->a_key_len = AKEY_LEN(c_key_len); + } else if (msg->calg == WCRYPTO_CIPHER_SM4) { + sqe->a_alg = A_ALG_SM4_GMAC; + sqe->a_key_len = SM4_AKEY_LEN; + } +} + +static void fill_gcm_first_bd3(struct wcrypto_aead_msg *msg, struct hisi_sec_bd3_sqe *sqe) +{ + sqe->ai_gen = AI_GEN_INNER; + sqe->stream_scene.auth_pad = AUTHPAD_NOPAD; + sqe->cipher = NO_CIPHER; + sqe->auth = AUTH_MAC_CALCULATE; + sqe->mac_len = MAC_LEN; + fill_gcm_akey_len(msg, sqe); + sqe->c_len = 0; + sqe->auth_ivin_offset = 0; + sqe->auth_key_iv.a_ivin_addr_h = 0; + sqe->auth_key_iv.a_ivin_addr_l = 0; + sqe->auth_src_offset = 0; + sqe->a_len = msg->assoc_bytes; + sqe->stream_scene.c_ivin_addr_l = sqe->ipsec_scene.c_ivin_addr_l; + sqe->stream_scene.c_ivin_addr_h = sqe->ipsec_scene.c_ivin_addr_h; + sqe->auth_key_iv.a_key_addr_l = sqe->c_key_addr_l; + sqe->auth_key_iv.a_key_addr_h = sqe->c_key_addr_h; +} + +static void fill_gcm_middle_bd3(struct wd_queue *q, struct wcrypto_aead_msg *msg, + struct hisi_sec_bd3_sqe *sqe) +{ + sqe->ai_gen = AI_GEN_IVIN_ADDR; + sqe->stream_scene.auth_pad = AUTHPAD_NOPAD; + sqe->auth = NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe); + sqe->a_len = 0; + sqe->stream_scene.c_ivin_addr_l = sqe->ipsec_scene.c_ivin_addr_l; + sqe->stream_scene.c_ivin_addr_h = sqe->ipsec_scene.c_ivin_addr_h; + sqe->auth_key_iv.a_key_addr_l = sqe->c_key_addr_l; + sqe->auth_key_iv.a_key_addr_h = sqe->c_key_addr_h; + + sqe->auth_key_iv.a_ivin_addr_l = sqe->mac_addr_l; + sqe->auth_key_iv.a_ivin_addr_h = sqe->mac_addr_h; +} + +static void get_galois_vector_s(struct wcrypto_aead_msg *msg, __u8 *s) +{ + __u8 a_c[AES_BLOCK_SIZE] = {0}; + __u64 cipher_len, aad_len; + __u32 i; + + aad_len = msg->assoc_bytes * BYTE_BITS; + memcpy(&a_c[BYTE_BITS], &aad_len, sizeof(__u64)); + + cipher_len = msg->long_data_len * BYTE_BITS; + memcpy(&a_c[0], &cipher_len, sizeof(__u64)); + + /* Based the little-endian operation */ + for (i = 0; i < AES_BLOCK_SIZE; i++) + s[i] = a_c[i] ^ msg->aiv[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; +} + +static int gcm_do_soft_mac(struct wcrypto_aead_msg *msg) +{ + typedef void (*enc_ops)(__u8 *, __u32, __u8 *, __u8 *); + __u8 ctr_r[AES_BLOCK_SIZE] = {0}; + __u8 data[AES_BLOCK_SIZE] = {0}; + __u8 H[AES_BLOCK_SIZE] = {0}; + __u8 K[AES_BLOCK_SIZE] = {0}; + __u8 S[AES_BLOCK_SIZE] = {0}; + __u8 g[AES_BLOCK_SIZE] = {0}; + __u8 G[AES_BLOCK_SIZE] = {0}; + __u32 i, len, block, offset; + enc_ops enc_func; + __u8 *out; + int ret; + + if (msg->calg == WCRYPTO_CIPHER_AES) + enc_func = aes_encrypt; + else if (msg->calg == WCRYPTO_CIPHER_SM4) + enc_func = sm4_encrypt; + else + return -WD_EINVAL; + + enc_func(msg->ckey, msg->ckey_bytes, data, H); + + len = msg->in_bytes; + offset = 0; + while (len) { + memset(data, 0, AES_BLOCK_SIZE); + block = len >= AES_BLOCK_SIZE ? AES_BLOCK_SIZE : len; + memcpy(data, msg->in + offset, block); + ctr_iv_inc(msg->iv, AES_BLOCK_SIZE >> CTR_MODE_LEN_SHIFT, msg->data_fmt); + enc_func(msg->ckey, msg->ckey_bytes, msg->iv, K); + out = msg->out + offset; + for (i = 0; i < block; i++) + out[i] = K[i] ^ data[i]; + + if (msg->op_type == WCRYPTO_CIPHER_ENCRYPTION_DIGEST) + memcpy(data, out, block); + + /* + * Mac and data is based on big-endian, the first argument of galois_compute + * must be converted to little-endian. + */ + for (i = 0; i < AES_BLOCK_SIZE; i++) + G[i] = data[GCM_BLOCK_OFFSET - i] ^ + msg->aiv[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; + + galois_compute(G, H, msg->aiv + GCM_STREAM_MAC_OFFSET, AES_BLOCK_SIZE); + len -= block; + offset += block; + } + + get_galois_vector_s(msg, S); + + galois_compute(S, H, g, AES_BLOCK_SIZE); + enc_func(msg->ckey, msg->ckey_bytes, msg->aiv, ctr_r); + + /* Get the GMAC tag final */ + for (i = 0; i < AES_BLOCK_SIZE; i++) + msg->mac[i] = g[i] ^ ctr_r[i]; + + if (msg->op_type == WCRYPTO_CIPHER_DECRYPTION_DIGEST) { + ret = memcmp(msg->mac, msg->in + msg->in_bytes, msg->auth_bytes); + if (ret) { + msg->result = WD_IN_EPARA; + WD_ERR("failed to do the gcm authentication!\n"); + return -WD_EINVAL; + } + } + + msg->result = WD_SUCCESS; + + return 0; +} + +static void gcm_auth_ivin(struct wcrypto_aead_msg *msg) +{ + __u32 final_counter = GCM_FINAL_COUNTER; + + /* auth_ivin = {cipher_ivin(16B), null(16B), auth_mac(16B), null(16B)} */ + memset(msg->aiv, 0, AIV_STREAM_LEN); + + memcpy(msg->aiv, msg->iv, GCM_IV_SIZE); + /* The last 4 bytes of c_ivin are counters */ + memcpy(msg->aiv + GCM_IV_SIZE, &final_counter, GCM_FINAL_COUNTER_LEN); + + /* Fill auth_ivin with the mac of last MIDDLE BD */ + memcpy(msg->aiv + GCM_STREAM_MAC_OFFSET, msg->mac, GCM_FULL_MAC_LEN); + + /* Use the user's origin mac for decrypt icv check */ + if (msg->op_type == WCRYPTO_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac, msg->in + msg->in_bytes, msg->auth_bytes); +} + +static int fill_aead_stream_bd3(struct wd_queue *q, struct wcrypto_aead_msg *msg, + struct hisi_sec_bd3_sqe *sqe) +{ + int ret; + + if (msg->msg_state != WCRYPTO_AEAD_MSG_BLOCK) { + sqe->scene = SCENE_STREAM; + if (msg->data_fmt == WD_SGL_BUF) { + WD_ERR("invalid data format for aead stream mode!\n"); + return -WD_EINVAL; + } + ret = map_addr(q, msg->mac, msg->auth_bytes, &sqe->mac_addr_l, + &sqe->mac_addr_h, msg->data_fmt); + if (unlikely(ret)) { + WD_ERR("fail to get aead mac dma address!\n"); + return -WD_EINVAL; + } + } + + switch (msg->msg_state) { + case WCRYPTO_AEAD_MSG_FIRST: + if (msg->cmode == WCRYPTO_CIPHER_GCM) + fill_gcm_first_bd3(msg, sqe); + break; + case WCRYPTO_AEAD_MSG_MIDDLE: + if (msg->cmode == WCRYPTO_CIPHER_GCM) + fill_gcm_middle_bd3(q, msg, sqe); + break; + case WCRYPTO_AEAD_MSG_END: + if (msg->cmode == WCRYPTO_CIPHER_GCM) { + gcm_auth_ivin(msg); + ret = gcm_do_soft_mac(msg); + if (unlikely(ret)) + goto out; + /* + * Avoids recalculation of data and ensure that + * the message sending and receiving process is + * compatible with the framework. + */ + sqe->invalid = 1; + } + break; + default: + /* Do nothing for the block messages. */ + break; + } + + return 0; + +out: + unmap_addr(q, msg->mac, msg->auth_bytes, sqe->mac_addr_l, + sqe->mac_addr_h, msg->data_fmt); + return ret; +} + static int fill_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_aead_msg *msg, struct wcrypto_aead_tag *tag) { @@ -2455,6 +2698,10 @@ static int fill_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, return ret; }
+ ret = fill_aead_stream_bd3(q, msg, sqe); + if (unlikely(ret)) + return ret; + if (tag) sqe->tag_l = tag->wcrypto_tag.ctx_id;
@@ -2599,6 +2846,15 @@ static void parse_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe3, return; }
+ if (msg->msg_state == WCRYPTO_AEAD_MSG_FIRST) + msg->iv[AES_BLOCK_SIZE - 1] = 0x1; + else if (msg->msg_state == WCRYPTO_AEAD_MSG_MIDDLE) + ctr_iv_inc(msg->iv, msg->in_bytes >> CTR_MODE_LEN_SHIFT, msg->data_fmt); + + if (msg->op_type == WCRYPTO_CIPHER_ENCRYPTION_DIGEST && + msg->msg_state == WCRYPTO_AEAD_MSG_END) + memcpy(msg->out + msg->in_bytes, msg->mac, msg->auth_bytes); + dma_addr = DMA_ADDR(sqe3->data_src_addr_h, sqe3->data_src_addr_l); drv_iova_unmap(q, msg->in, (void *)(uintptr_t)dma_addr, msg->in_bytes); dma_addr = DMA_ADDR(sqe3->data_dst_addr_h, sqe3->data_dst_addr_l); @@ -2611,6 +2867,9 @@ static void parse_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe3, sqe3->ipsec_scene.c_ivin_addr_h, msg->data_fmt); unmap_addr(q, msg->aiv, msg->iv_bytes, sqe3->auth_key_iv.a_ivin_addr_l, sqe3->auth_key_iv.a_ivin_addr_h, msg->data_fmt); + if (msg->msg_state != WCRYPTO_AEAD_MSG_BLOCK) + unmap_addr(q, msg->mac, msg->auth_bytes, sqe3->mac_addr_l, + sqe3->mac_addr_h, msg->data_fmt); }
/* diff --git a/v1/drv/hisi_sec_udrv.h b/v1/drv/hisi_sec_udrv.h index c266e960..250fd369 100644 --- a/v1/drv/hisi_sec_udrv.h +++ b/v1/drv/hisi_sec_udrv.h @@ -37,7 +37,7 @@ struct hisi_sec_sqe_type1 { __u32 c_key_type:2; __u32 a_key_type:2; __u32 rsvd0:10; - __u32 inveld:1; + __u32 invalid:1; __u32 mac_len:5; __u32 a_key_len:6; __u32 a_alg:6; @@ -116,7 +116,7 @@ struct hisi_sec_sqe_type2 { __u32 cal_iv_addr_en:1; __u32 tls_up:1; __u32 rsvd0:5; - __u32 inveld:1; + __u32 invalid:1; __u32 mac_len:5; __u32 a_key_len:6; __u32 a_alg:6; @@ -328,7 +328,7 @@ struct bd3_tls_type_back { /* the hw v2 sence */ struct hisi_sec_bd3_sqe { __u32 type:4; - __u32 inveld:1; + __u32 invalid:1; __u32 scene:4; __u32 de:2; __u32 src_addr_type:3; @@ -445,7 +445,8 @@ enum A_ALG { A_ALG_AES_CMAC = 0x21, A_ALG_AES_GMAC = 0x22, A_ALG_SM3 = 0x25, - A_ALG_HMAC_SM3 = 0x26 + A_ALG_HMAC_SM3 = 0x26, + A_ALG_SM4_GMAC = 0x31, };
enum C_MODE { diff --git a/v1/wd_aead.c b/v1/wd_aead.c index 884c6c71..3ebe1899 100644 --- a/v1/wd_aead.c +++ b/v1/wd_aead.c @@ -52,6 +52,9 @@ struct wcrypto_aead_ctx { __u16 iv_blk_size; struct wd_queue *q; struct wcrypto_aead_ctx_setup setup; + __u64 long_data_len; + __u8 *civ; + __u8 *mac; };
static void del_ctx_key(struct wcrypto_aead_ctx *ctx) @@ -83,6 +86,10 @@ static void del_ctx_key(struct wcrypto_aead_ctx *ctx) br->free(br->usr, ctx->ckey); if (ctx->akey) br->free(br->usr, ctx->akey); + if (ctx->civ) + br->free(br->usr, ctx->civ); + if (ctx->mac) + br->free(br->usr, ctx->mac); } }
@@ -228,17 +235,30 @@ void *wcrypto_create_aead_ctx(struct wd_queue *q, ctx->akey = setup->br.alloc(setup->br.usr, MAX_AEAD_KEY_SIZE); if (!ctx->akey) { WD_ERR("fail to alloc authenticate ctx key!\n"); - setup->br.free(setup->br.usr, ctx->ckey); goto free_ctx_ckey; } + ctx->civ = setup->br.alloc(setup->br.usr, AES_BLOCK_SIZE); + if (!ctx->civ) { + WD_ERR("fail to alloc civ for aead ctx!\n"); + goto free_ctx_akey; + } + ctx->mac = setup->br.alloc(setup->br.usr, MAX_AEAD_AUTH_SIZE); + if (!ctx->mac) { + WD_ERR("fail to alloc mac for aead ctx!\n"); + goto free_ctx_civ; + }
ctx->iv_blk_size = get_iv_block_size(setup->cmode); ret = init_aead_cookie(ctx, setup); if (ret) - goto free_ctx_akey; + goto free_ctx_mac;
return ctx;
+free_ctx_mac: + setup->br.free(setup->br.usr, ctx->mac); +free_ctx_civ: + setup->br.free(setup->br.usr, ctx->civ); free_ctx_akey: setup->br.free(setup->br.usr, ctx->akey); free_ctx_ckey: @@ -430,10 +450,51 @@ static int check_op_data(struct wcrypto_aead_op_data **op, return -WD_EINVAL; } } + if (unlikely(op[idx]->state >= WCRYPTO_AEAD_MSG_INVALID)) { + WD_ERR("fail to check message state: %d, idx: %u!\n", + op[idx]->state, idx); + return -WD_EINVAL; + } else if (idx && op[idx]->state != WCRYPTO_AEAD_MSG_BLOCK) { + WD_ERR("fail to send multiple messages for stream mode!\n"); + return -WD_EINVAL; + }
return 0; }
+static void fill_stream_msg(struct wcrypto_aead_msg *req, + struct wcrypto_aead_op_data *op, + struct wcrypto_aead_ctx *ctx) +{ + req->msg_state = op->state; + req->mac = ctx->mac; + switch (op->state) { + case WCRYPTO_AEAD_MSG_FIRST: + if (req->cmode == WCRYPTO_CIPHER_GCM) { + req->iv = ctx->civ; + memset(ctx->civ, 0, WCRYPTO_CCM_GCM_LEN); + memcpy(ctx->civ, op->iv, op->iv_bytes); + } + break; + case WCRYPTO_AEAD_MSG_MIDDLE: + if (req->cmode == WCRYPTO_CIPHER_GCM) { + req->iv = ctx->civ; + ctx->long_data_len += op->in_bytes; + req->long_data_len = ctx->long_data_len; + } + break; + case WCRYPTO_AEAD_MSG_END: + if (req->cmode == WCRYPTO_CIPHER_GCM) { + req->iv = ctx->civ; + req->long_data_len = ctx->long_data_len + op->in_bytes; + ctx->long_data_len = 0; + } + break; + default: + return; + } +} + static int aead_requests_init(struct wcrypto_aead_msg **req, struct wcrypto_aead_op_data **op, struct wcrypto_aead_ctx *ctx, __u32 num) @@ -473,6 +534,8 @@ static int aead_requests_init(struct wcrypto_aead_msg **req, } }
+ fill_stream_msg(req[0], op[0], ctx); + return WD_SUCCESS;
err_uninit_requests: diff --git a/v1/wd_aead.h b/v1/wd_aead.h index ae5697b4..5b0c417a 100644 --- a/v1/wd_aead.h +++ b/v1/wd_aead.h @@ -28,6 +28,14 @@ extern "C" { #endif
+enum wcrypto_aead_msg_state { + WCRYPTO_AEAD_MSG_BLOCK = 0x0, + WCRYPTO_AEAD_MSG_FIRST, + WCRYPTO_AEAD_MSG_MIDDLE, + WCRYPTO_AEAD_MSG_END, + WCRYPTO_AEAD_MSG_INVALID, +}; + enum wcrypto_aead_op_type { WCRYPTO_CIPHER_ENCRYPTION_DIGEST, WCRYPTO_CIPHER_DECRYPTION_DIGEST, @@ -98,6 +106,7 @@ struct wcrypto_aead_op_data { __u16 iv_bytes; __u16 assoc_size; void *priv; + enum wcrypto_aead_msg_state state; };
/* AEAD message format of Warpdrive */ @@ -126,6 +135,9 @@ struct wcrypto_aead_msg { __u8 *in; /* Input data VA pointer, should be DMA buffer */ __u8 *out; /* Output data VA pointer, should be DMA buffer */ __u64 usr_data; /* user identifier: struct wcrypto_cb_tag */ + __u8 *mac; + __u64 long_data_len; + enum wcrypto_aead_msg_state msg_state; };
/**