May 2023 - Acc - mailweb.openeuler.org

AccLib SIG例会
by openEuler conference 31 May '23

31 May '23

您好！ sig-AccLib SIG 邀请您参加 2023-05-31 11:00 召开的Zoom会议会议主题：AccLib SIG例会会议内容：固定会议链接: https://linaro-org.zoom.us/j/91879279131 会议链接：https://us06web.zoom.us/j/82737022321?pwd=REFadTJnZDllZ3EzK0xWMTUzQ1ZnQT09 会议纪要：https://etherpad.openeuler.org/p/sig-AccLib-meetings 温馨提醒：建议接入会议后修改参会人的姓名，也可以使用您在gitee.com的ID 更多资讯尽在：https://openeuler.org/zh/ Hello! openEuler sig-AccLib SIG invites you to attend the Zoom conference will be held at 2023-05-31 11:00, The subject of the conference is AccLib SIG例会, Summary: 固定会议链接: https://linaro-org.zoom.us/j/91879279131 You can join the meeting at https://us06web.zoom.us/j/82737022321?pwd=REFadTJnZDllZ3EzK0xWMTUzQ1ZnQT09. Add topics at https://etherpad.openeuler.org/p/sig-AccLib-meetings. Note: You are advised to change the participant name after joining the conference or use your ID at gitee.com. More information: https://openeuler.org/en/

1 0

[PATCH] uadk: fix error incorrect parameters
by Wenkai Lin 19 May '23

19 May '23

From: Weili Qian <qianweili(a)huawei.com> The dlclose() parameter must be 'dlhandle' instead of 'dlnode'. Modify the input parameter of the function. Signed-off-by: Weili Qian <qianweili(a)huawei.com> --- wd_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wd_util.c b/wd_util.c index d8d6482..2b4a53c 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2033,7 +2033,7 @@ void wd_dlclose_drv(void *dlh_list) while (dlhead) { dlnode = dlhead; dlhead = dlhead->next; - dlclose(dlnode); + dlclose(dlnode->dlhandle); free(dlnode); } } -- 2.30.0

1 0

[PATCH] uadk/aead: supports aes-gcm stream mode
by Wenkai Lin 18 May '23

18 May '23

In stream processing encryption mode, a long file needs to be encrypted. When the accelerator is invoked, the encryption result of each block is assembled. The assembled result is the same as the result of encrypting the entire file at a time. For hisi_sec, AAD data is completed by the first BD, plaintext data are done with middle BDs and end BD. In an encrypted stream, the first BD and end BD are unique and must be delivered to hardware. Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- Makefile.am | 3 +- drv/hisi_sec.c | 392 ++++++++++++++++++++++++++++++++++-- include/crypto/aes.h | 32 +++ include/crypto/galois.h | 19 ++ include/drv/wd_aead_drv.h | 7 + include/wd.h | 1 + include/wd_aead.h | 15 ++ lib/crypto/aes.c | 405 ++++++++++++++++++++++++++++++++++++++ lib/crypto/galois.c | 94 +++++++++ wd_aead.c | 44 +++++ 10 files changed, 992 insertions(+), 20 deletions(-) create mode 100644 include/crypto/aes.h create mode 100644 include/crypto/galois.h create mode 100644 lib/crypto/aes.c create mode 100644 lib/crypto/galois.c diff --git a/Makefile.am b/Makefile.am index f6b73c1..37ac9b0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -75,7 +75,8 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_sched.c wd_sched.h libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h + lib/crypto/aes.c lib/crypto/galois.c \ + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h galois.h libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h wd_hpre_drv.h diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 41285ba..ff78419 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1,14 +1,17 @@ /* SPDX-License-Identifier: Apache-2.0 */ /* Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. */ -#include "../include/drv/wd_cipher_drv.h" -#include "../include/drv/wd_digest_drv.h" -#include "../include/drv/wd_aead_drv.h" +#include "drv/wd_cipher_drv.h" +#include "drv/wd_digest_drv.h" +#include "drv/wd_aead_drv.h" +#include "crypto/aes.h" +#include "crypto/galois.h" #include "hisi_qm_udrv.h" #include "wd_cipher.h" #include "wd_digest.h" #include "wd_aead.h" +#define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 #define WORD_ALIGNMENT_MASK 0x3 #define CTR_MODE_LEN_SHIFT 4 @@ -68,6 +71,15 @@ #define DES3_BLOCK_SIZE 8 #define AES_BLOCK_SIZE 16 #define CTR_128BIT_COUNTER 16 +#define GCM_FINAL_COUNTER 0x1000000 +#define GCM_FINAL_COUNTER_LEN 4 +#define GCM_STREAM_MAC_OFFSET 32 +#define GCM_FULL_MAC_LEN 16 +#define GCM_AUTH_MAC_OFFSET 47 +#define GCM_BLOCK_SIZE AES_BLOCK_SIZE +#define AKEY_LEN(c_key_len) (2 * (c_key_len) + 0x4) +#define MAC_LEN 4 +#define LONG_AUTH_DATA_OFFSET 24 /* The max BD data length is 16M-512B */ #define MAX_INPUT_DATA_LEN 0xFFFE00 @@ -175,8 +187,10 @@ enum { }; enum sec_cipher_dir { + SEC_NO_CIPHER = 0x0, SEC_CIPHER_ENC = 0x1, SEC_CIPHER_DEC = 0x2, + SEC_CIPHER_COPY = 0x3, }; enum sec_bd_type { @@ -369,7 +383,8 @@ struct bd3_stream_scene { /* * auth_pad: 0~1 bits * stream_protocol: 2~4 bits - * reserved: 5~7 bits + * mac_sel: 5 bits + * reserved: 6~7 bits */ __u8 stream_auth_pad; __u8 plaintext_type; @@ -591,6 +606,7 @@ static void dump_sec_msg(void *msg, const char *alg) dmsg->key_bytes, dmsg->iv_bytes, dmsg->in_bytes, dmsg->out_bytes); } else if (!strcmp(alg, "aead")) { amsg = (struct wd_aead_msg *)msg; + WD_ERR("MSG_STATE:%u\n", amsg->msg_state); WD_ERR("type:%u calg:%u op_type:%u cmode:%u\n", amsg->alg_type, amsg->calg, amsg->op_type, amsg->cmode); WD_ERR("data_fmt:%u ckey_bytes:%u auth_bytes:%u\n", @@ -2095,6 +2111,10 @@ static int fill_aead_bd2_mode(struct wd_aead_msg *msg, case WD_CIPHER_GCM: c_mode = C_MODE_GCM; sqe->type_auth_cipher &= SEC_AUTH_MASK; + if ((msg->msg_state == AEAD_MSG_FIRST) && !msg->assoc_bytes) { + WD_ERR("invalid: first bd assoc bytes is 0!\n"); + return -WD_EINVAL; + } sqe->type2.alen_ivllen = msg->assoc_bytes; sqe->type2.icvw_kmode |= msg->auth_bytes; break; @@ -2170,8 +2190,22 @@ static void fill_aead_bd2_addr(struct wd_aead_msg *msg, sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; } -static int aead_len_check(struct wd_aead_msg *msg) +static int aead_len_check(struct wd_aead_msg *msg, enum sec_bd_type type) { + if (msg->msg_state == AEAD_MSG_MIDDLE) { + if ((!msg->in_bytes || (msg->in_bytes & (AES_BLOCK_SIZE - 1)))) { + WD_ERR("invalid: middle bd input size is 0 or not 16 bytes aligned!\n"); + return -WD_EINVAL; + } + } + + if (msg->cmode == WD_CIPHER_GCM || msg->cmode == WD_CIPHER_CCM) { + if (msg->msg_state == AEAD_MSG_BLOCK && type == BD_TYPE2 && !msg->in_bytes) { + WD_ERR("invalid: ccm/gcm block mode input size is 0 for hw_v2!\n"); + return -WD_EINVAL; + } + } + if (unlikely(msg->in_bytes + msg->assoc_bytes > MAX_INPUT_DATA_LEN)) { WD_ERR("aead input data length is too long, size = %u\n", msg->in_bytes + msg->assoc_bytes); @@ -2188,16 +2222,184 @@ static int aead_len_check(struct wd_aead_msg *msg) return 0; } +static void fill_gcm_akey_len(struct wd_aead_msg *msg, void *sqe, enum sec_bd_type type) +{ + struct hisi_sec_sqe3 *sqe3; + struct hisi_sec_sqe *sqe2; + __u8 c_key_len = 0; + + aead_get_aes_key_len(msg, &c_key_len); + + if (type == BD_TYPE2) { + sqe2 = (struct hisi_sec_sqe *)sqe; + sqe2->type2.mac_key_alg |= (__u32)AKEY_LEN(c_key_len) << MAC_LEN_OFFSET; + } else if (type == BD_TYPE3) { + sqe3 = (struct hisi_sec_sqe3 *)sqe; + sqe3->auth_mac_key |= (__u32)AKEY_LEN(c_key_len) << SEC_AKEY_OFFSET_V3; + } +} + +static void gcm_auth_ivin(struct wd_aead_msg *msg) +{ + __u32 final_counter = GCM_FINAL_COUNTER; + + /* auth_ivin = {cipher_ivin(16B), null(16B), auth_mac(16B), null(16B)} */ + memset(msg->aiv_stream, 0, AIV_STREAM_LEN); + + memcpy(msg->aiv_stream, msg->iv, GCM_IV_SIZE); + /* The last 4 bytes of c_ivin are counters */ + memcpy(msg->aiv_stream + GCM_IV_SIZE, &final_counter, GCM_FINAL_COUNTER_LEN); + + /* Fill auth_ivin with the mac of last MIDDLE BD */ + memcpy(msg->aiv_stream + GCM_STREAM_MAC_OFFSET, msg->mac, GCM_FULL_MAC_LEN); + + /* Use the user's origin mac for decrypt icv check */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac, msg->mac_bak, msg->auth_bytes); +} + +static void fill_gcm_first_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_INNER; + sqe->ai_apd_cs |= AUTHPAD_NOPAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher &= ~(SEC_CIPHER_COPY << SEC_CIPHER_OFFSET); + sqe->type_auth_cipher |= AUTH_HMAC_CALCULATE << AUTHTYPE_OFFSET; + sqe->type2.mac_key_alg = MAC_LEN; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.mac_key_alg |= A_ALG_AES_GMAC << AUTH_ALG_OFFSET; + sqe->type2.clen_ivhlen = 0; + sqe->type2.icvw_kmode = 0; + sqe->type2.a_ivin_addr = 0; + sqe->type2.c_key_addr = 0; + sqe->type2.c_alg = 0; + sqe->type2.auth_src_offset = 0; + sqe->type2.alen_ivllen = msg->assoc_bytes; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_middle_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_IVIN_ADDR; + sqe->ai_apd_cs |= AUTHPAD_NOPAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher |= NO_AUTH << AUTHTYPE_OFFSET; + sqe->type2.cipher_src_offset = 0; + sqe->type2.auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.alen_ivllen = 0; + sqe->type2.a_ivin_addr = sqe->type2.mac_addr; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_final_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + sqe->ai_apd_cs = AI_GEN_IVIN_ADDR; + sqe->ai_apd_cs |= AUTHPAD_PAD << AUTHPAD_OFFSET; + sqe->type_auth_cipher |= NO_AUTH << AUTHTYPE_OFFSET; + sqe->type2.cipher_src_offset = 0; + sqe->type2.auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE2); + sqe->type2.alen_ivllen = 0; + sqe->type2.long_a_data_len = msg->assoc_bytes; + sqe->type2.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; +} + +static void get_galois_vector_s(struct wd_aead_msg *msg, __u8 *s) +{ + unsigned int aad_len, cipher_len; + __u8 a_c[GCM_BLOCK_SIZE] = {0}; + int i; + + aad_len = msg->assoc_bytes * BYTE_BITS; + memcpy(&a_c[BYTE_BITS], &aad_len, sizeof(unsigned int)); + + cipher_len = msg->long_data_len * BYTE_BITS; + memcpy(&a_c[0], &cipher_len, sizeof(unsigned int)); + + /* Based the little-endian operation */ + for (i = 0; i < GCM_BLOCK_SIZE; i++) + s[i] = a_c[i] ^ msg->aiv_stream[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; +} + +static int gcm_do_soft_mac(struct wd_aead_msg *msg) +{ + __u8 ctr_r[GCM_BLOCK_SIZE] = {0}; + __u8 data[GCM_BLOCK_SIZE] = {0}; + __u8 H[GCM_BLOCK_SIZE] = {0}; + __u8 S[GCM_BLOCK_SIZE] = {0}; + __u8 g[GCM_BLOCK_SIZE] = {0}; + int i, ret; + + aes_encrypt(msg->ckey, msg->ckey_bytes, data, H); + + get_galois_vector_s(msg, S); + + galois_compute(S, H, g, GCM_BLOCK_SIZE); + + /* Encrypt ctr0 based on AES_ECB */ + aes_encrypt(msg->ckey, msg->ckey_bytes, msg->aiv_stream, ctr_r); + + /* Get the GMAC tag final */ + for (i = 0; i < GCM_BLOCK_SIZE; i++) + msg->mac[i] = g[i] ^ ctr_r[i]; + + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) { + ret = memcmp(msg->mac, msg->mac_bak, msg->auth_bytes); + if (ret) { + msg->result = WD_IN_EPARA; + WD_ERR("failed to do the gcm authentication!\n"); + return -WD_EINVAL; + } + } + + msg->result = WD_SUCCESS; + + return 0; +} + +static int fill_stream_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + int ret = 0; + + switch (msg->msg_state) { + case AEAD_MSG_FIRST: + fill_gcm_first_bd2(msg, sqe); + break; + case AEAD_MSG_MIDDLE: + fill_gcm_middle_bd2(msg, sqe); + break; + case AEAD_MSG_END: + gcm_auth_ivin(msg); + if (msg->in_bytes) { + fill_gcm_final_bd2(msg, sqe); + } else { + ret = gcm_do_soft_mac(msg); + if (!ret) + ret = WD_SOFT_COMPUTING; + } + break; + default: + break; + } + + return ret; +} + static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) { __u8 scene, cipher, de; int ret; - /* config BD type */ sqe->type_auth_cipher = BD_TYPE2; - /* config scene */ - scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET; - de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET; + + if (msg->msg_state == AEAD_MSG_BLOCK) + scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET; + else + scene = SEC_STREAM_SCENE << SEC_SCENE_OFFSET; if (msg->op_type == WD_CIPHER_ENCRYPTION_DIGEST) { cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; @@ -2213,6 +2415,8 @@ static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) WD_ERR("failed to check aead op type, op = %u\n", msg->op_type); return -WD_EINVAL; } + + de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET; sqe->sds_sa_type |= (__u8)(de | scene); sqe->type_auth_cipher |= cipher; @@ -2235,6 +2439,23 @@ static int fill_aead_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) return 0; } +int aead_msg_state_check(struct wd_aead_msg *msg) +{ + if (msg->cmode == WD_CIPHER_GCM) { + if (unlikely(msg->msg_state >= AEAD_MSG_INVALID)) { + WD_ERR("invalid: GCM input msg state is wrong!\n"); + return -WD_EINVAL; + } + } else { + if (unlikely(msg->msg_state != AEAD_MSG_BLOCK)) { + WD_ERR("invalid: cmode not support stream msg state!\n"); + return -WD_EINVAL; + } + } + + return 0; +} + int hisi_sec_aead_send(handle_t ctx, void *aead_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); @@ -2248,12 +2469,11 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) return -WD_EINVAL; } - if (unlikely(msg->cmode != WD_CIPHER_CBC && !msg->in_bytes)) { - WD_ERR("ccm or gcm not supports 0 packet size at hw_v2!\n"); - return -WD_EINVAL; - } + ret = aead_msg_state_check(msg); + if (unlikely(ret)) + return ret; - ret = aead_len_check(msg); + ret = aead_len_check(msg, BD_TYPE2); if (unlikely(ret)) return ret; @@ -2270,6 +2490,13 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) } fill_aead_bd2_addr(msg, &sqe); + + ret = fill_stream_bd2(msg, &sqe); + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret; + hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.tag = (__u16)msg->tag; @@ -2288,6 +2515,20 @@ int hisi_sec_aead_send(handle_t ctx, void *aead_msg) return 0; } +static void update_stream_counter(struct wd_aead_msg *recv_msg) +{ + /* + * The counter of the first middle BD is set to 1. + * Other middle BDs and tail BD are set based on + * cipher_len and the counter of the previous BD. + */ + if (recv_msg->msg_state == AEAD_MSG_FIRST) { + recv_msg->iv[MAX_IV_SIZE - 1] = 0x1; + } else if (recv_msg->msg_state == AEAD_MSG_MIDDLE) { + ctr_iv_inc(recv_msg->iv, recv_msg->in_bytes >> CTR_MODE_LEN_SHIFT); + } +} + static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { @@ -2324,10 +2565,19 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, temp_msg = recv_msg; } + update_stream_counter(recv_msg); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); } +static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) +{ + /* Asynchronous mode does not use the sent message, so ignores it */ + return (msg->msg_state == AEAD_MSG_END) && !msg->in_bytes && + qp->q_info.qp_mode == CTX_MODE_SYNC; +} + int hisi_sec_aead_recv(handle_t ctx, void *aead_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); @@ -2336,6 +2586,9 @@ int hisi_sec_aead_recv(handle_t ctx, void *aead_msg) __u16 count = 0; int ret; + if (soft_compute_check((struct hisi_qp *)h_qp, recv_msg)) + return 0; + ret = hisi_qm_recv(h_qp, &sqe, 1, &count); if (ret < 0) return ret; @@ -2436,6 +2689,10 @@ static int fill_aead_bd3_mode(struct wd_aead_msg *msg, case WD_CIPHER_GCM: sqe->c_mode_alg |= C_MODE_GCM; sqe->auth_mac_key &= SEC_AUTH_MASK_V3; + if ((msg->msg_state == AEAD_MSG_FIRST) && !msg->assoc_bytes) { + WD_ERR("invalid: first bd assoc bytes is 0!\n"); + return -WD_EINVAL; + } sqe->a_len_key = msg->assoc_bytes; sqe->c_icv_key |= msg->auth_bytes << SEC_MAC_OFFSET_V3; break; @@ -2465,17 +2722,97 @@ static void fill_aead_bd3_addr(struct wd_aead_msg *msg, sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; } +static void fill_gcm_first_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_INNER << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_NOPAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->c_icv_key &= ~0x3; + sqe->auth_mac_key = AUTH_HMAC_CALCULATE; + sqe->auth_mac_key |= MAC_LEN << SEC_MAC_OFFSET_V3; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->auth_mac_key |= A_ALG_AES_GMAC << SEC_AUTH_ALG_OFFSET_V3; + sqe->c_len_ivin = 0; + sqe->auth_ivin.a_ivin_addr = 0; + sqe->c_key_addr = 0; + sqe->c_mode_alg &= ~(0x7 << SEC_CALG_OFFSET_V3); + sqe->auth_src_offset = 0; + sqe->a_len_key = msg->assoc_bytes; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_middle_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_NOPAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->auth_mac_key |= NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->a_len_key = 0; + sqe->auth_ivin.a_ivin_addr = sqe->mac_addr; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; +} + +static void fill_gcm_final_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; + sqe->stream_scene.stream_auth_pad = AUTHPAD_PAD; + sqe->stream_scene.stream_auth_pad |= BIT(5); + sqe->auth_mac_key |= NO_AUTH; + sqe->cipher_src_offset = 0; + sqe->auth_src_offset = 0; + fill_gcm_akey_len(msg, sqe, BD_TYPE3); + sqe->a_len_key = 0; + sqe->stream_scene.long_a_data_len = msg->assoc_bytes; + sqe->stream_scene.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; +} + +static int fill_stream_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + int ret = 0; + + switch (msg->msg_state) { + case AEAD_MSG_FIRST: + fill_gcm_first_bd3(msg, sqe); + break; + case AEAD_MSG_MIDDLE: + fill_gcm_middle_bd3(msg, sqe); + break; + case AEAD_MSG_END: + gcm_auth_ivin(msg); + if (msg->in_bytes) { + fill_gcm_final_bd3(msg, sqe); + } else { + ret = gcm_do_soft_mac(msg); + if (!ret) + ret = WD_SOFT_COMPUTING; + } + break; + default: + break; + } + + return ret; +} static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) { __u16 scene, de; int ret; - /* config BD type */ sqe->bd_param = BD_TYPE3; - /* config scene */ - scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET_V3; - de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET_V3; + + if (msg->msg_state == AEAD_MSG_BLOCK) + scene = SEC_IPSEC_SCENE << SEC_SCENE_OFFSET_V3; + else + scene = SEC_STREAM_SCENE << SEC_SCENE_OFFSET_V3; if (msg->op_type == WD_CIPHER_ENCRYPTION_DIGEST) { sqe->c_icv_key = SEC_CIPHER_ENC; @@ -2489,6 +2826,8 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) WD_ERR("failed to check aead op type, op = %u\n", msg->op_type); return -WD_EINVAL; } + + de = DATA_DST_ADDR_ENABLE << SEC_DE_OFFSET_V3; sqe->bd_param |= (__u16)(de | scene); sqe->c_len_ivin = msg->in_bytes; @@ -2526,7 +2865,11 @@ int hisi_sec_aead_send_v3(handle_t ctx, void *aead_msg) return -WD_EINVAL; } - ret = aead_len_check(msg); + ret = aead_msg_state_check(msg); + if (unlikely(ret)) + return ret; + + ret = aead_len_check(msg, BD_TYPE3); if (unlikely(ret)) return ret; @@ -2544,6 +2887,12 @@ int hisi_sec_aead_send_v3(handle_t ctx, void *aead_msg) fill_aead_bd3_addr(msg, &sqe); + ret = fill_stream_bd3(msg, &sqe); + if (ret == WD_SOFT_COMPUTING) + return 0; + else if (unlikely(ret)) + return ret; + hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = msg->tag; ret = hisi_qm_send(h_qp, &sqe, 1, &count); @@ -2598,6 +2947,8 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, temp_msg = recv_msg; } + update_stream_counter(recv_msg); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); } @@ -2610,6 +2961,9 @@ int hisi_sec_aead_recv_v3(handle_t ctx, void *aead_msg) __u16 count = 0; int ret; + if (soft_compute_check((struct hisi_qp *)h_qp, recv_msg)) + return 0; + ret = hisi_qm_recv(h_qp, &sqe, 1, &count); if (ret < 0) return ret; diff --git a/include/crypto/aes.h b/include/crypto/aes.h new file mode 100644 index 0000000..eca58ba --- /dev/null +++ b/include/crypto/aes.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_AES_H__ +#define __WD_AES_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define UINT_B_CNT 8 +#define AES_MAXNR 14 + +struct aes_key { + unsigned int rd_key[4 * (AES_MAXNR + 1)]; + __u8 rounds; +}; + +union uni { + unsigned char b[UINT_B_CNT]; + __u32 w[2]; + __u64 d; +}; + +void aes_encrypt(__u8 *key, __u32 key_len, __u8 *src, __u8 *dst); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/crypto/galois.h b/include/crypto/galois.h new file mode 100644 index 0000000..07723be --- /dev/null +++ b/include/crypto/galois.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#ifndef __WD_GALOIS_H__ +#define __WD_GALOIS_H__ + +#include <linux/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +void galois_compute(__u8 *S, __u8 *H, __u8 *g, __u32 len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 195205c..b43bb36 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -57,12 +57,19 @@ struct wd_aead_msg { __u8 *iv; /* input auth iv pointer */ __u8 aiv[MAX_IV_SIZE]; + /* input auth iv pointer for stream mode */ + __u8 aiv_stream[AIV_STREAM_LEN]; /* input data pointer */ __u8 *in; /* output data pointer */ __u8 *out; /* mac */ __u8 *mac; + /* mac data pointer for decrypto as stream mode */ + __u8 mac_bak[AES_BLOCK_SIZE]; + /* total of data for stream mode */ + __u64 long_data_len; + enum wd_aead_msg_state msg_state; }; struct wd_aead_driver { diff --git a/include/wd.h b/include/wd.h index da3cfc6..97956e2 100644 --- a/include/wd.h +++ b/include/wd.h @@ -71,6 +71,7 @@ typedef void (*wd_log)(const char *format, ...); #define WD_SUCCESS 0 #define WD_STREAM_END 1 #define WD_STREAM_START 2 +#define WD_SOFT_COMPUTING 3 #define WD_EIO EIO #define WD_EAGAIN EAGAIN #define WD_ENOMEM ENOMEM diff --git a/include/wd_aead.h b/include/wd_aead.h index 9f6f4e8..eefbed3 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -17,6 +17,7 @@ extern "C" { #endif +#define AIV_STREAM_LEN 64 /** * wd_aead_op_type - Algorithm type of option */ @@ -43,6 +44,18 @@ struct wd_aead_sess_setup { void *sched_param; }; +/** + * wd_aead_msg_state - Notify the message state + * zero is message for block mode, non-zero is message for stream mode. + */ +enum wd_aead_msg_state { + AEAD_MSG_BLOCK = 0x0, + AEAD_MSG_FIRST, + AEAD_MSG_MIDDLE, + AEAD_MSG_END, + AEAD_MSG_INVALID, +}; + struct wd_aead_req; typedef void *wd_alg_aead_cb_t(struct wd_aead_req *req, void *cb_param); @@ -84,6 +97,8 @@ struct wd_aead_req { __u8 data_fmt; wd_alg_aead_cb_t *cb; void *cb_param; + + enum wd_aead_msg_state msg_state; }; /** diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c new file mode 100644 index 0000000..2145d24 --- /dev/null +++ b/lib/crypto/aes.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include "crypto/aes.h" + +#define WORD(n) (0x##n##n##n##n) +#define LONG(n) (0x##n##n##n##n##n##n##n##n) + +#define STATE_CNT 2 + +static void xtimeword(__u32 *w) +{ + __u32 a, b; + + a = *w; + b = a & WORD(80); + a ^= b; + b -= b >> 0x7; + b &= WORD(1B); + b ^= a << 0x1; + *w = b; +} + +static void xtimelong(__u64 *w) +{ + __u64 a, b; + + a = *w; + b = a & LONG(80); + a ^= b; + b -= b >> 0x7; + b &= LONG(1B); + b ^= a << 0x1; + *w = b; +} + +static __u32 caculate_x_final(__u32 x_1) +{ + __u32 x, y; + + x = x_1; + y = ((x & WORD(FE)) >> 0x1) | ((x & WORD(01)) << 0x7); + x &= WORD(39); + x ^= y & WORD(3F); + y = ((y & WORD(FC)) >> 0x2) | ((y & WORD(03)) << 0x6); + x ^= y & WORD(97); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(9B); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(3C); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(DD); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(72); + x ^= WORD(63); + + return x; +} + +static __u32 caculate_x(__u32 *w) +{ + __u32 x, y; + + x = *w; + y = ((x & WORD(FE)) >> 0x1) | ((x & WORD(01)) << 0x7); + x &= WORD(DD); + x ^= y & WORD(57); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(1C); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(4A); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(42); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(64); + y = ((y & WORD(FE)) >> 0x1) | ((y & WORD(01)) << 0x7); + x ^= y & WORD(E0); + return x; +} + +static void subword(__u32 *w) +{ +#define SPECIAL_WORD 0x0A0A0A0A0Au + __u32 x, a1, a2, a3, a4, a5, a6; + + x = caculate_x(w); + a1 = x ^ (x & WORD(F0)) >> 0x4; + a2 = ((x & WORD(CC)) >> 0x2) | ((x & WORD(33)) << 0x2); + a3 = (x & a1) ^ ((x & a1) & WORD(AA)) >> 0x1; + a3 ^= (((x << 0x1) & a1) ^ ((a1 << 0x1) & x)) & WORD(AA); + a4 = (a2 & a1) ^ ((a2 & a1) & WORD(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a1) ^ ((a1 << 0x1) & a2)) & WORD(AA); + a5 = (a3 & WORD(CC)) >> 0x2; + a3 ^= ((a4 << 0x2) ^ a4) & WORD(CC); + a4 = (a5 & WORD(22)) | (a5 & WORD(22)) >> 0x1; + a4 ^= (a5 << 0x1) & WORD(22); + a3 ^= a4; + a5 = (a3 & WORD(A0)) | (a3 & WORD(A0)) >> 0x1; + a5 ^= (a3 << 0x1) & WORD(A0); + a4 = a5 & WORD(C0); + a6 = a4 >> 0x2; + a4 ^= (a5 << 0x2) & WORD(C0); + a5 = (a6 & WORD(20)) | (a6 & WORD(20)) >> 0x1; + a5 ^= (a6 << 0x1) & WORD(20); + a4 |= a5; + a3 ^= a4 >> 0x4; + a3 &= WORD(0F); + a2 = a3 ^ (a3 & WORD(0C)) >> 0x2; + a4 = a3 & a2; + a4 ^= (a4 & SPECIAL_WORD) >> 0x1; + a4 ^= (((a3 << 0x1) & a2) ^ ((a2 << 0x1) & a3)) & WORD(0A); + a5 = (a4 & WORD(08)) | (a4 & WORD(08)) >> 0x1; + a5 ^= (a4 << 0x1) & WORD(08); + a4 ^= a5 >> 0x2; + a4 &= WORD(03); + a4 ^= (a4 & WORD(02)) >> 0x1; + a4 |= a4 << 0x2; + a3 = (a2 & a4) ^ ((a2 & a4) & WORD(0A)) >> 0x1; + a3 ^= (((a2 << 0x1) & a4) ^ ((a4 << 0x1) & a2)) & WORD(0A); + a3 |= a3 << 0x4; + a2 = ((a1 & WORD(CC)) >> 0x2) | ((a1 & WORD(33)) << 0x2); + x = (a1 & a3) ^ ((a1 & a3) & WORD(AA)) >> 0x1; + x ^= (((a1 << 0x1) & a3) ^ ((a3 << 0x1) & a1)) & WORD(AA); + a4 = (a2 & a3) ^ ((a2 & a3) & WORD(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a3) ^ ((a3 << 0x1) & a2)) & WORD(AA); + a5 = (x & WORD(CC)) >> 0x2; + x ^= ((a4 << 0x2) ^ a4) & WORD(CC); + a4 = (a5 & WORD(22)) | (a5 & WORD(22)) >> 0x1; + a4 ^= (a5 << 0x1) & WORD(22); + *w = caculate_x_final(x ^ a4); +} + +static __u64 caculate_long_x_final(__u64 x_1) +{ + __u64 x, y; + + x = x_1; + y = ((x & LONG(FE)) >> 0x1) | ((x & LONG(01)) << 0x7); + x &= LONG(39); + x ^= y & LONG(3F); + y = ((y & LONG(FC)) >> 0x2) | ((y & LONG(03)) << 0x6); + x ^= y & LONG(97); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(9B); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(3C); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(DD); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(72); + x ^= LONG(63); + + return x; +} + +static __u64 caculate_long_x(__u64 *w) +{ + __u64 x, y; + + x = *w; + y = ((x & LONG(FE)) >> 0x1) | ((x & LONG(01)) << 0x7); + x &= LONG(DD); + x ^= y & LONG(57); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(1C); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(4A); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(42); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(64); + y = ((y & LONG(FE)) >> 0x1) | ((y & LONG(01)) << 0x7); + x ^= y & LONG(E0); + + return x; +} + +static void sublong(__u64 *w) +{ + __u64 x, a1, a2, a3, a4, a5, a6; + + x = caculate_long_x(w); + a1 = x ^ (x & LONG(F0)) >> 0x4; + a2 = ((x & LONG(CC)) >> 0x2) | ((x & LONG(33)) << 0x2); + a3 = (x & a1) ^ ((x & a1) & LONG(AA)) >> 0x1; + a3 ^= (((x << 0x1) & a1) ^ ((a1 << 0x1) & x)) & LONG(AA); + a4 = (a2 & a1) ^ ((a2 & a1) & LONG(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a1) ^ ((a1 << 0x1) & a2)) & LONG(AA); + a5 = (a3 & LONG(CC)) >> 0x2; + a3 ^= ((a4 << 0x2) ^ a4) & LONG(CC); + a4 = (a5 & LONG(22)) | (a5 & LONG(22)) >> 0x1; + a4 ^= (a5 << 0x1) & LONG(22); + a3 ^= a4; + a5 = (a3 & LONG(A0)) | (a3 & LONG(A0)) >> 0x1; + a5 ^= (a3 << 0x1) & LONG(A0); + a4 = a5 & LONG(C0); + a6 = a4 >> 0x2; + a4 ^= (a5 << 0x2) & LONG(C0); + a5 = (a6 & LONG(20)) | (a6 & LONG(20)) >> 0x1; + a5 ^= (a6 << 0x1) & LONG(20); + a4 |= a5; + a3 ^= a4 >> 0x4; + a3 &= LONG(0F); + a2 = a3 ^ (a3 & LONG(0C)) >> 0x2; + a4 = a3 & a2; + a4 ^= (a4 & LONG(0A)) >> 0x1; + a4 ^= (((a3 << 0x1) & a2) ^ ((a2 << 0x1) & a3)) & LONG(0A); + a5 = (a4 & LONG(08)) | (a4 & LONG(08)) >> 0x1; + a5 ^= (a4 << 0x1) & LONG(08); + a4 ^= a5 >> 0x2; + a4 &= LONG(03); + a4 ^= (a4 & LONG(02)) >> 0x1; + a4 |= a4 << 0x2; + a3 = (a2 & a4) ^ ((a2 & a4) & LONG(0A)) >> 0x1; + a3 ^= (((a2 << 0x1) & a4) ^ ((a4 << 0x1) & a2)) & LONG(0A); + a3 |= a3 << 0x4; + a2 = ((a1 & LONG(CC)) >> 0x2) | ((a1 & LONG(33)) << 0x2); + x = (a1 & a3) ^ ((a1 & a3) & LONG(AA)) >> 0x1; + x ^= (((a1 << 0x1) & a3) ^ ((a3 << 0x1) & a1)) & LONG(AA); + a4 = (a2 & a3) ^ ((a2 & a3) & LONG(AA)) >> 0x1; + a4 ^= (((a2 << 0x1) & a3) ^ ((a3 << 0x1) & a2)) & LONG(AA); + a5 = (x & LONG(CC)) >> 0x2; + x ^= ((a4 << 0x2) ^ a4) & LONG(CC); + a4 = (a5 & LONG(22)) | (a5 & LONG(22)) >> 0x1; + a4 ^= (a5 << 0x1) & LONG(22); + *w = caculate_long_x_final(x ^ a4); +} + +static void shift_rows(__u64 *state) +{ +#define S_CNT 4 + unsigned char s[S_CNT]; + unsigned char *s0; + __u8 r, i; + + s0 = (unsigned char *)state; + for (r = 0; r < S_CNT; r++) { + for (i = 0; i < S_CNT; i++) + s[i] = s0[i * S_CNT + r]; + + for (i = 0; i < S_CNT; i++) + s0[i * S_CNT + r] = s[(r + i) % S_CNT]; + } +} + +static void mix_columns(__u64 *state) +{ +#define REVERT_OFFSET 0x3 +#define A 0xFFFF0000FFFF0000uLL +#define B 0xFF00FF00FF00FF00uLL + union uni s1; + union uni s; + __u8 c, i; + + for (c = 0; c < STATE_CNT; c++) { + s1.d = state[c]; + s.d = s1.d; + s.d ^= ((s.d & A) >> 0x10) + | ((s.d & ~A) << 0x10); + s.d ^= ((s.d & B) >> 0x8) + | ((s.d & ~B) << 0x8); + s.d ^= s1.d; + xtimelong(&s1.d); + s.d ^= s1.d; + for (i = 0; i < UINT_B_CNT; i++) { + if (i == UINT_B_CNT - 1 || i == (UINT_B_CNT >> 0x1) - 1) + s.b[i] ^= s1.b[i - REVERT_OFFSET]; + else + s.b[i] ^= s1.b[i + 1]; + } + state[c] = s.d; + } +} + +static void add_round_key(__u64 *state, const __u64 *w) +{ + state[0] ^= w[0]; + state[1] ^= w[1]; +} + +static void cipher(const unsigned char *in, unsigned char *out, + const __u64 *w, __u8 nr) +{ +#define STATE_BYTE 16 + __u64 state[STATE_CNT]; + __u8 i; + + memcpy(state, in, STATE_BYTE); + + add_round_key(state, w); + + for (i = 1; i < nr; i++) { + sublong(&state[0]); + sublong(&state[1]); + shift_rows(state); + mix_columns(state); + add_round_key(state, w + i * STATE_CNT); + } + + sublong(&state[0]); + sublong(&state[1]); + shift_rows(state); + add_round_key(state, w + nr * STATE_CNT); + + memcpy(out, state, STATE_BYTE); +} + +static void rotword(__u32 *x) +{ +#define WORDBYTE 4 + unsigned char *w0; + unsigned char tmp; + __u8 i; + + w0 = (unsigned char *)x; + tmp = w0[0]; + for (i = 0; i < WORDBYTE - 1; i++) + w0[i] = w0[i + 1]; + + w0[WORDBYTE - 1] = tmp; +} + +static void key_expansion(const unsigned char *key, __u64 *w, __u8 nr, __u8 nk) +{ +#define RCON_LEN 4 +#define N 2 +#define NK 6 + __u32 rcon; + union uni prev; + __u32 temp; + __u8 i, n, m; + + memcpy(w, key, nk * RCON_LEN); + memcpy(&rcon, "\1\0\0\0", RCON_LEN); + n = nk / N; + m = (nr + 1) * N; + prev.d = w[n - 1]; + for (i = n; i < m; i++) { + temp = prev.w[1]; + if (i % n == 0) { + rotword(&temp); + subword(&temp); + temp ^= rcon; + xtimeword(&rcon); + } else if (nk > NK && i % n == N) { + subword(&temp); + } + prev.d = w[i - n]; + prev.w[0] ^= temp; + prev.w[1] ^= prev.w[0]; + w[i] = prev.d; + } +} + +static int aes_set_encrypt_key(const unsigned char *userkey, const int bits, + struct aes_key *key) +{ +#define AES_128_BIT 128 +#define AES_192_BIT 192 +#define AES_256_BIT 256 +#define AES_128_ROUNDS 10 +#define AES_192_ROUNDS 12 +#define AES_256_ROUNDS 14 +#define KEY_NK 32 + __u64 *rk; + + if (!userkey || !key) + return -1; + if (bits != AES_128_BIT && bits != AES_192_BIT && bits != AES_256_BIT) + return -1; + + rk = (__u64 *)key->rd_key; + + if (bits == AES_128_BIT) + key->rounds = AES_128_ROUNDS; + else if (bits == AES_192_BIT) + key->rounds = AES_192_ROUNDS; + else + key->rounds = AES_256_ROUNDS; + + key_expansion(userkey, rk, key->rounds, bits / KEY_NK); + return 0; +} + +static void aes_encrypt_(const __u8 *in, __u8 *out, const struct aes_key *key) +{ + const __u64 *rk; + + rk = (__u64 *)key->rd_key; + + cipher(in, out, rk, key->rounds); +} + +void aes_encrypt(__u8 *key, __u32 key_len, __u8 *src, __u8 *dst) +{ + struct aes_key local_key; + int ret; + + ret = aes_set_encrypt_key(key, key_len << 0x3, &local_key); + if (ret) + return; + + aes_encrypt_(src, dst, &local_key); +} diff --git a/lib/crypto/galois.c b/lib/crypto/galois.c new file mode 100644 index 0000000..94f97fe --- /dev/null +++ b/lib/crypto/galois.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2023 Huawei Technologies Co.,Ltd. All rights reserved. */ + +#include <string.h> +#include "crypto/galois.h" + +#define GF_LSB1_MASK 0x1 +#define GF_R 0xe1000000 +#define GALOIS_BITS 128 +#define UINT_BITS (sizeof(unsigned int) * 8) + +#define GALOIS_UL_COUNT 4 +#define GALOIS_UL_COUNT_H 12 + +#define GALOIS_PARA_TRANS_S(S, i) (((unsigned int)(S)[(i) + 3] << 24) | \ + ((unsigned int)(S)[(i) + 2] << 16) | \ + ((unsigned int)(S)[(i) + 1] << 8) | \ + ((unsigned int)(S)[(i)])) +#define GALOIS_PARA_TRANS_H(H, i) (((unsigned int)(H)[(i)] << 24) | \ + ((unsigned int)(H)[(i) + 1] << 16) | \ + ((unsigned int)(H)[(i) + 2] << 8) | \ + ((unsigned int)(H)[(i) + 3])) + +#define GALOIS_UINT_TRANS_CHAR(src_addr) \ + do { \ + unsigned int temp = *(src_addr); \ + *(src_addr) = ((temp & 0xff) << 24) | ((temp & 0xff00) << 8) \ + | ((temp & 0xff0000) >> 8) | ((temp & 0xff000000) >> 24); \ + } while (0) + +static void galois_xtime_128(unsigned int *src, unsigned int *dst) +{ + unsigned int tmp; + __u8 i; + + /* Based the NIST Special Publication 800-38D */ + for (i = 0; i < GALOIS_UL_COUNT; i++) { + tmp = src[i] >> 0x1; + dst[i] = tmp | (src[i + 1] << (UINT_BITS - 1)); + if (i == GALOIS_UL_COUNT - 1) { + if (src[0] & GF_LSB1_MASK) + dst[i] = tmp ^ GF_R; + else + dst[i] = tmp; + } + } +} + +static void galois_multi(unsigned int *input_a, unsigned int *input_b, + unsigned int *mul, unsigned int array_size) +{ + /* 4 * (unsigned int), 4 * 32bit = 128bit */ + unsigned int gf_V[GALOIS_BITS][GALOIS_UL_COUNT] = {0}; + unsigned int *gf_Z = mul; + __u8 i, j, k; + + memcpy(gf_V[0], input_a, sizeof(unsigned int) * array_size); + + for (i = 1; i < GALOIS_BITS; i++) + galois_xtime_128(gf_V[i - 1], gf_V[i]); + + for (i = 0; i < GALOIS_BITS; i++) { + k = GALOIS_BITS - i - 1; + if ((input_b[(k / UINT_BITS)] >> (k % UINT_BITS)) & 0x1) { + for (j = 0; j < array_size; j++) + gf_Z[j] ^= gf_V[i][j]; + } + } +} + +void galois_compute(__u8 *S, __u8 *H, __u8 *g, __u32 len) +{ + unsigned int SL[GALOIS_UL_COUNT] = {0}; + unsigned int HL[GALOIS_UL_COUNT] = {0}; + unsigned int G[GALOIS_UL_COUNT] = {0}; + __u8 i, j; + + /* Do galois multiplication operation on blocks: G = S x H */ + for (i = 0; i < GALOIS_UL_COUNT; i++) { + j = i * GALOIS_UL_COUNT; + SL[i] = GALOIS_PARA_TRANS_S(S, j); + j = GALOIS_UL_COUNT_H - j; + HL[i] = GALOIS_PARA_TRANS_H(H, j); + } + + galois_multi(SL, HL, G, GALOIS_UL_COUNT); + + j = len - GALOIS_UL_COUNT; + for (i = 0; i < GALOIS_UL_COUNT; i++) { + GALOIS_UINT_TRANS_CHAR(&G[i]); + memcpy(&g[j], &G[i], sizeof(unsigned int)); + j -= GALOIS_UL_COUNT; + } +} diff --git a/wd_aead.c b/wd_aead.c index 38d3f8f..c1fc354 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -48,11 +48,17 @@ struct wd_aead_sess { enum wd_digest_mode dmode; unsigned char ckey[MAX_CIPHER_KEY_SIZE]; unsigned char akey[MAX_HMAC_KEY_SIZE]; + /* Mac data pointer for decrypto as stream mode */ + unsigned char mac_bak[WD_AEAD_CCM_GCM_MAX]; __u16 ckey_bytes; __u16 akey_bytes; __u16 auth_bytes; void *priv; void *sched_key; + /* Stored the counter for gcm stream mode */ + __u8 iv[MAX_IV_SIZE]; + /* Total of data for stream mode */ + __u64 long_data_len; }; struct wd_env_config wd_aead_env_config; @@ -485,6 +491,41 @@ void wd_aead_uninit(void) wd_alg_clear_init(&wd_aead_setting.status); } +static void fill_stream_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, + struct wd_aead_sess *sess) +{ + switch (req->msg_state) { + case AEAD_MSG_FIRST: + /* Stream iv is extended to 16 bytes and last 4 bytes must be zero */ + memset(sess->iv, 0, MAX_IV_SIZE); + memcpy(sess->iv, req->iv, GCM_IV_SIZE); + + /* Store the original mac of first message to session */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(sess->mac_bak, req->mac, sess->auth_bytes); + break; + case AEAD_MSG_MIDDLE: + /* Middle messages need to store the stream's total length to session */ + sess->long_data_len += req->in_bytes; + + msg->long_data_len = sess->long_data_len; + break; + case AEAD_MSG_END: + /* Sets the original mac for final message */ + if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) + memcpy(msg->mac_bak, sess->mac_bak, sess->auth_bytes); + + msg->long_data_len = sess->long_data_len + req->in_bytes; + /* Reset the session's long_data_len */ + sess->long_data_len = 0; + break; + default: + return; + } + + msg->iv = sess->iv; +} + static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, struct wd_aead_sess *sess) { @@ -510,6 +551,9 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, msg->mac = req->mac; msg->auth_bytes = sess->auth_bytes; msg->data_fmt = req->data_fmt; + + msg->msg_state = req->msg_state; + fill_stream_msg(msg, req, sess); } static int send_recv_sync(struct wd_ctx_internal *ctx, -- 2.30.0

1 10

AccLib SIG例会
by openEuler conference 16 May '23

16 May '23

您好！ sig-AccLib SIG 邀请您参加 2023-05-17 11:00 召开的Zoom会议会议主题：AccLib SIG例会会议内容： https://linaro-org.zoom.us/j/91879279131 会议链接：https://us06web.zoom.us/j/89840749174?pwd=MGl2VWkwOFVBOUsyRmg3UFNSbFJEQT09 会议纪要：https://etherpad.openeuler.org/p/sig-AccLib-meetings 温馨提醒：建议接入会议后修改参会人的姓名，也可以使用您在gitee.com的ID 更多资讯尽在：https://openeuler.org/zh/ Hello! openEuler sig-AccLib SIG invites you to attend the Zoom conference will be held at 2023-05-17 11:00, The subject of the conference is AccLib SIG例会, Summary: https://linaro-org.zoom.us/j/91879279131 You can join the meeting at https://us06web.zoom.us/j/89840749174?pwd=MGl2VWkwOFVBOUsyRmg3UFNSbFJEQT09. Add topics at https://etherpad.openeuler.org/p/sig-AccLib-meetings. Note: You are advised to change the participant name after joining the conference or use your ID at gitee.com. More information: https://openeuler.org/en/

1 0

[PATCH] uadk: aead and digest support env init
by Wenkai Lin 08 May '23

08 May '23

aead and digest support env init for init2. Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- include/wd_util.h | 2 ++ wd_aead.c | 8 ++++++-- wd_cipher.c | 19 +++++-------------- wd_comp.c | 20 +++++--------------- wd_dh.c | 18 +++++------------- wd_digest.c | 10 +++++++--- wd_ecc.c | 19 +++++-------------- wd_rsa.c | 18 +++++------------- wd_util.c | 17 ++++++++++++++++- 9 files changed, 56 insertions(+), 75 deletions(-) diff --git a/include/wd_util.h b/include/wd_util.h index ce76852..3265946 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -444,6 +444,8 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_alg_driver *driver, enum wd_type type, int max_op_type); +void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); + /** * wd_alg_attrs_init() - Request the ctxs and initialize the sched_domain * with the given devices list, ctxs number and numa mask. diff --git a/wd_aead.c b/wd_aead.c index 9db2480..1e6011d 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -599,8 +599,9 @@ res_retry: goto out_dlopen; } + aead_ctx_params.ctx_set_num = aead_ctx_num; ret = wd_ctx_param_init(&aead_ctx_params, ctx_params, - aead_ctx_num, wd_aead_setting.driver, + wd_aead_setting.driver, WD_AEAD_TYPE, WD_DIGEST_CIPHER_DECRYPTION + 1); if (ret) { if (ret == -WD_EAGAIN) { @@ -625,13 +626,16 @@ res_retry: goto res_retry; } WD_ERR("fail to init alg attrs.\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_aead_setting.status); + wd_ctx_param_uninit(&aead_ctx_params); return 0; +out_ctx_uninit: + wd_ctx_param_uninit(&aead_ctx_params); out_driver: wd_alg_drv_unbind(wd_aead_setting.driver); out_dlopen: diff --git a/wd_cipher.c b/wd_cipher.c index e4f3d6b..7ba4de4 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -384,7 +384,6 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p { struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; - struct bitmask *inner_bmp; int ret = 0; bool flag; @@ -412,15 +411,6 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p goto out_uninit; } - inner_bmp = numa_allocate_nodemask(); - if (!inner_bmp) { - WD_ERR("fail to allocate nodemask.\n"); - ret = -WD_ENOMEM; - goto out_dlopen; - } - - cipher_ctx_params.bmp = inner_bmp; - res_retry: memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); @@ -429,7 +419,7 @@ res_retry: if (!wd_cipher_setting.driver) { WD_ERR("fail to bind a valid driver.\n"); ret = -WD_EINVAL; - goto out_bmp; + goto out_dlopen; } cipher_ctx_params.ctx_set_num = cipher_ctx_num; @@ -459,17 +449,18 @@ res_retry: goto res_retry; } WD_ERR("fail to init alg attrs.\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_cipher_setting.status); + wd_ctx_param_uninit(&cipher_ctx_params); return 0; +out_ctx_uninit: + wd_ctx_param_uninit(&cipher_ctx_params); out_driver: wd_alg_drv_unbind(wd_cipher_setting.driver); -out_bmp: - numa_free_nodemask(inner_bmp); out_dlopen: wd_dlclose_drv(wd_cipher_setting.dlh_list); out_uninit: diff --git a/wd_comp.c b/wd_comp.c index 7b79864..3d3deae 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -206,7 +206,6 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par { struct wd_ctx_nums comp_ctx_num[WD_DIR_MAX] = {0}; struct wd_ctx_params comp_ctx_params; - struct bitmask *inner_bmp; int ret = 0; bool flag; @@ -234,14 +233,6 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par goto out_uninit; } - inner_bmp = numa_allocate_nodemask(); - if (!inner_bmp) { - WD_ERR("fail to allocate nodemask.\n"); - ret = -WD_ENOMEM; - goto out_dlopen; - } - comp_ctx_params.bmp = inner_bmp; - res_retry: memset(&wd_comp_setting.config, 0, sizeof(struct wd_ctx_config_internal)); @@ -249,7 +240,7 @@ res_retry: wd_comp_setting.driver = wd_alg_drv_bind(task_type, alg); if (!wd_comp_setting.driver) { WD_ERR("fail to bind a valid driver.\n"); - goto out_bmp; + goto out_dlopen; } comp_ctx_params.ctx_set_num = comp_ctx_num; @@ -278,19 +269,18 @@ res_retry: goto res_retry; } WD_ERR("fail to init alg attrs.\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_comp_setting.status); - - numa_free_nodemask(inner_bmp); + wd_ctx_param_uninit(&comp_ctx_params); return 0; +out_ctx_uninit: + wd_ctx_param_uninit(&comp_ctx_params); out_driver: wd_alg_drv_unbind(wd_comp_setting.driver); -out_bmp: - numa_free_nodemask(inner_bmp); out_dlopen: wd_dlclose_drv(wd_comp_setting.dlh_list); out_uninit: diff --git a/wd_dh.c b/wd_dh.c index 2f6978b..82e7624 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -197,7 +197,6 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param { struct wd_ctx_nums dh_ctx_num[WD_DH_PHASE2] = {0}; struct wd_ctx_params dh_ctx_params = {0}; - struct bitmask *inner_bmp; bool flag; int ret; @@ -225,14 +224,6 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param goto out_clear_init; } - inner_bmp = numa_allocate_nodemask(); - if (!inner_bmp) { - ret = -WD_ENOMEM; - WD_ERR("fail to allocate nodemask.\n"); - goto out_dlopen; - } - dh_ctx_params.bmp = inner_bmp; - res_retry: memset(&wd_dh_setting.config, 0, sizeof(struct wd_ctx_config_internal)); @@ -241,7 +232,7 @@ res_retry: if (!wd_dh_setting.driver) { WD_ERR("fail to bind a valid driver.\n"); ret = -WD_EINVAL; - goto out_bmp; + goto out_dlopen; } dh_ctx_params.ctx_set_num = dh_ctx_num; @@ -270,17 +261,18 @@ res_retry: goto res_retry; } WD_ERR("failed to init alg attrs!\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_dh_setting.status); + wd_ctx_param_uninit(&dh_ctx_params); return 0; +out_ctx_uninit: + wd_ctx_param_uninit(&dh_ctx_params); out_driver: wd_alg_drv_unbind(wd_dh_setting.driver); -out_bmp: - numa_free_nodemask(inner_bmp); out_dlopen: wd_dlclose_drv(wd_dh_setting.dlh_list); out_clear_init: diff --git a/wd_digest.c b/wd_digest.c index c57e7d6..2e70f27 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -340,8 +340,8 @@ static bool wd_digest_algs_check(const char *alg) int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) { - struct wd_ctx_nums digest_ctx_num[1] = {0}; struct wd_ctx_params digest_ctx_params = {0}; + struct wd_ctx_nums digest_ctx_num[1] = {0}; int ret = 0; bool flag; @@ -385,8 +385,9 @@ res_retry: goto out_dlopen; } + digest_ctx_params.ctx_set_num = digest_ctx_num; ret = wd_ctx_param_init(&digest_ctx_params, ctx_params, - digest_ctx_num, wd_digest_setting.driver, 1); + wd_digest_setting.driver, WD_DIGEST_TYPE, 1); if (ret) { if (ret == -WD_EAGAIN) { wd_disable_drv(wd_digest_setting.driver); @@ -410,13 +411,16 @@ res_retry: goto res_retry; } WD_ERR("fail to init alg attrs.\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_digest_setting.status); + wd_ctx_param_uninit(&digest_ctx_params); return 0; +out_ctx_uninit: + wd_ctx_param_uninit(&digest_ctx_params); out_driver: wd_alg_drv_unbind(wd_digest_setting.driver); out_dlopen: diff --git a/wd_ecc.c b/wd_ecc.c index 183d2bd..21f7c30 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -253,7 +253,6 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para { struct wd_ctx_nums ecc_ctx_num[WD_EC_OP_MAX] = {0}; struct wd_ctx_params ecc_ctx_params = {0}; - struct bitmask *inner_bmp; bool flag; int ret; @@ -281,14 +280,6 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; } - inner_bmp = numa_allocate_nodemask(); - if (!inner_bmp) { - WD_ERR("fail to allocate nodemask.\n"); - ret = -WD_ENOMEM; - goto out_dlopen; - } - ecc_ctx_params.bmp = inner_bmp; - res_retry: memset(&wd_ecc_setting.config, 0, sizeof(struct wd_ctx_config_internal)); @@ -297,7 +288,7 @@ res_retry: if (!wd_ecc_setting.driver) { WD_ERR("failed to bind a valid driver!\n"); ret = -WD_EINVAL; - goto out_bmp; + goto out_dlopen; } ecc_ctx_params.ctx_set_num = ecc_ctx_num; @@ -326,17 +317,17 @@ res_retry: goto res_retry; } WD_ERR("failed to init alg attrs!\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_ecc_setting.status); + wd_ctx_param_uninit(&ecc_ctx_params); return 0; - +out_ctx_uninit: + wd_ctx_param_uninit(&ecc_ctx_params); out_driver: wd_alg_drv_unbind(wd_ecc_setting.driver); -out_bmp: - numa_free_nodemask(inner_bmp); out_dlopen: wd_dlclose_drv(wd_ecc_setting.dlh_list); out_clear_init: diff --git a/wd_rsa.c b/wd_rsa.c index c51c7b1..e685628 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -238,7 +238,6 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para { struct wd_ctx_nums rsa_ctx_num[WD_RSA_GENKEY] = {0}; struct wd_ctx_params rsa_ctx_params = {0}; - struct bitmask *inner_bmp; bool flag; int ret; @@ -266,14 +265,6 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; } - inner_bmp = numa_allocate_nodemask(); - if (!inner_bmp) { - WD_ERR("fail to allocate nodemask.\n"); - ret = -WD_ENOMEM; - goto out_dlopen; - } - rsa_ctx_params.bmp = inner_bmp; - res_retry: memset(&wd_rsa_setting.config, 0, sizeof(struct wd_ctx_config_internal)); @@ -282,7 +273,7 @@ res_retry: if (!wd_rsa_setting.driver) { WD_ERR("failed to bind a valid driver!\n"); ret = -WD_EINVAL; - goto out_bmp; + goto out_dlopen; } rsa_ctx_params.ctx_set_num = rsa_ctx_num; @@ -311,17 +302,18 @@ res_retry: goto res_retry; } WD_ERR("failed to init alg attrs!\n"); - goto out_driver; + goto out_ctx_uninit; } wd_alg_set_init(&wd_rsa_setting.status); + wd_ctx_param_uninit(&rsa_ctx_params); return 0; +out_ctx_uninit: + wd_ctx_param_uninit(&rsa_ctx_params); out_driver: wd_alg_drv_unbind(wd_rsa_setting.driver); -out_bmp: - numa_free_nodemask(inner_bmp); out_dlopen: wd_dlclose_drv(wd_rsa_setting.dlh_list); out_clear_init: diff --git a/wd_util.c b/wd_util.c index 9b5486b..e80cca0 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2084,6 +2084,11 @@ out_free_str: return ret; } +void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params) +{ + numa_free_nodemask(ctx_params->bmp); +} + int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, struct wd_alg_driver *driver, @@ -2093,22 +2098,30 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, const char *var_s; int i, ret; + ctx_params->bmp = numa_allocate_nodemask(); + if (!ctx_params->bmp) { + WD_ERR("fail to allocate nodemask.\n"); + return -WD_ENOMEM; + } + var_s = secure_getenv(env_name); if (var_s && strlen(var_s)) { /* environment variable has the highest priority */ ret = wd_env_set_ctx_nums(env_name, var_s, ctx_params, max_op_type); if (ret) { WD_ERR("fail to init ctx nums from %s!\n", env_name); + numa_free_nodemask(ctx_params->bmp); return -WD_EINVAL; } } else { /* environment variable is not set, try to use user_ctx_params first */ if (user_ctx_params) { - ctx_params->bmp = user_ctx_params->bmp; + copy_bitmask_to_bitmask(user_ctx_params->bmp, ctx_params->bmp); ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; ctx_params->op_type_num = user_ctx_params->op_type_num; if (ctx_params->op_type_num > max_op_type) { WD_ERR("fail to check user op type numbers.\n"); + numa_free_nodemask(ctx_params->bmp); return -WD_EINVAL; } @@ -2116,6 +2129,7 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, } /* user_ctx_params is also not set, use driver's defalut queue_num */ + numa_free_nodemask(ctx_params->bmp); ctx_params->bmp = NULL; for (i = 0; i < driver->op_type_num; i++) { ctx_params->ctx_set_num[i].sync_ctx_num = driver->queue_num; @@ -2126,6 +2140,7 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, ctx_params->op_type_num = driver->op_type_num; if (ctx_params->op_type_num > max_op_type) { WD_ERR("fail to check driver op type numbers.\n"); + numa_free_nodemask(ctx_params->bmp); return -WD_EAGAIN; } -- 2.30.0

1 0