[PATCH] uadk: add hpre sva and no-sva interface normalization func
From: lizhi <lizhi206@huawei.com> The hpre v2 interface needs to support the no-sva business model within the new evolution framework. This ensures that users switching to the v2 interface can utilize both the sva mode and the no-sva mode. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_comp.c | 3 +- drv/hisi_hpre.c | 650 ++++++++++++++++++++++++++++----------- drv/hisi_qm_udrv.c | 15 +- include/drv/wd_dh_drv.h | 3 + include/drv/wd_ecc_drv.h | 3 + include/drv/wd_rsa_drv.h | 3 + include/wd_dh.h | 2 + include/wd_ecc.h | 2 + include/wd_rsa.h | 2 + wd_dh.c | 22 +- wd_ecc.c | 53 ++-- wd_rsa.c | 56 +++- 12 files changed, 580 insertions(+), 234 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 87d2103..8474e17 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -267,7 +267,8 @@ static __u32 copy_to_out(struct wd_comp_msg *msg, struct hisi_comp_buf *buf, __u struct wd_comp_req *req = &msg->req; struct wd_datalist *node = req->list_dst; __u32 sgl_restlen, copy_len; - __u32 len = 0, sgl_cplen = 0; + __u32 sgl_cplen = 0; + __u32 len = 0; copy_len = total_len > req->dst_len ? req->dst_len : total_len; diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 7ce842b..f3f2451 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -34,14 +34,17 @@ #define SM2_PONIT_SIZE 64 #define MAX_HASH_LENS BITS_TO_BYTES(521) #define HW_PLAINTEXT_BYTES_MAX BITS_TO_BYTES(4096) -#define HPRE_CTX_Q_NUM_DEF 1 +#define HPRE_CTX_Q_NUM_DEF 1 +#define MAP_PAIR_NUM_MAX 6 #define CRT_PARAMS_SZ(key_size) ((5 * (key_size)) >> 1) #define CRT_GEN_PARAMS_SZ(key_size) ((7 * (key_size)) >> 1) #define GEN_PARAMS_SZ(key_size) ((key_size) << 1) #define CRT_PARAM_SZ(key_size) ((key_size) >> 1) -#define WD_TRANS_FAIL 0 +#define GEN_PARAMS_SZ_UL(key_size) ((unsigned long)(key_size) << 1) +#define DMA_ADDR(hi, lo) ((__u64)(((__u64)(hi) << 32) | (__u64)(lo))) +#define WD_TRANS_FAIL 0 #define CURVE_PARAM_NUM 6 #define SECP256R1_KEY_SIZE 32 @@ -78,6 +81,21 @@ enum hpre_alg_name { WD_ECC }; +enum hpre_hw_msg_field { + HW_MSG_IN, + HW_MSG_OUT, + HW_MSG_KEY, +}; + +struct map_info_cache { + struct map_pair { + void *addr; + uintptr_t pa; + size_t size; + } pairs[MAP_PAIR_NUM_MAX]; + size_t cnt; +}; + /* put vendor hardware message as a user interface is not suitable here */ struct hisi_hpre_sqe { __u32 alg : 5; @@ -113,12 +131,122 @@ struct hisi_hpre_sqe { struct hisi_hpre_ctx { struct wd_ctx_config_internal config; + struct wd_mm_ops *mm_ops; + handle_t rsv_mem_ctx; }; struct hpre_ecc_ctx { __u32 enable_hpcore; }; +static void add_map_info(struct map_info_cache *cache, void *addr, uintptr_t dma, size_t size) +{ + /* The cnt is guaranteed not to exceed MAP_PAIR_NUM_MAX within hpre. */ + cache->pairs[cache->cnt].addr = addr; + cache->pairs[cache->cnt].pa = dma; + cache->pairs[cache->cnt].size = size; + cache->cnt++; +} + +static void unmap_addr_in_cache(struct wd_mm_ops *mm_ops, struct map_info_cache *cache) +{ + size_t i; + + if (mm_ops->sva_mode) + return; + + /* The cnt is guaranteed not to exceed MAP_PAIR_NUM_MAX within hpre. */ + for (i = 0; i < cache->cnt; i++) + mm_ops->iova_unmap(mm_ops->usr, cache->pairs[i].addr, + (void *)cache->pairs[i].pa, cache->pairs[i].size); +} + +static void unsetup_hw_msg_addr(struct wd_mm_ops *mm_ops, enum hpre_hw_msg_field t_type, + struct hisi_hpre_sqe *hw_msg, void *va, size_t data_sz) +{ + void *addr; + + if (!mm_ops || mm_ops->sva_mode || !va || !data_sz) + return; + + switch (t_type) { + case HW_MSG_KEY: + addr = VA_ADDR(hw_msg->hi_key, hw_msg->low_key); + break; + case HW_MSG_IN: + addr = VA_ADDR(hw_msg->hi_in, hw_msg->low_in); + break; + case HW_MSG_OUT: + addr = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); + break; + default: + return; + } + + if (!addr) + return; + + mm_ops->iova_unmap(mm_ops->usr, va, (void *)addr, data_sz); +} + +static uintptr_t select_addr_by_sva_mode(struct wd_mm_ops *mm_ops, void *data, + size_t data_sz, struct map_info_cache *cache) +{ + uintptr_t addr; + + if (!mm_ops->sva_mode) { + addr = (uintptr_t)mm_ops->iova_map(mm_ops->usr, data, data_sz); + if (!addr) { + WD_ERR("Failed to get mappped DMA address for hardware.\n"); + return 0; + } + add_map_info(cache, data, addr, data_sz); + } else { + addr = (uintptr_t)data; + } + + return addr; +} + +static void fill_hw_msg_addr(enum hpre_hw_msg_field t_type, struct hisi_hpre_sqe *hw_msg, + uintptr_t addr) +{ + switch (t_type) { + case HW_MSG_KEY: + hw_msg->low_key = LW_U32(addr); + hw_msg->hi_key = HI_U32(addr); + break; + case HW_MSG_IN: + hw_msg->low_in = LW_U32(addr); + hw_msg->hi_in = HI_U32(addr); + break; + case HW_MSG_OUT: + hw_msg->low_out = LW_U32(addr); + hw_msg->hi_out = HI_U32(addr); + break; + default: + return; + } +} + +static int check_hpre_mem_params(struct wd_mm_ops *mm_ops, enum wd_mem_type mm_type) +{ + if (!mm_ops) { + WD_ERR("Memory operation functions are null.\n"); + return -WD_EINVAL; + } + + if (mm_type == UADK_MEM_AUTO && !mm_ops->sva_mode) { + WD_ERR("No-SVA in auto mode is not supported yet.\n"); + return -WD_EINVAL; + } else if (mm_type > UADK_MEM_PROXY) { + WD_ERR("failed to check memory type.\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} + static void dump_hpre_msg(void *msg, int alg) { struct wd_rsa_msg *rsa_msg; @@ -269,7 +397,8 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey, *data = wd_dq->data; - return WD_SUCCESS; + return (int)(wd_dq->bsize + wd_qinv->bsize + wd_p->bsize + + wd_q->bsize + wd_dp->bsize); } static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data) @@ -292,7 +421,7 @@ static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data) *data = wd_d->data; - return WD_SUCCESS; + return (int)(wd_n->bsize + wd_d->bsize); } static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data) @@ -315,7 +444,7 @@ static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data) *data = wd_e->data; - return WD_SUCCESS; + return (int)(wd_n->bsize + wd_e->bsize); } static int fill_rsa_genkey_in(struct wd_rsa_kg_in *genkey) @@ -378,22 +507,21 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg, { struct wd_rsa_req *req = &msg->req; struct wd_rsa_kg_out *key = req->dst; + struct wd_rsa_msg *target_msg; __u16 kbytes = msg->key_bytes; struct wd_dtb qinv, dq, dp; struct wd_dtb d, n; - void *data; int ret; - if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD) { - /* async */ - if (qp_mode == CTX_MODE_ASYNC) { - data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); - key = container_of(data, struct wd_rsa_kg_out, data); - } else { - key = req->dst; - } + target_msg = (struct wd_rsa_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag); + if (!target_msg) { + WD_ERR("failed to get correct rsa send msg from hardware!\n"); + return -WD_ADDR_ERR; } + if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD) + key = target_msg->req.dst; + msg->result = WD_SUCCESS; if (hw_msg->alg == HPRE_ALG_KG_CRT) { req->dst_bytes = CRT_GEN_PARAMS_SZ(kbytes); @@ -424,37 +552,38 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg, return WD_SUCCESS; } -static int rsa_prepare_key(struct wd_rsa_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int rsa_prepare_key(struct wd_rsa_msg *msg, struct hisi_hpre_sqe *hw_msg, + struct map_info_cache *cache) { struct wd_rsa_req *req = &msg->req; + uintptr_t addr; + int ret, len; void *data; - int ret; if (req->op_type == WD_RSA_SIGN) { if (hw_msg->alg == HPRE_ALG_NC_CRT) { - ret = fill_rsa_crt_prikey2((void *)msg->key, &data); - if (ret) - return ret; + len = fill_rsa_crt_prikey2((void *)msg->key, &data); + if (len <= 0) + return -WD_EINVAL; } else { - ret = fill_rsa_prikey1((void *)msg->key, &data); - if (ret) - return ret; + len = fill_rsa_prikey1((void *)msg->key, &data); + if (len <= 0) + return -WD_EINVAL; hw_msg->alg = HPRE_ALG_NC_NCRT; } } else if (req->op_type == WD_RSA_VERIFY) { - ret = fill_rsa_pubkey((void *)msg->key, &data); - if (ret) - return ret; + len = fill_rsa_pubkey((void *)msg->key, &data); + if (len <= 0) + return -WD_EINVAL; hw_msg->alg = HPRE_ALG_NC_NCRT; } else if (req->op_type == WD_RSA_GENKEY) { ret = fill_rsa_genkey_in((void *)msg->key); if (ret) return ret; - ret = wd_rsa_kg_in_data((void *)msg->key, (char **)&data); - if (ret < 0) { + len = wd_rsa_kg_in_data((void *)msg->key, (char **)&data); + if (len < 0) { WD_ERR("failed to get rsa gen key data!\n"); - return ret; + return -WD_EINVAL; } if (hw_msg->alg == HPRE_ALG_NC_CRT) hw_msg->alg = HPRE_ALG_KG_CRT; @@ -465,36 +594,53 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg, return -WD_EINVAL; } - hw_msg->low_key = LW_U32((uintptr_t)data); - hw_msg->hi_key = HI_U32((uintptr_t)data); + addr = select_addr_by_sva_mode(msg->mm_ops, data, len, cache); + if (!addr) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr); - return WD_SUCCESS; + return ret; } -static int rsa_prepare_iot(struct wd_rsa_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int rsa_prepare_iot(struct wd_rsa_msg *msg, struct hisi_hpre_sqe *hw_msg, + struct map_info_cache *cache) { struct wd_rsa_req *req = &msg->req; struct wd_rsa_kg_out *kout = req->dst; int ret = WD_SUCCESS; - void *out = NULL; + uintptr_t phy, out; if (req->op_type != WD_RSA_GENKEY) { - hw_msg->low_in = LW_U32((uintptr_t)req->src); - hw_msg->hi_in = HI_U32((uintptr_t)req->src); - out = req->dst; + phy = select_addr_by_sva_mode(msg->mm_ops, req->src, req->src_bytes, cache); + if (!phy) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_IN, hw_msg, phy); + phy = select_addr_by_sva_mode(msg->mm_ops, req->dst, req->dst_bytes, cache); + if (!phy) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_OUT, hw_msg, phy); } else { hw_msg->low_in = 0; hw_msg->hi_in = 0; ret = wd_rsa_kg_out_data(kout, (char **)&out); if (ret < 0) return ret; - } - hw_msg->low_out = LW_U32((uintptr_t)out); - hw_msg->hi_out = HI_U32((uintptr_t)out); + if (!msg->mm_ops->sva_mode) { + phy = (uintptr_t)msg->mm_ops->iova_map(msg->mm_ops->usr, kout, ret); + if (!phy) { + WD_ERR("Failed to get DMA address for rsa output!\n"); + return -WD_ENOMEM; + } + add_map_info(cache, kout, phy, ret); + out = phy + out - (uintptr_t)kout; + } + + hw_msg->low_out = LW_U32(out); + hw_msg->hi_out = HI_U32(out); + } - return ret; + return WD_SUCCESS; } static int hpre_init_qm_priv(struct wd_ctx_config_internal *config, @@ -615,10 +761,17 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_rsa_msg *msg = rsa_msg; + struct map_info_cache cache = {0}; struct hisi_hpre_sqe hw_msg; __u16 send_cnt = 0; int ret; + ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type); + if (ret) { + WD_ERR("rsa memory parmas is err, and ret is %d!\n", ret); + return ret; + } + memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe)); if (msg->key_type == WD_RSA_PRIKEY1 || @@ -631,21 +784,30 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) hw_msg.task_len1 = msg->key_bytes / BYTE_BITS - 0x1; - ret = rsa_prepare_key(msg, &hw_msg); - if (ret < 0) - return ret; + ret = rsa_prepare_key(msg, &hw_msg, &cache); + if (ret) + goto rsa_fail; /* prepare in/out put */ - ret = rsa_prepare_iot(msg, &hw_msg); - if (ret < 0) - return ret; + ret = rsa_prepare_iot(msg, &hw_msg, &cache); + if (ret) + goto rsa_fail; hisi_set_msg_id(h_qp, &msg->tag); hw_msg.done = 0x1; hw_msg.etype = 0x0; - hw_msg.low_tag = msg->tag; + hw_msg.low_tag = LW_U32((uintptr_t)msg); + hw_msg.hi_tag = HI_U32((uintptr_t)msg); - return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); + ret = hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); + if (unlikely(ret)) + goto rsa_fail; + + return ret; + +rsa_fail: + unmap_addr_in_cache(msg->mm_ops, &cache); + return ret; } static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, @@ -671,32 +833,29 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; - struct hisi_hpre_sqe hw_msg = {0}; + struct wd_rsa_msg *target_msg; struct wd_rsa_msg *msg = rsa_msg; - struct wd_rsa_msg *temp_msg; + struct hisi_hpre_sqe hw_msg = {0}; + size_t ilen = 0, olen = 0; __u16 recv_cnt = 0; + __u16 kbytes; int ret; ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt); if (ret < 0) return ret; - ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag); + target_msg = (struct wd_rsa_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag); + if (!target_msg) { + WD_ERR("failed to get correct send msg from hardware!\n"); + return -WD_ADDR_ERR; + } + + ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag); if (ret) return ret; - msg->tag = LW_U16(hw_msg.low_tag); - if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag); - if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); - return -WD_EINVAL; - } - } else { - temp_msg = msg; - } - + msg->tag = target_msg->tag; hpre_result_check(&hw_msg, &msg->result); if (!msg->result) { ret = rsa_out_transfer(msg, &hw_msg, qp->q_info.qp_mode); @@ -707,15 +866,36 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) } if (unlikely(msg->result != WD_SUCCESS)) - dump_hpre_msg(temp_msg, WD_RSA); + dump_hpre_msg(target_msg, WD_RSA); + + if (!target_msg->mm_ops->sva_mode) { + kbytes = target_msg->key_bytes; + if (hw_msg.alg == HPRE_ALG_KG_CRT) { + olen = CRT_GEN_PARAMS_SZ(kbytes); + ilen = GEN_PARAMS_SZ_UL(kbytes); + } else if (hw_msg.alg == HPRE_ALG_KG_STD) { + olen = GEN_PARAMS_SZ_UL(kbytes); + ilen = GEN_PARAMS_SZ_UL(kbytes); + } else { + olen = kbytes; + ilen = kbytes; + } + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, &hw_msg, + target_msg->req.src, ilen); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, &hw_msg, + target_msg->req.dst, olen); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, &hw_msg, + target_msg->key, target_msg->key_bytes); + } return WD_SUCCESS; } -static int fill_dh_xp_params(struct wd_dh_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int fill_dh_xp_params(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg, + struct map_info_cache *cache) { struct wd_dh_req *req = &msg->req; + uintptr_t addr; void *x, *p; int ret; @@ -735,26 +915,30 @@ static int fill_dh_xp_params(struct wd_dh_msg *msg, return ret; } - hw_msg->low_key = LW_U32((uintptr_t)x); - hw_msg->hi_key = HI_U32((uintptr_t)x); + addr = select_addr_by_sva_mode(msg->mm_ops, x, GEN_PARAMS_SZ_UL(msg->key_bytes), cache); + if (!addr) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr); - return WD_SUCCESS; + return ret; } -static int dh_out_transfer(struct wd_dh_msg *msg, - struct hisi_hpre_sqe *hw_msg, __u8 qp_mode) +static int dh_out_transfer(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg, + __u8 qp_mode) { __u16 key_bytes = (hw_msg->task_len1 + 1) * BYTE_BITS; struct wd_dh_req *req = &msg->req; + struct wd_dh_msg *target_msg; void *out; int ret; - /* async */ - if (qp_mode == CTX_MODE_ASYNC) - out = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); - else - out = req->pri; + target_msg = (struct wd_dh_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag); + if (!target_msg) { + WD_ERR("failed to get correct send msg from hardware!\n"); + return -WD_ADDR_ERR; + } + out = target_msg->req.pri; ret = hpre_bin_to_crypto_bin((char *)out, (const char *)out, key_bytes, "dh out"); if (!ret) @@ -768,12 +952,20 @@ static int dh_out_transfer(struct wd_dh_msg *msg, static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); + struct map_info_cache cache = {0}; struct wd_dh_msg *msg = dh_msg; struct wd_dh_req *req = &msg->req; struct hisi_hpre_sqe hw_msg; __u16 send_cnt = 0; + uintptr_t addr; int ret; + ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type); + if (ret) { + WD_ERR("dh memory parmas is err, and ret is %d!\n", ret); + return ret; + } + memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe)); if (msg->is_g2 && req->op_type != WD_DH_PHASE2) @@ -791,32 +983,50 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) WD_ERR("failed to transfer dh g para format to hpre bin!\n"); return ret; } - - hw_msg.low_in = LW_U32((uintptr_t)msg->g); - hw_msg.hi_in = HI_U32((uintptr_t)msg->g); + addr = select_addr_by_sva_mode(msg->mm_ops, msg->g, msg->key_bytes, &cache); + if (!addr) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_IN, &hw_msg, addr); } - ret = fill_dh_xp_params(msg, &hw_msg); - if (ret) - return ret; + ret = fill_dh_xp_params(msg, &hw_msg, &cache); + if (ret) { + WD_ERR("failed to fill dh x or p para!\n"); + goto dh_fail; + } hisi_set_msg_id(h_qp, &msg->tag); - hw_msg.low_out = LW_U32((uintptr_t)req->pri); - hw_msg.hi_out = HI_U32((uintptr_t)req->pri); hw_msg.done = 0x1; hw_msg.etype = 0x0; - hw_msg.low_tag = msg->tag; - return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); + hw_msg.low_tag = LW_U32((uintptr_t)msg); + hw_msg.hi_tag = HI_U32((uintptr_t)msg); + + addr = select_addr_by_sva_mode(msg->mm_ops, req->pri, req->pri_bytes, &cache); + if (!addr) { + ret = -WD_ENOMEM; + goto dh_fail; + } + fill_hw_msg_addr(HW_MSG_OUT, &hw_msg, addr); + + ret = hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); + if (unlikely(ret)) + goto dh_fail; + + return ret; + +dh_fail: + unmap_addr_in_cache(msg->mm_ops, &cache); + return ret; } static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; - struct wd_dh_msg *msg = dh_msg; struct hisi_hpre_sqe hw_msg = {0}; - struct wd_dh_msg *temp_msg; + struct wd_dh_msg *msg = dh_msg; + struct wd_dh_msg *target_msg; __u16 recv_cnt = 0; int ret; @@ -824,22 +1034,17 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) if (ret < 0) return ret; - ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag); + target_msg = (struct wd_dh_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag); + if (!target_msg) { + WD_ERR("failed to get correct send msg from hardware!\n"); + return -WD_ADDR_ERR; + } + + ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag); if (ret) return ret; - msg->tag = LW_U16(hw_msg.low_tag); - if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag); - if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); - return -WD_EINVAL; - } - } else { - temp_msg = msg; - } - + msg->tag = target_msg->tag; hpre_result_check(&hw_msg, &msg->result); if (!msg->result) { ret = dh_out_transfer(msg, &hw_msg, qp->q_info.qp_mode); @@ -850,7 +1055,16 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) } if (unlikely(msg->result != WD_SUCCESS)) - dump_hpre_msg(temp_msg, WD_DH); + dump_hpre_msg(target_msg, WD_DH); + + if (!target_msg->mm_ops->sva_mode) { + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, &hw_msg, + target_msg->req.pri, target_msg->req.pri_bytes); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, &hw_msg, + target_msg->req.x_p, GEN_PARAMS_SZ_UL(target_msg->key_bytes)); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, &hw_msg, + target_msg->g, target_msg->key_bytes); + } return WD_SUCCESS; } @@ -1081,30 +1295,49 @@ static bool is_prikey_used(__u8 op_type) op_type == HPRE_SM2_DEC; } -static int ecc_prepare_key(struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static __u32 ecc_get_prikey_size(struct wd_ecc_msg *msg) +{ + if (msg->req.op_type == WD_SM2_SIGN || + msg->req.op_type == WD_ECDSA_SIGN || + msg->req.op_type == WD_SM2_DECRYPT) + return ECC_PRIKEY_SZ(msg->key_bytes); + else if (msg->curve_id == WD_X25519 || + msg->curve_id == WD_X448) + return X_DH_HW_KEY_SZ(msg->key_bytes); + else + return ECDH_HW_KEY_SZ(msg->key_bytes); +} + +static int ecc_prepare_key(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, + struct map_info_cache *cache) { void *data = NULL; + uintptr_t addr; + size_t ksz; int ret; if (is_prikey_used(msg->req.op_type)) { + ksz = ecc_get_prikey_size(msg); ret = ecc_prepare_prikey((void *)msg->key, &data, msg->curve_id); if (ret) return ret; } else { + ksz = ECC_PUBKEY_SZ(msg->key_bytes); ret = ecc_prepare_pubkey((void *)msg->key, &data); if (ret) return ret; } - hw_msg->low_key = LW_U32((uintptr_t)data); - hw_msg->hi_key = HI_U32((uintptr_t)data); + addr = select_addr_by_sva_mode(msg->mm_ops, data, ksz, cache); + if (!addr) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr); - return WD_SUCCESS; + return ret; } -static void ecc_get_io_len(__u32 atype, __u32 hsz, size_t *ilen, - size_t *olen) +static void ecc_get_io_len(__u32 atype, __u32 hsz, + size_t *ilen, size_t *olen) { if (atype == HPRE_ALG_ECDH_MULTIPLY) { *olen = ECDH_OUT_PARAMS_SZ(hsz); @@ -1467,12 +1700,12 @@ static int ecc_prepare_out(struct wd_ecc_msg *msg, void **data) } /* prepare in/out hw message */ -static int ecc_prepare_iot(struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int ecc_prepare_iot(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, + struct map_info_cache *cache) { + size_t i_sz, o_sz; void *data = NULL; - size_t i_sz = 0; - size_t o_sz = 0; + uintptr_t addr; __u16 kbytes; int ret; @@ -1483,8 +1716,11 @@ static int ecc_prepare_iot(struct wd_ecc_msg *msg, WD_ERR("failed to prepare ecc in!\n"); return ret; } - hw_msg->low_in = LW_U32((uintptr_t)data); - hw_msg->hi_in = HI_U32((uintptr_t)data); + + addr = select_addr_by_sva_mode(msg->mm_ops, data, i_sz, cache); + if (!addr) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_IN, hw_msg, addr); ret = ecc_prepare_out(msg, &data); if (ret) { @@ -1496,8 +1732,10 @@ static int ecc_prepare_iot(struct wd_ecc_msg *msg, if (!data) return WD_SUCCESS; - hw_msg->low_out = LW_U32((uintptr_t)data); - hw_msg->hi_out = HI_U32((uintptr_t)data); + addr = select_addr_by_sva_mode(msg->mm_ops, data, o_sz, cache); + if (!addr) + return -WD_ENOMEM; + fill_hw_msg_addr(HW_MSG_OUT, hw_msg, addr); return WD_SUCCESS; } @@ -1613,7 +1851,7 @@ static struct wd_ecc_out *create_ecdh_out(struct wd_ecc_msg *msg) return NULL; } - out = malloc(len); + out = msg->mm_ops->alloc(msg->mm_ops->usr, len); if (!out) { WD_ERR("failed to alloc out memory, sz = %u!\n", len); return NULL; @@ -1677,7 +1915,7 @@ static struct wd_ecc_msg *create_req(struct wd_ecc_msg *src, __u8 req_idx) prikey = (struct wd_ecc_prikey *)(ecc_key + 1); ecc_key->prikey = prikey; - prikey->data = malloc(ECC_PRIKEY_SZ(src->key_bytes)); + prikey->data = src->mm_ops->alloc(src->mm_ops->usr, ECC_PRIKEY_SZ(src->key_bytes)); if (unlikely(!prikey->data)) { WD_ERR("failed to alloc prikey data!\n"); goto fail_alloc_key_data; @@ -1696,7 +1934,7 @@ static struct wd_ecc_msg *create_req(struct wd_ecc_msg *src, __u8 req_idx) return dst; fail_set_prikey: - free(prikey->data); + src->mm_ops->free(src->mm_ops->usr, prikey->data); fail_alloc_key_data: free(ecc_key); fail_alloc_key: @@ -1709,9 +1947,9 @@ static void free_req(struct wd_ecc_msg *msg) { struct wd_ecc_key *key = (void *)msg->key; - free(key->prikey->data); + msg->mm_ops->free(msg->mm_ops->usr, key->prikey->data); free(key); - free(msg->req.dst); + msg->mm_ops->free(msg->mm_ops->usr, msg->req.dst); free(msg); } @@ -1732,7 +1970,8 @@ static int split_req(struct wd_ecc_msg *src, struct wd_ecc_msg **dst) return WD_SUCCESS; } -static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) +static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, + struct map_info_cache *cache) { __u32 hw_sz = get_hw_keysz(msg->key_bytes); __u8 op_type = msg->req.op_type; @@ -1757,39 +1996,42 @@ static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) return ret; /* prepare key */ - ret = ecc_prepare_key(msg, hw_msg); + ret = ecc_prepare_key(msg, hw_msg, cache); if (ret) return ret; /* prepare in/out put */ - ret = ecc_prepare_iot(msg, hw_msg); + ret = ecc_prepare_iot(msg, hw_msg, cache); if (ret) return ret; hw_msg->done = 0x1; hw_msg->etype = 0x0; - hw_msg->low_tag = msg->tag; + + hw_msg->low_tag = LW_U32((uintptr_t)msg); + hw_msg->hi_tag = HI_U32((uintptr_t)msg); hw_msg->task_len1 = hw_sz / BYTE_BITS - 0x1; return ret; } -static int ecc_general_send(handle_t ctx, struct wd_ecc_msg *msg) +static int ecc_general_send(handle_t ctx, struct wd_ecc_msg *msg, + struct map_info_cache *cache) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_hpre_sqe hw_msg; __u16 send_cnt = 0; int ret; - ret = ecc_fill(msg, &hw_msg); + ret = ecc_fill(msg, &hw_msg, cache); if (ret) return ret; return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt); } - -static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg) +static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg, + struct map_info_cache *cache) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_sm2_enc_in *ein = msg->req.src; @@ -1801,7 +2043,7 @@ static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg) if (ein->plaintext.dsize <= HW_PLAINTEXT_BYTES_MAX && hash->type == WD_HASH_SM3) - return ecc_general_send(ctx, msg); + return ecc_general_send(ctx, msg, cache); if (unlikely(!ein->k_set)) { WD_ERR("invalid: k not set!\n"); @@ -1824,13 +2066,13 @@ static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg) return ret; } - ret = ecc_fill(msg_dst[0], &hw_msg[0]); + ret = ecc_fill(msg_dst[0], &hw_msg[0], cache); if (unlikely(ret)) { WD_ERR("failed to fill 1th sqe, ret = %d!\n", ret); goto fail_fill_sqe; } - ret = ecc_fill(msg_dst[1], &hw_msg[1]); + ret = ecc_fill(msg_dst[1], &hw_msg[1], cache); if (unlikely(ret)) { WD_ERR("failed to fill 2th sqe, ret = %d!\n", ret); goto fail_fill_sqe; @@ -1855,7 +2097,8 @@ fail_fill_sqe: return ret; } -static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg) +static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg, + struct map_info_cache *cache) { struct wd_sm2_dec_in *din = (void *)msg->req.src; struct wd_hash_mt *hash = &msg->hash; @@ -1865,7 +2108,7 @@ static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg) /* c2 data lens <= 4096 bit */ if (din->c2.dsize <= BITS_TO_BYTES(4096) && hash->type == WD_HASH_SM3) - return ecc_general_send(ctx, msg); + return ecc_general_send(ctx, msg, cache); if (unlikely(!hash->cb || hash->type >= WD_HASH_MAX)) { WD_ERR("invalid: input hash type %u is error!\n", hash->type); @@ -1891,14 +2134,14 @@ static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg) goto free_dst; } - ret = ecc_general_send(ctx, dst); + ret = ecc_general_send(ctx, dst, cache); if (unlikely(ret)) goto free_req_dst; return ret; free_req_dst: - free(dst->req.dst); + msg->mm_ops->free(msg->mm_ops->usr, dst->req.dst); free_dst: free(dst); return ret; @@ -1908,14 +2151,37 @@ static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; + struct map_info_cache cache = {0}; + int ret; + + ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type); + if (ret) { + WD_ERR("ecc memory parmas is err, and ret is %d!\n", ret); + return ret; + } hisi_set_msg_id(h_qp, &msg->tag); - if (msg->req.op_type == WD_SM2_ENCRYPT) - return sm2_enc_send(ctx, msg); - else if (msg->req.op_type == WD_SM2_DECRYPT) - return sm2_dec_send(ctx, msg); + if (msg->req.op_type == WD_SM2_ENCRYPT) { + ret = sm2_enc_send(ctx, msg, &cache); + if (ret) + goto ecc_fail; + return ret; + } else if (msg->req.op_type == WD_SM2_DECRYPT) { + ret = sm2_dec_send(ctx, msg, &cache); + if (ret) + goto ecc_fail; + return ret; + } - return ecc_general_send(ctx, msg); + ret = ecc_general_send(ctx, msg, &cache); + if (ret) + goto ecc_fail; + + return ret; + +ecc_fail: + unmap_addr_in_cache(msg->mm_ops, &cache); + return ret; } static int ecdh_out_transfer(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) @@ -2020,14 +2286,14 @@ static int sm2_enc_out_transfer(struct wd_ecc_msg *msg, static int ecc_out_transfer(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, __u8 qp_mode) { + struct wd_ecc_msg *target_msg; int ret = -WD_EINVAL; - void *va; + + target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag); /* async */ - if (qp_mode == CTX_MODE_ASYNC) { - va = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); - msg->req.dst = container_of(va, struct wd_ecc_out, data); - } + if (qp_mode == CTX_MODE_ASYNC) + msg->req.dst = target_msg->req.dst; if (hw_msg->alg == HPRE_ALG_SM2_SIGN || hw_msg->alg == HPRE_ALG_ECDSA_SIGN) @@ -2319,19 +2585,16 @@ static int sm2_convert_dec_out(struct wd_ecc_msg *src, static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { - struct wd_ecc_msg *temp_msg; + struct wd_ecc_msg *target_msg; + size_t ilen = 0; + size_t olen = 0; + __u16 kbytes; int ret; - msg->tag = LW_U16(hw_msg->low_tag); - if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag); - if (!temp_msg) { - WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", - qp->q_info.idx, msg->tag); - return -WD_EINVAL; - } - } else { - temp_msg = msg; + target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag); + if (!target_msg) { + WD_ERR("failed to get correct ecc send msg from hardware!\n"); + return -WD_ADDR_ERR; } hpre_result_check(hw_msg, &msg->result); @@ -2347,10 +2610,20 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, goto dump_err_msg; } - return ret; + return WD_SUCCESS; dump_err_msg: - dump_hpre_msg(temp_msg, WD_ECC); + kbytes = target_msg->key_bytes; + if (!target_msg->mm_ops->sva_mode) { + ecc_get_io_len(hw_msg->alg, kbytes, &ilen, &olen); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, hw_msg, + target_msg->req.dst, olen); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, hw_msg, + target_msg->key, kbytes); + unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, hw_msg, + target_msg->req.src, ilen); + } + dump_hpre_msg(target_msg, WD_ECC); return ret; } @@ -2363,8 +2636,6 @@ static int parse_second_sqe(handle_t h_qp, struct wd_ecc_msg *dst; __u16 recv_cnt = 0; int cnt = 0; - void *data; - __u32 hsz; int ret; while (1) { @@ -2380,10 +2651,12 @@ static int parse_second_sqe(handle_t h_qp, break; } - data = VA_ADDR(hw_msg.hi_out, hw_msg.low_out); - hsz = (hw_msg.task_len1 + 1) * BYTE_BITS; - dst = *(struct wd_ecc_msg **)((uintptr_t)data + - hsz * ECDH_OUT_PARAM_NUM); + dst = (struct wd_ecc_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag); + if (!dst) { + WD_ERR("failed to get correct sm2 enc second send msg from hardware!\n"); + return -WD_ADDR_ERR; + } + ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg); msg->result = dst->result; *second = dst; @@ -2394,19 +2667,17 @@ static int parse_second_sqe(handle_t h_qp, static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { - __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *second = NULL; struct wd_ecc_msg *first; struct wd_ecc_msg src; - void *data; - __u32 hsz; int ret; - msg->tag = tag; - data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); - hsz = (hw_msg->task_len1 + 1) * BYTE_BITS; - first = *(struct wd_ecc_msg **)((uintptr_t)data + - hsz * ECDH_OUT_PARAM_NUM); + first = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag); + if (!first) { + WD_ERR("failed to get correct sm2 enc msg from hardware!\n"); + return -WD_ADDR_ERR; + } + memcpy(&src, first + 1, sizeof(src)); /* parse first sqe */ @@ -2440,17 +2711,15 @@ free_first: static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { - __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *dst; struct wd_ecc_msg src; - void *data; - __u32 hsz; int ret; - data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out); - hsz = (hw_msg->task_len1 + 1) * BYTE_BITS; - dst = *(struct wd_ecc_msg **)((uintptr_t)data + - hsz * ECDH_OUT_PARAM_NUM); + dst = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag); + if (!dst) { + WD_ERR("failed to get correct sm2 dec msg from hardware!\n"); + return -WD_ADDR_ERR; + } memcpy(&src, dst + 1, sizeof(src)); /* parse first sqe */ @@ -2460,7 +2729,6 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, goto fail; } msg->result = dst->result; - msg->tag = tag; ret = sm2_convert_dec_out(&src, dst); if (unlikely(ret)) { @@ -2469,7 +2737,7 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, } fail: - free(dst->req.dst); + dst->mm_ops->free(dst->mm_ops->usr, dst->req.dst); free(dst); return ret; @@ -2479,6 +2747,7 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; + struct wd_ecc_msg *target_msg; struct hisi_hpre_sqe hw_msg; __u16 recv_cnt = 0; int ret; @@ -2487,10 +2756,17 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) if (ret) return ret; - ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag); + target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag); + if (!target_msg) { + WD_ERR("failed to get correct send msg from hardware!\n"); + return -WD_ADDR_ERR; + } + + ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag); if (ret) return ret; + msg->tag = target_msg->tag; if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_ENC) return sm2_enc_parse(h_qp, msg, &hw_msg); @@ -2549,7 +2825,7 @@ static bool is_valid_hw_type(struct wd_alg_driver *drv) hpre_ctx = (struct hisi_hpre_ctx *)drv->priv; qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->config.ctxs[0].ctx); - if (qp->q_info.hw_type < HISI_QM_API_VER3_BASE) + if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE) return false; return true; } diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 9251b4c..216b80a 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -473,7 +473,7 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count) { struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct hisi_qm_queue_info *q_info; - __u16 free_num, send_num; + __u16 free_num; __u16 tail; if (unlikely(!qp || !req || !count)) @@ -488,11 +488,14 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count) return -WD_EBUSY; } - send_num = expect > free_num ? free_num : expect; + if (expect > free_num) { + pthread_spin_unlock(&q_info->sd_lock); + return -WD_EBUSY; + } tail = q_info->sq_tail_index; - hisi_qm_fill_sqe(req, q_info, tail, send_num); - tail = (tail + send_num) % q_info->sq_depth; + hisi_qm_fill_sqe(req, q_info, tail, expect); + tail = (tail + expect) % q_info->sq_depth; /* * Before sending doorbell, check the queue status, @@ -510,9 +513,9 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count) q_info->sq_tail_index = tail; /* Make sure used_num is changed before the next thread gets free sqe. */ - __atomic_add_fetch(&q_info->used_num, send_num, __ATOMIC_RELAXED); + __atomic_add_fetch(&q_info->used_num, expect, __ATOMIC_RELAXED); pthread_spin_unlock(&q_info->sd_lock); - *count = send_num; + *count = expect; return 0; } diff --git a/include/drv/wd_dh_drv.h b/include/drv/wd_dh_drv.h index d205dc4..d2a6157 100644 --- a/include/drv/wd_dh_drv.h +++ b/include/drv/wd_dh_drv.h @@ -16,12 +16,15 @@ extern "C" { /* DH message format */ struct wd_dh_msg { struct wd_dh_req req; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; __u32 tag; /* User-defined request identifier */ void *g; __u16 gbytes; __u16 key_bytes; /* Input key bytes */ __u8 is_g2; __u8 result; /* Data format, denoted by WD error code */ + __u8 *rsv_out; /* reserved output data pointer */ }; struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag); diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index 6193c8b..b123a9b 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -48,6 +48,8 @@ extern "C" { /* ECC message format */ struct wd_ecc_msg { struct wd_ecc_req req; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; struct wd_hash_mt hash; __u32 tag; /* User-defined request identifier */ __u8 *key; /* Input key VA, should be DMA buffer */ @@ -55,6 +57,7 @@ struct wd_ecc_msg { __u8 curve_id; /* Ec curve denoted by enum wd_ecc_curve_type */ __u8 result; /* alg op error code */ void *drv_cfg; /* internal driver configuration */ + __u8 *rsv_out; /* reserved output data pointer */ }; struct wd_ecc_pubkey { diff --git a/include/drv/wd_rsa_drv.h b/include/drv/wd_rsa_drv.h index d231ecf..c12f3e0 100644 --- a/include/drv/wd_rsa_drv.h +++ b/include/drv/wd_rsa_drv.h @@ -42,11 +42,14 @@ struct wd_rsa_kg_out { /* RSA message format */ struct wd_rsa_msg { struct wd_rsa_req req; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; __u32 tag; /* User-defined request identifier */ __u16 key_bytes; /* Input key bytes */ __u8 key_type; /* Denoted by enum wd_rsa_key_type */ __u8 result; /* Data format, denoted by WD error code */ __u8 *key; /* Input key VA pointer, should be DMA buffer */ + __u8 *rsv_out; /* reserved output data pointer */ }; struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag); diff --git a/include/wd_dh.h b/include/wd_dh.h index afc2f7c..235c602 100644 --- a/include/wd_dh.h +++ b/include/wd_dh.h @@ -27,6 +27,8 @@ struct wd_dh_sess_setup { __u16 key_bits; /* DH key bites */ bool is_g2; /* is g2 mode or not */ void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_dh_req { diff --git a/include/wd_ecc.h b/include/wd_ecc.h index 6f670e2..18c1c0d 100644 --- a/include/wd_ecc.h +++ b/include/wd_ecc.h @@ -116,6 +116,8 @@ struct wd_ecc_sess_setup { struct wd_rand_mt rand; /* rand method from user */ struct wd_hash_mt hash; /* hash method from user */ void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_ecc_req { diff --git a/include/wd_rsa.h b/include/wd_rsa.h index 2f4e589..9c91432 100644 --- a/include/wd_rsa.h +++ b/include/wd_rsa.h @@ -60,6 +60,8 @@ struct wd_rsa_sess_setup { __u16 key_bits; /* RSA key bits */ bool is_crt; /* CRT mode or not */ void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; bool wd_rsa_is_crt(handle_t sess); diff --git a/wd_dh.c b/wd_dh.c index 221322f..0c1372c 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -26,6 +26,8 @@ struct wd_dh_sess { struct wd_dtb g; struct wd_dh_sess_setup setup; void *sched_key; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; static struct wd_dh_setting { @@ -326,6 +328,8 @@ static int fill_dh_msg(struct wd_dh_msg *msg, struct wd_dh_req *req, memcpy(&msg->req, req, sizeof(*req)); msg->result = WD_EINVAL; msg->key_bytes = sess->key_size; + msg->mm_ops = &sess->mm_ops; + msg->mm_type = sess->mm_type; if (unlikely(req->pri_bytes < sess->key_size)) { WD_ERR("invalid: req pri bytes %hu is error!\n", req->pri_bytes); @@ -581,6 +585,7 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g) handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) { struct wd_dh_sess *sess; + int ret; if (!setup) { WD_ERR("invalid: alloc dh sess setup NULL!\n"); @@ -606,10 +611,19 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT; - sess->g.data = malloc(sess->key_size); - if (!sess->g.data) + ret = wd_mem_ops_init(wd_dh_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); goto sess_err; + } + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + sess->mm_type = setup->mm_type; + sess->g.data = sess->mm_ops.alloc(sess->mm_ops.usr, sess->key_size); + if (!sess->g.data) { + WD_ERR("failed to malloc sess g param memory!\n"); + goto sess_err; + } sess->g.bsize = sess->key_size; /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_dh_setting.sched.sched_init( @@ -622,7 +636,7 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) return (handle_t)sess; sched_err: - free(sess->g.data); + sess->mm_ops.free(sess->mm_ops.usr, sess->g.data); sess_err: free(sess); return (handle_t)0; @@ -638,7 +652,7 @@ void wd_dh_free_sess(handle_t sess) } if (sess_t->g.data) - free(sess_t->g.data); + sess_t->mm_ops.free(sess_t->mm_ops.usr, sess_t->g.data); if (sess_t->sched_key) free(sess_t->sched_key); diff --git a/wd_ecc.c b/wd_ecc.c index b1971b9..2c88d0a 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -52,6 +52,8 @@ struct wd_ecc_sess { struct wd_ecc_sess_setup setup; struct wd_ecc_extend_ops eops; void *sched_key; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_ecc_curve_list { @@ -494,7 +496,7 @@ static void release_ecc_prikey(struct wd_ecc_sess *sess) struct wd_ecc_prikey *prikey = sess->key.prikey; wd_memset_zero(prikey->data, prikey->size); - free(prikey->data); + sess->mm_ops.free(sess->mm_ops.usr, prikey->data); free(prikey); sess->key.prikey = NULL; } @@ -503,7 +505,7 @@ static void release_ecc_pubkey(struct wd_ecc_sess *sess) { struct wd_ecc_pubkey *pubkey = sess->key.pubkey; - free(pubkey->data); + sess->mm_ops.free(sess->mm_ops.usr, pubkey->data); free(pubkey); sess->key.pubkey = NULL; } @@ -522,7 +524,7 @@ static struct wd_ecc_prikey *create_ecc_prikey(struct wd_ecc_sess *sess) } dsz = ECC_PRIKEY_SZ(hsz); - data = malloc(dsz); + data = sess->mm_ops.alloc(sess->mm_ops.usr, dsz); if (!data) { WD_ERR("failed to malloc prikey data, sz = %u!\n", dsz); free(prikey); @@ -551,7 +553,7 @@ static struct wd_ecc_pubkey *create_ecc_pubkey(struct wd_ecc_sess *sess) } dsz = ECC_PUBKEY_SZ(hsz); - data = malloc(dsz); + data = sess->mm_ops.alloc(sess->mm_ops.usr, dsz); if (!data) { WD_ERR("failed to malloc pubkey data, sz = %u!\n", dsz); free(pubkey); @@ -570,7 +572,7 @@ static void release_ecc_in(struct wd_ecc_sess *sess, struct wd_ecc_in *ecc_in) { wd_memset_zero(ecc_in->data, ecc_in->size); - free(ecc_in); + sess->mm_ops.free(sess->mm_ops.usr, ecc_in); } static struct wd_ecc_in *create_ecc_in(struct wd_ecc_sess *sess, __u32 num) @@ -585,7 +587,7 @@ static struct wd_ecc_in *create_ecc_in(struct wd_ecc_sess *sess, __u32 num) hsz = get_key_bsz(sess->key_size); len = sizeof(struct wd_ecc_in) + hsz * num; - in = malloc(len); + in = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!in) { WD_ERR("failed to malloc ecc in, sz = %u!\n", len); return NULL; @@ -613,7 +615,7 @@ static struct wd_ecc_in *create_sm2_sign_in(struct wd_ecc_sess *sess, len = sizeof(struct wd_ecc_in) + ECC_SIGN_IN_PARAM_NUM * ksz + m_len; - in = malloc(len); + in = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!in) { WD_ERR("failed to malloc sm2 sign in, sz = %llu!\n", len); return NULL; @@ -653,7 +655,7 @@ static struct wd_ecc_in *create_sm2_enc_in(struct wd_ecc_sess *sess, } len = sizeof(struct wd_ecc_in) + ksz + m_len; - in = malloc(len); + in = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!in) { WD_ERR("failed to malloc sm2 enc in, sz = %llu!\n", len); return NULL; @@ -697,7 +699,7 @@ static void *create_sm2_ciphertext(struct wd_ecc_sess *sess, __u32 m_len, *len = (__u64)st_sz + ECC_POINT_PARAM_NUM * (__u64)sess->key_size + (__u64)m_len + (__u64)h_byts; - start = malloc(*len); + start = sess->mm_ops.alloc(sess->mm_ops.usr, *len); if (unlikely(!start)) { WD_ERR("failed to alloc start, sz = %llu!\n", *len); return NULL; @@ -745,7 +747,7 @@ static struct wd_ecc_out *create_ecc_out(struct wd_ecc_sess *sess, __u32 num) hsz = get_key_bsz(sess->key_size); len = sizeof(struct wd_ecc_out) + hsz * num; - out = malloc(len); + out = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!out) { WD_ERR("failed to malloc out, sz = %u!\n", len); return NULL; @@ -1149,13 +1151,13 @@ static void del_sess_key(struct wd_ecc_sess *sess) { if (sess->key.prikey) { wd_memset_zero(sess->key.prikey->data, sess->key.prikey->size); - free(sess->key.prikey->data); + sess->mm_ops.free(sess->mm_ops.usr, sess->key.prikey->data); free(sess->key.prikey); sess->key.prikey = NULL; } if (sess->key.pubkey) { - free(sess->key.pubkey->data); + sess->mm_ops.free(sess->mm_ops.usr, sess->key.pubkey->data); free(sess->key.pubkey); sess->key.pubkey = NULL; } @@ -1227,6 +1229,15 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = BITS_TO_BYTES(setup->key_bits); + /* Memory type set */ + ret = wd_mem_ops_init(wd_ecc_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); + goto sess_err; + } + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + sess->mm_type = setup->mm_type; + if (wd_ecc_setting.driver->get_extend_ops) { ret = wd_ecc_setting.driver->get_extend_ops(&sess->eops); if (ret) { @@ -1508,9 +1519,10 @@ void wd_ecxdh_get_out_params(struct wd_ecc_out *out, struct wd_ecc_point **pbk) void wd_ecc_del_in(handle_t sess, struct wd_ecc_in *in) { + struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess; __u32 bsz; - if (!in) { + if (!sess_t || !in) { WD_ERR("invalid: del ecc in parameter error!\n"); return; } @@ -1522,14 +1534,15 @@ void wd_ecc_del_in(handle_t sess, struct wd_ecc_in *in) } wd_memset_zero(in->data, bsz); - free(in); + sess_t->mm_ops.free(sess_t->mm_ops.usr, in); } -void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out) +void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out) { + struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess; __u32 bsz; - if (!out) { + if (!sess_t || !out) { WD_ERR("invalid: del ecc out parameter error!\n"); return; } @@ -1541,7 +1554,7 @@ void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out) } wd_memset_zero(out->data, bsz); - free(out); + sess_t->mm_ops.free(sess_t->mm_ops.usr, out); } static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req, @@ -1551,6 +1564,8 @@ static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req, memcpy(&msg->req, req, sizeof(msg->req)); memcpy(&msg->hash, &sess->setup.hash, sizeof(msg->hash)); + msg->mm_ops = &sess->mm_ops; + msg->mm_type = sess->mm_type; msg->key_bytes = sess->key_size; msg->curve_id = sess->setup.cv.cfg.id; msg->drv_cfg = sess->eops.params; @@ -1922,7 +1937,7 @@ static struct wd_ecc_in *create_sm2_verf_in(struct wd_ecc_sess *sess, hsz = get_key_bsz(sess->key_size); len = sizeof(struct wd_ecc_in) + ECC_VERF_IN_PARAM_NUM * hsz + m_len; - in = malloc(len); + in = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!in) { WD_ERR("failed to malloc sm2 verf in, sz = %llu!\n", len); return NULL; @@ -2211,7 +2226,7 @@ struct wd_ecc_out *wd_sm2_new_dec_out(handle_t sess, __u32 plaintext_len) } len = sizeof(*ecc_out) + plaintext_len; - ecc_out = malloc(len); + ecc_out = sess_t->mm_ops.alloc(sess_t->mm_ops.usr, len); if (!ecc_out) { WD_ERR("failed to malloc ecc_out, sz = %llu!\n", len); return NULL; diff --git a/wd_rsa.c b/wd_rsa.c index cf9239c..bc78c6a 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -67,6 +67,8 @@ struct wd_rsa_sess { struct wd_rsa_prikey *prikey; struct wd_rsa_sess_setup setup; void *sched_key; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; static struct wd_rsa_setting { @@ -373,6 +375,8 @@ static int fill_rsa_msg(struct wd_rsa_msg *msg, struct wd_rsa_req *req, memcpy(&msg->req, req, sizeof(*req)); msg->key_bytes = sess->key_size; msg->result = WD_EINVAL; + msg->mm_ops = &sess->mm_ops; + msg->mm_type = sess->mm_type; switch (msg->req.op_type) { case WD_RSA_SIGN: @@ -641,7 +645,7 @@ struct wd_rsa_kg_in *wd_rsa_new_kg_in(handle_t sess, struct wd_dtb *e, } kg_in_size = (int)GEN_PARAMS_SZ(c->key_size); - kg_in = malloc(kg_in_size + sizeof(*kg_in)); + kg_in = c->mm_ops.alloc(c->mm_ops.usr, kg_in_size + sizeof(*kg_in)); if (!kg_in) { WD_ERR("failed to malloc kg_in memory!\n"); return NULL; @@ -681,19 +685,16 @@ void wd_rsa_get_kg_in_params(struct wd_rsa_kg_in *kin, struct wd_dtb *e, p->data = (void *)kin->p; } -static void del_kg(void *k) +void wd_rsa_del_kg_in(handle_t sess, struct wd_rsa_kg_in *ki) { - if (!k) { + struct wd_rsa_sess *c = (struct wd_rsa_sess *)sess; + + if (!c || !ki) { WD_ERR("invalid: del key generate params err!\n"); return; } - free(k); -} - -void wd_rsa_del_kg_in(handle_t sess, struct wd_rsa_kg_in *ki) -{ - del_kg(ki); + c->mm_ops.free(c->mm_ops.usr, ki); } struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess) @@ -719,7 +720,7 @@ struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess) else kg_out_size = (int)GEN_PARAMS_SZ(c->key_size); - kg_out = malloc(kg_out_size + sizeof(*kg_out)); + kg_out = c->mm_ops.alloc(c->mm_ops.usr, kg_out_size + sizeof(*kg_out)); if (!kg_out) { WD_ERR("failed to malloc kg_out memory!\n"); return NULL; @@ -741,13 +742,15 @@ struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess) void wd_rsa_del_kg_out(handle_t sess, struct wd_rsa_kg_out *kout) { - if (!kout) { + struct wd_rsa_sess *c = (struct wd_rsa_sess *)sess; + + if (!c || !kout) { WD_ERR("invalid: param null at del kg out!\n"); return; } wd_memset_zero(kout->data, kout->size); - del_kg(kout); + c->mm_ops.free(c->mm_ops.usr, kout); } void wd_rsa_get_kg_out_params(struct wd_rsa_kg_out *kout, struct wd_dtb *d, @@ -804,6 +807,11 @@ void wd_rsa_set_kg_out_crt_psz(struct wd_rsa_kg_out *kout, size_t dq_sz, size_t dp_sz) { + if (!kout) { + WD_ERR("invalid: input null when set kg out crt psz!\n"); + return; + } + kout->qinvbytes = qinv_sz; kout->dqbytes = dq_sz; kout->dpbytes = dp_sz; @@ -813,6 +821,11 @@ void wd_rsa_set_kg_out_psz(struct wd_rsa_kg_out *kout, size_t d_sz, size_t n_sz) { + if (!kout) { + WD_ERR("invalid: input null when set kg out psz!\n"); + return; + } + kout->dbytes = d_sz; kout->nbytes = n_sz; } @@ -862,7 +875,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup, if (setup->is_crt) { len = sizeof(struct wd_rsa_prikey) + (int)CRT_PARAMS_SZ(sess->key_size); - sess->prikey = malloc(len); + sess->prikey = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!sess->prikey) { WD_ERR("failed to alloc sess prikey2!\n"); return -WD_ENOMEM; @@ -873,7 +886,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup, } else { len = sizeof(struct wd_rsa_prikey) + (int)GEN_PARAMS_SZ(sess->key_size); - sess->prikey = malloc(len); + sess->prikey = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!sess->prikey) { WD_ERR("failed to alloc sess prikey1!\n"); return -WD_ENOMEM; @@ -885,7 +898,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup, len = sizeof(struct wd_rsa_pubkey) + (int)GEN_PARAMS_SZ(sess->key_size); - sess->pubkey = malloc(len); + sess->pubkey = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!sess->pubkey) { free(sess->prikey); WD_ERR("failed to alloc sess pubkey!\n"); @@ -912,8 +925,8 @@ static void del_sess_key(struct wd_rsa_sess *sess) wd_memset_zero(prk->pkey.pkey2.data, CRT_PARAMS_SZ(sess->key_size)); else wd_memset_zero(prk->pkey.pkey1.data, GEN_PARAMS_SZ(sess->key_size)); - free(sess->prikey); - free(sess->pubkey); + sess->mm_ops.free(sess->mm_ops.usr, sess->prikey); + sess->mm_ops.free(sess->mm_ops.usr, sess->pubkey); } static void del_sess(struct wd_rsa_sess *c) @@ -948,6 +961,15 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT; + /* Memory type set */ + ret = wd_mem_ops_init(wd_rsa_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); + goto sess_err; + } + + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + sess->mm_type = setup->mm_type; ret = create_sess_key(setup, sess); if (ret) { WD_ERR("failed to create rsa sess keys!\n"); -- 2.33.0
From: Wenkai Lin <linwenkai6@hisilicon.com> 1. Remove lock from hashjoin ctx. 2. Avoid to causes numeric value overflow. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_dae_join_gather.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/drv/hisi_dae_join_gather.c b/drv/hisi_dae_join_gather.c index 9b9c07d..63c7670 100644 --- a/drv/hisi_dae_join_gather.c +++ b/drv/hisi_dae_join_gather.c @@ -86,7 +86,6 @@ struct join_gather_ctx { struct join_gather_col_data cols_data; struct hash_table_data table_data; struct hash_table_data rehash_table; - pthread_spinlock_t lock; __u32 hash_table_row_size; __u32 batch_row_size[DAE_MAX_TABLE_NUM]; }; @@ -365,7 +364,8 @@ static int check_join_gather_param(struct wd_join_gather_msg *msg) { struct wd_probe_out_info *output; struct wd_gather_req *greq; - __u64 row_num, size; + __u32 row_num; + __u64 size; if (!msg) { WD_ERR("invalid: input join gather msg is NULL!\n"); @@ -398,10 +398,10 @@ static int check_join_gather_param(struct wd_join_gather_msg *msg) return -WD_EINVAL; } if (msg->index_type == WD_BATCH_ADDR_INDEX) { - row_num = msg->req.output_row_num << DAE_ADDR_INDEX_SHIFT; - if (output->build_index.row_num < row_num) { - WD_ERR("build index row number is less than: %llu\n", - row_num); + row_num = output->build_index.row_num >> DAE_ADDR_INDEX_SHIFT; + if (row_num < msg->req.output_row_num) { + WD_ERR("build index row number(%u) is less than needed\n", + output->build_index.row_num); return -WD_EINVAL; } } @@ -438,10 +438,10 @@ static int check_join_gather_param(struct wd_join_gather_msg *msg) return -WD_EINVAL; } } else { - row_num = msg->req.output_row_num << DAE_ADDR_INDEX_SHIFT; - if (greq->index.row_num < row_num) { - WD_ERR("build index row number is less than: %llu\n", - row_num); + row_num = greq->index.row_num >> DAE_ADDR_INDEX_SHIFT; + if (row_num < msg->req.output_row_num) { + WD_ERR("build index row number(%u) is less than needed\n", + greq->index.row_num); return -WD_EINVAL; } } @@ -898,7 +898,6 @@ static void join_gather_sess_priv_uninit(struct wd_alg_driver *drv, void *priv) return; } - pthread_spin_destroy(&ctx->lock); free(ctx); } @@ -930,10 +929,6 @@ static int join_gather_sess_priv_init(struct wd_alg_driver *drv, if (ret) goto free_ctx; - ret = pthread_spin_init(&ctx->lock, PTHREAD_PROCESS_SHARED); - if (ret) - goto free_ctx; - *priv = ctx; return WD_SUCCESS; -- 2.33.0
From: Chenghai Huang <huangchenghai2@huawei.com> The current UADK v2 supports the nosav feature, which allows business operations to be executed when the driver is loaded with uacce_mode=2. However, it requires using the reserved memory allocated through uacce. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_comp.c | 261 +++++++++++++++++++++++++------------- include/drv/wd_comp_drv.h | 13 ++ include/wd_comp.h | 2 + wd_comp.c | 43 ++++++- 4 files changed, 228 insertions(+), 91 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 2ad4c00..65dcf8e 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -77,6 +77,7 @@ #define max_in_data_size(outl) ((__u32)(((__u64)(outl) << 3) / 9) & 0xfffffffc) #define HZ_MAX_SIZE (8 * 1024 * 1024) +#define HW_CTX_SIZE 0x10000 #define RSV_OFFSET 64 #define CTX_DW1_OFFSET 4 @@ -244,15 +245,6 @@ struct hisi_zip_ctx { struct wd_ctx_config_internal config; }; -struct comp_sgl { - void *in; - void *out; - void *out_seq; - struct wd_datalist *list_src; - struct wd_datalist *list_dst; - struct wd_datalist *seq_start; -}; - static void dump_zip_msg(struct wd_comp_msg *msg) { WD_ERR("dump zip message after a task error occurs.\n"); @@ -377,11 +369,11 @@ static int check_enable_store_buf(struct wd_comp_msg *msg, __u32 out_size, int h return 0; } -static int get_sgl_from_pool(handle_t h_qp, struct comp_sgl *c_sgl) +static int get_sgl_from_pool(handle_t h_qp, struct comp_sgl *c_sgl, struct wd_mm_ops *mm_ops) { handle_t h_sgl_pool; - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, mm_ops); if (unlikely(!h_sgl_pool)) { WD_ERR("failed to get sglpool!\n"); return -WD_EINVAL; @@ -416,6 +408,23 @@ err_free_sgl_in: return -WD_ENOMEM; } +static void free_hw_sgl(handle_t h_qp, struct comp_sgl *c_sgl, struct wd_mm_ops *mm_ops) +{ + handle_t h_sgl_pool; + + h_sgl_pool = hisi_qm_get_sglpool(h_qp, mm_ops); + if (unlikely(!h_sgl_pool)) { + WD_ERR("failed to get sglpool!\n"); + return; + } + + hisi_qm_put_hw_sgl(h_sgl_pool, c_sgl->in); + hisi_qm_put_hw_sgl(h_sgl_pool, c_sgl->out); + + if (c_sgl->seq_start) + hisi_qm_put_hw_sgl(h_sgl_pool, c_sgl->out_seq); +} + static void fill_comp_buf_size(struct hisi_zip_sqe *sqe, __u32 in_size, __u32 out_size) { @@ -423,8 +432,124 @@ static void fill_comp_buf_size(struct hisi_zip_sqe *sqe, __u32 in_size, sqe->dest_avail_out = out_size; } -static void fill_buf_addr(struct hisi_zip_sqe *sqe, struct hisi_comp_sqe_addr *addr) +static int zip_mem_map(struct wd_mm_ops *mm_ops, struct hisi_zip_sqe *sqe, + struct hisi_comp_sqe_addr *addr) { + void *phy_dst, *phy_ctx, *mempool; + void *phy_lit = NULL; + void *phy_src = NULL; + + mempool = mm_ops->usr; + /* When the src len is 0, map is not required */ + if (sqe->input_data_length) { + phy_src = mm_ops->iova_map(mempool, addr->src_addr, sqe->input_data_length); + if (!phy_src) { + WD_ERR("get zip src dma address fail!\n"); + return -WD_ENOMEM; + } + + sqe->source_addr_l = lower_32_bits(phy_src); + sqe->source_addr_h = upper_32_bits(phy_src); + } + + if (addr->lit_addr) { + phy_lit = mm_ops->iova_map(mempool, addr->lit_addr, sqe->dw13); + if (!phy_lit) { + WD_ERR("get zip lits dma address fail!\n"); + goto unmap_src; + } + + sqe->literals_addr_l = lower_32_bits(phy_lit); + sqe->literals_addr_h = upper_32_bits(phy_lit); + } + + phy_dst = mm_ops->iova_map(mempool, addr->dst_addr, sqe->dest_avail_out); + if (!phy_dst) { + WD_ERR("get zip dst dma address fail!\n"); + goto unmap_lit; + } + + sqe->dest_addr_l = lower_32_bits(phy_dst); + sqe->dest_addr_h = upper_32_bits(phy_dst); + + /* The source addr needs to be filled with a valid address when length is 0 */ + if (!sqe->input_data_length) { + sqe->source_addr_l = sqe->dest_addr_l; + sqe->source_addr_h = sqe->dest_addr_h; + } + + if (addr->ctx_addr) { + phy_ctx = mm_ops->iova_map(mempool, addr->ctx_addr, HW_CTX_SIZE); + if (!phy_ctx) { + WD_ERR("get zip ctx dma address fail!\n"); + goto unmap_dst; + } + + sqe->stream_ctx_addr_l = lower_32_bits(phy_ctx); + sqe->stream_ctx_addr_h = upper_32_bits(phy_ctx); + } + + return 0; + +unmap_dst: + mm_ops->iova_unmap(mempool, addr->dst_addr, phy_dst, sqe->dest_avail_out); +unmap_lit: + if (addr->lit_addr) + mm_ops->iova_unmap(mempool, addr->lit_addr, phy_lit, sqe->dw13); +unmap_src: + if (sqe->input_data_length) + mm_ops->iova_unmap(mempool, addr->src_addr, phy_src, sqe->input_data_length); + return -WD_ENOMEM; +} + +static void zip_mem_unmap(struct wd_comp_msg *msg, struct hisi_zip_sqe *sqe) +{ + void *dma_addr, *src_addr, *seq_addr, *lit_addr; + struct wd_mm_ops *mm_ops = msg->mm_ops; + struct wd_comp_req *req = &msg->req; + void *mempool = mm_ops->usr; + + if (msg->data_fmt == WD_SGL_BUF) + src_addr = msg->c_sgl.in; + else + src_addr = req->src; + + if (sqe->input_data_length) { + dma_addr = VA_ADDR(sqe->source_addr_h, sqe->source_addr_l); + mm_ops->iova_unmap(mempool, src_addr, dma_addr, sqe->input_data_length); + } + + if (msg->alg_type == WD_LZ77_ZSTD || msg->alg_type == WD_LZ77_ONLY) { + if (msg->data_fmt == WD_SGL_BUF) { + seq_addr = msg->c_sgl.out_seq; + lit_addr = msg->c_sgl.out; + } else { + seq_addr = req->dst + sqe->dw13; + lit_addr = req->dst; + } + + dma_addr = VA_ADDR(sqe->literals_addr_h, sqe->literals_addr_l); + mm_ops->iova_unmap(mempool, lit_addr, dma_addr, sqe->dw13); + dma_addr = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); + mm_ops->iova_unmap(mempool, seq_addr, dma_addr, sqe->dest_avail_out); + } else { + dma_addr = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); + mm_ops->iova_unmap(mempool, req->dst, dma_addr, sqe->dest_avail_out); + } + + if (msg->stream_mode == WD_COMP_STATEFUL) { + dma_addr = VA_ADDR(sqe->stream_ctx_addr_h, sqe->stream_ctx_addr_l); + mm_ops->iova_unmap(mempool, msg->ctx_buf, dma_addr, HW_CTX_SIZE); + } +} + +static int fill_buf_addr(struct hisi_zip_sqe *sqe, struct hisi_comp_sqe_addr *addr, + struct wd_mm_ops *mm_ops) +{ + /* No-SVA mode and Memory is USER mode or PROXY mode */ + if (!mm_ops->sva_mode) + return zip_mem_map(mm_ops, sqe, addr); + sqe->source_addr_l = lower_32_bits(addr->src_addr); sqe->source_addr_h = upper_32_bits(addr->src_addr); sqe->dest_addr_l = lower_32_bits(addr->dst_addr); @@ -439,6 +564,8 @@ static void fill_buf_addr(struct hisi_zip_sqe *sqe, struct hisi_comp_sqe_addr *a sqe->stream_ctx_addr_l = lower_32_bits(addr->ctx_addr); sqe->stream_ctx_addr_h = upper_32_bits(addr->ctx_addr); } + + return 0; } static int fill_buf_deflate_generic(struct hisi_zip_sqe *sqe, @@ -498,15 +625,13 @@ static int fill_buf_deflate_generic(struct hisi_zip_sqe *sqe, fill_comp_buf_size(sqe, in_size, out_size); - if (msg->ctx_buf) + if (msg->stream_mode == WD_COMP_STATEFUL) ctx_buf = msg->ctx_buf + RSV_OFFSET; addr.src_addr = src; addr.dst_addr = dst; addr.ctx_addr = ctx_buf; - fill_buf_addr(sqe, &addr); - - return 0; + return fill_buf_addr(sqe, &addr, msg->mm_ops); } static int fill_buf_deflate(handle_t h_qp, struct hisi_zip_sqe *sqe, @@ -570,9 +695,7 @@ static int fill_buf_lz4(handle_t h_qp, struct hisi_zip_sqe *sqe, addr.src_addr = src; addr.dst_addr = dst; - fill_buf_addr(sqe, &addr); - - return 0; + return fill_buf_addr(sqe, &addr, msg->mm_ops); } static void fill_buf_type_sgl(struct hisi_zip_sqe *sqe) @@ -585,25 +708,28 @@ static void fill_buf_type_sgl(struct hisi_zip_sqe *sqe) } static int fill_buf_addr_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, - struct wd_datalist *list_src, struct wd_datalist *list_dst) + struct comp_sgl *c_sgl, struct wd_mm_ops *mm_ops) { struct hisi_comp_sqe_addr addr = {0}; - struct comp_sgl c_sgl; int ret; - c_sgl.list_src = list_src; - c_sgl.list_dst = list_dst; - c_sgl.seq_start = NULL; - - ret = get_sgl_from_pool(h_qp, &c_sgl); + ret = get_sgl_from_pool(h_qp, c_sgl, mm_ops); if (unlikely(ret)) return ret; - addr.src_addr = c_sgl.in; - addr.dst_addr = c_sgl.out; - fill_buf_addr(sqe, &addr); + addr.src_addr = c_sgl->in; + if (c_sgl->seq_start) { + addr.lit_addr = c_sgl->out; + addr.dst_addr = c_sgl->out_seq; + } else { + addr.dst_addr = c_sgl->out; + } + + ret = fill_buf_addr(sqe, &addr, mm_ops); + if (unlikely(ret)) + free_hw_sgl(h_qp, c_sgl, mm_ops); - return 0; + return ret; } static void fill_buf_sgl_skip(struct hisi_zip_sqe *sqe, __u32 src_skip, @@ -625,7 +751,6 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, int head_size) { struct wd_comp_req *req = &msg->req; - struct wd_datalist *list_src = req->list_src; struct wd_datalist *list_dst = req->list_dst; __u32 out_size = msg->avail_out; __u32 in_size = req->src_len; @@ -655,10 +780,6 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_buf_type_sgl(sqe); - ret = fill_buf_addr_sgl(h_qp, sqe, list_src, list_dst); - if (unlikely(ret)) - return ret; - if (head != NULL && msg->req.op_type == WD_DIR_COMPRESS) { memcpy(req->list_dst->data, head, head_size); dst_skip = head_size; @@ -682,7 +803,10 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_comp_buf_size(sqe, in_size, out_size); - return 0; + msg->c_sgl.list_src = req->list_src; + msg->c_sgl.list_dst = list_dst; + msg->c_sgl.seq_start = NULL; + return fill_buf_addr_sgl(h_qp, sqe, &msg->c_sgl, msg->mm_ops); } static int fill_buf_deflate_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, @@ -827,7 +951,7 @@ static int fill_buf_lz77_zstd(handle_t h_qp, struct hisi_zip_sqe *sqe, if (unlikely(seq_avail_out > HZ_MAX_SIZE)) seq_avail_out = HZ_MAX_SIZE; - if (msg->ctx_buf) { + if (msg->stream_mode == WD_COMP_STATEFUL) { ctx_buf = msg->ctx_buf + RSV_OFFSET; if (msg->alg_type == WD_LZ77_ZSTD) { if (data->blk_type != COMP_BLK) @@ -852,9 +976,7 @@ static int fill_buf_lz77_zstd(handle_t h_qp, struct hisi_zip_sqe *sqe, addr.dst_addr = req->dst + lits_size; addr.lit_addr = req->dst; addr.ctx_addr = ctx_buf; - fill_buf_addr(sqe, &addr); - - return 0; + return fill_buf_addr(sqe, &addr, msg->mm_ops); } static struct wd_datalist *get_seq_start_list(struct wd_comp_req *req) @@ -972,13 +1094,11 @@ static int lz77_buf_check_sgl(struct wd_comp_msg *msg, __u32 lits_size) static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, struct wd_comp_msg *msg) { - struct hisi_comp_sqe_addr addr = {0}; struct wd_comp_req *req = &msg->req; struct wd_lz77_zstd_data *data = req->priv; __u32 in_size = msg->req.src_len; __u32 out_size = msg->avail_out; struct wd_datalist *seq_start; - struct comp_sgl c_sgl; __u32 lits_size; int ret; @@ -1004,27 +1124,15 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_buf_size_lz77_zstd(sqe, in_size, lits_size, out_size - lits_size); - c_sgl.list_src = req->list_src; - c_sgl.list_dst = req->list_dst; - c_sgl.seq_start = seq_start; - - ret = get_sgl_from_pool(h_qp, &c_sgl); - if (unlikely(ret)) - return ret; - - addr.src_addr = c_sgl.in; - addr.dst_addr = c_sgl.out_seq; - addr.lit_addr = c_sgl.out; - fill_buf_addr(sqe, &addr); - - return 0; + msg->c_sgl.list_src = req->list_src; + msg->c_sgl.list_dst = req->list_dst; + msg->c_sgl.seq_start = seq_start; + return fill_buf_addr_sgl(h_qp, sqe, &msg->c_sgl, msg->mm_ops); } static int fill_buf_lz4_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, struct wd_comp_msg *msg) { - struct wd_datalist *list_src = msg->req.list_src; - struct wd_datalist *list_dst = msg->req.list_dst; int ret; ret = check_lz4_msg(msg, WD_SGL_BUF); @@ -1035,7 +1143,10 @@ static int fill_buf_lz4_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_comp_buf_size(sqe, msg->req.src_len, msg->avail_out); - return fill_buf_addr_sgl(h_qp, sqe, list_src, list_dst); + msg->c_sgl.list_src = msg->req.list_src; + msg->c_sgl.list_dst = msg->req.list_dst; + msg->c_sgl.seq_start = NULL; + return fill_buf_addr_sgl(h_qp, sqe, &msg->c_sgl, msg->mm_ops); } static void fill_sqe_type_v1(struct hisi_zip_sqe *sqe) @@ -1420,31 +1531,6 @@ static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, return 0; } -static void free_hw_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, - enum wd_comp_alg_type alg_type) -{ - void *hw_sgl_in, *hw_sgl_out; - handle_t h_sgl_pool; - - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); - if (unlikely(!h_sgl_pool)) { - WD_ERR("failed to get sglpool to free hw sgl!\n"); - return; - } - - hw_sgl_in = VA_ADDR(sqe->source_addr_h, sqe->source_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in); - - hw_sgl_out = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); - - if (alg_type == WD_LZ77_ZSTD || alg_type == WD_LZ77_ONLY) { - hw_sgl_out = VA_ADDR(sqe->literals_addr_h, - sqe->literals_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); - } -} - static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); @@ -1468,7 +1554,7 @@ static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *com ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (unlikely(ret < 0)) { if (msg->req.data_fmt == WD_SGL_BUF) - free_hw_sgl(h_qp, &sqe, msg->alg_type); + free_hw_sgl(h_qp, &msg->c_sgl, msg->mm_ops); if (ret != -WD_EBUSY) WD_ERR("failed to send to hardware, ret = %d!\n", ret); @@ -1620,8 +1706,11 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, recv_msg->checksum = sqe->dw31; recv_msg->alg_type = alg_type; + if (recv_msg->mm_ops && !recv_msg->mm_ops->sva_mode) + zip_mem_unmap(recv_msg, sqe); + if (buf_type == WD_SGL_BUF) - free_hw_sgl((handle_t)qp, sqe, alg_type); + free_hw_sgl((handle_t)qp, &recv_msg->c_sgl, recv_msg->mm_ops); if (unlikely(recv_msg->req.status == WD_IN_EPARA)) dump_zip_msg(recv_msg); diff --git a/include/drv/wd_comp_drv.h b/include/drv/wd_comp_drv.h index 213cf2d..95f7fb7 100644 --- a/include/drv/wd_comp_drv.h +++ b/include/drv/wd_comp_drv.h @@ -24,9 +24,22 @@ enum wd_comp_state { WD_COMP_STATELESS, }; +struct comp_sgl { + void *in; + void *out; + void *out_seq; + struct wd_datalist *list_src; + struct wd_datalist *list_dst; + struct wd_datalist *seq_start; +}; + /* fixme wd_comp_msg */ struct wd_comp_msg { struct wd_comp_req req; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; + /* Store sgl addr for nosva */ + struct comp_sgl c_sgl; /* Denoted HW ctx cache, for stream mode */ void *ctx_buf; /* Denoted by enum wd_comp_alg_type */ diff --git a/include/wd_comp.h b/include/wd_comp.h index 8e056d1..5d09536 100644 --- a/include/wd_comp.h +++ b/include/wd_comp.h @@ -151,6 +151,8 @@ struct wd_comp_sess_setup { enum wd_comp_winsz_type win_sz; /* Denoted by enum wd_comp_winsz_type */ enum wd_comp_op_type op_type; /* Denoted by enum wd_comp_op_type */ void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; /** diff --git a/wd_comp.c b/wd_comp.c index 5e62462..c67b7f1 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -15,7 +15,7 @@ #include "drv/wd_comp_drv.h" #include "wd_comp.h" -#define HW_CTX_SIZE (64 * 1024) +#define HW_CTX_SIZE 0x10000 #define STREAM_CHUNK (128 * 1024) #define WD_ZLIB_HEADER_SZ 2 #define WD_GZIP_HEADER_SZ 10 @@ -41,6 +41,8 @@ struct wd_comp_sess { __u32 checksum; __u8 *ctx_buf; void *sched_key; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_comp_setting { @@ -437,6 +439,24 @@ static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) return WD_SUCCESS; } +static int wd_alloc_ctx_buf(struct wd_mm_ops *mm_ops, struct wd_comp_sess *sess) +{ + + sess->ctx_buf = mm_ops->alloc(mm_ops->usr, HW_CTX_SIZE); + if (!sess->ctx_buf) + return -WD_ENOMEM; + + memset(sess->ctx_buf, 0, HW_CTX_SIZE); + + return WD_SUCCESS; +} + +static void wd_free_ctx_buf(struct wd_mm_ops *mm_ops, struct wd_comp_sess *sess) +{ + mm_ops->free(mm_ops->usr, sess->ctx_buf); + sess->ctx_buf = NULL; +} + handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) { struct wd_comp_sess *sess; @@ -453,8 +473,15 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) if (!sess) return (handle_t)0; - sess->ctx_buf = calloc(1, HW_CTX_SIZE); - if (!sess->ctx_buf) + /* Memory type set */ + ret = wd_mem_ops_init(wd_comp_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); + goto sess_err; + } + + ret = wd_alloc_ctx_buf(&setup->mm_ops, sess); + if (ret) goto sess_err; sess->alg_type = setup->alg_type; @@ -462,6 +489,9 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) sess->win_sz = setup->win_sz; sess->stream_pos = WD_COMP_STREAM_NEW; + sess->mm_type = setup->mm_type; + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_comp_setting.sched.sched_init( wd_comp_setting.sched.h_sched_ctx, setup->sched_param); @@ -473,7 +503,7 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) return (handle_t)sess; sched_err: - free(sess->ctx_buf); + wd_free_ctx_buf(&setup->mm_ops, sess); sess_err: free(sess); return (handle_t)0; @@ -487,7 +517,7 @@ void wd_comp_free_sess(handle_t h_sess) return; if (sess->ctx_buf) - free(sess->ctx_buf); + wd_free_ctx_buf(&sess->mm_ops, sess); if (sess->sched_key) free(sess->sched_key); @@ -522,6 +552,9 @@ static void fill_comp_msg(struct wd_comp_sess *sess, struct wd_comp_msg *msg, msg->win_sz = sess->win_sz; msg->avail_out = req->dst_len; + msg->mm_type = sess->mm_type; + msg->mm_ops = &sess->mm_ops; + msg->req.last = 1; } -- 2.33.0
participants (1)
-
ZongYu Wu