Acc
Threads by month
- ----- 2025 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
November 2025
- 2 participants
- 9 discussions
您好！
sig-AccLib 邀请您参加 2025-11-26 11:00 召开的Zoom会议
会议主题：【2025/11/26 AccLIb SIG 双周例会 11:00 - 12:00】
会议内容：
会议链接：linaro-org.zoom.us/j/91879279131
会议纪要：etherpad.openeuler.org/p/sig-AccLib-meet
会议链接：https://us06web.zoom.us/j/81738189438?pwd=pBbpGzVXKm85xvTcYVseW7cijBuaQp.1
会议纪要：https://etherpad.openeuler.org/p/sig-AccLib-meetings
更多资讯尽在：https://www.openeuler.org/zh/
Hello!
sig-AccLib invites you to attend the Zoom conference will be held at 2025-11-26 11:00,
The subject of the conference is 【2025/11/26 AccLIb SIG 双周例会 11:00 - 12:00】
Summary:
会议链接：linaro-org.zoom.us/j/91879279131
会议纪要：etherpad.openeuler.org/p/sig-AccLib-meet
You can join the meeting at https://us06web.zoom.us/j/81738189438?pwd=pBbpGzVXKm85xvTcYVseW7cijBuaQp.1
Add topics at https://etherpad.openeuler.org/p/sig-AccLib-meetings
More information: https://www.openeuler.org/en/
1
0
18 Nov '25
From: lizhi <lizhi206(a)huawei.com>
The hpre v2 interface needs to support the no-sva business model
within the new evolution framework. This ensures that users
switching to the v2 interface can utilize both the sva mode
and the no-sva mode.
Signed-off-by: lizhi <lizhi206(a)huawei.com>
Signed-off-by: Zongyu Wu <wuzongyu1(a)huawei.com>
---
drv/hisi_comp.c | 3 +-
drv/hisi_hpre.c | 650 ++++++++++++++++++++++++++++-----------
drv/hisi_qm_udrv.c | 15 +-
include/drv/wd_dh_drv.h | 3 +
include/drv/wd_ecc_drv.h | 3 +
include/drv/wd_rsa_drv.h | 3 +
include/wd_dh.h | 2 +
include/wd_ecc.h | 2 +
include/wd_rsa.h | 2 +
wd_dh.c | 22 +-
wd_ecc.c | 53 ++--
wd_rsa.c | 56 +++-
12 files changed, 580 insertions(+), 234 deletions(-)
diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c
index 87d2103..8474e17 100644
--- a/drv/hisi_comp.c
+++ b/drv/hisi_comp.c
@@ -267,7 +267,8 @@ static __u32 copy_to_out(struct wd_comp_msg *msg, struct hisi_comp_buf *buf, __u
struct wd_comp_req *req = &msg->req;
struct wd_datalist *node = req->list_dst;
__u32 sgl_restlen, copy_len;
- __u32 len = 0, sgl_cplen = 0;
+ __u32 sgl_cplen = 0;
+ __u32 len = 0;
copy_len = total_len > req->dst_len ?
req->dst_len : total_len;
diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c
index 7ce842b..f3f2451 100644
--- a/drv/hisi_hpre.c
+++ b/drv/hisi_hpre.c
@@ -34,14 +34,17 @@
#define SM2_PONIT_SIZE 64
#define MAX_HASH_LENS BITS_TO_BYTES(521)
#define HW_PLAINTEXT_BYTES_MAX BITS_TO_BYTES(4096)
-#define HPRE_CTX_Q_NUM_DEF 1
+#define HPRE_CTX_Q_NUM_DEF 1
+#define MAP_PAIR_NUM_MAX 6
#define CRT_PARAMS_SZ(key_size) ((5 * (key_size)) >> 1)
#define CRT_GEN_PARAMS_SZ(key_size) ((7 * (key_size)) >> 1)
#define GEN_PARAMS_SZ(key_size) ((key_size) << 1)
#define CRT_PARAM_SZ(key_size) ((key_size) >> 1)
-#define WD_TRANS_FAIL 0
+#define GEN_PARAMS_SZ_UL(key_size) ((unsigned long)(key_size) << 1)
+#define DMA_ADDR(hi, lo) ((__u64)(((__u64)(hi) << 32) | (__u64)(lo)))
+#define WD_TRANS_FAIL 0
#define CURVE_PARAM_NUM 6
#define SECP256R1_KEY_SIZE 32
@@ -78,6 +81,21 @@ enum hpre_alg_name {
WD_ECC
};
+enum hpre_hw_msg_field {
+ HW_MSG_IN,
+ HW_MSG_OUT,
+ HW_MSG_KEY,
+};
+
+struct map_info_cache {
+ struct map_pair {
+ void *addr;
+ uintptr_t pa;
+ size_t size;
+ } pairs[MAP_PAIR_NUM_MAX];
+ size_t cnt;
+};
+
/* put vendor hardware message as a user interface is not suitable here */
struct hisi_hpre_sqe {
__u32 alg : 5;
@@ -113,12 +131,122 @@ struct hisi_hpre_sqe {
struct hisi_hpre_ctx {
struct wd_ctx_config_internal config;
+ struct wd_mm_ops *mm_ops;
+ handle_t rsv_mem_ctx;
};
struct hpre_ecc_ctx {
__u32 enable_hpcore;
};
+static void add_map_info(struct map_info_cache *cache, void *addr, uintptr_t dma, size_t size)
+{
+ /* The cnt is guaranteed not to exceed MAP_PAIR_NUM_MAX within hpre. */
+ cache->pairs[cache->cnt].addr = addr;
+ cache->pairs[cache->cnt].pa = dma;
+ cache->pairs[cache->cnt].size = size;
+ cache->cnt++;
+}
+
+static void unmap_addr_in_cache(struct wd_mm_ops *mm_ops, struct map_info_cache *cache)
+{
+ size_t i;
+
+ if (mm_ops->sva_mode)
+ return;
+
+ /* The cnt is guaranteed not to exceed MAP_PAIR_NUM_MAX within hpre. */
+ for (i = 0; i < cache->cnt; i++)
+ mm_ops->iova_unmap(mm_ops->usr, cache->pairs[i].addr,
+ (void *)cache->pairs[i].pa, cache->pairs[i].size);
+}
+
+static void unsetup_hw_msg_addr(struct wd_mm_ops *mm_ops, enum hpre_hw_msg_field t_type,
+ struct hisi_hpre_sqe *hw_msg, void *va, size_t data_sz)
+{
+ void *addr;
+
+ if (!mm_ops || mm_ops->sva_mode || !va || !data_sz)
+ return;
+
+ switch (t_type) {
+ case HW_MSG_KEY:
+ addr = VA_ADDR(hw_msg->hi_key, hw_msg->low_key);
+ break;
+ case HW_MSG_IN:
+ addr = VA_ADDR(hw_msg->hi_in, hw_msg->low_in);
+ break;
+ case HW_MSG_OUT:
+ addr = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
+ break;
+ default:
+ return;
+ }
+
+ if (!addr)
+ return;
+
+ mm_ops->iova_unmap(mm_ops->usr, va, (void *)addr, data_sz);
+}
+
+static uintptr_t select_addr_by_sva_mode(struct wd_mm_ops *mm_ops, void *data,
+ size_t data_sz, struct map_info_cache *cache)
+{
+ uintptr_t addr;
+
+ if (!mm_ops->sva_mode) {
+ addr = (uintptr_t)mm_ops->iova_map(mm_ops->usr, data, data_sz);
+ if (!addr) {
+ WD_ERR("Failed to get mappped DMA address for hardware.\n");
+ return 0;
+ }
+ add_map_info(cache, data, addr, data_sz);
+ } else {
+ addr = (uintptr_t)data;
+ }
+
+ return addr;
+}
+
+static void fill_hw_msg_addr(enum hpre_hw_msg_field t_type, struct hisi_hpre_sqe *hw_msg,
+ uintptr_t addr)
+{
+ switch (t_type) {
+ case HW_MSG_KEY:
+ hw_msg->low_key = LW_U32(addr);
+ hw_msg->hi_key = HI_U32(addr);
+ break;
+ case HW_MSG_IN:
+ hw_msg->low_in = LW_U32(addr);
+ hw_msg->hi_in = HI_U32(addr);
+ break;
+ case HW_MSG_OUT:
+ hw_msg->low_out = LW_U32(addr);
+ hw_msg->hi_out = HI_U32(addr);
+ break;
+ default:
+ return;
+ }
+}
+
+static int check_hpre_mem_params(struct wd_mm_ops *mm_ops, enum wd_mem_type mm_type)
+{
+ if (!mm_ops) {
+ WD_ERR("Memory operation functions are null.\n");
+ return -WD_EINVAL;
+ }
+
+ if (mm_type == UADK_MEM_AUTO && !mm_ops->sva_mode) {
+ WD_ERR("No-SVA in auto mode is not supported yet.\n");
+ return -WD_EINVAL;
+ } else if (mm_type > UADK_MEM_PROXY) {
+ WD_ERR("failed to check memory type.\n");
+ return -WD_EINVAL;
+ }
+
+ return WD_SUCCESS;
+}
+
static void dump_hpre_msg(void *msg, int alg)
{
struct wd_rsa_msg *rsa_msg;
@@ -269,7 +397,8 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey,
*data = wd_dq->data;
- return WD_SUCCESS;
+ return (int)(wd_dq->bsize + wd_qinv->bsize + wd_p->bsize +
+ wd_q->bsize + wd_dp->bsize);
}
static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data)
@@ -292,7 +421,7 @@ static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data)
*data = wd_d->data;
- return WD_SUCCESS;
+ return (int)(wd_n->bsize + wd_d->bsize);
}
static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data)
@@ -315,7 +444,7 @@ static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data)
*data = wd_e->data;
- return WD_SUCCESS;
+ return (int)(wd_n->bsize + wd_e->bsize);
}
static int fill_rsa_genkey_in(struct wd_rsa_kg_in *genkey)
@@ -378,22 +507,21 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg,
{
struct wd_rsa_req *req = &msg->req;
struct wd_rsa_kg_out *key = req->dst;
+ struct wd_rsa_msg *target_msg;
__u16 kbytes = msg->key_bytes;
struct wd_dtb qinv, dq, dp;
struct wd_dtb d, n;
- void *data;
int ret;
- if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD) {
- /* async */
- if (qp_mode == CTX_MODE_ASYNC) {
- data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- key = container_of(data, struct wd_rsa_kg_out, data);
- } else {
- key = req->dst;
- }
+ target_msg = (struct wd_rsa_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct rsa send msg from hardware!\n");
+ return -WD_ADDR_ERR;
}
+ if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD)
+ key = target_msg->req.dst;
+
msg->result = WD_SUCCESS;
if (hw_msg->alg == HPRE_ALG_KG_CRT) {
req->dst_bytes = CRT_GEN_PARAMS_SZ(kbytes);
@@ -424,37 +552,38 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg,
return WD_SUCCESS;
}
-static int rsa_prepare_key(struct wd_rsa_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int rsa_prepare_key(struct wd_rsa_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
struct wd_rsa_req *req = &msg->req;
+ uintptr_t addr;
+ int ret, len;
void *data;
- int ret;
if (req->op_type == WD_RSA_SIGN) {
if (hw_msg->alg == HPRE_ALG_NC_CRT) {
- ret = fill_rsa_crt_prikey2((void *)msg->key, &data);
- if (ret)
- return ret;
+ len = fill_rsa_crt_prikey2((void *)msg->key, &data);
+ if (len <= 0)
+ return -WD_EINVAL;
} else {
- ret = fill_rsa_prikey1((void *)msg->key, &data);
- if (ret)
- return ret;
+ len = fill_rsa_prikey1((void *)msg->key, &data);
+ if (len <= 0)
+ return -WD_EINVAL;
hw_msg->alg = HPRE_ALG_NC_NCRT;
}
} else if (req->op_type == WD_RSA_VERIFY) {
- ret = fill_rsa_pubkey((void *)msg->key, &data);
- if (ret)
- return ret;
+ len = fill_rsa_pubkey((void *)msg->key, &data);
+ if (len <= 0)
+ return -WD_EINVAL;
hw_msg->alg = HPRE_ALG_NC_NCRT;
} else if (req->op_type == WD_RSA_GENKEY) {
ret = fill_rsa_genkey_in((void *)msg->key);
if (ret)
return ret;
- ret = wd_rsa_kg_in_data((void *)msg->key, (char **)&data);
- if (ret < 0) {
+ len = wd_rsa_kg_in_data((void *)msg->key, (char **)&data);
+ if (len < 0) {
WD_ERR("failed to get rsa gen key data!\n");
- return ret;
+ return -WD_EINVAL;
}
if (hw_msg->alg == HPRE_ALG_NC_CRT)
hw_msg->alg = HPRE_ALG_KG_CRT;
@@ -465,36 +594,53 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg,
return -WD_EINVAL;
}
- hw_msg->low_key = LW_U32((uintptr_t)data);
- hw_msg->hi_key = HI_U32((uintptr_t)data);
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, len, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr);
- return WD_SUCCESS;
+ return ret;
}
-static int rsa_prepare_iot(struct wd_rsa_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int rsa_prepare_iot(struct wd_rsa_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
struct wd_rsa_req *req = &msg->req;
struct wd_rsa_kg_out *kout = req->dst;
int ret = WD_SUCCESS;
- void *out = NULL;
+ uintptr_t phy, out;
if (req->op_type != WD_RSA_GENKEY) {
- hw_msg->low_in = LW_U32((uintptr_t)req->src);
- hw_msg->hi_in = HI_U32((uintptr_t)req->src);
- out = req->dst;
+ phy = select_addr_by_sva_mode(msg->mm_ops, req->src, req->src_bytes, cache);
+ if (!phy)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_IN, hw_msg, phy);
+ phy = select_addr_by_sva_mode(msg->mm_ops, req->dst, req->dst_bytes, cache);
+ if (!phy)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_OUT, hw_msg, phy);
} else {
hw_msg->low_in = 0;
hw_msg->hi_in = 0;
ret = wd_rsa_kg_out_data(kout, (char **)&out);
if (ret < 0)
return ret;
- }
- hw_msg->low_out = LW_U32((uintptr_t)out);
- hw_msg->hi_out = HI_U32((uintptr_t)out);
+ if (!msg->mm_ops->sva_mode) {
+ phy = (uintptr_t)msg->mm_ops->iova_map(msg->mm_ops->usr, kout, ret);
+ if (!phy) {
+ WD_ERR("Failed to get DMA address for rsa output!\n");
+ return -WD_ENOMEM;
+ }
+ add_map_info(cache, kout, phy, ret);
+ out = phy + out - (uintptr_t)kout;
+ }
+
+ hw_msg->low_out = LW_U32(out);
+ hw_msg->hi_out = HI_U32(out);
+ }
- return ret;
+ return WD_SUCCESS;
}
static int hpre_init_qm_priv(struct wd_ctx_config_internal *config,
@@ -615,10 +761,17 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_rsa_msg *msg = rsa_msg;
+ struct map_info_cache cache = {0};
struct hisi_hpre_sqe hw_msg;
__u16 send_cnt = 0;
int ret;
+ ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type);
+ if (ret) {
+ WD_ERR("rsa memory parmas is err, and ret is %d!\n", ret);
+ return ret;
+ }
+
memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe));
if (msg->key_type == WD_RSA_PRIKEY1 ||
@@ -631,21 +784,30 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
hw_msg.task_len1 = msg->key_bytes / BYTE_BITS - 0x1;
- ret = rsa_prepare_key(msg, &hw_msg);
- if (ret < 0)
- return ret;
+ ret = rsa_prepare_key(msg, &hw_msg, &cache);
+ if (ret)
+ goto rsa_fail;
/* prepare in/out put */
- ret = rsa_prepare_iot(msg, &hw_msg);
- if (ret < 0)
- return ret;
+ ret = rsa_prepare_iot(msg, &hw_msg, &cache);
+ if (ret)
+ goto rsa_fail;
hisi_set_msg_id(h_qp, &msg->tag);
hw_msg.done = 0x1;
hw_msg.etype = 0x0;
- hw_msg.low_tag = msg->tag;
+ hw_msg.low_tag = LW_U32((uintptr_t)msg);
+ hw_msg.hi_tag = HI_U32((uintptr_t)msg);
- return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ ret = hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ if (unlikely(ret))
+ goto rsa_fail;
+
+ return ret;
+
+rsa_fail:
+ unmap_addr_in_cache(msg->mm_ops, &cache);
+ return ret;
}
static void hpre_result_check(struct hisi_hpre_sqe *hw_msg,
@@ -671,32 +833,29 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct hisi_qp *qp = (struct hisi_qp *)h_qp;
- struct hisi_hpre_sqe hw_msg = {0};
+ struct wd_rsa_msg *target_msg;
struct wd_rsa_msg *msg = rsa_msg;
- struct wd_rsa_msg *temp_msg;
+ struct hisi_hpre_sqe hw_msg = {0};
+ size_t ilen = 0, olen = 0;
__u16 recv_cnt = 0;
+ __u16 kbytes;
int ret;
ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt);
if (ret < 0)
return ret;
- ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag);
+ target_msg = (struct wd_rsa_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
+ ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag);
if (ret)
return ret;
- msg->tag = LW_U16(hw_msg.low_tag);
- if (qp->q_info.qp_mode == CTX_MODE_ASYNC) {
- temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag);
- if (!temp_msg) {
- WD_ERR("failed to get send msg! idx = %u, tag = %u.\n",
- qp->q_info.idx, msg->tag);
- return -WD_EINVAL;
- }
- } else {
- temp_msg = msg;
- }
-
+ msg->tag = target_msg->tag;
hpre_result_check(&hw_msg, &msg->result);
if (!msg->result) {
ret = rsa_out_transfer(msg, &hw_msg, qp->q_info.qp_mode);
@@ -707,15 +866,36 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
}
if (unlikely(msg->result != WD_SUCCESS))
- dump_hpre_msg(temp_msg, WD_RSA);
+ dump_hpre_msg(target_msg, WD_RSA);
+
+ if (!target_msg->mm_ops->sva_mode) {
+ kbytes = target_msg->key_bytes;
+ if (hw_msg.alg == HPRE_ALG_KG_CRT) {
+ olen = CRT_GEN_PARAMS_SZ(kbytes);
+ ilen = GEN_PARAMS_SZ_UL(kbytes);
+ } else if (hw_msg.alg == HPRE_ALG_KG_STD) {
+ olen = GEN_PARAMS_SZ_UL(kbytes);
+ ilen = GEN_PARAMS_SZ_UL(kbytes);
+ } else {
+ olen = kbytes;
+ ilen = kbytes;
+ }
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, &hw_msg,
+ target_msg->req.src, ilen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, &hw_msg,
+ target_msg->req.dst, olen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, &hw_msg,
+ target_msg->key, target_msg->key_bytes);
+ }
return WD_SUCCESS;
}
-static int fill_dh_xp_params(struct wd_dh_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int fill_dh_xp_params(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
struct wd_dh_req *req = &msg->req;
+ uintptr_t addr;
void *x, *p;
int ret;
@@ -735,26 +915,30 @@ static int fill_dh_xp_params(struct wd_dh_msg *msg,
return ret;
}
- hw_msg->low_key = LW_U32((uintptr_t)x);
- hw_msg->hi_key = HI_U32((uintptr_t)x);
+ addr = select_addr_by_sva_mode(msg->mm_ops, x, GEN_PARAMS_SZ_UL(msg->key_bytes), cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr);
- return WD_SUCCESS;
+ return ret;
}
-static int dh_out_transfer(struct wd_dh_msg *msg,
- struct hisi_hpre_sqe *hw_msg, __u8 qp_mode)
+static int dh_out_transfer(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ __u8 qp_mode)
{
__u16 key_bytes = (hw_msg->task_len1 + 1) * BYTE_BITS;
struct wd_dh_req *req = &msg->req;
+ struct wd_dh_msg *target_msg;
void *out;
int ret;
- /* async */
- if (qp_mode == CTX_MODE_ASYNC)
- out = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- else
- out = req->pri;
+ target_msg = (struct wd_dh_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+ out = target_msg->req.pri;
ret = hpre_bin_to_crypto_bin((char *)out,
(const char *)out, key_bytes, "dh out");
if (!ret)
@@ -768,12 +952,20 @@ static int dh_out_transfer(struct wd_dh_msg *msg,
static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
+ struct map_info_cache cache = {0};
struct wd_dh_msg *msg = dh_msg;
struct wd_dh_req *req = &msg->req;
struct hisi_hpre_sqe hw_msg;
__u16 send_cnt = 0;
+ uintptr_t addr;
int ret;
+ ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type);
+ if (ret) {
+ WD_ERR("dh memory parmas is err, and ret is %d!\n", ret);
+ return ret;
+ }
+
memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe));
if (msg->is_g2 && req->op_type != WD_DH_PHASE2)
@@ -791,32 +983,50 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
WD_ERR("failed to transfer dh g para format to hpre bin!\n");
return ret;
}
-
- hw_msg.low_in = LW_U32((uintptr_t)msg->g);
- hw_msg.hi_in = HI_U32((uintptr_t)msg->g);
+ addr = select_addr_by_sva_mode(msg->mm_ops, msg->g, msg->key_bytes, &cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_IN, &hw_msg, addr);
}
- ret = fill_dh_xp_params(msg, &hw_msg);
- if (ret)
- return ret;
+ ret = fill_dh_xp_params(msg, &hw_msg, &cache);
+ if (ret) {
+ WD_ERR("failed to fill dh x or p para!\n");
+ goto dh_fail;
+ }
hisi_set_msg_id(h_qp, &msg->tag);
- hw_msg.low_out = LW_U32((uintptr_t)req->pri);
- hw_msg.hi_out = HI_U32((uintptr_t)req->pri);
hw_msg.done = 0x1;
hw_msg.etype = 0x0;
- hw_msg.low_tag = msg->tag;
- return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ hw_msg.low_tag = LW_U32((uintptr_t)msg);
+ hw_msg.hi_tag = HI_U32((uintptr_t)msg);
+
+ addr = select_addr_by_sva_mode(msg->mm_ops, req->pri, req->pri_bytes, &cache);
+ if (!addr) {
+ ret = -WD_ENOMEM;
+ goto dh_fail;
+ }
+ fill_hw_msg_addr(HW_MSG_OUT, &hw_msg, addr);
+
+ ret = hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ if (unlikely(ret))
+ goto dh_fail;
+
+ return ret;
+
+dh_fail:
+ unmap_addr_in_cache(msg->mm_ops, &cache);
+ return ret;
}
static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct hisi_qp *qp = (struct hisi_qp *)h_qp;
- struct wd_dh_msg *msg = dh_msg;
struct hisi_hpre_sqe hw_msg = {0};
- struct wd_dh_msg *temp_msg;
+ struct wd_dh_msg *msg = dh_msg;
+ struct wd_dh_msg *target_msg;
__u16 recv_cnt = 0;
int ret;
@@ -824,22 +1034,17 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
if (ret < 0)
return ret;
- ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag);
+ target_msg = (struct wd_dh_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
+ ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag);
if (ret)
return ret;
- msg->tag = LW_U16(hw_msg.low_tag);
- if (qp->q_info.qp_mode == CTX_MODE_ASYNC) {
- temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag);
- if (!temp_msg) {
- WD_ERR("failed to get send msg! idx = %u, tag = %u.\n",
- qp->q_info.idx, msg->tag);
- return -WD_EINVAL;
- }
- } else {
- temp_msg = msg;
- }
-
+ msg->tag = target_msg->tag;
hpre_result_check(&hw_msg, &msg->result);
if (!msg->result) {
ret = dh_out_transfer(msg, &hw_msg, qp->q_info.qp_mode);
@@ -850,7 +1055,16 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
}
if (unlikely(msg->result != WD_SUCCESS))
- dump_hpre_msg(temp_msg, WD_DH);
+ dump_hpre_msg(target_msg, WD_DH);
+
+ if (!target_msg->mm_ops->sva_mode) {
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, &hw_msg,
+ target_msg->req.pri, target_msg->req.pri_bytes);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, &hw_msg,
+ target_msg->req.x_p, GEN_PARAMS_SZ_UL(target_msg->key_bytes));
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, &hw_msg,
+ target_msg->g, target_msg->key_bytes);
+ }
return WD_SUCCESS;
}
@@ -1081,30 +1295,49 @@ static bool is_prikey_used(__u8 op_type)
op_type == HPRE_SM2_DEC;
}
-static int ecc_prepare_key(struct wd_ecc_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static __u32 ecc_get_prikey_size(struct wd_ecc_msg *msg)
+{
+ if (msg->req.op_type == WD_SM2_SIGN ||
+ msg->req.op_type == WD_ECDSA_SIGN ||
+ msg->req.op_type == WD_SM2_DECRYPT)
+ return ECC_PRIKEY_SZ(msg->key_bytes);
+ else if (msg->curve_id == WD_X25519 ||
+ msg->curve_id == WD_X448)
+ return X_DH_HW_KEY_SZ(msg->key_bytes);
+ else
+ return ECDH_HW_KEY_SZ(msg->key_bytes);
+}
+
+static int ecc_prepare_key(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
void *data = NULL;
+ uintptr_t addr;
+ size_t ksz;
int ret;
if (is_prikey_used(msg->req.op_type)) {
+ ksz = ecc_get_prikey_size(msg);
ret = ecc_prepare_prikey((void *)msg->key, &data, msg->curve_id);
if (ret)
return ret;
} else {
+ ksz = ECC_PUBKEY_SZ(msg->key_bytes);
ret = ecc_prepare_pubkey((void *)msg->key, &data);
if (ret)
return ret;
}
- hw_msg->low_key = LW_U32((uintptr_t)data);
- hw_msg->hi_key = HI_U32((uintptr_t)data);
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, ksz, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr);
- return WD_SUCCESS;
+ return ret;
}
-static void ecc_get_io_len(__u32 atype, __u32 hsz, size_t *ilen,
- size_t *olen)
+static void ecc_get_io_len(__u32 atype, __u32 hsz,
+ size_t *ilen, size_t *olen)
{
if (atype == HPRE_ALG_ECDH_MULTIPLY) {
*olen = ECDH_OUT_PARAMS_SZ(hsz);
@@ -1467,12 +1700,12 @@ static int ecc_prepare_out(struct wd_ecc_msg *msg, void **data)
}
/* prepare in/out hw message */
-static int ecc_prepare_iot(struct wd_ecc_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int ecc_prepare_iot(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
+ size_t i_sz, o_sz;
void *data = NULL;
- size_t i_sz = 0;
- size_t o_sz = 0;
+ uintptr_t addr;
__u16 kbytes;
int ret;
@@ -1483,8 +1716,11 @@ static int ecc_prepare_iot(struct wd_ecc_msg *msg,
WD_ERR("failed to prepare ecc in!\n");
return ret;
}
- hw_msg->low_in = LW_U32((uintptr_t)data);
- hw_msg->hi_in = HI_U32((uintptr_t)data);
+
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, i_sz, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_IN, hw_msg, addr);
ret = ecc_prepare_out(msg, &data);
if (ret) {
@@ -1496,8 +1732,10 @@ static int ecc_prepare_iot(struct wd_ecc_msg *msg,
if (!data)
return WD_SUCCESS;
- hw_msg->low_out = LW_U32((uintptr_t)data);
- hw_msg->hi_out = HI_U32((uintptr_t)data);
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, o_sz, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_OUT, hw_msg, addr);
return WD_SUCCESS;
}
@@ -1613,7 +1851,7 @@ static struct wd_ecc_out *create_ecdh_out(struct wd_ecc_msg *msg)
return NULL;
}
- out = malloc(len);
+ out = msg->mm_ops->alloc(msg->mm_ops->usr, len);
if (!out) {
WD_ERR("failed to alloc out memory, sz = %u!\n", len);
return NULL;
@@ -1677,7 +1915,7 @@ static struct wd_ecc_msg *create_req(struct wd_ecc_msg *src, __u8 req_idx)
prikey = (struct wd_ecc_prikey *)(ecc_key + 1);
ecc_key->prikey = prikey;
- prikey->data = malloc(ECC_PRIKEY_SZ(src->key_bytes));
+ prikey->data = src->mm_ops->alloc(src->mm_ops->usr, ECC_PRIKEY_SZ(src->key_bytes));
if (unlikely(!prikey->data)) {
WD_ERR("failed to alloc prikey data!\n");
goto fail_alloc_key_data;
@@ -1696,7 +1934,7 @@ static struct wd_ecc_msg *create_req(struct wd_ecc_msg *src, __u8 req_idx)
return dst;
fail_set_prikey:
- free(prikey->data);
+ src->mm_ops->free(src->mm_ops->usr, prikey->data);
fail_alloc_key_data:
free(ecc_key);
fail_alloc_key:
@@ -1709,9 +1947,9 @@ static void free_req(struct wd_ecc_msg *msg)
{
struct wd_ecc_key *key = (void *)msg->key;
- free(key->prikey->data);
+ msg->mm_ops->free(msg->mm_ops->usr, key->prikey->data);
free(key);
- free(msg->req.dst);
+ msg->mm_ops->free(msg->mm_ops->usr, msg->req.dst);
free(msg);
}
@@ -1732,7 +1970,8 @@ static int split_req(struct wd_ecc_msg *src, struct wd_ecc_msg **dst)
return WD_SUCCESS;
}
-static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg)
+static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
__u32 hw_sz = get_hw_keysz(msg->key_bytes);
__u8 op_type = msg->req.op_type;
@@ -1757,39 +1996,42 @@ static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg)
return ret;
/* prepare key */
- ret = ecc_prepare_key(msg, hw_msg);
+ ret = ecc_prepare_key(msg, hw_msg, cache);
if (ret)
return ret;
/* prepare in/out put */
- ret = ecc_prepare_iot(msg, hw_msg);
+ ret = ecc_prepare_iot(msg, hw_msg, cache);
if (ret)
return ret;
hw_msg->done = 0x1;
hw_msg->etype = 0x0;
- hw_msg->low_tag = msg->tag;
+
+ hw_msg->low_tag = LW_U32((uintptr_t)msg);
+ hw_msg->hi_tag = HI_U32((uintptr_t)msg);
hw_msg->task_len1 = hw_sz / BYTE_BITS - 0x1;
return ret;
}
-static int ecc_general_send(handle_t ctx, struct wd_ecc_msg *msg)
+static int ecc_general_send(handle_t ctx, struct wd_ecc_msg *msg,
+ struct map_info_cache *cache)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct hisi_hpre_sqe hw_msg;
__u16 send_cnt = 0;
int ret;
- ret = ecc_fill(msg, &hw_msg);
+ ret = ecc_fill(msg, &hw_msg, cache);
if (ret)
return ret;
return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
}
-
-static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg)
+static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg,
+ struct map_info_cache *cache)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_sm2_enc_in *ein = msg->req.src;
@@ -1801,7 +2043,7 @@ static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg)
if (ein->plaintext.dsize <= HW_PLAINTEXT_BYTES_MAX &&
hash->type == WD_HASH_SM3)
- return ecc_general_send(ctx, msg);
+ return ecc_general_send(ctx, msg, cache);
if (unlikely(!ein->k_set)) {
WD_ERR("invalid: k not set!\n");
@@ -1824,13 +2066,13 @@ static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg)
return ret;
}
- ret = ecc_fill(msg_dst[0], &hw_msg[0]);
+ ret = ecc_fill(msg_dst[0], &hw_msg[0], cache);
if (unlikely(ret)) {
WD_ERR("failed to fill 1th sqe, ret = %d!\n", ret);
goto fail_fill_sqe;
}
- ret = ecc_fill(msg_dst[1], &hw_msg[1]);
+ ret = ecc_fill(msg_dst[1], &hw_msg[1], cache);
if (unlikely(ret)) {
WD_ERR("failed to fill 2th sqe, ret = %d!\n", ret);
goto fail_fill_sqe;
@@ -1855,7 +2097,8 @@ fail_fill_sqe:
return ret;
}
-static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg)
+static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg,
+ struct map_info_cache *cache)
{
struct wd_sm2_dec_in *din = (void *)msg->req.src;
struct wd_hash_mt *hash = &msg->hash;
@@ -1865,7 +2108,7 @@ static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg)
/* c2 data lens <= 4096 bit */
if (din->c2.dsize <= BITS_TO_BYTES(4096) &&
hash->type == WD_HASH_SM3)
- return ecc_general_send(ctx, msg);
+ return ecc_general_send(ctx, msg, cache);
if (unlikely(!hash->cb || hash->type >= WD_HASH_MAX)) {
WD_ERR("invalid: input hash type %u is error!\n", hash->type);
@@ -1891,14 +2134,14 @@ static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg)
goto free_dst;
}
- ret = ecc_general_send(ctx, dst);
+ ret = ecc_general_send(ctx, dst, cache);
if (unlikely(ret))
goto free_req_dst;
return ret;
free_req_dst:
- free(dst->req.dst);
+ msg->mm_ops->free(msg->mm_ops->usr, dst->req.dst);
free_dst:
free(dst);
return ret;
@@ -1908,14 +2151,37 @@ static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_ecc_msg *msg = ecc_msg;
+ struct map_info_cache cache = {0};
+ int ret;
+
+ ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type);
+ if (ret) {
+ WD_ERR("ecc memory parmas is err, and ret is %d!\n", ret);
+ return ret;
+ }
hisi_set_msg_id(h_qp, &msg->tag);
- if (msg->req.op_type == WD_SM2_ENCRYPT)
- return sm2_enc_send(ctx, msg);
- else if (msg->req.op_type == WD_SM2_DECRYPT)
- return sm2_dec_send(ctx, msg);
+ if (msg->req.op_type == WD_SM2_ENCRYPT) {
+ ret = sm2_enc_send(ctx, msg, &cache);
+ if (ret)
+ goto ecc_fail;
+ return ret;
+ } else if (msg->req.op_type == WD_SM2_DECRYPT) {
+ ret = sm2_dec_send(ctx, msg, &cache);
+ if (ret)
+ goto ecc_fail;
+ return ret;
+ }
- return ecc_general_send(ctx, msg);
+ ret = ecc_general_send(ctx, msg, &cache);
+ if (ret)
+ goto ecc_fail;
+
+ return ret;
+
+ecc_fail:
+ unmap_addr_in_cache(msg->mm_ops, &cache);
+ return ret;
}
static int ecdh_out_transfer(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg)
@@ -2020,14 +2286,14 @@ static int sm2_enc_out_transfer(struct wd_ecc_msg *msg,
static int ecc_out_transfer(struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg, __u8 qp_mode)
{
+ struct wd_ecc_msg *target_msg;
int ret = -WD_EINVAL;
- void *va;
+
+ target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
/* async */
- if (qp_mode == CTX_MODE_ASYNC) {
- va = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- msg->req.dst = container_of(va, struct wd_ecc_out, data);
- }
+ if (qp_mode == CTX_MODE_ASYNC)
+ msg->req.dst = target_msg->req.dst;
if (hw_msg->alg == HPRE_ALG_SM2_SIGN ||
hw_msg->alg == HPRE_ALG_ECDSA_SIGN)
@@ -2319,19 +2585,16 @@ static int sm2_convert_dec_out(struct wd_ecc_msg *src,
static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg)
{
- struct wd_ecc_msg *temp_msg;
+ struct wd_ecc_msg *target_msg;
+ size_t ilen = 0;
+ size_t olen = 0;
+ __u16 kbytes;
int ret;
- msg->tag = LW_U16(hw_msg->low_tag);
- if (qp->q_info.qp_mode == CTX_MODE_ASYNC) {
- temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag);
- if (!temp_msg) {
- WD_ERR("failed to get send msg! idx = %u, tag = %u.\n",
- qp->q_info.idx, msg->tag);
- return -WD_EINVAL;
- }
- } else {
- temp_msg = msg;
+ target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct ecc send msg from hardware!\n");
+ return -WD_ADDR_ERR;
}
hpre_result_check(hw_msg, &msg->result);
@@ -2347,10 +2610,20 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg,
goto dump_err_msg;
}
- return ret;
+ return WD_SUCCESS;
dump_err_msg:
- dump_hpre_msg(temp_msg, WD_ECC);
+ kbytes = target_msg->key_bytes;
+ if (!target_msg->mm_ops->sva_mode) {
+ ecc_get_io_len(hw_msg->alg, kbytes, &ilen, &olen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, hw_msg,
+ target_msg->req.dst, olen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, hw_msg,
+ target_msg->key, kbytes);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, hw_msg,
+ target_msg->req.src, ilen);
+ }
+ dump_hpre_msg(target_msg, WD_ECC);
return ret;
}
@@ -2363,8 +2636,6 @@ static int parse_second_sqe(handle_t h_qp,
struct wd_ecc_msg *dst;
__u16 recv_cnt = 0;
int cnt = 0;
- void *data;
- __u32 hsz;
int ret;
while (1) {
@@ -2380,10 +2651,12 @@ static int parse_second_sqe(handle_t h_qp,
break;
}
- data = VA_ADDR(hw_msg.hi_out, hw_msg.low_out);
- hsz = (hw_msg.task_len1 + 1) * BYTE_BITS;
- dst = *(struct wd_ecc_msg **)((uintptr_t)data +
- hsz * ECDH_OUT_PARAM_NUM);
+ dst = (struct wd_ecc_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!dst) {
+ WD_ERR("failed to get correct sm2 enc second send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg);
msg->result = dst->result;
*second = dst;
@@ -2394,19 +2667,17 @@ static int parse_second_sqe(handle_t h_qp,
static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg)
{
- __u16 tag = LW_U16(hw_msg->low_tag);
struct wd_ecc_msg *second = NULL;
struct wd_ecc_msg *first;
struct wd_ecc_msg src;
- void *data;
- __u32 hsz;
int ret;
- msg->tag = tag;
- data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- hsz = (hw_msg->task_len1 + 1) * BYTE_BITS;
- first = *(struct wd_ecc_msg **)((uintptr_t)data +
- hsz * ECDH_OUT_PARAM_NUM);
+ first = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!first) {
+ WD_ERR("failed to get correct sm2 enc msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
memcpy(&src, first + 1, sizeof(src));
/* parse first sqe */
@@ -2440,17 +2711,15 @@ free_first:
static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg)
{
- __u16 tag = LW_U16(hw_msg->low_tag);
struct wd_ecc_msg *dst;
struct wd_ecc_msg src;
- void *data;
- __u32 hsz;
int ret;
- data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- hsz = (hw_msg->task_len1 + 1) * BYTE_BITS;
- dst = *(struct wd_ecc_msg **)((uintptr_t)data +
- hsz * ECDH_OUT_PARAM_NUM);
+ dst = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!dst) {
+ WD_ERR("failed to get correct sm2 dec msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
memcpy(&src, dst + 1, sizeof(src));
/* parse first sqe */
@@ -2460,7 +2729,6 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg,
goto fail;
}
msg->result = dst->result;
- msg->tag = tag;
ret = sm2_convert_dec_out(&src, dst);
if (unlikely(ret)) {
@@ -2469,7 +2737,7 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg,
}
fail:
- free(dst->req.dst);
+ dst->mm_ops->free(dst->mm_ops->usr, dst->req.dst);
free(dst);
return ret;
@@ -2479,6 +2747,7 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_ecc_msg *msg = ecc_msg;
+ struct wd_ecc_msg *target_msg;
struct hisi_hpre_sqe hw_msg;
__u16 recv_cnt = 0;
int ret;
@@ -2487,10 +2756,17 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg)
if (ret)
return ret;
- ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag);
+ target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
+ ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag);
if (ret)
return ret;
+ msg->tag = target_msg->tag;
if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY &&
hw_msg.sm2_mlen == HPRE_SM2_ENC)
return sm2_enc_parse(h_qp, msg, &hw_msg);
@@ -2549,7 +2825,7 @@ static bool is_valid_hw_type(struct wd_alg_driver *drv)
hpre_ctx = (struct hisi_hpre_ctx *)drv->priv;
qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->config.ctxs[0].ctx);
- if (qp->q_info.hw_type < HISI_QM_API_VER3_BASE)
+ if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE)
return false;
return true;
}
diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c
index 9251b4c..216b80a 100644
--- a/drv/hisi_qm_udrv.c
+++ b/drv/hisi_qm_udrv.c
@@ -473,7 +473,7 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count)
{
struct hisi_qp *qp = (struct hisi_qp *)h_qp;
struct hisi_qm_queue_info *q_info;
- __u16 free_num, send_num;
+ __u16 free_num;
__u16 tail;
if (unlikely(!qp || !req || !count))
@@ -488,11 +488,14 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count)
return -WD_EBUSY;
}
- send_num = expect > free_num ? free_num : expect;
+ if (expect > free_num) {
+ pthread_spin_unlock(&q_info->sd_lock);
+ return -WD_EBUSY;
+ }
tail = q_info->sq_tail_index;
- hisi_qm_fill_sqe(req, q_info, tail, send_num);
- tail = (tail + send_num) % q_info->sq_depth;
+ hisi_qm_fill_sqe(req, q_info, tail, expect);
+ tail = (tail + expect) % q_info->sq_depth;
/*
* Before sending doorbell, check the queue status,
@@ -510,9 +513,9 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count)
q_info->sq_tail_index = tail;
/* Make sure used_num is changed before the next thread gets free sqe. */
- __atomic_add_fetch(&q_info->used_num, send_num, __ATOMIC_RELAXED);
+ __atomic_add_fetch(&q_info->used_num, expect, __ATOMIC_RELAXED);
pthread_spin_unlock(&q_info->sd_lock);
- *count = send_num;
+ *count = expect;
return 0;
}
diff --git a/include/drv/wd_dh_drv.h b/include/drv/wd_dh_drv.h
index d205dc4..d2a6157 100644
--- a/include/drv/wd_dh_drv.h
+++ b/include/drv/wd_dh_drv.h
@@ -16,12 +16,15 @@ extern "C" {
/* DH message format */
struct wd_dh_msg {
struct wd_dh_req req;
+ struct wd_mm_ops *mm_ops;
+ enum wd_mem_type mm_type;
__u32 tag; /* User-defined request identifier */
void *g;
__u16 gbytes;
__u16 key_bytes; /* Input key bytes */
__u8 is_g2;
__u8 result; /* Data format, denoted by WD error code */
+ __u8 *rsv_out; /* reserved output data pointer */
};
struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag);
diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h
index 6193c8b..b123a9b 100644
--- a/include/drv/wd_ecc_drv.h
+++ b/include/drv/wd_ecc_drv.h
@@ -48,6 +48,8 @@ extern "C" {
/* ECC message format */
struct wd_ecc_msg {
struct wd_ecc_req req;
+ struct wd_mm_ops *mm_ops;
+ enum wd_mem_type mm_type;
struct wd_hash_mt hash;
__u32 tag; /* User-defined request identifier */
__u8 *key; /* Input key VA, should be DMA buffer */
@@ -55,6 +57,7 @@ struct wd_ecc_msg {
__u8 curve_id; /* Ec curve denoted by enum wd_ecc_curve_type */
__u8 result; /* alg op error code */
void *drv_cfg; /* internal driver configuration */
+ __u8 *rsv_out; /* reserved output data pointer */
};
struct wd_ecc_pubkey {
diff --git a/include/drv/wd_rsa_drv.h b/include/drv/wd_rsa_drv.h
index d231ecf..c12f3e0 100644
--- a/include/drv/wd_rsa_drv.h
+++ b/include/drv/wd_rsa_drv.h
@@ -42,11 +42,14 @@ struct wd_rsa_kg_out {
/* RSA message format */
struct wd_rsa_msg {
struct wd_rsa_req req;
+ struct wd_mm_ops *mm_ops;
+ enum wd_mem_type mm_type;
__u32 tag; /* User-defined request identifier */
__u16 key_bytes; /* Input key bytes */
__u8 key_type; /* Denoted by enum wd_rsa_key_type */
__u8 result; /* Data format, denoted by WD error code */
__u8 *key; /* Input key VA pointer, should be DMA buffer */
+ __u8 *rsv_out; /* reserved output data pointer */
};
struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag);
diff --git a/include/wd_dh.h b/include/wd_dh.h
index afc2f7c..235c602 100644
--- a/include/wd_dh.h
+++ b/include/wd_dh.h
@@ -27,6 +27,8 @@ struct wd_dh_sess_setup {
__u16 key_bits; /* DH key bites */
bool is_g2; /* is g2 mode or not */
void *sched_param;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
struct wd_dh_req {
diff --git a/include/wd_ecc.h b/include/wd_ecc.h
index 6f670e2..18c1c0d 100644
--- a/include/wd_ecc.h
+++ b/include/wd_ecc.h
@@ -116,6 +116,8 @@ struct wd_ecc_sess_setup {
struct wd_rand_mt rand; /* rand method from user */
struct wd_hash_mt hash; /* hash method from user */
void *sched_param;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
struct wd_ecc_req {
diff --git a/include/wd_rsa.h b/include/wd_rsa.h
index 2f4e589..9c91432 100644
--- a/include/wd_rsa.h
+++ b/include/wd_rsa.h
@@ -60,6 +60,8 @@ struct wd_rsa_sess_setup {
__u16 key_bits; /* RSA key bits */
bool is_crt; /* CRT mode or not */
void *sched_param;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
bool wd_rsa_is_crt(handle_t sess);
diff --git a/wd_dh.c b/wd_dh.c
index 221322f..0c1372c 100644
--- a/wd_dh.c
+++ b/wd_dh.c
@@ -26,6 +26,8 @@ struct wd_dh_sess {
struct wd_dtb g;
struct wd_dh_sess_setup setup;
void *sched_key;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
static struct wd_dh_setting {
@@ -326,6 +328,8 @@ static int fill_dh_msg(struct wd_dh_msg *msg, struct wd_dh_req *req,
memcpy(&msg->req, req, sizeof(*req));
msg->result = WD_EINVAL;
msg->key_bytes = sess->key_size;
+ msg->mm_ops = &sess->mm_ops;
+ msg->mm_type = sess->mm_type;
if (unlikely(req->pri_bytes < sess->key_size)) {
WD_ERR("invalid: req pri bytes %hu is error!\n", req->pri_bytes);
@@ -581,6 +585,7 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g)
handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup)
{
struct wd_dh_sess *sess;
+ int ret;
if (!setup) {
WD_ERR("invalid: alloc dh sess setup NULL!\n");
@@ -606,10 +611,19 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup)
memcpy(&sess->setup, setup, sizeof(*setup));
sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT;
- sess->g.data = malloc(sess->key_size);
- if (!sess->g.data)
+ ret = wd_mem_ops_init(wd_dh_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type);
+ if (ret) {
+ WD_ERR("failed to init memory ops!\n");
goto sess_err;
+ }
+ memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops));
+ sess->mm_type = setup->mm_type;
+ sess->g.data = sess->mm_ops.alloc(sess->mm_ops.usr, sess->key_size);
+ if (!sess->g.data) {
+ WD_ERR("failed to malloc sess g param memory!\n");
+ goto sess_err;
+ }
sess->g.bsize = sess->key_size;
/* Some simple scheduler don't need scheduling parameters */
sess->sched_key = (void *)wd_dh_setting.sched.sched_init(
@@ -622,7 +636,7 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup)
return (handle_t)sess;
sched_err:
- free(sess->g.data);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->g.data);
sess_err:
free(sess);
return (handle_t)0;
@@ -638,7 +652,7 @@ void wd_dh_free_sess(handle_t sess)
}
if (sess_t->g.data)
- free(sess_t->g.data);
+ sess_t->mm_ops.free(sess_t->mm_ops.usr, sess_t->g.data);
if (sess_t->sched_key)
free(sess_t->sched_key);
diff --git a/wd_ecc.c b/wd_ecc.c
index b1971b9..2c88d0a 100644
--- a/wd_ecc.c
+++ b/wd_ecc.c
@@ -52,6 +52,8 @@ struct wd_ecc_sess {
struct wd_ecc_sess_setup setup;
struct wd_ecc_extend_ops eops;
void *sched_key;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
struct wd_ecc_curve_list {
@@ -494,7 +496,7 @@ static void release_ecc_prikey(struct wd_ecc_sess *sess)
struct wd_ecc_prikey *prikey = sess->key.prikey;
wd_memset_zero(prikey->data, prikey->size);
- free(prikey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, prikey->data);
free(prikey);
sess->key.prikey = NULL;
}
@@ -503,7 +505,7 @@ static void release_ecc_pubkey(struct wd_ecc_sess *sess)
{
struct wd_ecc_pubkey *pubkey = sess->key.pubkey;
- free(pubkey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, pubkey->data);
free(pubkey);
sess->key.pubkey = NULL;
}
@@ -522,7 +524,7 @@ static struct wd_ecc_prikey *create_ecc_prikey(struct wd_ecc_sess *sess)
}
dsz = ECC_PRIKEY_SZ(hsz);
- data = malloc(dsz);
+ data = sess->mm_ops.alloc(sess->mm_ops.usr, dsz);
if (!data) {
WD_ERR("failed to malloc prikey data, sz = %u!\n", dsz);
free(prikey);
@@ -551,7 +553,7 @@ static struct wd_ecc_pubkey *create_ecc_pubkey(struct wd_ecc_sess *sess)
}
dsz = ECC_PUBKEY_SZ(hsz);
- data = malloc(dsz);
+ data = sess->mm_ops.alloc(sess->mm_ops.usr, dsz);
if (!data) {
WD_ERR("failed to malloc pubkey data, sz = %u!\n", dsz);
free(pubkey);
@@ -570,7 +572,7 @@ static void release_ecc_in(struct wd_ecc_sess *sess,
struct wd_ecc_in *ecc_in)
{
wd_memset_zero(ecc_in->data, ecc_in->size);
- free(ecc_in);
+ sess->mm_ops.free(sess->mm_ops.usr, ecc_in);
}
static struct wd_ecc_in *create_ecc_in(struct wd_ecc_sess *sess, __u32 num)
@@ -585,7 +587,7 @@ static struct wd_ecc_in *create_ecc_in(struct wd_ecc_sess *sess, __u32 num)
hsz = get_key_bsz(sess->key_size);
len = sizeof(struct wd_ecc_in) + hsz * num;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc ecc in, sz = %u!\n", len);
return NULL;
@@ -613,7 +615,7 @@ static struct wd_ecc_in *create_sm2_sign_in(struct wd_ecc_sess *sess,
len = sizeof(struct wd_ecc_in)
+ ECC_SIGN_IN_PARAM_NUM * ksz + m_len;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc sm2 sign in, sz = %llu!\n", len);
return NULL;
@@ -653,7 +655,7 @@ static struct wd_ecc_in *create_sm2_enc_in(struct wd_ecc_sess *sess,
}
len = sizeof(struct wd_ecc_in) + ksz + m_len;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc sm2 enc in, sz = %llu!\n", len);
return NULL;
@@ -697,7 +699,7 @@ static void *create_sm2_ciphertext(struct wd_ecc_sess *sess, __u32 m_len,
*len = (__u64)st_sz + ECC_POINT_PARAM_NUM * (__u64)sess->key_size +
(__u64)m_len + (__u64)h_byts;
- start = malloc(*len);
+ start = sess->mm_ops.alloc(sess->mm_ops.usr, *len);
if (unlikely(!start)) {
WD_ERR("failed to alloc start, sz = %llu!\n", *len);
return NULL;
@@ -745,7 +747,7 @@ static struct wd_ecc_out *create_ecc_out(struct wd_ecc_sess *sess, __u32 num)
hsz = get_key_bsz(sess->key_size);
len = sizeof(struct wd_ecc_out) + hsz * num;
- out = malloc(len);
+ out = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!out) {
WD_ERR("failed to malloc out, sz = %u!\n", len);
return NULL;
@@ -1149,13 +1151,13 @@ static void del_sess_key(struct wd_ecc_sess *sess)
{
if (sess->key.prikey) {
wd_memset_zero(sess->key.prikey->data, sess->key.prikey->size);
- free(sess->key.prikey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->key.prikey->data);
free(sess->key.prikey);
sess->key.prikey = NULL;
}
if (sess->key.pubkey) {
- free(sess->key.pubkey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->key.pubkey->data);
free(sess->key.pubkey);
sess->key.pubkey = NULL;
}
@@ -1227,6 +1229,15 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup)
memcpy(&sess->setup, setup, sizeof(*setup));
sess->key_size = BITS_TO_BYTES(setup->key_bits);
+ /* Memory type set */
+ ret = wd_mem_ops_init(wd_ecc_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type);
+ if (ret) {
+ WD_ERR("failed to init memory ops!\n");
+ goto sess_err;
+ }
+ memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops));
+ sess->mm_type = setup->mm_type;
+
if (wd_ecc_setting.driver->get_extend_ops) {
ret = wd_ecc_setting.driver->get_extend_ops(&sess->eops);
if (ret) {
@@ -1508,9 +1519,10 @@ void wd_ecxdh_get_out_params(struct wd_ecc_out *out, struct wd_ecc_point **pbk)
void wd_ecc_del_in(handle_t sess, struct wd_ecc_in *in)
{
+ struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess;
__u32 bsz;
- if (!in) {
+ if (!sess_t || !in) {
WD_ERR("invalid: del ecc in parameter error!\n");
return;
}
@@ -1522,14 +1534,15 @@ void wd_ecc_del_in(handle_t sess, struct wd_ecc_in *in)
}
wd_memset_zero(in->data, bsz);
- free(in);
+ sess_t->mm_ops.free(sess_t->mm_ops.usr, in);
}
-void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out)
+void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out)
{
+ struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess;
__u32 bsz;
- if (!out) {
+ if (!sess_t || !out) {
WD_ERR("invalid: del ecc out parameter error!\n");
return;
}
@@ -1541,7 +1554,7 @@ void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out)
}
wd_memset_zero(out->data, bsz);
- free(out);
+ sess_t->mm_ops.free(sess_t->mm_ops.usr, out);
}
static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req,
@@ -1551,6 +1564,8 @@ static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req,
memcpy(&msg->req, req, sizeof(msg->req));
memcpy(&msg->hash, &sess->setup.hash, sizeof(msg->hash));
+ msg->mm_ops = &sess->mm_ops;
+ msg->mm_type = sess->mm_type;
msg->key_bytes = sess->key_size;
msg->curve_id = sess->setup.cv.cfg.id;
msg->drv_cfg = sess->eops.params;
@@ -1922,7 +1937,7 @@ static struct wd_ecc_in *create_sm2_verf_in(struct wd_ecc_sess *sess,
hsz = get_key_bsz(sess->key_size);
len = sizeof(struct wd_ecc_in) + ECC_VERF_IN_PARAM_NUM * hsz +
m_len;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc sm2 verf in, sz = %llu!\n", len);
return NULL;
@@ -2211,7 +2226,7 @@ struct wd_ecc_out *wd_sm2_new_dec_out(handle_t sess, __u32 plaintext_len)
}
len = sizeof(*ecc_out) + plaintext_len;
- ecc_out = malloc(len);
+ ecc_out = sess_t->mm_ops.alloc(sess_t->mm_ops.usr, len);
if (!ecc_out) {
WD_ERR("failed to malloc ecc_out, sz = %llu!\n", len);
return NULL;
diff --git a/wd_rsa.c b/wd_rsa.c
index cf9239c..bc78c6a 100644
--- a/wd_rsa.c
+++ b/wd_rsa.c
@@ -67,6 +67,8 @@ struct wd_rsa_sess {
struct wd_rsa_prikey *prikey;
struct wd_rsa_sess_setup setup;
void *sched_key;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
static struct wd_rsa_setting {
@@ -373,6 +375,8 @@ static int fill_rsa_msg(struct wd_rsa_msg *msg, struct wd_rsa_req *req,
memcpy(&msg->req, req, sizeof(*req));
msg->key_bytes = sess->key_size;
msg->result = WD_EINVAL;
+ msg->mm_ops = &sess->mm_ops;
+ msg->mm_type = sess->mm_type;
switch (msg->req.op_type) {
case WD_RSA_SIGN:
@@ -641,7 +645,7 @@ struct wd_rsa_kg_in *wd_rsa_new_kg_in(handle_t sess, struct wd_dtb *e,
}
kg_in_size = (int)GEN_PARAMS_SZ(c->key_size);
- kg_in = malloc(kg_in_size + sizeof(*kg_in));
+ kg_in = c->mm_ops.alloc(c->mm_ops.usr, kg_in_size + sizeof(*kg_in));
if (!kg_in) {
WD_ERR("failed to malloc kg_in memory!\n");
return NULL;
@@ -681,19 +685,16 @@ void wd_rsa_get_kg_in_params(struct wd_rsa_kg_in *kin, struct wd_dtb *e,
p->data = (void *)kin->p;
}
-static void del_kg(void *k)
+void wd_rsa_del_kg_in(handle_t sess, struct wd_rsa_kg_in *ki)
{
- if (!k) {
+ struct wd_rsa_sess *c = (struct wd_rsa_sess *)sess;
+
+ if (!c || !ki) {
WD_ERR("invalid: del key generate params err!\n");
return;
}
- free(k);
-}
-
-void wd_rsa_del_kg_in(handle_t sess, struct wd_rsa_kg_in *ki)
-{
- del_kg(ki);
+ c->mm_ops.free(c->mm_ops.usr, ki);
}
struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess)
@@ -719,7 +720,7 @@ struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess)
else
kg_out_size = (int)GEN_PARAMS_SZ(c->key_size);
- kg_out = malloc(kg_out_size + sizeof(*kg_out));
+ kg_out = c->mm_ops.alloc(c->mm_ops.usr, kg_out_size + sizeof(*kg_out));
if (!kg_out) {
WD_ERR("failed to malloc kg_out memory!\n");
return NULL;
@@ -741,13 +742,15 @@ struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess)
void wd_rsa_del_kg_out(handle_t sess, struct wd_rsa_kg_out *kout)
{
- if (!kout) {
+ struct wd_rsa_sess *c = (struct wd_rsa_sess *)sess;
+
+ if (!c || !kout) {
WD_ERR("invalid: param null at del kg out!\n");
return;
}
wd_memset_zero(kout->data, kout->size);
- del_kg(kout);
+ c->mm_ops.free(c->mm_ops.usr, kout);
}
void wd_rsa_get_kg_out_params(struct wd_rsa_kg_out *kout, struct wd_dtb *d,
@@ -804,6 +807,11 @@ void wd_rsa_set_kg_out_crt_psz(struct wd_rsa_kg_out *kout,
size_t dq_sz,
size_t dp_sz)
{
+ if (!kout) {
+ WD_ERR("invalid: input null when set kg out crt psz!\n");
+ return;
+ }
+
kout->qinvbytes = qinv_sz;
kout->dqbytes = dq_sz;
kout->dpbytes = dp_sz;
@@ -813,6 +821,11 @@ void wd_rsa_set_kg_out_psz(struct wd_rsa_kg_out *kout,
size_t d_sz,
size_t n_sz)
{
+ if (!kout) {
+ WD_ERR("invalid: input null when set kg out psz!\n");
+ return;
+ }
+
kout->dbytes = d_sz;
kout->nbytes = n_sz;
}
@@ -862,7 +875,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup,
if (setup->is_crt) {
len = sizeof(struct wd_rsa_prikey) +
(int)CRT_PARAMS_SZ(sess->key_size);
- sess->prikey = malloc(len);
+ sess->prikey = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!sess->prikey) {
WD_ERR("failed to alloc sess prikey2!\n");
return -WD_ENOMEM;
@@ -873,7 +886,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup,
} else {
len = sizeof(struct wd_rsa_prikey) +
(int)GEN_PARAMS_SZ(sess->key_size);
- sess->prikey = malloc(len);
+ sess->prikey = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!sess->prikey) {
WD_ERR("failed to alloc sess prikey1!\n");
return -WD_ENOMEM;
@@ -885,7 +898,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup,
len = sizeof(struct wd_rsa_pubkey) +
(int)GEN_PARAMS_SZ(sess->key_size);
- sess->pubkey = malloc(len);
+ sess->pubkey = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!sess->pubkey) {
free(sess->prikey);
WD_ERR("failed to alloc sess pubkey!\n");
@@ -912,8 +925,8 @@ static void del_sess_key(struct wd_rsa_sess *sess)
wd_memset_zero(prk->pkey.pkey2.data, CRT_PARAMS_SZ(sess->key_size));
else
wd_memset_zero(prk->pkey.pkey1.data, GEN_PARAMS_SZ(sess->key_size));
- free(sess->prikey);
- free(sess->pubkey);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->prikey);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->pubkey);
}
static void del_sess(struct wd_rsa_sess *c)
@@ -948,6 +961,15 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup)
memcpy(&sess->setup, setup, sizeof(*setup));
sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT;
+ /* Memory type set */
+ ret = wd_mem_ops_init(wd_rsa_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type);
+ if (ret) {
+ WD_ERR("failed to init memory ops!\n");
+ goto sess_err;
+ }
+
+ memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops));
+ sess->mm_type = setup->mm_type;
ret = create_sess_key(setup, sess);
if (ret) {
WD_ERR("failed to create rsa sess keys!\n");
--
2.33.0
1
0
From: Wenkai Lin <linwenkai6(a)hisilicon.com>
Extract a common function fill_hashagg_data_info.
Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com>
Signed-off-by: Zongyu Wu <wuzongyu1(a)huawei.com>
---
drv/hisi_dae.c | 56 +++++++++++++++++++-------------------------------
1 file changed, 21 insertions(+), 35 deletions(-)
diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c
index 4f4d13c..49387aa 100644
--- a/drv/hisi_dae.c
+++ b/drv/hisi_dae.c
@@ -26,7 +26,7 @@
#define DAE_VCHAR_OFFSET_SIZE 2
#define DAE_COL_BIT_NUM 4
#define DAE_AGG_START_COL 16
-#define DAE_HASHAGG_MAX_ROW_NUN 50000
+#define DAE_HASHAGG_MAX_ROW_NUM 50000
/* align size */
#define DAE_CHAR_ALIGN_SIZE 4
@@ -294,23 +294,8 @@ static void fill_hashagg_merge_key_data(struct dae_sqe *sqe, struct dae_ext_sqe
}
}
-static void fill_hashagg_normal_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
- struct hashagg_col_data *cols_data, __u32 agg_cols_num)
-{
- struct hw_agg_data *agg_data = cols_data->input_data;
- __u32 i;
-
- for (i = 0; i < agg_cols_num; i++) {
- sqe->agg_data_type[i] = agg_data[i].hw_type;
- sqe->agg_data_type[i] |= agg_data[i].sum_outtype << DAE_COL_BIT_NUM;
- ext_sqe->agg_data_info[i] = agg_data[i].data_info;
- }
-
- sqe->agg_col_bitmap = GENMASK(agg_cols_num + DAE_AGG_START_COL - 1, DAE_AGG_START_COL);
-}
-
-static void fill_hashagg_rehash_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
- struct hw_agg_data *agg_data, __u32 agg_cols_num)
+static void fill_hashagg_data_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
+ struct hw_agg_data *agg_data, __u32 agg_cols_num)
{
__u32 i;
@@ -344,14 +329,14 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext
hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.agg_cols;
agg_col_num = msg->agg_cols_num;
- fill_hashagg_normal_info(sqe, ext_sqe, cols_data, agg_col_num);
+ fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num);
break;
case WD_AGG_REHASH_INPUT:
agg_data = cols_data->output_data;
hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.agg_cols;
agg_col_num = cols_data->output_num;
- fill_hashagg_rehash_info(sqe, ext_sqe, agg_data, agg_col_num);
+ fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num);
break;
case WD_AGG_STREAM_OUTPUT:
case WD_AGG_REHASH_OUTPUT:
@@ -359,7 +344,7 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext
hw_agg_addr = &addr_list->output_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.out_agg_cols;
agg_col_num = cols_data->output_num;
- fill_hashagg_normal_info(sqe, ext_sqe, cols_data, cols_data->input_num);
+ fill_hashagg_data_info(sqe, ext_sqe, cols_data->input_data, cols_data->input_num);
break;
}
@@ -385,7 +370,7 @@ static void fill_hashagg_merge_input_data(struct dae_sqe *sqe, struct dae_ext_sq
struct hashagg_ctx *agg_ctx = msg->priv;
struct hashagg_col_data *cols_data = &agg_ctx->cols_data;
- fill_hashagg_rehash_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num);
+ fill_hashagg_data_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num);
}
static void fill_hashagg_ext_addr(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
@@ -422,26 +407,15 @@ static void fill_hashagg_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
static int check_hashagg_param(struct wd_agg_msg *msg)
{
- struct hashagg_col_data *cols_data;
- struct hashagg_ctx *agg_ctx;
-
if (!msg) {
WD_ERR("invalid: input hashagg msg is NULL!\n");
return -WD_EINVAL;
}
- agg_ctx = msg->priv;
- cols_data = &agg_ctx->cols_data;
- if (cols_data->output_num > DAE_MAX_OUTPUT_COLS) {
- WD_ERR("invalid: input hashagg output num %u is more than %d!\n",
- cols_data->output_num, DAE_MAX_OUTPUT_COLS);
- return -WD_EINVAL;
- }
-
if ((msg->pos == WD_AGG_STREAM_INPUT || msg->pos == WD_AGG_REHASH_INPUT) &&
- msg->row_count > DAE_HASHAGG_MAX_ROW_NUN) {
+ msg->row_count > DAE_HASHAGG_MAX_ROW_NUM) {
WD_ERR("invalid: input hashagg row count %u is more than %d!\n",
- msg->row_count, DAE_HASHAGG_MAX_ROW_NUN);
+ msg->row_count, DAE_HASHAGG_MAX_ROW_NUM);
return -WD_EINVAL;
}
@@ -1005,6 +979,7 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols,
struct hw_agg_data *user_output_data,
__u32 cols_num, __u32 *output_num)
{
+ __u32 tmp = *output_num;
__u32 i, j, k = 0;
int ret;
@@ -1013,7 +988,15 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols,
WD_ERR("invalid: col alg num(%u) more than 2!\n", agg_cols[i].col_alg_num);
return -WD_EINVAL;
}
+ tmp += agg_cols[i].col_alg_num;
+ }
+ if (tmp > DAE_MAX_OUTPUT_COLS) {
+ WD_ERR("invalid: output col num is more than %d!\n", DAE_MAX_OUTPUT_COLS);
+ return -WD_EINVAL;
+ }
+
+ for (i = 0; i < cols_num; i++) {
for (j = 0; j < agg_cols[i].col_alg_num; j++) {
ret = hashagg_check_input_data(&agg_cols[i], &user_input_data[i],
&user_output_data[k], j);
@@ -1137,6 +1120,9 @@ static int transfer_data_to_hw_type(struct hashagg_col_data *cols_data,
struct wd_agg_col_info *agg_cols = setup->agg_cols_info;
int ret;
+ if (setup->is_count_all)
+ cols_data->output_num++;
+
ret = transfer_input_col_info(agg_cols, user_input_data, user_output_data,
setup->agg_cols_num, &cols_data->output_num);
if (ret)
--
2.33.0
1
0
From: Wenkai Lin <linwenkai6(a)hisilicon.com>
Extract a common function fill_hashagg_data_info.
Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com>
Signed-off-by: Zongyu Wu <wuzongyu1(a)huawei.com>
---
drv/hisi_dae.c | 56 +++++++++++++++++++-------------------------------
1 file changed, 21 insertions(+), 35 deletions(-)
diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c
index 4f4d13c..49387aa 100644
--- a/drv/hisi_dae.c
+++ b/drv/hisi_dae.c
@@ -26,7 +26,7 @@
#define DAE_VCHAR_OFFSET_SIZE 2
#define DAE_COL_BIT_NUM 4
#define DAE_AGG_START_COL 16
-#define DAE_HASHAGG_MAX_ROW_NUN 50000
+#define DAE_HASHAGG_MAX_ROW_NUM 50000
/* align size */
#define DAE_CHAR_ALIGN_SIZE 4
@@ -294,23 +294,8 @@ static void fill_hashagg_merge_key_data(struct dae_sqe *sqe, struct dae_ext_sqe
}
}
-static void fill_hashagg_normal_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
- struct hashagg_col_data *cols_data, __u32 agg_cols_num)
-{
- struct hw_agg_data *agg_data = cols_data->input_data;
- __u32 i;
-
- for (i = 0; i < agg_cols_num; i++) {
- sqe->agg_data_type[i] = agg_data[i].hw_type;
- sqe->agg_data_type[i] |= agg_data[i].sum_outtype << DAE_COL_BIT_NUM;
- ext_sqe->agg_data_info[i] = agg_data[i].data_info;
- }
-
- sqe->agg_col_bitmap = GENMASK(agg_cols_num + DAE_AGG_START_COL - 1, DAE_AGG_START_COL);
-}
-
-static void fill_hashagg_rehash_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
- struct hw_agg_data *agg_data, __u32 agg_cols_num)
+static void fill_hashagg_data_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
+ struct hw_agg_data *agg_data, __u32 agg_cols_num)
{
__u32 i;
@@ -344,14 +329,14 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext
hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.agg_cols;
agg_col_num = msg->agg_cols_num;
- fill_hashagg_normal_info(sqe, ext_sqe, cols_data, agg_col_num);
+ fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num);
break;
case WD_AGG_REHASH_INPUT:
agg_data = cols_data->output_data;
hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.agg_cols;
agg_col_num = cols_data->output_num;
- fill_hashagg_rehash_info(sqe, ext_sqe, agg_data, agg_col_num);
+ fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num);
break;
case WD_AGG_STREAM_OUTPUT:
case WD_AGG_REHASH_OUTPUT:
@@ -359,7 +344,7 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext
hw_agg_addr = &addr_list->output_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.out_agg_cols;
agg_col_num = cols_data->output_num;
- fill_hashagg_normal_info(sqe, ext_sqe, cols_data, cols_data->input_num);
+ fill_hashagg_data_info(sqe, ext_sqe, cols_data->input_data, cols_data->input_num);
break;
}
@@ -385,7 +370,7 @@ static void fill_hashagg_merge_input_data(struct dae_sqe *sqe, struct dae_ext_sq
struct hashagg_ctx *agg_ctx = msg->priv;
struct hashagg_col_data *cols_data = &agg_ctx->cols_data;
- fill_hashagg_rehash_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num);
+ fill_hashagg_data_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num);
}
static void fill_hashagg_ext_addr(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
@@ -422,26 +407,15 @@ static void fill_hashagg_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
static int check_hashagg_param(struct wd_agg_msg *msg)
{
- struct hashagg_col_data *cols_data;
- struct hashagg_ctx *agg_ctx;
-
if (!msg) {
WD_ERR("invalid: input hashagg msg is NULL!\n");
return -WD_EINVAL;
}
- agg_ctx = msg->priv;
- cols_data = &agg_ctx->cols_data;
- if (cols_data->output_num > DAE_MAX_OUTPUT_COLS) {
- WD_ERR("invalid: input hashagg output num %u is more than %d!\n",
- cols_data->output_num, DAE_MAX_OUTPUT_COLS);
- return -WD_EINVAL;
- }
-
if ((msg->pos == WD_AGG_STREAM_INPUT || msg->pos == WD_AGG_REHASH_INPUT) &&
- msg->row_count > DAE_HASHAGG_MAX_ROW_NUN) {
+ msg->row_count > DAE_HASHAGG_MAX_ROW_NUM) {
WD_ERR("invalid: input hashagg row count %u is more than %d!\n",
- msg->row_count, DAE_HASHAGG_MAX_ROW_NUN);
+ msg->row_count, DAE_HASHAGG_MAX_ROW_NUM);
return -WD_EINVAL;
}
@@ -1005,6 +979,7 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols,
struct hw_agg_data *user_output_data,
__u32 cols_num, __u32 *output_num)
{
+ __u32 tmp = *output_num;
__u32 i, j, k = 0;
int ret;
@@ -1013,7 +988,15 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols,
WD_ERR("invalid: col alg num(%u) more than 2!\n", agg_cols[i].col_alg_num);
return -WD_EINVAL;
}
+ tmp += agg_cols[i].col_alg_num;
+ }
+ if (tmp > DAE_MAX_OUTPUT_COLS) {
+ WD_ERR("invalid: output col num is more than %d!\n", DAE_MAX_OUTPUT_COLS);
+ return -WD_EINVAL;
+ }
+
+ for (i = 0; i < cols_num; i++) {
for (j = 0; j < agg_cols[i].col_alg_num; j++) {
ret = hashagg_check_input_data(&agg_cols[i], &user_input_data[i],
&user_output_data[k], j);
@@ -1137,6 +1120,9 @@ static int transfer_data_to_hw_type(struct hashagg_col_data *cols_data,
struct wd_agg_col_info *agg_cols = setup->agg_cols_info;
int ret;
+ if (setup->is_count_all)
+ cols_data->output_num++;
+
ret = transfer_input_col_info(agg_cols, user_input_data, user_output_data,
setup->agg_cols_num, &cols_data->output_num);
if (ret)
--
2.33.0
1
0
18 Nov '25
From: lizhi <lizhi206(a)huawei.com>
The hpre v2 interface needs to support the no-sva business model
within the new evolution framework. This ensures that users
switching to the v2 interface can utilize both the sva mode
and the no-sva mode.
Signed-off-by: lizhi <lizhi206(a)huawei.com>
Signed-off-by: Zongyu Wu <wuzongyu1(a)huawei.com>
---
drv/hisi_comp.c | 3 +-
drv/hisi_hpre.c | 650 ++++++++++++++++++++++++++++-----------
drv/hisi_qm_udrv.c | 15 +-
include/drv/wd_dh_drv.h | 3 +
include/drv/wd_ecc_drv.h | 3 +
include/drv/wd_rsa_drv.h | 3 +
include/wd_dh.h | 2 +
include/wd_ecc.h | 2 +
include/wd_rsa.h | 2 +
wd_dh.c | 22 +-
wd_ecc.c | 53 ++--
wd_rsa.c | 56 +++-
12 files changed, 580 insertions(+), 234 deletions(-)
diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c
index 87d2103..8474e17 100644
--- a/drv/hisi_comp.c
+++ b/drv/hisi_comp.c
@@ -267,7 +267,8 @@ static __u32 copy_to_out(struct wd_comp_msg *msg, struct hisi_comp_buf *buf, __u
struct wd_comp_req *req = &msg->req;
struct wd_datalist *node = req->list_dst;
__u32 sgl_restlen, copy_len;
- __u32 len = 0, sgl_cplen = 0;
+ __u32 sgl_cplen = 0;
+ __u32 len = 0;
copy_len = total_len > req->dst_len ?
req->dst_len : total_len;
diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c
index 7ce842b..f3f2451 100644
--- a/drv/hisi_hpre.c
+++ b/drv/hisi_hpre.c
@@ -34,14 +34,17 @@
#define SM2_PONIT_SIZE 64
#define MAX_HASH_LENS BITS_TO_BYTES(521)
#define HW_PLAINTEXT_BYTES_MAX BITS_TO_BYTES(4096)
-#define HPRE_CTX_Q_NUM_DEF 1
+#define HPRE_CTX_Q_NUM_DEF 1
+#define MAP_PAIR_NUM_MAX 6
#define CRT_PARAMS_SZ(key_size) ((5 * (key_size)) >> 1)
#define CRT_GEN_PARAMS_SZ(key_size) ((7 * (key_size)) >> 1)
#define GEN_PARAMS_SZ(key_size) ((key_size) << 1)
#define CRT_PARAM_SZ(key_size) ((key_size) >> 1)
-#define WD_TRANS_FAIL 0
+#define GEN_PARAMS_SZ_UL(key_size) ((unsigned long)(key_size) << 1)
+#define DMA_ADDR(hi, lo) ((__u64)(((__u64)(hi) << 32) | (__u64)(lo)))
+#define WD_TRANS_FAIL 0
#define CURVE_PARAM_NUM 6
#define SECP256R1_KEY_SIZE 32
@@ -78,6 +81,21 @@ enum hpre_alg_name {
WD_ECC
};
+enum hpre_hw_msg_field {
+ HW_MSG_IN,
+ HW_MSG_OUT,
+ HW_MSG_KEY,
+};
+
+struct map_info_cache {
+ struct map_pair {
+ void *addr;
+ uintptr_t pa;
+ size_t size;
+ } pairs[MAP_PAIR_NUM_MAX];
+ size_t cnt;
+};
+
/* put vendor hardware message as a user interface is not suitable here */
struct hisi_hpre_sqe {
__u32 alg : 5;
@@ -113,12 +131,122 @@ struct hisi_hpre_sqe {
struct hisi_hpre_ctx {
struct wd_ctx_config_internal config;
+ struct wd_mm_ops *mm_ops;
+ handle_t rsv_mem_ctx;
};
struct hpre_ecc_ctx {
__u32 enable_hpcore;
};
+static void add_map_info(struct map_info_cache *cache, void *addr, uintptr_t dma, size_t size)
+{
+ /* The cnt is guaranteed not to exceed MAP_PAIR_NUM_MAX within hpre. */
+ cache->pairs[cache->cnt].addr = addr;
+ cache->pairs[cache->cnt].pa = dma;
+ cache->pairs[cache->cnt].size = size;
+ cache->cnt++;
+}
+
+static void unmap_addr_in_cache(struct wd_mm_ops *mm_ops, struct map_info_cache *cache)
+{
+ size_t i;
+
+ if (mm_ops->sva_mode)
+ return;
+
+ /* The cnt is guaranteed not to exceed MAP_PAIR_NUM_MAX within hpre. */
+ for (i = 0; i < cache->cnt; i++)
+ mm_ops->iova_unmap(mm_ops->usr, cache->pairs[i].addr,
+ (void *)cache->pairs[i].pa, cache->pairs[i].size);
+}
+
+static void unsetup_hw_msg_addr(struct wd_mm_ops *mm_ops, enum hpre_hw_msg_field t_type,
+ struct hisi_hpre_sqe *hw_msg, void *va, size_t data_sz)
+{
+ void *addr;
+
+ if (!mm_ops || mm_ops->sva_mode || !va || !data_sz)
+ return;
+
+ switch (t_type) {
+ case HW_MSG_KEY:
+ addr = VA_ADDR(hw_msg->hi_key, hw_msg->low_key);
+ break;
+ case HW_MSG_IN:
+ addr = VA_ADDR(hw_msg->hi_in, hw_msg->low_in);
+ break;
+ case HW_MSG_OUT:
+ addr = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
+ break;
+ default:
+ return;
+ }
+
+ if (!addr)
+ return;
+
+ mm_ops->iova_unmap(mm_ops->usr, va, (void *)addr, data_sz);
+}
+
+static uintptr_t select_addr_by_sva_mode(struct wd_mm_ops *mm_ops, void *data,
+ size_t data_sz, struct map_info_cache *cache)
+{
+ uintptr_t addr;
+
+ if (!mm_ops->sva_mode) {
+ addr = (uintptr_t)mm_ops->iova_map(mm_ops->usr, data, data_sz);
+ if (!addr) {
+ WD_ERR("Failed to get mappped DMA address for hardware.\n");
+ return 0;
+ }
+ add_map_info(cache, data, addr, data_sz);
+ } else {
+ addr = (uintptr_t)data;
+ }
+
+ return addr;
+}
+
+static void fill_hw_msg_addr(enum hpre_hw_msg_field t_type, struct hisi_hpre_sqe *hw_msg,
+ uintptr_t addr)
+{
+ switch (t_type) {
+ case HW_MSG_KEY:
+ hw_msg->low_key = LW_U32(addr);
+ hw_msg->hi_key = HI_U32(addr);
+ break;
+ case HW_MSG_IN:
+ hw_msg->low_in = LW_U32(addr);
+ hw_msg->hi_in = HI_U32(addr);
+ break;
+ case HW_MSG_OUT:
+ hw_msg->low_out = LW_U32(addr);
+ hw_msg->hi_out = HI_U32(addr);
+ break;
+ default:
+ return;
+ }
+}
+
+static int check_hpre_mem_params(struct wd_mm_ops *mm_ops, enum wd_mem_type mm_type)
+{
+ if (!mm_ops) {
+ WD_ERR("Memory operation functions are null.\n");
+ return -WD_EINVAL;
+ }
+
+ if (mm_type == UADK_MEM_AUTO && !mm_ops->sva_mode) {
+ WD_ERR("No-SVA in auto mode is not supported yet.\n");
+ return -WD_EINVAL;
+ } else if (mm_type > UADK_MEM_PROXY) {
+ WD_ERR("failed to check memory type.\n");
+ return -WD_EINVAL;
+ }
+
+ return WD_SUCCESS;
+}
+
static void dump_hpre_msg(void *msg, int alg)
{
struct wd_rsa_msg *rsa_msg;
@@ -269,7 +397,8 @@ static int fill_rsa_crt_prikey2(struct wd_rsa_prikey *prikey,
*data = wd_dq->data;
- return WD_SUCCESS;
+ return (int)(wd_dq->bsize + wd_qinv->bsize + wd_p->bsize +
+ wd_q->bsize + wd_dp->bsize);
}
static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data)
@@ -292,7 +421,7 @@ static int fill_rsa_prikey1(struct wd_rsa_prikey *prikey, void **data)
*data = wd_d->data;
- return WD_SUCCESS;
+ return (int)(wd_n->bsize + wd_d->bsize);
}
static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data)
@@ -315,7 +444,7 @@ static int fill_rsa_pubkey(struct wd_rsa_pubkey *pubkey, void **data)
*data = wd_e->data;
- return WD_SUCCESS;
+ return (int)(wd_n->bsize + wd_e->bsize);
}
static int fill_rsa_genkey_in(struct wd_rsa_kg_in *genkey)
@@ -378,22 +507,21 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg,
{
struct wd_rsa_req *req = &msg->req;
struct wd_rsa_kg_out *key = req->dst;
+ struct wd_rsa_msg *target_msg;
__u16 kbytes = msg->key_bytes;
struct wd_dtb qinv, dq, dp;
struct wd_dtb d, n;
- void *data;
int ret;
- if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD) {
- /* async */
- if (qp_mode == CTX_MODE_ASYNC) {
- data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- key = container_of(data, struct wd_rsa_kg_out, data);
- } else {
- key = req->dst;
- }
+ target_msg = (struct wd_rsa_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct rsa send msg from hardware!\n");
+ return -WD_ADDR_ERR;
}
+ if (hw_msg->alg == HPRE_ALG_KG_CRT || hw_msg->alg == HPRE_ALG_KG_STD)
+ key = target_msg->req.dst;
+
msg->result = WD_SUCCESS;
if (hw_msg->alg == HPRE_ALG_KG_CRT) {
req->dst_bytes = CRT_GEN_PARAMS_SZ(kbytes);
@@ -424,37 +552,38 @@ static int rsa_out_transfer(struct wd_rsa_msg *msg,
return WD_SUCCESS;
}
-static int rsa_prepare_key(struct wd_rsa_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int rsa_prepare_key(struct wd_rsa_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
struct wd_rsa_req *req = &msg->req;
+ uintptr_t addr;
+ int ret, len;
void *data;
- int ret;
if (req->op_type == WD_RSA_SIGN) {
if (hw_msg->alg == HPRE_ALG_NC_CRT) {
- ret = fill_rsa_crt_prikey2((void *)msg->key, &data);
- if (ret)
- return ret;
+ len = fill_rsa_crt_prikey2((void *)msg->key, &data);
+ if (len <= 0)
+ return -WD_EINVAL;
} else {
- ret = fill_rsa_prikey1((void *)msg->key, &data);
- if (ret)
- return ret;
+ len = fill_rsa_prikey1((void *)msg->key, &data);
+ if (len <= 0)
+ return -WD_EINVAL;
hw_msg->alg = HPRE_ALG_NC_NCRT;
}
} else if (req->op_type == WD_RSA_VERIFY) {
- ret = fill_rsa_pubkey((void *)msg->key, &data);
- if (ret)
- return ret;
+ len = fill_rsa_pubkey((void *)msg->key, &data);
+ if (len <= 0)
+ return -WD_EINVAL;
hw_msg->alg = HPRE_ALG_NC_NCRT;
} else if (req->op_type == WD_RSA_GENKEY) {
ret = fill_rsa_genkey_in((void *)msg->key);
if (ret)
return ret;
- ret = wd_rsa_kg_in_data((void *)msg->key, (char **)&data);
- if (ret < 0) {
+ len = wd_rsa_kg_in_data((void *)msg->key, (char **)&data);
+ if (len < 0) {
WD_ERR("failed to get rsa gen key data!\n");
- return ret;
+ return -WD_EINVAL;
}
if (hw_msg->alg == HPRE_ALG_NC_CRT)
hw_msg->alg = HPRE_ALG_KG_CRT;
@@ -465,36 +594,53 @@ static int rsa_prepare_key(struct wd_rsa_msg *msg,
return -WD_EINVAL;
}
- hw_msg->low_key = LW_U32((uintptr_t)data);
- hw_msg->hi_key = HI_U32((uintptr_t)data);
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, len, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr);
- return WD_SUCCESS;
+ return ret;
}
-static int rsa_prepare_iot(struct wd_rsa_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int rsa_prepare_iot(struct wd_rsa_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
struct wd_rsa_req *req = &msg->req;
struct wd_rsa_kg_out *kout = req->dst;
int ret = WD_SUCCESS;
- void *out = NULL;
+ uintptr_t phy, out;
if (req->op_type != WD_RSA_GENKEY) {
- hw_msg->low_in = LW_U32((uintptr_t)req->src);
- hw_msg->hi_in = HI_U32((uintptr_t)req->src);
- out = req->dst;
+ phy = select_addr_by_sva_mode(msg->mm_ops, req->src, req->src_bytes, cache);
+ if (!phy)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_IN, hw_msg, phy);
+ phy = select_addr_by_sva_mode(msg->mm_ops, req->dst, req->dst_bytes, cache);
+ if (!phy)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_OUT, hw_msg, phy);
} else {
hw_msg->low_in = 0;
hw_msg->hi_in = 0;
ret = wd_rsa_kg_out_data(kout, (char **)&out);
if (ret < 0)
return ret;
- }
- hw_msg->low_out = LW_U32((uintptr_t)out);
- hw_msg->hi_out = HI_U32((uintptr_t)out);
+ if (!msg->mm_ops->sva_mode) {
+ phy = (uintptr_t)msg->mm_ops->iova_map(msg->mm_ops->usr, kout, ret);
+ if (!phy) {
+ WD_ERR("Failed to get DMA address for rsa output!\n");
+ return -WD_ENOMEM;
+ }
+ add_map_info(cache, kout, phy, ret);
+ out = phy + out - (uintptr_t)kout;
+ }
+
+ hw_msg->low_out = LW_U32(out);
+ hw_msg->hi_out = HI_U32(out);
+ }
- return ret;
+ return WD_SUCCESS;
}
static int hpre_init_qm_priv(struct wd_ctx_config_internal *config,
@@ -615,10 +761,17 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_rsa_msg *msg = rsa_msg;
+ struct map_info_cache cache = {0};
struct hisi_hpre_sqe hw_msg;
__u16 send_cnt = 0;
int ret;
+ ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type);
+ if (ret) {
+ WD_ERR("rsa memory parmas is err, and ret is %d!\n", ret);
+ return ret;
+ }
+
memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe));
if (msg->key_type == WD_RSA_PRIKEY1 ||
@@ -631,21 +784,30 @@ static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
hw_msg.task_len1 = msg->key_bytes / BYTE_BITS - 0x1;
- ret = rsa_prepare_key(msg, &hw_msg);
- if (ret < 0)
- return ret;
+ ret = rsa_prepare_key(msg, &hw_msg, &cache);
+ if (ret)
+ goto rsa_fail;
/* prepare in/out put */
- ret = rsa_prepare_iot(msg, &hw_msg);
- if (ret < 0)
- return ret;
+ ret = rsa_prepare_iot(msg, &hw_msg, &cache);
+ if (ret)
+ goto rsa_fail;
hisi_set_msg_id(h_qp, &msg->tag);
hw_msg.done = 0x1;
hw_msg.etype = 0x0;
- hw_msg.low_tag = msg->tag;
+ hw_msg.low_tag = LW_U32((uintptr_t)msg);
+ hw_msg.hi_tag = HI_U32((uintptr_t)msg);
- return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ ret = hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ if (unlikely(ret))
+ goto rsa_fail;
+
+ return ret;
+
+rsa_fail:
+ unmap_addr_in_cache(msg->mm_ops, &cache);
+ return ret;
}
static void hpre_result_check(struct hisi_hpre_sqe *hw_msg,
@@ -671,32 +833,29 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct hisi_qp *qp = (struct hisi_qp *)h_qp;
- struct hisi_hpre_sqe hw_msg = {0};
+ struct wd_rsa_msg *target_msg;
struct wd_rsa_msg *msg = rsa_msg;
- struct wd_rsa_msg *temp_msg;
+ struct hisi_hpre_sqe hw_msg = {0};
+ size_t ilen = 0, olen = 0;
__u16 recv_cnt = 0;
+ __u16 kbytes;
int ret;
ret = hisi_qm_recv(h_qp, &hw_msg, 1, &recv_cnt);
if (ret < 0)
return ret;
- ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag);
+ target_msg = (struct wd_rsa_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
+ ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag);
if (ret)
return ret;
- msg->tag = LW_U16(hw_msg.low_tag);
- if (qp->q_info.qp_mode == CTX_MODE_ASYNC) {
- temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag);
- if (!temp_msg) {
- WD_ERR("failed to get send msg! idx = %u, tag = %u.\n",
- qp->q_info.idx, msg->tag);
- return -WD_EINVAL;
- }
- } else {
- temp_msg = msg;
- }
-
+ msg->tag = target_msg->tag;
hpre_result_check(&hw_msg, &msg->result);
if (!msg->result) {
ret = rsa_out_transfer(msg, &hw_msg, qp->q_info.qp_mode);
@@ -707,15 +866,36 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg)
}
if (unlikely(msg->result != WD_SUCCESS))
- dump_hpre_msg(temp_msg, WD_RSA);
+ dump_hpre_msg(target_msg, WD_RSA);
+
+ if (!target_msg->mm_ops->sva_mode) {
+ kbytes = target_msg->key_bytes;
+ if (hw_msg.alg == HPRE_ALG_KG_CRT) {
+ olen = CRT_GEN_PARAMS_SZ(kbytes);
+ ilen = GEN_PARAMS_SZ_UL(kbytes);
+ } else if (hw_msg.alg == HPRE_ALG_KG_STD) {
+ olen = GEN_PARAMS_SZ_UL(kbytes);
+ ilen = GEN_PARAMS_SZ_UL(kbytes);
+ } else {
+ olen = kbytes;
+ ilen = kbytes;
+ }
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, &hw_msg,
+ target_msg->req.src, ilen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, &hw_msg,
+ target_msg->req.dst, olen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, &hw_msg,
+ target_msg->key, target_msg->key_bytes);
+ }
return WD_SUCCESS;
}
-static int fill_dh_xp_params(struct wd_dh_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int fill_dh_xp_params(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
struct wd_dh_req *req = &msg->req;
+ uintptr_t addr;
void *x, *p;
int ret;
@@ -735,26 +915,30 @@ static int fill_dh_xp_params(struct wd_dh_msg *msg,
return ret;
}
- hw_msg->low_key = LW_U32((uintptr_t)x);
- hw_msg->hi_key = HI_U32((uintptr_t)x);
+ addr = select_addr_by_sva_mode(msg->mm_ops, x, GEN_PARAMS_SZ_UL(msg->key_bytes), cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr);
- return WD_SUCCESS;
+ return ret;
}
-static int dh_out_transfer(struct wd_dh_msg *msg,
- struct hisi_hpre_sqe *hw_msg, __u8 qp_mode)
+static int dh_out_transfer(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ __u8 qp_mode)
{
__u16 key_bytes = (hw_msg->task_len1 + 1) * BYTE_BITS;
struct wd_dh_req *req = &msg->req;
+ struct wd_dh_msg *target_msg;
void *out;
int ret;
- /* async */
- if (qp_mode == CTX_MODE_ASYNC)
- out = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- else
- out = req->pri;
+ target_msg = (struct wd_dh_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+ out = target_msg->req.pri;
ret = hpre_bin_to_crypto_bin((char *)out,
(const char *)out, key_bytes, "dh out");
if (!ret)
@@ -768,12 +952,20 @@ static int dh_out_transfer(struct wd_dh_msg *msg,
static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
+ struct map_info_cache cache = {0};
struct wd_dh_msg *msg = dh_msg;
struct wd_dh_req *req = &msg->req;
struct hisi_hpre_sqe hw_msg;
__u16 send_cnt = 0;
+ uintptr_t addr;
int ret;
+ ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type);
+ if (ret) {
+ WD_ERR("dh memory parmas is err, and ret is %d!\n", ret);
+ return ret;
+ }
+
memset(&hw_msg, 0, sizeof(struct hisi_hpre_sqe));
if (msg->is_g2 && req->op_type != WD_DH_PHASE2)
@@ -791,32 +983,50 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
WD_ERR("failed to transfer dh g para format to hpre bin!\n");
return ret;
}
-
- hw_msg.low_in = LW_U32((uintptr_t)msg->g);
- hw_msg.hi_in = HI_U32((uintptr_t)msg->g);
+ addr = select_addr_by_sva_mode(msg->mm_ops, msg->g, msg->key_bytes, &cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_IN, &hw_msg, addr);
}
- ret = fill_dh_xp_params(msg, &hw_msg);
- if (ret)
- return ret;
+ ret = fill_dh_xp_params(msg, &hw_msg, &cache);
+ if (ret) {
+ WD_ERR("failed to fill dh x or p para!\n");
+ goto dh_fail;
+ }
hisi_set_msg_id(h_qp, &msg->tag);
- hw_msg.low_out = LW_U32((uintptr_t)req->pri);
- hw_msg.hi_out = HI_U32((uintptr_t)req->pri);
hw_msg.done = 0x1;
hw_msg.etype = 0x0;
- hw_msg.low_tag = msg->tag;
- return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ hw_msg.low_tag = LW_U32((uintptr_t)msg);
+ hw_msg.hi_tag = HI_U32((uintptr_t)msg);
+
+ addr = select_addr_by_sva_mode(msg->mm_ops, req->pri, req->pri_bytes, &cache);
+ if (!addr) {
+ ret = -WD_ENOMEM;
+ goto dh_fail;
+ }
+ fill_hw_msg_addr(HW_MSG_OUT, &hw_msg, addr);
+
+ ret = hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
+ if (unlikely(ret))
+ goto dh_fail;
+
+ return ret;
+
+dh_fail:
+ unmap_addr_in_cache(msg->mm_ops, &cache);
+ return ret;
}
static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct hisi_qp *qp = (struct hisi_qp *)h_qp;
- struct wd_dh_msg *msg = dh_msg;
struct hisi_hpre_sqe hw_msg = {0};
- struct wd_dh_msg *temp_msg;
+ struct wd_dh_msg *msg = dh_msg;
+ struct wd_dh_msg *target_msg;
__u16 recv_cnt = 0;
int ret;
@@ -824,22 +1034,17 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
if (ret < 0)
return ret;
- ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag);
+ target_msg = (struct wd_dh_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
+ ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag);
if (ret)
return ret;
- msg->tag = LW_U16(hw_msg.low_tag);
- if (qp->q_info.qp_mode == CTX_MODE_ASYNC) {
- temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag);
- if (!temp_msg) {
- WD_ERR("failed to get send msg! idx = %u, tag = %u.\n",
- qp->q_info.idx, msg->tag);
- return -WD_EINVAL;
- }
- } else {
- temp_msg = msg;
- }
-
+ msg->tag = target_msg->tag;
hpre_result_check(&hw_msg, &msg->result);
if (!msg->result) {
ret = dh_out_transfer(msg, &hw_msg, qp->q_info.qp_mode);
@@ -850,7 +1055,16 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg)
}
if (unlikely(msg->result != WD_SUCCESS))
- dump_hpre_msg(temp_msg, WD_DH);
+ dump_hpre_msg(target_msg, WD_DH);
+
+ if (!target_msg->mm_ops->sva_mode) {
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, &hw_msg,
+ target_msg->req.pri, target_msg->req.pri_bytes);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, &hw_msg,
+ target_msg->req.x_p, GEN_PARAMS_SZ_UL(target_msg->key_bytes));
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, &hw_msg,
+ target_msg->g, target_msg->key_bytes);
+ }
return WD_SUCCESS;
}
@@ -1081,30 +1295,49 @@ static bool is_prikey_used(__u8 op_type)
op_type == HPRE_SM2_DEC;
}
-static int ecc_prepare_key(struct wd_ecc_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static __u32 ecc_get_prikey_size(struct wd_ecc_msg *msg)
+{
+ if (msg->req.op_type == WD_SM2_SIGN ||
+ msg->req.op_type == WD_ECDSA_SIGN ||
+ msg->req.op_type == WD_SM2_DECRYPT)
+ return ECC_PRIKEY_SZ(msg->key_bytes);
+ else if (msg->curve_id == WD_X25519 ||
+ msg->curve_id == WD_X448)
+ return X_DH_HW_KEY_SZ(msg->key_bytes);
+ else
+ return ECDH_HW_KEY_SZ(msg->key_bytes);
+}
+
+static int ecc_prepare_key(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
void *data = NULL;
+ uintptr_t addr;
+ size_t ksz;
int ret;
if (is_prikey_used(msg->req.op_type)) {
+ ksz = ecc_get_prikey_size(msg);
ret = ecc_prepare_prikey((void *)msg->key, &data, msg->curve_id);
if (ret)
return ret;
} else {
+ ksz = ECC_PUBKEY_SZ(msg->key_bytes);
ret = ecc_prepare_pubkey((void *)msg->key, &data);
if (ret)
return ret;
}
- hw_msg->low_key = LW_U32((uintptr_t)data);
- hw_msg->hi_key = HI_U32((uintptr_t)data);
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, ksz, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_KEY, hw_msg, addr);
- return WD_SUCCESS;
+ return ret;
}
-static void ecc_get_io_len(__u32 atype, __u32 hsz, size_t *ilen,
- size_t *olen)
+static void ecc_get_io_len(__u32 atype, __u32 hsz,
+ size_t *ilen, size_t *olen)
{
if (atype == HPRE_ALG_ECDH_MULTIPLY) {
*olen = ECDH_OUT_PARAMS_SZ(hsz);
@@ -1467,12 +1700,12 @@ static int ecc_prepare_out(struct wd_ecc_msg *msg, void **data)
}
/* prepare in/out hw message */
-static int ecc_prepare_iot(struct wd_ecc_msg *msg,
- struct hisi_hpre_sqe *hw_msg)
+static int ecc_prepare_iot(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
+ size_t i_sz, o_sz;
void *data = NULL;
- size_t i_sz = 0;
- size_t o_sz = 0;
+ uintptr_t addr;
__u16 kbytes;
int ret;
@@ -1483,8 +1716,11 @@ static int ecc_prepare_iot(struct wd_ecc_msg *msg,
WD_ERR("failed to prepare ecc in!\n");
return ret;
}
- hw_msg->low_in = LW_U32((uintptr_t)data);
- hw_msg->hi_in = HI_U32((uintptr_t)data);
+
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, i_sz, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_IN, hw_msg, addr);
ret = ecc_prepare_out(msg, &data);
if (ret) {
@@ -1496,8 +1732,10 @@ static int ecc_prepare_iot(struct wd_ecc_msg *msg,
if (!data)
return WD_SUCCESS;
- hw_msg->low_out = LW_U32((uintptr_t)data);
- hw_msg->hi_out = HI_U32((uintptr_t)data);
+ addr = select_addr_by_sva_mode(msg->mm_ops, data, o_sz, cache);
+ if (!addr)
+ return -WD_ENOMEM;
+ fill_hw_msg_addr(HW_MSG_OUT, hw_msg, addr);
return WD_SUCCESS;
}
@@ -1613,7 +1851,7 @@ static struct wd_ecc_out *create_ecdh_out(struct wd_ecc_msg *msg)
return NULL;
}
- out = malloc(len);
+ out = msg->mm_ops->alloc(msg->mm_ops->usr, len);
if (!out) {
WD_ERR("failed to alloc out memory, sz = %u!\n", len);
return NULL;
@@ -1677,7 +1915,7 @@ static struct wd_ecc_msg *create_req(struct wd_ecc_msg *src, __u8 req_idx)
prikey = (struct wd_ecc_prikey *)(ecc_key + 1);
ecc_key->prikey = prikey;
- prikey->data = malloc(ECC_PRIKEY_SZ(src->key_bytes));
+ prikey->data = src->mm_ops->alloc(src->mm_ops->usr, ECC_PRIKEY_SZ(src->key_bytes));
if (unlikely(!prikey->data)) {
WD_ERR("failed to alloc prikey data!\n");
goto fail_alloc_key_data;
@@ -1696,7 +1934,7 @@ static struct wd_ecc_msg *create_req(struct wd_ecc_msg *src, __u8 req_idx)
return dst;
fail_set_prikey:
- free(prikey->data);
+ src->mm_ops->free(src->mm_ops->usr, prikey->data);
fail_alloc_key_data:
free(ecc_key);
fail_alloc_key:
@@ -1709,9 +1947,9 @@ static void free_req(struct wd_ecc_msg *msg)
{
struct wd_ecc_key *key = (void *)msg->key;
- free(key->prikey->data);
+ msg->mm_ops->free(msg->mm_ops->usr, key->prikey->data);
free(key);
- free(msg->req.dst);
+ msg->mm_ops->free(msg->mm_ops->usr, msg->req.dst);
free(msg);
}
@@ -1732,7 +1970,8 @@ static int split_req(struct wd_ecc_msg *src, struct wd_ecc_msg **dst)
return WD_SUCCESS;
}
-static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg)
+static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg,
+ struct map_info_cache *cache)
{
__u32 hw_sz = get_hw_keysz(msg->key_bytes);
__u8 op_type = msg->req.op_type;
@@ -1757,39 +1996,42 @@ static int ecc_fill(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg)
return ret;
/* prepare key */
- ret = ecc_prepare_key(msg, hw_msg);
+ ret = ecc_prepare_key(msg, hw_msg, cache);
if (ret)
return ret;
/* prepare in/out put */
- ret = ecc_prepare_iot(msg, hw_msg);
+ ret = ecc_prepare_iot(msg, hw_msg, cache);
if (ret)
return ret;
hw_msg->done = 0x1;
hw_msg->etype = 0x0;
- hw_msg->low_tag = msg->tag;
+
+ hw_msg->low_tag = LW_U32((uintptr_t)msg);
+ hw_msg->hi_tag = HI_U32((uintptr_t)msg);
hw_msg->task_len1 = hw_sz / BYTE_BITS - 0x1;
return ret;
}
-static int ecc_general_send(handle_t ctx, struct wd_ecc_msg *msg)
+static int ecc_general_send(handle_t ctx, struct wd_ecc_msg *msg,
+ struct map_info_cache *cache)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct hisi_hpre_sqe hw_msg;
__u16 send_cnt = 0;
int ret;
- ret = ecc_fill(msg, &hw_msg);
+ ret = ecc_fill(msg, &hw_msg, cache);
if (ret)
return ret;
return hisi_qm_send(h_qp, &hw_msg, 1, &send_cnt);
}
-
-static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg)
+static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg,
+ struct map_info_cache *cache)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_sm2_enc_in *ein = msg->req.src;
@@ -1801,7 +2043,7 @@ static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg)
if (ein->plaintext.dsize <= HW_PLAINTEXT_BYTES_MAX &&
hash->type == WD_HASH_SM3)
- return ecc_general_send(ctx, msg);
+ return ecc_general_send(ctx, msg, cache);
if (unlikely(!ein->k_set)) {
WD_ERR("invalid: k not set!\n");
@@ -1824,13 +2066,13 @@ static int sm2_enc_send(handle_t ctx, struct wd_ecc_msg *msg)
return ret;
}
- ret = ecc_fill(msg_dst[0], &hw_msg[0]);
+ ret = ecc_fill(msg_dst[0], &hw_msg[0], cache);
if (unlikely(ret)) {
WD_ERR("failed to fill 1th sqe, ret = %d!\n", ret);
goto fail_fill_sqe;
}
- ret = ecc_fill(msg_dst[1], &hw_msg[1]);
+ ret = ecc_fill(msg_dst[1], &hw_msg[1], cache);
if (unlikely(ret)) {
WD_ERR("failed to fill 2th sqe, ret = %d!\n", ret);
goto fail_fill_sqe;
@@ -1855,7 +2097,8 @@ fail_fill_sqe:
return ret;
}
-static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg)
+static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg,
+ struct map_info_cache *cache)
{
struct wd_sm2_dec_in *din = (void *)msg->req.src;
struct wd_hash_mt *hash = &msg->hash;
@@ -1865,7 +2108,7 @@ static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg)
/* c2 data lens <= 4096 bit */
if (din->c2.dsize <= BITS_TO_BYTES(4096) &&
hash->type == WD_HASH_SM3)
- return ecc_general_send(ctx, msg);
+ return ecc_general_send(ctx, msg, cache);
if (unlikely(!hash->cb || hash->type >= WD_HASH_MAX)) {
WD_ERR("invalid: input hash type %u is error!\n", hash->type);
@@ -1891,14 +2134,14 @@ static int sm2_dec_send(handle_t ctx, struct wd_ecc_msg *msg)
goto free_dst;
}
- ret = ecc_general_send(ctx, dst);
+ ret = ecc_general_send(ctx, dst, cache);
if (unlikely(ret))
goto free_req_dst;
return ret;
free_req_dst:
- free(dst->req.dst);
+ msg->mm_ops->free(msg->mm_ops->usr, dst->req.dst);
free_dst:
free(dst);
return ret;
@@ -1908,14 +2151,37 @@ static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_ecc_msg *msg = ecc_msg;
+ struct map_info_cache cache = {0};
+ int ret;
+
+ ret = check_hpre_mem_params(msg->mm_ops, msg->mm_type);
+ if (ret) {
+ WD_ERR("ecc memory parmas is err, and ret is %d!\n", ret);
+ return ret;
+ }
hisi_set_msg_id(h_qp, &msg->tag);
- if (msg->req.op_type == WD_SM2_ENCRYPT)
- return sm2_enc_send(ctx, msg);
- else if (msg->req.op_type == WD_SM2_DECRYPT)
- return sm2_dec_send(ctx, msg);
+ if (msg->req.op_type == WD_SM2_ENCRYPT) {
+ ret = sm2_enc_send(ctx, msg, &cache);
+ if (ret)
+ goto ecc_fail;
+ return ret;
+ } else if (msg->req.op_type == WD_SM2_DECRYPT) {
+ ret = sm2_dec_send(ctx, msg, &cache);
+ if (ret)
+ goto ecc_fail;
+ return ret;
+ }
- return ecc_general_send(ctx, msg);
+ ret = ecc_general_send(ctx, msg, &cache);
+ if (ret)
+ goto ecc_fail;
+
+ return ret;
+
+ecc_fail:
+ unmap_addr_in_cache(msg->mm_ops, &cache);
+ return ret;
}
static int ecdh_out_transfer(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg)
@@ -2020,14 +2286,14 @@ static int sm2_enc_out_transfer(struct wd_ecc_msg *msg,
static int ecc_out_transfer(struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg, __u8 qp_mode)
{
+ struct wd_ecc_msg *target_msg;
int ret = -WD_EINVAL;
- void *va;
+
+ target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
/* async */
- if (qp_mode == CTX_MODE_ASYNC) {
- va = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- msg->req.dst = container_of(va, struct wd_ecc_out, data);
- }
+ if (qp_mode == CTX_MODE_ASYNC)
+ msg->req.dst = target_msg->req.dst;
if (hw_msg->alg == HPRE_ALG_SM2_SIGN ||
hw_msg->alg == HPRE_ALG_ECDSA_SIGN)
@@ -2319,19 +2585,16 @@ static int sm2_convert_dec_out(struct wd_ecc_msg *src,
static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg)
{
- struct wd_ecc_msg *temp_msg;
+ struct wd_ecc_msg *target_msg;
+ size_t ilen = 0;
+ size_t olen = 0;
+ __u16 kbytes;
int ret;
- msg->tag = LW_U16(hw_msg->low_tag);
- if (qp->q_info.qp_mode == CTX_MODE_ASYNC) {
- temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag);
- if (!temp_msg) {
- WD_ERR("failed to get send msg! idx = %u, tag = %u.\n",
- qp->q_info.idx, msg->tag);
- return -WD_EINVAL;
- }
- } else {
- temp_msg = msg;
+ target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct ecc send msg from hardware!\n");
+ return -WD_ADDR_ERR;
}
hpre_result_check(hw_msg, &msg->result);
@@ -2347,10 +2610,20 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg,
goto dump_err_msg;
}
- return ret;
+ return WD_SUCCESS;
dump_err_msg:
- dump_hpre_msg(temp_msg, WD_ECC);
+ kbytes = target_msg->key_bytes;
+ if (!target_msg->mm_ops->sva_mode) {
+ ecc_get_io_len(hw_msg->alg, kbytes, &ilen, &olen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_OUT, hw_msg,
+ target_msg->req.dst, olen);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_KEY, hw_msg,
+ target_msg->key, kbytes);
+ unsetup_hw_msg_addr(target_msg->mm_ops, HW_MSG_IN, hw_msg,
+ target_msg->req.src, ilen);
+ }
+ dump_hpre_msg(target_msg, WD_ECC);
return ret;
}
@@ -2363,8 +2636,6 @@ static int parse_second_sqe(handle_t h_qp,
struct wd_ecc_msg *dst;
__u16 recv_cnt = 0;
int cnt = 0;
- void *data;
- __u32 hsz;
int ret;
while (1) {
@@ -2380,10 +2651,12 @@ static int parse_second_sqe(handle_t h_qp,
break;
}
- data = VA_ADDR(hw_msg.hi_out, hw_msg.low_out);
- hsz = (hw_msg.task_len1 + 1) * BYTE_BITS;
- dst = *(struct wd_ecc_msg **)((uintptr_t)data +
- hsz * ECDH_OUT_PARAM_NUM);
+ dst = (struct wd_ecc_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!dst) {
+ WD_ERR("failed to get correct sm2 enc second send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg);
msg->result = dst->result;
*second = dst;
@@ -2394,19 +2667,17 @@ static int parse_second_sqe(handle_t h_qp,
static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg)
{
- __u16 tag = LW_U16(hw_msg->low_tag);
struct wd_ecc_msg *second = NULL;
struct wd_ecc_msg *first;
struct wd_ecc_msg src;
- void *data;
- __u32 hsz;
int ret;
- msg->tag = tag;
- data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- hsz = (hw_msg->task_len1 + 1) * BYTE_BITS;
- first = *(struct wd_ecc_msg **)((uintptr_t)data +
- hsz * ECDH_OUT_PARAM_NUM);
+ first = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!first) {
+ WD_ERR("failed to get correct sm2 enc msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
memcpy(&src, first + 1, sizeof(src));
/* parse first sqe */
@@ -2440,17 +2711,15 @@ free_first:
static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg,
struct hisi_hpre_sqe *hw_msg)
{
- __u16 tag = LW_U16(hw_msg->low_tag);
struct wd_ecc_msg *dst;
struct wd_ecc_msg src;
- void *data;
- __u32 hsz;
int ret;
- data = VA_ADDR(hw_msg->hi_out, hw_msg->low_out);
- hsz = (hw_msg->task_len1 + 1) * BYTE_BITS;
- dst = *(struct wd_ecc_msg **)((uintptr_t)data +
- hsz * ECDH_OUT_PARAM_NUM);
+ dst = (struct wd_ecc_msg *)VA_ADDR(hw_msg->hi_tag, hw_msg->low_tag);
+ if (!dst) {
+ WD_ERR("failed to get correct sm2 dec msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
memcpy(&src, dst + 1, sizeof(src));
/* parse first sqe */
@@ -2460,7 +2729,6 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg,
goto fail;
}
msg->result = dst->result;
- msg->tag = tag;
ret = sm2_convert_dec_out(&src, dst);
if (unlikely(ret)) {
@@ -2469,7 +2737,7 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg,
}
fail:
- free(dst->req.dst);
+ dst->mm_ops->free(dst->mm_ops->usr, dst->req.dst);
free(dst);
return ret;
@@ -2479,6 +2747,7 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg)
{
handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx);
struct wd_ecc_msg *msg = ecc_msg;
+ struct wd_ecc_msg *target_msg;
struct hisi_hpre_sqe hw_msg;
__u16 recv_cnt = 0;
int ret;
@@ -2487,10 +2756,17 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg)
if (ret)
return ret;
- ret = hisi_check_bd_id(h_qp, msg->tag, hw_msg.low_tag);
+ target_msg = (struct wd_ecc_msg *)VA_ADDR(hw_msg.hi_tag, hw_msg.low_tag);
+ if (!target_msg) {
+ WD_ERR("failed to get correct send msg from hardware!\n");
+ return -WD_ADDR_ERR;
+ }
+
+ ret = hisi_check_bd_id(h_qp, msg->tag, target_msg->tag);
if (ret)
return ret;
+ msg->tag = target_msg->tag;
if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY &&
hw_msg.sm2_mlen == HPRE_SM2_ENC)
return sm2_enc_parse(h_qp, msg, &hw_msg);
@@ -2549,7 +2825,7 @@ static bool is_valid_hw_type(struct wd_alg_driver *drv)
hpre_ctx = (struct hisi_hpre_ctx *)drv->priv;
qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->config.ctxs[0].ctx);
- if (qp->q_info.hw_type < HISI_QM_API_VER3_BASE)
+ if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE)
return false;
return true;
}
diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c
index 9251b4c..216b80a 100644
--- a/drv/hisi_qm_udrv.c
+++ b/drv/hisi_qm_udrv.c
@@ -473,7 +473,7 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count)
{
struct hisi_qp *qp = (struct hisi_qp *)h_qp;
struct hisi_qm_queue_info *q_info;
- __u16 free_num, send_num;
+ __u16 free_num;
__u16 tail;
if (unlikely(!qp || !req || !count))
@@ -488,11 +488,14 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count)
return -WD_EBUSY;
}
- send_num = expect > free_num ? free_num : expect;
+ if (expect > free_num) {
+ pthread_spin_unlock(&q_info->sd_lock);
+ return -WD_EBUSY;
+ }
tail = q_info->sq_tail_index;
- hisi_qm_fill_sqe(req, q_info, tail, send_num);
- tail = (tail + send_num) % q_info->sq_depth;
+ hisi_qm_fill_sqe(req, q_info, tail, expect);
+ tail = (tail + expect) % q_info->sq_depth;
/*
* Before sending doorbell, check the queue status,
@@ -510,9 +513,9 @@ int hisi_qm_send(handle_t h_qp, const void *req, __u16 expect, __u16 *count)
q_info->sq_tail_index = tail;
/* Make sure used_num is changed before the next thread gets free sqe. */
- __atomic_add_fetch(&q_info->used_num, send_num, __ATOMIC_RELAXED);
+ __atomic_add_fetch(&q_info->used_num, expect, __ATOMIC_RELAXED);
pthread_spin_unlock(&q_info->sd_lock);
- *count = send_num;
+ *count = expect;
return 0;
}
diff --git a/include/drv/wd_dh_drv.h b/include/drv/wd_dh_drv.h
index d205dc4..d2a6157 100644
--- a/include/drv/wd_dh_drv.h
+++ b/include/drv/wd_dh_drv.h
@@ -16,12 +16,15 @@ extern "C" {
/* DH message format */
struct wd_dh_msg {
struct wd_dh_req req;
+ struct wd_mm_ops *mm_ops;
+ enum wd_mem_type mm_type;
__u32 tag; /* User-defined request identifier */
void *g;
__u16 gbytes;
__u16 key_bytes; /* Input key bytes */
__u8 is_g2;
__u8 result; /* Data format, denoted by WD error code */
+ __u8 *rsv_out; /* reserved output data pointer */
};
struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag);
diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h
index 6193c8b..b123a9b 100644
--- a/include/drv/wd_ecc_drv.h
+++ b/include/drv/wd_ecc_drv.h
@@ -48,6 +48,8 @@ extern "C" {
/* ECC message format */
struct wd_ecc_msg {
struct wd_ecc_req req;
+ struct wd_mm_ops *mm_ops;
+ enum wd_mem_type mm_type;
struct wd_hash_mt hash;
__u32 tag; /* User-defined request identifier */
__u8 *key; /* Input key VA, should be DMA buffer */
@@ -55,6 +57,7 @@ struct wd_ecc_msg {
__u8 curve_id; /* Ec curve denoted by enum wd_ecc_curve_type */
__u8 result; /* alg op error code */
void *drv_cfg; /* internal driver configuration */
+ __u8 *rsv_out; /* reserved output data pointer */
};
struct wd_ecc_pubkey {
diff --git a/include/drv/wd_rsa_drv.h b/include/drv/wd_rsa_drv.h
index d231ecf..c12f3e0 100644
--- a/include/drv/wd_rsa_drv.h
+++ b/include/drv/wd_rsa_drv.h
@@ -42,11 +42,14 @@ struct wd_rsa_kg_out {
/* RSA message format */
struct wd_rsa_msg {
struct wd_rsa_req req;
+ struct wd_mm_ops *mm_ops;
+ enum wd_mem_type mm_type;
__u32 tag; /* User-defined request identifier */
__u16 key_bytes; /* Input key bytes */
__u8 key_type; /* Denoted by enum wd_rsa_key_type */
__u8 result; /* Data format, denoted by WD error code */
__u8 *key; /* Input key VA pointer, should be DMA buffer */
+ __u8 *rsv_out; /* reserved output data pointer */
};
struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag);
diff --git a/include/wd_dh.h b/include/wd_dh.h
index afc2f7c..235c602 100644
--- a/include/wd_dh.h
+++ b/include/wd_dh.h
@@ -27,6 +27,8 @@ struct wd_dh_sess_setup {
__u16 key_bits; /* DH key bites */
bool is_g2; /* is g2 mode or not */
void *sched_param;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
struct wd_dh_req {
diff --git a/include/wd_ecc.h b/include/wd_ecc.h
index 6f670e2..18c1c0d 100644
--- a/include/wd_ecc.h
+++ b/include/wd_ecc.h
@@ -116,6 +116,8 @@ struct wd_ecc_sess_setup {
struct wd_rand_mt rand; /* rand method from user */
struct wd_hash_mt hash; /* hash method from user */
void *sched_param;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
struct wd_ecc_req {
diff --git a/include/wd_rsa.h b/include/wd_rsa.h
index 2f4e589..9c91432 100644
--- a/include/wd_rsa.h
+++ b/include/wd_rsa.h
@@ -60,6 +60,8 @@ struct wd_rsa_sess_setup {
__u16 key_bits; /* RSA key bits */
bool is_crt; /* CRT mode or not */
void *sched_param;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
bool wd_rsa_is_crt(handle_t sess);
diff --git a/wd_dh.c b/wd_dh.c
index 221322f..0c1372c 100644
--- a/wd_dh.c
+++ b/wd_dh.c
@@ -26,6 +26,8 @@ struct wd_dh_sess {
struct wd_dtb g;
struct wd_dh_sess_setup setup;
void *sched_key;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
static struct wd_dh_setting {
@@ -326,6 +328,8 @@ static int fill_dh_msg(struct wd_dh_msg *msg, struct wd_dh_req *req,
memcpy(&msg->req, req, sizeof(*req));
msg->result = WD_EINVAL;
msg->key_bytes = sess->key_size;
+ msg->mm_ops = &sess->mm_ops;
+ msg->mm_type = sess->mm_type;
if (unlikely(req->pri_bytes < sess->key_size)) {
WD_ERR("invalid: req pri bytes %hu is error!\n", req->pri_bytes);
@@ -581,6 +585,7 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g)
handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup)
{
struct wd_dh_sess *sess;
+ int ret;
if (!setup) {
WD_ERR("invalid: alloc dh sess setup NULL!\n");
@@ -606,10 +611,19 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup)
memcpy(&sess->setup, setup, sizeof(*setup));
sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT;
- sess->g.data = malloc(sess->key_size);
- if (!sess->g.data)
+ ret = wd_mem_ops_init(wd_dh_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type);
+ if (ret) {
+ WD_ERR("failed to init memory ops!\n");
goto sess_err;
+ }
+ memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops));
+ sess->mm_type = setup->mm_type;
+ sess->g.data = sess->mm_ops.alloc(sess->mm_ops.usr, sess->key_size);
+ if (!sess->g.data) {
+ WD_ERR("failed to malloc sess g param memory!\n");
+ goto sess_err;
+ }
sess->g.bsize = sess->key_size;
/* Some simple scheduler don't need scheduling parameters */
sess->sched_key = (void *)wd_dh_setting.sched.sched_init(
@@ -622,7 +636,7 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup)
return (handle_t)sess;
sched_err:
- free(sess->g.data);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->g.data);
sess_err:
free(sess);
return (handle_t)0;
@@ -638,7 +652,7 @@ void wd_dh_free_sess(handle_t sess)
}
if (sess_t->g.data)
- free(sess_t->g.data);
+ sess_t->mm_ops.free(sess_t->mm_ops.usr, sess_t->g.data);
if (sess_t->sched_key)
free(sess_t->sched_key);
diff --git a/wd_ecc.c b/wd_ecc.c
index b1971b9..2c88d0a 100644
--- a/wd_ecc.c
+++ b/wd_ecc.c
@@ -52,6 +52,8 @@ struct wd_ecc_sess {
struct wd_ecc_sess_setup setup;
struct wd_ecc_extend_ops eops;
void *sched_key;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
struct wd_ecc_curve_list {
@@ -494,7 +496,7 @@ static void release_ecc_prikey(struct wd_ecc_sess *sess)
struct wd_ecc_prikey *prikey = sess->key.prikey;
wd_memset_zero(prikey->data, prikey->size);
- free(prikey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, prikey->data);
free(prikey);
sess->key.prikey = NULL;
}
@@ -503,7 +505,7 @@ static void release_ecc_pubkey(struct wd_ecc_sess *sess)
{
struct wd_ecc_pubkey *pubkey = sess->key.pubkey;
- free(pubkey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, pubkey->data);
free(pubkey);
sess->key.pubkey = NULL;
}
@@ -522,7 +524,7 @@ static struct wd_ecc_prikey *create_ecc_prikey(struct wd_ecc_sess *sess)
}
dsz = ECC_PRIKEY_SZ(hsz);
- data = malloc(dsz);
+ data = sess->mm_ops.alloc(sess->mm_ops.usr, dsz);
if (!data) {
WD_ERR("failed to malloc prikey data, sz = %u!\n", dsz);
free(prikey);
@@ -551,7 +553,7 @@ static struct wd_ecc_pubkey *create_ecc_pubkey(struct wd_ecc_sess *sess)
}
dsz = ECC_PUBKEY_SZ(hsz);
- data = malloc(dsz);
+ data = sess->mm_ops.alloc(sess->mm_ops.usr, dsz);
if (!data) {
WD_ERR("failed to malloc pubkey data, sz = %u!\n", dsz);
free(pubkey);
@@ -570,7 +572,7 @@ static void release_ecc_in(struct wd_ecc_sess *sess,
struct wd_ecc_in *ecc_in)
{
wd_memset_zero(ecc_in->data, ecc_in->size);
- free(ecc_in);
+ sess->mm_ops.free(sess->mm_ops.usr, ecc_in);
}
static struct wd_ecc_in *create_ecc_in(struct wd_ecc_sess *sess, __u32 num)
@@ -585,7 +587,7 @@ static struct wd_ecc_in *create_ecc_in(struct wd_ecc_sess *sess, __u32 num)
hsz = get_key_bsz(sess->key_size);
len = sizeof(struct wd_ecc_in) + hsz * num;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc ecc in, sz = %u!\n", len);
return NULL;
@@ -613,7 +615,7 @@ static struct wd_ecc_in *create_sm2_sign_in(struct wd_ecc_sess *sess,
len = sizeof(struct wd_ecc_in)
+ ECC_SIGN_IN_PARAM_NUM * ksz + m_len;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc sm2 sign in, sz = %llu!\n", len);
return NULL;
@@ -653,7 +655,7 @@ static struct wd_ecc_in *create_sm2_enc_in(struct wd_ecc_sess *sess,
}
len = sizeof(struct wd_ecc_in) + ksz + m_len;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc sm2 enc in, sz = %llu!\n", len);
return NULL;
@@ -697,7 +699,7 @@ static void *create_sm2_ciphertext(struct wd_ecc_sess *sess, __u32 m_len,
*len = (__u64)st_sz + ECC_POINT_PARAM_NUM * (__u64)sess->key_size +
(__u64)m_len + (__u64)h_byts;
- start = malloc(*len);
+ start = sess->mm_ops.alloc(sess->mm_ops.usr, *len);
if (unlikely(!start)) {
WD_ERR("failed to alloc start, sz = %llu!\n", *len);
return NULL;
@@ -745,7 +747,7 @@ static struct wd_ecc_out *create_ecc_out(struct wd_ecc_sess *sess, __u32 num)
hsz = get_key_bsz(sess->key_size);
len = sizeof(struct wd_ecc_out) + hsz * num;
- out = malloc(len);
+ out = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!out) {
WD_ERR("failed to malloc out, sz = %u!\n", len);
return NULL;
@@ -1149,13 +1151,13 @@ static void del_sess_key(struct wd_ecc_sess *sess)
{
if (sess->key.prikey) {
wd_memset_zero(sess->key.prikey->data, sess->key.prikey->size);
- free(sess->key.prikey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->key.prikey->data);
free(sess->key.prikey);
sess->key.prikey = NULL;
}
if (sess->key.pubkey) {
- free(sess->key.pubkey->data);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->key.pubkey->data);
free(sess->key.pubkey);
sess->key.pubkey = NULL;
}
@@ -1227,6 +1229,15 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup)
memcpy(&sess->setup, setup, sizeof(*setup));
sess->key_size = BITS_TO_BYTES(setup->key_bits);
+ /* Memory type set */
+ ret = wd_mem_ops_init(wd_ecc_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type);
+ if (ret) {
+ WD_ERR("failed to init memory ops!\n");
+ goto sess_err;
+ }
+ memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops));
+ sess->mm_type = setup->mm_type;
+
if (wd_ecc_setting.driver->get_extend_ops) {
ret = wd_ecc_setting.driver->get_extend_ops(&sess->eops);
if (ret) {
@@ -1508,9 +1519,10 @@ void wd_ecxdh_get_out_params(struct wd_ecc_out *out, struct wd_ecc_point **pbk)
void wd_ecc_del_in(handle_t sess, struct wd_ecc_in *in)
{
+ struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess;
__u32 bsz;
- if (!in) {
+ if (!sess_t || !in) {
WD_ERR("invalid: del ecc in parameter error!\n");
return;
}
@@ -1522,14 +1534,15 @@ void wd_ecc_del_in(handle_t sess, struct wd_ecc_in *in)
}
wd_memset_zero(in->data, bsz);
- free(in);
+ sess_t->mm_ops.free(sess_t->mm_ops.usr, in);
}
-void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out)
+void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out)
{
+ struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess;
__u32 bsz;
- if (!out) {
+ if (!sess_t || !out) {
WD_ERR("invalid: del ecc out parameter error!\n");
return;
}
@@ -1541,7 +1554,7 @@ void wd_ecc_del_out(handle_t sess, struct wd_ecc_out *out)
}
wd_memset_zero(out->data, bsz);
- free(out);
+ sess_t->mm_ops.free(sess_t->mm_ops.usr, out);
}
static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req,
@@ -1551,6 +1564,8 @@ static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req,
memcpy(&msg->req, req, sizeof(msg->req));
memcpy(&msg->hash, &sess->setup.hash, sizeof(msg->hash));
+ msg->mm_ops = &sess->mm_ops;
+ msg->mm_type = sess->mm_type;
msg->key_bytes = sess->key_size;
msg->curve_id = sess->setup.cv.cfg.id;
msg->drv_cfg = sess->eops.params;
@@ -1922,7 +1937,7 @@ static struct wd_ecc_in *create_sm2_verf_in(struct wd_ecc_sess *sess,
hsz = get_key_bsz(sess->key_size);
len = sizeof(struct wd_ecc_in) + ECC_VERF_IN_PARAM_NUM * hsz +
m_len;
- in = malloc(len);
+ in = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!in) {
WD_ERR("failed to malloc sm2 verf in, sz = %llu!\n", len);
return NULL;
@@ -2211,7 +2226,7 @@ struct wd_ecc_out *wd_sm2_new_dec_out(handle_t sess, __u32 plaintext_len)
}
len = sizeof(*ecc_out) + plaintext_len;
- ecc_out = malloc(len);
+ ecc_out = sess_t->mm_ops.alloc(sess_t->mm_ops.usr, len);
if (!ecc_out) {
WD_ERR("failed to malloc ecc_out, sz = %llu!\n", len);
return NULL;
diff --git a/wd_rsa.c b/wd_rsa.c
index cf9239c..bc78c6a 100644
--- a/wd_rsa.c
+++ b/wd_rsa.c
@@ -67,6 +67,8 @@ struct wd_rsa_sess {
struct wd_rsa_prikey *prikey;
struct wd_rsa_sess_setup setup;
void *sched_key;
+ struct wd_mm_ops mm_ops;
+ enum wd_mem_type mm_type;
};
static struct wd_rsa_setting {
@@ -373,6 +375,8 @@ static int fill_rsa_msg(struct wd_rsa_msg *msg, struct wd_rsa_req *req,
memcpy(&msg->req, req, sizeof(*req));
msg->key_bytes = sess->key_size;
msg->result = WD_EINVAL;
+ msg->mm_ops = &sess->mm_ops;
+ msg->mm_type = sess->mm_type;
switch (msg->req.op_type) {
case WD_RSA_SIGN:
@@ -641,7 +645,7 @@ struct wd_rsa_kg_in *wd_rsa_new_kg_in(handle_t sess, struct wd_dtb *e,
}
kg_in_size = (int)GEN_PARAMS_SZ(c->key_size);
- kg_in = malloc(kg_in_size + sizeof(*kg_in));
+ kg_in = c->mm_ops.alloc(c->mm_ops.usr, kg_in_size + sizeof(*kg_in));
if (!kg_in) {
WD_ERR("failed to malloc kg_in memory!\n");
return NULL;
@@ -681,19 +685,16 @@ void wd_rsa_get_kg_in_params(struct wd_rsa_kg_in *kin, struct wd_dtb *e,
p->data = (void *)kin->p;
}
-static void del_kg(void *k)
+void wd_rsa_del_kg_in(handle_t sess, struct wd_rsa_kg_in *ki)
{
- if (!k) {
+ struct wd_rsa_sess *c = (struct wd_rsa_sess *)sess;
+
+ if (!c || !ki) {
WD_ERR("invalid: del key generate params err!\n");
return;
}
- free(k);
-}
-
-void wd_rsa_del_kg_in(handle_t sess, struct wd_rsa_kg_in *ki)
-{
- del_kg(ki);
+ c->mm_ops.free(c->mm_ops.usr, ki);
}
struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess)
@@ -719,7 +720,7 @@ struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess)
else
kg_out_size = (int)GEN_PARAMS_SZ(c->key_size);
- kg_out = malloc(kg_out_size + sizeof(*kg_out));
+ kg_out = c->mm_ops.alloc(c->mm_ops.usr, kg_out_size + sizeof(*kg_out));
if (!kg_out) {
WD_ERR("failed to malloc kg_out memory!\n");
return NULL;
@@ -741,13 +742,15 @@ struct wd_rsa_kg_out *wd_rsa_new_kg_out(handle_t sess)
void wd_rsa_del_kg_out(handle_t sess, struct wd_rsa_kg_out *kout)
{
- if (!kout) {
+ struct wd_rsa_sess *c = (struct wd_rsa_sess *)sess;
+
+ if (!c || !kout) {
WD_ERR("invalid: param null at del kg out!\n");
return;
}
wd_memset_zero(kout->data, kout->size);
- del_kg(kout);
+ c->mm_ops.free(c->mm_ops.usr, kout);
}
void wd_rsa_get_kg_out_params(struct wd_rsa_kg_out *kout, struct wd_dtb *d,
@@ -804,6 +807,11 @@ void wd_rsa_set_kg_out_crt_psz(struct wd_rsa_kg_out *kout,
size_t dq_sz,
size_t dp_sz)
{
+ if (!kout) {
+ WD_ERR("invalid: input null when set kg out crt psz!\n");
+ return;
+ }
+
kout->qinvbytes = qinv_sz;
kout->dqbytes = dq_sz;
kout->dpbytes = dp_sz;
@@ -813,6 +821,11 @@ void wd_rsa_set_kg_out_psz(struct wd_rsa_kg_out *kout,
size_t d_sz,
size_t n_sz)
{
+ if (!kout) {
+ WD_ERR("invalid: input null when set kg out psz!\n");
+ return;
+ }
+
kout->dbytes = d_sz;
kout->nbytes = n_sz;
}
@@ -862,7 +875,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup,
if (setup->is_crt) {
len = sizeof(struct wd_rsa_prikey) +
(int)CRT_PARAMS_SZ(sess->key_size);
- sess->prikey = malloc(len);
+ sess->prikey = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!sess->prikey) {
WD_ERR("failed to alloc sess prikey2!\n");
return -WD_ENOMEM;
@@ -873,7 +886,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup,
} else {
len = sizeof(struct wd_rsa_prikey) +
(int)GEN_PARAMS_SZ(sess->key_size);
- sess->prikey = malloc(len);
+ sess->prikey = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!sess->prikey) {
WD_ERR("failed to alloc sess prikey1!\n");
return -WD_ENOMEM;
@@ -885,7 +898,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup,
len = sizeof(struct wd_rsa_pubkey) +
(int)GEN_PARAMS_SZ(sess->key_size);
- sess->pubkey = malloc(len);
+ sess->pubkey = sess->mm_ops.alloc(sess->mm_ops.usr, len);
if (!sess->pubkey) {
free(sess->prikey);
WD_ERR("failed to alloc sess pubkey!\n");
@@ -912,8 +925,8 @@ static void del_sess_key(struct wd_rsa_sess *sess)
wd_memset_zero(prk->pkey.pkey2.data, CRT_PARAMS_SZ(sess->key_size));
else
wd_memset_zero(prk->pkey.pkey1.data, GEN_PARAMS_SZ(sess->key_size));
- free(sess->prikey);
- free(sess->pubkey);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->prikey);
+ sess->mm_ops.free(sess->mm_ops.usr, sess->pubkey);
}
static void del_sess(struct wd_rsa_sess *c)
@@ -948,6 +961,15 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup)
memcpy(&sess->setup, setup, sizeof(*setup));
sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT;
+ /* Memory type set */
+ ret = wd_mem_ops_init(wd_rsa_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type);
+ if (ret) {
+ WD_ERR("failed to init memory ops!\n");
+ goto sess_err;
+ }
+
+ memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops));
+ sess->mm_type = setup->mm_type;
ret = create_sess_key(setup, sess);
if (ret) {
WD_ERR("failed to create rsa sess keys!\n");
--
2.33.0
1
2
From: Wenkai Lin <linwenkai6(a)hisilicon.com>
Extract a common function fill_hashagg_data_info.
Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com>
Signed-off-by: Zongyu Wu <wuzongyu1(a)huawei.com>
---
drv/hisi_dae.c | 56 +++++++++++++++++++-------------------------------
1 file changed, 21 insertions(+), 35 deletions(-)
diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c
index 4f4d13c..49387aa 100644
--- a/drv/hisi_dae.c
+++ b/drv/hisi_dae.c
@@ -26,7 +26,7 @@
#define DAE_VCHAR_OFFSET_SIZE 2
#define DAE_COL_BIT_NUM 4
#define DAE_AGG_START_COL 16
-#define DAE_HASHAGG_MAX_ROW_NUN 50000
+#define DAE_HASHAGG_MAX_ROW_NUM 50000
/* align size */
#define DAE_CHAR_ALIGN_SIZE 4
@@ -294,23 +294,8 @@ static void fill_hashagg_merge_key_data(struct dae_sqe *sqe, struct dae_ext_sqe
}
}
-static void fill_hashagg_normal_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
- struct hashagg_col_data *cols_data, __u32 agg_cols_num)
-{
- struct hw_agg_data *agg_data = cols_data->input_data;
- __u32 i;
-
- for (i = 0; i < agg_cols_num; i++) {
- sqe->agg_data_type[i] = agg_data[i].hw_type;
- sqe->agg_data_type[i] |= agg_data[i].sum_outtype << DAE_COL_BIT_NUM;
- ext_sqe->agg_data_info[i] = agg_data[i].data_info;
- }
-
- sqe->agg_col_bitmap = GENMASK(agg_cols_num + DAE_AGG_START_COL - 1, DAE_AGG_START_COL);
-}
-
-static void fill_hashagg_rehash_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
- struct hw_agg_data *agg_data, __u32 agg_cols_num)
+static void fill_hashagg_data_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
+ struct hw_agg_data *agg_data, __u32 agg_cols_num)
{
__u32 i;
@@ -344,14 +329,14 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext
hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.agg_cols;
agg_col_num = msg->agg_cols_num;
- fill_hashagg_normal_info(sqe, ext_sqe, cols_data, agg_col_num);
+ fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num);
break;
case WD_AGG_REHASH_INPUT:
agg_data = cols_data->output_data;
hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.agg_cols;
agg_col_num = cols_data->output_num;
- fill_hashagg_rehash_info(sqe, ext_sqe, agg_data, agg_col_num);
+ fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num);
break;
case WD_AGG_STREAM_OUTPUT:
case WD_AGG_REHASH_OUTPUT:
@@ -359,7 +344,7 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext
hw_agg_addr = &addr_list->output_addr[DAE_AGG_START_COL];
usr_agg_addr = msg->req.out_agg_cols;
agg_col_num = cols_data->output_num;
- fill_hashagg_normal_info(sqe, ext_sqe, cols_data, cols_data->input_num);
+ fill_hashagg_data_info(sqe, ext_sqe, cols_data->input_data, cols_data->input_num);
break;
}
@@ -385,7 +370,7 @@ static void fill_hashagg_merge_input_data(struct dae_sqe *sqe, struct dae_ext_sq
struct hashagg_ctx *agg_ctx = msg->priv;
struct hashagg_col_data *cols_data = &agg_ctx->cols_data;
- fill_hashagg_rehash_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num);
+ fill_hashagg_data_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num);
}
static void fill_hashagg_ext_addr(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
@@ -422,26 +407,15 @@ static void fill_hashagg_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe,
static int check_hashagg_param(struct wd_agg_msg *msg)
{
- struct hashagg_col_data *cols_data;
- struct hashagg_ctx *agg_ctx;
-
if (!msg) {
WD_ERR("invalid: input hashagg msg is NULL!\n");
return -WD_EINVAL;
}
- agg_ctx = msg->priv;
- cols_data = &agg_ctx->cols_data;
- if (cols_data->output_num > DAE_MAX_OUTPUT_COLS) {
- WD_ERR("invalid: input hashagg output num %u is more than %d!\n",
- cols_data->output_num, DAE_MAX_OUTPUT_COLS);
- return -WD_EINVAL;
- }
-
if ((msg->pos == WD_AGG_STREAM_INPUT || msg->pos == WD_AGG_REHASH_INPUT) &&
- msg->row_count > DAE_HASHAGG_MAX_ROW_NUN) {
+ msg->row_count > DAE_HASHAGG_MAX_ROW_NUM) {
WD_ERR("invalid: input hashagg row count %u is more than %d!\n",
- msg->row_count, DAE_HASHAGG_MAX_ROW_NUN);
+ msg->row_count, DAE_HASHAGG_MAX_ROW_NUM);
return -WD_EINVAL;
}
@@ -1005,6 +979,7 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols,
struct hw_agg_data *user_output_data,
__u32 cols_num, __u32 *output_num)
{
+ __u32 tmp = *output_num;
__u32 i, j, k = 0;
int ret;
@@ -1013,7 +988,15 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols,
WD_ERR("invalid: col alg num(%u) more than 2!\n", agg_cols[i].col_alg_num);
return -WD_EINVAL;
}
+ tmp += agg_cols[i].col_alg_num;
+ }
+ if (tmp > DAE_MAX_OUTPUT_COLS) {
+ WD_ERR("invalid: output col num is more than %d!\n", DAE_MAX_OUTPUT_COLS);
+ return -WD_EINVAL;
+ }
+
+ for (i = 0; i < cols_num; i++) {
for (j = 0; j < agg_cols[i].col_alg_num; j++) {
ret = hashagg_check_input_data(&agg_cols[i], &user_input_data[i],
&user_output_data[k], j);
@@ -1137,6 +1120,9 @@ static int transfer_data_to_hw_type(struct hashagg_col_data *cols_data,
struct wd_agg_col_info *agg_cols = setup->agg_cols_info;
int ret;
+ if (setup->is_count_all)
+ cols_data->output_num++;
+
ret = transfer_input_col_info(agg_cols, user_input_data, user_output_data,
setup->agg_cols_num, &cols_data->output_num);
if (ret)
--
2.33.0
1
0
From: parm64 <parm64(a)huawei.com>
Zhushuai Yin (6):
uadk_provider: fix spelling errors and incorrect return value issues
uadk_provider: prov supports dynamic algorithm cropping func
uadk_provider: provide log configuration functionality
uadk_provider:fix the issue of algorithm prohibition failure for DH
uadk_provider: fix issues that may cause encryption to fail
uadk_engine: fix the issue of incorrect engine use the V2 interface
ZongYu Wu (1):
uadk_engine: fix uadk prov module spelling error
lizhi (2):
uadk_provider: fix poll timeout in abnormal scenarios
uadk_engine: clean up unused functions
src/uadk_aead.c | 2 +-
src/uadk_async.c | 9 +-
src/uadk_cipher.c | 3 +-
src/uadk_cipher_adapter.c | 3 +-
src/uadk_dh.c | 3 +-
src/uadk_digest.c | 2 +-
src/uadk_ec.c | 2 +-
src/uadk_ecx.c | 18 --
src/uadk_engine_init.c | 11 +-
src/uadk_pkey.c | 3 +-
src/uadk_prov.h | 23 +-
src/uadk_prov_aead.c | 105 +++----
src/uadk_prov_cipher.c | 120 ++++----
src/uadk_prov_dh.c | 194 ++++++------
src/uadk_prov_digest.c | 78 ++---
src/uadk_prov_ec_kmgmt.c | 60 ++--
src/uadk_prov_ecdh_exch.c | 73 ++---
src/uadk_prov_ecdsa.c | 118 ++++----
src/uadk_prov_ecx.c | 162 +++++-----
src/uadk_prov_ffc.c | 9 +-
src/uadk_prov_hmac.c | 76 ++---
src/uadk_prov_init.c | 607 +++++++++++++++++++++++++++++++++++---
src/uadk_prov_packet.c | 5 +-
src/uadk_prov_pkey.c | 87 +++---
src/uadk_prov_pkey.h | 3 -
src/uadk_prov_rsa.c | 111 ++++---
src/uadk_prov_sm2.c | 474 ++++++++++++++---------------
src/uadk_rsa.c | 3 +-
src/uadk_sm2.c | 9 -
src/uadk_utils.c | 14 +
src/uadk_utils.h | 27 ++
uadk_provider.cnf | 27 ++
32 files changed, 1507 insertions(+), 934 deletions(-)
--
2.43.0
1
9
14 Nov '25
From: Longfang Liu <liulongfang(a)huawei.com>
Add kernel-state reserved memory handling functionality to the
uadk SVA framework to adapt to No-SVA features, including functions
for applying for and initializing reserved memory pools, applying for,
using, and releasing memory within the memory pool.
Signed-off-by: Longfang Liu <liulongfang(a)huawei.com>
Signed-off-by: Zongyu Wu <wuzongyu1(a)huawei.com>
---
Makefile.am | 9 +-
include/uacce.h | 7 +
include/wd.h | 23 +
include/wd_alg_common.h | 36 +-
include/wd_bmm.h | 44 ++
include/wd_internal.h | 70 +++
include/wd_util.h | 2 +
libwd.map | 14 +
wd.c | 38 +-
wd_bmm.c | 1057 +++++++++++++++++++++++++++++++++++++++
wd_util.c | 107 +++-
11 files changed, 1355 insertions(+), 52 deletions(-)
create mode 100644 include/wd_bmm.h
create mode 100644 include/wd_internal.h
create mode 100644 wd_bmm.c
diff --git a/Makefile.am b/Makefile.am
index f897533..0e1203a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -32,12 +32,13 @@ YEAR = 2025
AM_CFLAGS+= -DUADK_VERSION_NUMBER="\"UADK version: ${MAJOR}.${MINOR}.${REVISION}\""
AM_CFLAGS+= -DUADK_RELEASED_TIME="\"Released ${MONTH} ${DAY}, ${YEAR}\""
-pkginclude_HEADERS = include/wd.h include/wd_cipher.h include/wd_aead.h \
+pkginclude_HEADERS = include/wd.h include/wd_internal.h include/wd_cipher.h include/wd_aead.h \
include/wd_comp.h include/wd_dh.h include/wd_digest.h \
include/wd_rsa.h include/uacce.h include/wd_alg_common.h \
include/wd_ecc.h include/wd_sched.h include/wd_alg.h \
include/wd_zlibwrapper.h include/wd_dae.h include/wd_agg.h \
- include/wd_udma.h include/wd_join_gather.h
+ include/wd_udma.h include/wd_join_gather.h \
+ include/wd_bmm.h
nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd_dh.h \
v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h
@@ -48,7 +49,7 @@ uadk_driversdir=$(libdir)/uadk
uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \
libisa_ce.la libisa_sve.la libhisi_dae.la libhisi_udma.la
-libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \
+libwd_la_SOURCES=wd.c wd_mempool.c wd_bmm.c wd_bmm.h wd.h wd_alg.c wd_alg.h \
v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \
v1/wd_rsa.c v1/wd_rsa.h \
v1/wd_aead.c v1/wd_aead.h \
@@ -126,7 +127,7 @@ libwd_comp_la_DEPENDENCIES = libwd.la
libhisi_zip_la_LIBADD = -ldl
-libwd_crypto_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma
+libwd_crypto_la_LIBADD = -lwd -ldl -lnuma -lm -lpthread
libwd_crypto_la_DEPENDENCIES = libwd.la
libwd_udma_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lm -lpthread
diff --git a/include/uacce.h b/include/uacce.h
index f7fae27..c6bb4fb 100644
--- a/include/uacce.h
+++ b/include/uacce.h
@@ -15,6 +15,12 @@ extern "C" {
#define UACCE_CMD_START _IO('W', 0)
#define UACCE_CMD_PUT_Q _IO('W', 1)
+#define UACCE_CMD_GET_SS_DMA _IOR('W', 3, unsigned long)
+
+/* Pass DMA SS region slice size by granularity 64KB */
+#define UACCE_GRAN_SIZE 0x10000ull
+#define UACCE_GRAN_SHIFT 16
+#define UACCE_GRAN_NUM_MASK 0xfffull
/**
* UACCE Device flags:
@@ -33,6 +39,7 @@ enum {
enum uacce_qfrt {
UACCE_QFRT_MMIO = 0, /* device mmio region */
UACCE_QFRT_DUS = 1, /* device user share */
+ UACCE_QFRT_SS, /* static share memory */
UACCE_QFRT_MAX,
};
diff --git a/include/wd.h b/include/wd.h
index b62d355..b97e5c7 100644
--- a/include/wd.h
+++ b/include/wd.h
@@ -38,6 +38,7 @@ typedef unsigned long long __u64;
/* Required compiler attributes */
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
#define handle_t uintptr_t
typedef struct wd_dev_mask wd_dev_mask_t;
@@ -115,6 +116,28 @@ enum wd_alg_type {
WD_AEAD,
};
+/* Memory APIs for UADK API Layer */
+typedef void *(*wd_alloc)(void *usr, size_t size);
+typedef void (*wd_free)(void *usr, void *va);
+
+ /* Memory VA to DMA address map and unmap */
+typedef void *(*wd_map)(void *usr, void *va, size_t sz);
+typedef void (*wd_unmap)(void *usr, void *va, void *dma, size_t sz);
+typedef __u32 (*wd_bufsize)(void *usr);
+
+/* Memory from user, it is given at ctx creating. */
+struct wd_mm_ops {
+ wd_alloc alloc; /* Memory allocation */
+ wd_free free; /* Memory free */
+ wd_map iova_map; /* Get iova from user space VA */
+
+ /* Destroy the mapping between the PA of VA and iova */
+ wd_unmap iova_unmap;
+ wd_bufsize get_bufsize; /* Optional */
+ void *usr; /* Data for the above operations */
+ bool sva_mode; /* Record whether the OS is SVA or No-SVA mode */
+};
+
/*
* If the actual size of data is inconsistent
* with dsize, undefined behavior occurs.
diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h
index fd77426..a294877 100644
--- a/include/wd_alg_common.h
+++ b/include/wd_alg_common.h
@@ -12,6 +12,7 @@
#include <numa.h>
#include "wd.h"
#include "wd_alg.h"
+#include "wd_internal.h"
#ifdef __cplusplus
extern "C" {
@@ -24,7 +25,6 @@ extern "C" {
#define BITS_TO_BYTES(bits) (((bits) + 7) >> 3)
#define BYTES_TO_BITS(bytes) ((bytes) << 3)
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
#define MAX_STR_LEN 256
#define CTX_TYPE_INVALID 9999
#define POLL_TIME 1000
@@ -60,6 +60,13 @@ enum wd_init_type {
WD_TYPE_V2,
};
+enum wd_mem_type {
+ UADK_MEM_AUTO,
+ UADK_MEM_USER,
+ UADK_MEM_PROXY,
+ UADK_MEM_MAX,
+};
+
/*
* struct wd_ctx - Define one ctx and related type.
* @ctx: The ctx itself.
@@ -132,27 +139,6 @@ struct wd_ctx_params {
struct wd_cap_config *cap;
};
-struct wd_soft_ctx {
- void *priv;
-};
-
-struct wd_ctx_internal {
- handle_t ctx;
- __u8 op_type;
- __u8 ctx_mode;
- __u16 sqn;
- pthread_spinlock_t lock;
-};
-
-struct wd_ctx_config_internal {
- __u32 ctx_num;
- int shmid;
- struct wd_ctx_internal *ctxs;
- void *priv;
- bool epoll_en;
- unsigned long *msg_cnt;
-};
-
/*
* struct wd_comp_sched - Define a scheduler.
* @name: Name of this scheduler.
@@ -181,12 +167,6 @@ struct wd_sched {
typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched);
typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count);
-struct wd_datalist {
- void *data;
- __u32 len;
- struct wd_datalist *next;
-};
-
#ifdef __cplusplus
}
#endif
diff --git a/include/wd_bmm.h b/include/wd_bmm.h
new file mode 100644
index 0000000..76b56a0
--- /dev/null
+++ b/include/wd_bmm.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/*
+ * Copyright 2025 Huawei Technologies Co.,Ltd. All rights reserved.
+ */
+
+#ifndef _WD_SVA_BMM_H
+#define _WD_SVA_BMM_H
+
+#include <stdint.h>
+#include "wd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Memory pool creating parameters */
+struct wd_mempool_setup {
+ __u32 block_size; /* Block buffer size */
+ __u32 block_num; /* Block buffer number */
+ __u32 align_size; /* Block buffer starting address align size */
+ struct wd_mm_ops ops; /* memory from user if don't use UADK memory */
+};
+
+void *wd_mempool_alloc(handle_t h_ctx, struct wd_mempool_setup *setup);
+void wd_mempool_free(handle_t h_ctx, void *pool);
+void *wd_mem_alloc(void *pool, size_t size);
+void wd_mem_free(void *pool, void *buf);
+
+void *wd_mem_map(void *pool, void *buf, size_t sz);
+void wd_mem_unmap(void *pool, void *buf_dma, void *buf, size_t sz);
+int wd_get_free_num(void *pool, __u32 *free_num);
+int wd_get_fail_num(void *pool, __u32 *fail_num);
+__u32 wd_get_bufsize(void *pool);
+
+handle_t wd_find_ctx(const char *alg_name);
+void wd_remove_ctx_list(void);
+int wd_insert_ctx_list(handle_t h_ctx, char *alg_name);
+__u32 wd_get_dev_id(void *pool);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _WD_SVA_BMM_H */
diff --git a/include/wd_internal.h b/include/wd_internal.h
new file mode 100644
index 0000000..cd90ebf
--- /dev/null
+++ b/include/wd_internal.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/*
+ * Copyright 2025 Huawei Technologies Co.,Ltd. All rights reserved.
+ */
+
+#ifndef WD_INTERNAL_H
+#define WD_INTERNAL_H
+
+#include <pthread.h>
+#include <stdbool.h>
+#include "wd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define DECIMAL_NUMBER 10
+#define MAX_FD_NUM 65535
+
+struct wd_ctx_h {
+ int fd;
+ char dev_path[MAX_DEV_NAME_LEN];
+ char *dev_name;
+ char *drv_name;
+ unsigned long qfrs_offs[UACCE_QFRT_MAX];
+ void *qfrs_base[UACCE_QFRT_MAX];
+ struct uacce_dev *dev;
+ void *priv;
+};
+
+struct wd_soft_ctx {
+ int fd;
+ void *priv;
+};
+
+struct wd_ce_ctx {
+ int fd;
+ char *drv_name;
+ void *priv;
+};
+
+struct wd_ctx_internal {
+ handle_t ctx;
+ __u8 op_type;
+ __u8 ctx_mode;
+ __u16 sqn;
+ pthread_spinlock_t lock;
+};
+
+struct wd_ctx_config_internal {
+ __u32 ctx_num;
+ int shmid;
+ struct wd_ctx_internal *ctxs;
+ void *priv;
+ bool epoll_en;
+ unsigned long *msg_cnt;
+ char *alg_name;
+};
+
+struct wd_datalist {
+ void *data;
+ __u32 len;
+ struct wd_datalist *next;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/wd_util.h b/include/wd_util.h
index 4a5204d..a337284 100644
--- a/include/wd_util.h
+++ b/include/wd_util.h
@@ -553,6 +553,8 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type)
pthread_spin_unlock(&ctx->lock);
}
+int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type);
+
#ifdef __cplusplus
}
#endif
diff --git a/libwd.map b/libwd.map
index 5522ec0..b1b90b3 100644
--- a/libwd.map
+++ b/libwd.map
@@ -49,5 +49,19 @@ global:
wd_enable_drv;
wd_disable_drv;
wd_get_alg_head;
+
+ wd_find_ctx;
+ wd_get_dev_id;
+ wd_remove_ctx_list;
+ wd_insert_ctx_list;
+ wd_mempool_alloc;
+ wd_mempool_free;
+ wd_mem_alloc;
+ wd_mem_free;
+ wd_mem_map;
+ wd_mem_unmap;
+ wd_get_free_num;
+ wd_get_fail_num;
+ wd_get_bufsize;
local: *;
};
diff --git a/wd.c b/wd.c
index c1cc282..3e867b6 100644
--- a/wd.c
+++ b/wd.c
@@ -20,6 +20,7 @@
#include "wd.h"
#include "wd_alg.h"
+#include "wd_internal.h"
#define SYS_CLASS_DIR "/sys/class/uacce"
#define FILE_MAX_SIZE (8 << 20)
@@ -33,16 +34,18 @@ enum UADK_LOG_LEVEL {
static int uadk_log_level = WD_LOG_INVALID;
-struct wd_ctx_h {
- int fd;
- char dev_path[MAX_DEV_NAME_LEN];
- char *dev_name;
- char *drv_name;
- unsigned long qfrs_offs[UACCE_QFRT_MAX];
- void *qfrs_base[UACCE_QFRT_MAX];
- struct uacce_dev *dev;
- void *priv;
-};
+static int wd_check_ctx_type(handle_t h_ctx)
+{
+ struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
+
+ /* A simple and efficient method to check the queue type */
+ if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) {
+ WD_INFO("Invalid: this ctx not HW ctx.\n");
+ return -WD_HW_EACCESS;
+ }
+
+ return 0;
+}
static void wd_parse_log_level(void)
{
@@ -446,7 +449,7 @@ void wd_release_ctx(handle_t h_ctx)
{
struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
- if (!ctx)
+ if (!ctx || wd_check_ctx_type(h_ctx))
return;
close(ctx->fd);
@@ -461,7 +464,7 @@ int wd_ctx_start(handle_t h_ctx)
struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
int ret;
- if (!ctx)
+ if (!ctx || wd_check_ctx_type(h_ctx))
return -WD_EINVAL;
ret = wd_ctx_set_io_cmd(h_ctx, UACCE_CMD_START, NULL);
@@ -527,6 +530,7 @@ void wd_ctx_unmap_qfr(handle_t h_ctx, enum uacce_qfrt qfrt)
unsigned long wd_ctx_get_region_size(handle_t h_ctx, enum uacce_qfrt qfrt)
{
struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
+
if (!ctx || qfrt >= UACCE_QFRT_MAX)
return 0;
return ctx->qfrs_offs[qfrt];
@@ -585,8 +589,16 @@ int wd_ctx_wait(handle_t h_ctx, __u16 ms)
int wd_is_sva(handle_t h_ctx)
{
struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
+ int ret;
- if (!ctx || !ctx->dev)
+ if (!ctx)
+ return -WD_EINVAL;
+
+ ret = wd_check_ctx_type(h_ctx);
+ if (ret)
+ return ret;
+
+ if (!ctx->dev)
return -WD_EINVAL;
if ((unsigned int)ctx->dev->flags & UACCE_DEV_SVA)
diff --git a/wd_bmm.c b/wd_bmm.c
new file mode 100644
index 0000000..21c46ca
--- /dev/null
+++ b/wd_bmm.c
@@ -0,0 +1,1057 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/*
+ * Copyright 2025 Huawei Technologies Co.,Ltd. All rights reserved.
+ */
+
+/* Block Memory Management (lib): Adapted for SVA mode */
+#define _GNU_SOURCE
+#include <dirent.h>
+#include <numa.h>
+#include <sched.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+#include "wd_internal.h"
+#include "wd_bmm.h"
+#include "uacce.h"
+#include "wd.h"
+
+#define __ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+#define ALIGN(x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#define UACCE_DEV_IOMMU (1<<7)
+
+#define TAG_FREE 0x12345678 /* block is free */
+#define TAG_USED 0x87654321 /* block is busy */
+#define MAX_ALIGN_SIZE 0x1000 /* 4KB */
+#define MAX_BLOCK_SIZE 0x10000000 /* 256MB */
+#define BLK_BALANCE_SZ 0x100000ul
+#define NUM_TIMES(x) (87 * (x) / 100)
+
+#define BYTE_SIZE 8
+#define BIT_SHIFT 3
+
+struct wd_ss_region {
+ unsigned long long pa;
+ void *va;
+ size_t size;
+ TAILQ_ENTRY(wd_ss_region) next;
+};
+TAILQ_HEAD(wd_ss_region_list, wd_ss_region);
+
+struct ctx_info {
+ int fd;
+ int iommu_type;
+ void *ss_va;
+ size_t ss_mm_size;
+ struct wd_ss_region_list ss_list;
+ struct wd_ss_region_list *head;
+ unsigned long qfrs_offset[UACCE_QFRT_MAX];
+};
+
+struct wd_blk_hd {
+ unsigned int blk_tag;
+ unsigned int blk_num;
+ void *blk_dma;
+ void *blk;
+};
+
+struct wd_blkpool {
+ pthread_spinlock_t pool_lock;
+ unsigned int free_blk_num;
+ unsigned int alloc_failures;
+ struct ctx_info *cinfo;
+ struct wd_blk_hd *blk_array; // memory blk array
+ unsigned int total_blocks; // total blk numbers
+ unsigned char *free_bitmap; // free blk bitmap, 0 mean unused
+ unsigned int bitmap_size; // bitmap's memory size
+ void *usr_mem_start;
+ void *act_start;
+ unsigned int act_hd_sz;
+ unsigned int act_blk_sz;
+ unsigned long act_mem_sz;
+ unsigned int dev_id;
+ struct wd_mempool_setup setup;
+};
+
+struct mem_ctx_node {
+ char alg_name[CRYPTO_MAX_ALG_NAME];
+ handle_t h_ctx;
+ int numa_id;
+ bool used;
+ TAILQ_ENTRY(mem_ctx_node) list_node;
+};
+static TAILQ_HEAD(, mem_ctx_node) g_mem_ctx_list = TAILQ_HEAD_INITIALIZER(g_mem_ctx_list);
+static pthread_mutex_t g_mem_ctx_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+handle_t wd_find_ctx(const char *alg_name)
+{
+ struct mem_ctx_node *close_node = NULL;
+ struct mem_ctx_node *node;
+ int min_distance = 0xFFFF;
+ int cpu = sched_getcpu();
+ int nid = numa_node_of_cpu(cpu);
+ handle_t h_ctx = 0;
+ int numa_dis;
+
+ if (!alg_name) {
+ WD_ERR("Invalid: alg_name is NULL!\n");
+ return 0;
+ }
+
+ pthread_mutex_lock(&g_mem_ctx_mutex);
+ TAILQ_FOREACH(node, &g_mem_ctx_list, list_node) {
+ if (node->used == false && strstr(node->alg_name, alg_name)) {
+ if (node->numa_id == nid) {
+ h_ctx = node->h_ctx;
+ node->used = true;
+ break;
+ }
+
+ /* Query the queue with the shortest NUMA distance */
+ numa_dis = numa_distance(nid, node->numa_id);
+ if (numa_dis < min_distance) {
+ min_distance = numa_dis;
+ close_node = node;
+ }
+ }
+ }
+
+ /* If no ctx matching the NUMA ID, use the shortest distance instead ctx */
+ if (!h_ctx && close_node) {
+ h_ctx = close_node->h_ctx;
+ close_node->used = true;
+ }
+ pthread_mutex_unlock(&g_mem_ctx_mutex);
+
+ if (!h_ctx)
+ WD_ERR("Failed to find mem ctx for alg: %s\n", alg_name);
+
+ return h_ctx;
+}
+
+void wd_remove_ctx_list(void)
+{
+ struct mem_ctx_node *node;
+
+ pthread_mutex_lock(&g_mem_ctx_mutex);
+ /* Free all list node */
+ while ((node = TAILQ_FIRST(&g_mem_ctx_list)) != NULL) {
+ /* Use TAILQ_REMOVE to remove list node */
+ TAILQ_REMOVE(&g_mem_ctx_list, node, list_node);
+ free(node);
+ }
+
+ pthread_mutex_unlock(&g_mem_ctx_mutex);
+}
+
+int wd_insert_ctx_list(handle_t h_ctx, char *alg_name)
+{
+ struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
+ struct mem_ctx_node *new_node;
+ int numa_id;
+
+ if (!alg_name || !h_ctx) {
+ WD_ERR("Invalid: input params is NULL!\n");
+ return -WD_EINVAL;
+ }
+
+ /* A simple and efficient method to check the queue type */
+ if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) {
+ WD_INFO("Invalid ctx: this ctx not HW ctx.\n");
+ return 0;
+ }
+
+ numa_id = ctx->dev->numa_id;
+ new_node = malloc(sizeof(struct mem_ctx_node));
+ if (new_node) {
+ pthread_mutex_lock(&g_mem_ctx_mutex);
+ strncpy(new_node->alg_name, alg_name, CRYPTO_MAX_ALG_NAME - 1);
+ new_node->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0';
+ new_node->numa_id = numa_id;
+ new_node->h_ctx = h_ctx;
+ new_node->used = false;
+ TAILQ_INSERT_TAIL(&g_mem_ctx_list, new_node, list_node);
+ pthread_mutex_unlock(&g_mem_ctx_mutex);
+ return 0;
+ }
+
+ return -WD_ENOMEM;
+}
+
+static void wd_free_slice(struct ctx_info *cinfo)
+{
+ struct wd_ss_region *rgn;
+
+ while (true) {
+ rgn = TAILQ_FIRST(&cinfo->ss_list);
+ if (!rgn)
+ break;
+ TAILQ_REMOVE(&cinfo->ss_list, rgn, next);
+ free(rgn);
+ }
+}
+
+static void wd_add_slice(struct ctx_info *cinfo, struct wd_ss_region *rgn)
+{
+ struct wd_ss_region *rg;
+
+ rg = TAILQ_LAST(&cinfo->ss_list, wd_ss_region_list);
+ if (rg) {
+ if (rg->pa + rg->size == rgn->pa) {
+ rg->size += rgn->size;
+ free(rgn);
+ return;
+ }
+ }
+
+ TAILQ_INSERT_TAIL(&cinfo->ss_list, rgn, next);
+}
+
+static void wd_show_ss_slices(struct ctx_info *cinfo)
+{
+ struct wd_ss_region *rgn;
+ int i = 0;
+
+ TAILQ_FOREACH(rgn, cinfo->head, next) {
+ WD_ERR("slice-%d:size = 0x%lx\n", i, rgn->size);
+ i++;
+ }
+}
+
+static void bitmap_set_bit(unsigned char *bitmap, unsigned int bit_index)
+{
+ if (!bitmap)
+ return;
+
+ bitmap[bit_index >> BIT_SHIFT] |= (1 << (bit_index % BYTE_SIZE));
+}
+
+static void bitmap_clear_bit(unsigned char *bitmap, unsigned int bit_index)
+{
+ if (!bitmap)
+ return;
+
+ bitmap[bit_index >> BIT_SHIFT] &= ~(1 << (bit_index % BYTE_SIZE));
+}
+
+static bool bitmap_test_bit(const unsigned char *bitmap, unsigned int bit_index)
+{
+ if (!bitmap)
+ return false;
+
+ /* bit is 1, it indicates that the block has already been used and is not free */
+ if ((bitmap[bit_index >> BIT_SHIFT] >> (bit_index % BYTE_SIZE)) & 0x1)
+ return false;
+
+ return true;
+}
+
+static void *wd_mmap_qfr(struct ctx_info *cinfo, enum uacce_qfrt qfrt, size_t size)
+{
+ off_t off;
+
+ off = qfrt * getpagesize();
+
+ return mmap(0, size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, cinfo->fd, off);
+}
+
+static void wd_unmap_reserve_mem(void *addr, size_t size)
+{
+ int ret;
+
+ if (!addr)
+ return;
+
+ ret = munmap(addr, size);
+ if (ret)
+ WD_ERR("wd qfr unmap failed!\n");
+}
+
+static void *wd_map_reserve_mem(struct wd_blkpool *pool, size_t size)
+{
+ struct ctx_info *cinfo = pool->cinfo;
+ struct wd_ss_region *rgn;
+ unsigned long info;
+ size_t tmp = size;
+ unsigned long i = 0;
+ void *ptr;
+ int ret = 1;
+
+ if (!cinfo) {
+ WD_ERR("ctx queue information is NULL!\n");
+ return NULL;
+ }
+
+ /* Make sure memory map granularity size align */
+ if (!cinfo->iommu_type)
+ tmp = ALIGN(tmp, UACCE_GRAN_SIZE);
+
+ ptr = wd_mmap_qfr(cinfo, UACCE_QFRT_SS, tmp);
+ if (ptr == MAP_FAILED) {
+ WD_ERR("wd drv mmap fail!(err = %d)\n", errno);
+ return NULL;
+ }
+
+ cinfo->ss_va = ptr;
+ cinfo->ss_mm_size = tmp;
+ tmp = 0;
+ while (ret > 0) {
+ info = i;
+ ret = ioctl(cinfo->fd, UACCE_CMD_GET_SS_DMA, &info);
+ if (ret < 0) {
+ wd_show_ss_slices(cinfo);
+ WD_ERR("get DMA fail!\n");
+ goto err_out;
+ }
+
+ rgn = malloc(sizeof(*rgn));
+ if (!rgn) {
+ WD_ERR("alloc ss region fail!\n");
+ goto err_out;
+ }
+ memset(rgn, 0, sizeof(*rgn));
+
+ if (cinfo->iommu_type)
+ rgn->size = cinfo->ss_mm_size;
+ else
+ rgn->size = (info & UACCE_GRAN_NUM_MASK) <<
+ UACCE_GRAN_SHIFT;
+ rgn->pa = info & (~UACCE_GRAN_NUM_MASK);
+ rgn->va = ptr + tmp;
+ tmp += rgn->size;
+ wd_add_slice(cinfo, rgn);
+
+ i++;
+ }
+
+ return ptr;
+
+err_out:
+ wd_free_slice(cinfo);
+ wd_unmap_reserve_mem(cinfo->ss_va, cinfo->ss_mm_size);
+
+ return NULL;
+}
+
+static int wd_pool_params_check(struct wd_mempool_setup *setup)
+{
+ if (!setup->block_num || !setup->block_size ||
+ setup->block_size > MAX_BLOCK_SIZE) {
+ WD_ERR("Invalid: block_size or block_num(%x, %u)!\n",
+ setup->block_size, setup->block_num);
+ return -WD_EINVAL;
+ }
+
+ /* Check parameters, and align_size must be 2^N */
+ if (setup->align_size <= 0x1 || setup->align_size > MAX_ALIGN_SIZE ||
+ (setup->align_size & (setup->align_size - 0x1))) {
+ WD_ERR("Invalid align_size.\n");
+ return -WD_EINVAL;
+ }
+
+ return WD_SUCCESS;
+}
+
+static int wd_ctx_info_init(struct wd_ctx_h *ctx, struct wd_blkpool *p)
+{
+ struct ctx_info *cinfo;
+
+ cinfo = calloc(1, sizeof(struct ctx_info));
+ if (!cinfo) {
+ WD_ERR("failed to alloc ctx info memory.\n");
+ return -WD_ENOMEM;
+ }
+
+ cinfo->fd = ctx->fd;
+ cinfo->iommu_type = (unsigned int)ctx->dev->flags & UACCE_DEV_IOMMU;
+ cinfo->head = &cinfo->ss_list;
+ TAILQ_INIT(&cinfo->ss_list);
+ (void)memcpy(cinfo->qfrs_offset, ctx->qfrs_offs,
+ sizeof(cinfo->qfrs_offset));
+ p->cinfo = (void *)cinfo;
+
+ return 0;
+}
+
+static int wd_pool_pre_layout(handle_t h_ctx,
+ struct wd_blkpool *p,
+ struct wd_mempool_setup *sp)
+{
+ struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
+ struct ctx_info *cinfo = NULL;
+ unsigned int asz;
+ int ret;
+
+ if (!ctx && !sp->ops.alloc) {
+ WD_ERR("ctx is NULL!\n");
+ return -WD_EINVAL;
+ }
+
+ if (!sp->ops.alloc) {
+ ret = wd_ctx_info_init(ctx, p);
+ if (ret) {
+ WD_ERR("failed to init ctx info.\n");
+ return ret;
+ }
+ cinfo = p->cinfo;
+ }
+
+ ret = wd_pool_params_check(sp);
+ if (ret) {
+ free(p->cinfo);
+ p->cinfo = NULL;
+ return ret;
+ }
+
+ asz = sp->align_size;
+
+ /* Get actual value by align */
+ p->act_hd_sz = ALIGN(sizeof(struct wd_blk_hd), asz);
+ p->act_blk_sz = ALIGN(sp->block_size, asz);
+ p->act_mem_sz = (p->act_hd_sz + p->act_blk_sz) *
+ (unsigned long)sp->block_num + asz;
+
+ /*
+ * When we use WD reserve memory and the blk_sz is larger than 1M,
+ * in order to ensure the mem_pool to be success,
+ * ensure that the allocated memory is an integer multiple of 1M.
+ */
+ if (!sp->ops.alloc && (cinfo && !cinfo->iommu_type))
+ p->act_mem_sz = ((p->act_mem_sz + BLK_BALANCE_SZ - 1) & ~(BLK_BALANCE_SZ - 1)) << 1;
+
+ return WD_SUCCESS;
+}
+
+/**
+ * wd_iova_map - Map virtual address to physical address
+ * @cinfo: context information
+ * @va: virtual address to map
+ * @sz: size of the mapping (not used in current implementation)
+ *
+ * When IOMMU is enabled, the PA is actually an IOVA; userspace still sees it
+ * as consistent and contiguous with the VA.
+ * When IOMMU is disabled, the PA refers to the kernel's physical address, which
+ * must be physically contiguous to be allocated by the kernel.
+ * Therefore, the PA address can be obtained from the offset of the VA.
+ *
+ */
+static void *wd_iova_map(struct ctx_info *cinfo, void *va, size_t sz)
+{
+ struct wd_ss_region *rgn;
+ unsigned long offset;
+ void *dma_addr;
+
+ if (!cinfo || !va) {
+ WD_ERR("wd iova map: parameter err!\n");
+ return NULL;
+ }
+
+ /* Search through all memory regions to find where va belongs */
+ TAILQ_FOREACH(rgn, cinfo->head, next) {
+ if (rgn->va <= va && va < rgn->va + rgn->size) {
+ /* Calculate offset within the region */
+ offset = (uintptr_t)va - (uintptr_t)rgn->va;
+ /* Add base physical address of the region */
+ dma_addr = (void *)((uintptr_t)rgn->pa + offset);
+ return dma_addr;
+ }
+ }
+
+ WD_ERR("wd iova map: va not found in any region\n");
+ return NULL;
+}
+
+/**
+ * wd_iova_unmap - Unmap physical address (no-op in non-IOMMU mode)
+ * @cinfo: context information
+ * @va: virtual address
+ * @dma: physical address
+ * @sz: size of the mapping (not used in current implementation)
+ *
+ * In non-IOMMU mode, this function does nothing as there's no need to unmap.
+ * In IOMMU mode, this would typically involve unmapping the DMA address.
+ */
+static void wd_iova_unmap(struct ctx_info *cinfo, void *va, void *dma, size_t sz)
+{
+ /* For no-iommu, dma-unmap doing nothing */
+}
+
+static void wd_pool_uninit(struct wd_blkpool *p)
+{
+ struct ctx_info *cinfo = p->cinfo;
+ struct wd_blk_hd *fhd = NULL;
+ unsigned long block_size;
+ unsigned int i;
+
+ block_size = (unsigned long)p->act_hd_sz + p->act_blk_sz;
+ /* Clean up the allocated resources. */
+ for (i = 0; i < p->total_blocks; i++) {
+ /* Release the previously allocated blocks. */
+ fhd = &p->blk_array[i];
+ wd_iova_unmap(cinfo, fhd->blk, fhd->blk_dma, block_size);
+ }
+
+ free(p->free_bitmap);
+ p->free_bitmap = NULL;
+ free(p->blk_array);
+ p->blk_array = NULL;
+}
+
+static int wd_pool_init(struct wd_blkpool *p)
+{
+ struct ctx_info *cinfo = p->cinfo;
+ __u32 blk_size = p->setup.block_size;
+ void *dma_start, *dma_end, *va;
+ struct wd_blk_hd *fhd = NULL;
+ struct wd_blk_hd *hd = NULL;
+ unsigned int i, j, act_num;
+ unsigned long block_size;
+ unsigned int dma_num = 0;
+
+ p->act_start = (void *)ALIGN((uintptr_t)p->usr_mem_start,
+ p->setup.align_size);
+
+ /* Calculate the actual number of allocatable blocks */
+ block_size = (unsigned long)(p->act_hd_sz + p->act_blk_sz);
+ if (block_size == 0) {
+ WD_ERR("Invalid block size with header.\n");
+ return -WD_EINVAL;
+ }
+ act_num = p->act_mem_sz / block_size;
+ if (!act_num) {
+ WD_ERR("Invalid memory size.\n");
+ return -WD_EINVAL;
+ }
+
+ /* Allocate block array */
+ p->blk_array = (struct wd_blk_hd *)malloc(act_num * sizeof(struct wd_blk_hd));
+ if (!p->blk_array) {
+ WD_ERR("Failed to allocate block array.\n");
+ return -WD_ENOMEM;
+ }
+
+ /* Allocate bitmap */
+ p->total_blocks = act_num;
+ p->bitmap_size = (act_num + BYTE_SIZE - 1) >> BIT_SHIFT;
+ p->free_bitmap = (unsigned char *)calloc(1, p->bitmap_size);
+ if (!p->free_bitmap) {
+ WD_ERR("Failed to allocate free bitmap.\n");
+ goto bitmap_error;
+ }
+
+ /* Initialize all blocks. */
+ for (i = 0; i < act_num; i++) {
+ /* Calculate the virtual address of the current block. */
+ va = (void *)((uintptr_t)p->act_start + block_size * i);
+
+ /* Get the physical address. */
+ dma_start = wd_iova_map(cinfo, va, 0);
+ dma_end = wd_iova_map(cinfo, va + blk_size - 1, 0);
+ if (!dma_start || !dma_end) {
+ WD_ERR("wd_iova_map err.\n");
+ /* Clean up the allocated resources. */
+ goto init_blk_error;
+ }
+
+ /* Check whether the physical addresses are contiguous. */
+ if ((uintptr_t)dma_end - (uintptr_t)dma_start != blk_size - 1) {
+ /* If OS kernel is not open SMMU, need to check dma address */
+ WD_INFO("wd dma address not continuous.\n");
+ /* Mark as unavailable, bit value is 1. */
+ bitmap_set_bit(p->free_bitmap, i);
+ continue;
+ }
+
+ /* Initialize the block. */
+ hd = &p->blk_array[i];
+ hd->blk_dma = dma_start;
+ hd->blk = va;
+ hd->blk_tag = TAG_FREE;
+ hd->blk_num = 0;
+
+ dma_num++;
+ }
+
+ /*
+ * if dma_num <= (1 / 1.15) * user's block_num, we think the pool
+ * is created with failure.
+ */
+ if (dma_num <= NUM_TIMES(p->setup.block_num)) {
+ WD_ERR("dma_num = %u, not enough.\n", dma_num);
+ goto init_blk_error;
+ }
+
+ p->free_blk_num = dma_num;
+ p->setup.block_num = dma_num;
+
+ return WD_SUCCESS;
+
+init_blk_error:
+ /* Clean up the allocated resources. */
+ for (j = 0; j < i; j++) {
+ /* Release the previously allocated blocks. */
+ fhd = &p->blk_array[j];
+ wd_iova_unmap(cinfo, fhd->blk, fhd->blk_dma, block_size);
+ }
+ free(p->free_bitmap);
+
+bitmap_error:
+ free(p->blk_array);
+
+ return -WD_ENOMEM;
+}
+
+static int usr_pool_init(struct wd_blkpool *p)
+{
+ struct wd_mempool_setup *sp = &p->setup;
+ __u32 blk_size = sp->block_size;
+ struct wd_blk_hd *hd = NULL;
+ __u32 i;
+
+ p->act_start = (void *)ALIGN((uintptr_t)p->usr_mem_start,
+ sp->align_size);
+ for (i = 0; i < sp->block_num; i++) {
+ hd = (void *)((uintptr_t)p->act_start + (p->act_hd_sz + p->act_blk_sz) * i);
+ hd->blk = (void *)((uintptr_t)hd + p->act_hd_sz);
+ hd->blk_dma = sp->ops.iova_map(sp->ops.usr, hd->blk, blk_size);
+ if (!hd->blk_dma) {
+ WD_ERR("failed to map usr blk.\n");
+ return -WD_ENOMEM;
+ }
+ hd->blk_tag = TAG_FREE;
+ }
+
+ p->free_blk_num = sp->block_num;
+
+ return WD_SUCCESS;
+}
+
+static int wd_parse_dev_id(char *dev_name)
+{
+ char *last_dash = NULL;
+ char *endptr;
+ int dev_id;
+
+ if (!dev_name)
+ return -WD_EINVAL;
+
+ /* Find the last '-' in the string. */
+ last_dash = strrchr(dev_name, '-');
+ if (!last_dash || *(last_dash + 1) == '\0')
+ return -WD_EINVAL;
+
+ /* Parse the following number */
+ dev_id = strtol(last_dash + 1, &endptr, DECIMAL_NUMBER);
+ /* Check whether it is truly all digits */
+ if (*endptr != '\0' || dev_id < 0)
+ return -WD_EINVAL;
+
+ return dev_id;
+}
+
+static int wd_mempool_init(handle_t h_ctx, struct wd_blkpool *pool,
+ struct wd_mempool_setup *setup)
+{
+ struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx;
+ struct ctx_info *cinfo = pool->cinfo;
+ void *addr = NULL;
+ int ret;
+
+ /* Use user's memory, and its ops alloc function */
+ if (setup->ops.alloc && setup->ops.free && setup->ops.iova_map) {
+ addr = setup->ops.alloc(setup->ops.usr, pool->act_mem_sz);
+ if (!addr) {
+ WD_ERR("failed to allocate memory in user pool.\n");
+ return -WD_EINVAL;
+ }
+
+ pool->usr_mem_start = addr;
+ if (usr_pool_init(pool)) {
+ WD_ERR("failed to initialize user pool.\n");
+ setup->ops.free(setup->ops.usr, addr);
+ return -WD_EINVAL;
+ }
+ } else {
+ /* Use wd to reserve memory */
+ addr = wd_map_reserve_mem(pool, pool->act_mem_sz);
+ if (!addr) {
+ WD_ERR("wd pool failed to reserve memory.\n");
+ return -WD_ENOMEM;
+ }
+
+ pool->usr_mem_start = addr;
+ if (wd_pool_init(pool)) {
+ WD_ERR("failed to initialize wd pool.\n");
+ goto err_out;
+ }
+ setup->block_num = pool->setup.block_num;
+ }
+
+ ret = wd_parse_dev_id(ctx->dev_path);
+ if (ret < 0) {
+ wd_pool_uninit(pool);
+ goto err_out;
+ }
+ pool->dev_id = ret;
+
+ return WD_SUCCESS;
+
+err_out:
+ if (pool->cinfo) {
+ wd_free_slice(cinfo);
+ wd_unmap_reserve_mem(cinfo->ss_va, cinfo->ss_mm_size);
+ pool->cinfo = NULL;
+ }
+ return -WD_EINVAL;
+}
+
+void *wd_mempool_alloc(handle_t h_ctx, struct wd_mempool_setup *setup)
+{
+ struct wd_blkpool *pool = NULL;
+ int ret;
+
+ if (!setup || !h_ctx) {
+ WD_ERR("Input param is NULL!\n");
+ return NULL;
+ }
+
+ ret = wd_is_sva(h_ctx);
+ if (ret < 0) {
+ WD_ERR("failed to check device ctx!\n");
+ return NULL;
+ } else if (ret == UACCE_DEV_SVA) {
+ WD_ERR("the device is SVA mode!\n");
+ return NULL;
+ }
+
+ pool = calloc(1, sizeof(*pool));
+ if (!pool) {
+ WD_ERR("failed to malloc pool.\n");
+ return NULL;
+ }
+ ret = pthread_spin_init(&pool->pool_lock, PTHREAD_PROCESS_PRIVATE);
+ if (ret)
+ goto err_pool_alloc;
+
+ memcpy(&pool->setup, setup, sizeof(pool->setup));
+
+ ret = wd_pool_pre_layout(h_ctx, pool, setup);
+ if (ret)
+ goto err_pool_layout;
+
+ ret = wd_mempool_init(h_ctx, pool, setup);
+ if (ret)
+ goto err_pool_init;
+
+ return pool;
+
+err_pool_init:
+ if (pool->cinfo) {
+ free(pool->cinfo);
+ pool->cinfo = NULL;
+ }
+err_pool_layout:
+ pthread_spin_destroy(&pool->pool_lock);
+err_pool_alloc:
+ free(pool);
+
+ return NULL;
+}
+
+void wd_mempool_free(handle_t h_ctx, void *pool)
+{
+ struct wd_mempool_setup *setup;
+ struct wd_blkpool *p = pool;
+
+ if (!p || !h_ctx) {
+ WD_ERR("pool destroy err, pool or ctx is NULL.\n");
+ return;
+ }
+
+ setup = &p->setup;
+ if (p->free_blk_num != setup->block_num) {
+ WD_ERR("Can not destroy blk pool, as it's in use.\n");
+ return;
+ }
+
+ if (setup->ops.free)
+ setup->ops.free(setup->ops.usr, p->usr_mem_start);
+
+ if (p->cinfo) {
+ /* Free block array memory */
+ if (p->blk_array)
+ free(p->blk_array);
+
+ if (p->free_bitmap)
+ free(p->free_bitmap);
+
+ wd_free_slice(p->cinfo);
+ wd_unmap_reserve_mem(p->cinfo->ss_va, p->cinfo->ss_mm_size);
+ free(p->cinfo);
+ p->cinfo = NULL;
+ }
+
+ pthread_spin_destroy(&p->pool_lock);
+ free(p);
+}
+
+void wd_mem_free(void *pool, void *buf)
+{
+ struct wd_blkpool *p = pool;
+ struct wd_blk_hd *current_hd;
+ struct wd_blk_hd *hd;
+ unsigned int current_idx;
+ unsigned int blk_idx;
+ unsigned long offset;
+ unsigned int i, num;
+ unsigned long sz;
+
+ if (unlikely(!p || !buf)) {
+ WD_ERR("free blk parameters err!\n");
+ return;
+ }
+
+ sz = p->act_hd_sz + p->act_blk_sz;
+ if (!sz) {
+ WD_ERR("memory pool blk size is zero!\n");
+ return;
+ }
+
+ if ((uintptr_t)buf < (uintptr_t)p->act_start) {
+ WD_ERR("free block addr is error.\n");
+ return;
+ }
+
+ /* Calculate the block index. */
+ offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start);
+ blk_idx = offset / sz;
+
+ /* Check if the index is valid. */
+ if (blk_idx >= p->total_blocks) {
+ WD_ERR("Invalid block index<%u>.\n", blk_idx);
+ return;
+ }
+
+ /* Get the block header. */
+ hd = &p->blk_array[blk_idx];
+ num = hd->blk_num;
+
+ pthread_spin_lock(&p->pool_lock);
+ /* Release all related blocks. */
+ for (i = 0; i < num; i++) {
+ // Recalculate the index (since it is contiguous).
+ current_idx = blk_idx + i;
+ current_hd = &p->blk_array[current_idx];
+ current_hd->blk_tag = TAG_FREE;
+ current_hd->blk_num = 0;
+ bitmap_clear_bit(p->free_bitmap, current_idx);
+ }
+ p->free_blk_num += num;
+ pthread_spin_unlock(&p->pool_lock);
+}
+
+static int wd_find_contiguous_blocks(struct wd_blkpool *p,
+ unsigned int required_blocks,
+ unsigned int *start_block)
+{
+#define MAX_SKIP_ATTEMPTS 10
+ unsigned int consecutive_count = 0;
+ unsigned int skip_attempts = 0;
+ struct wd_blk_hd *hd, *tl;
+ unsigned int i;
+
+ if (required_blocks == 0 || required_blocks > p->total_blocks)
+ return -WD_EINVAL;
+
+ for (i = 0; i < p->total_blocks; i++) {
+ if (!bitmap_test_bit(p->free_bitmap, i)) {
+ consecutive_count = 0;
+ continue;
+ }
+
+ if (consecutive_count == 0)
+ *start_block = i;
+ consecutive_count++;
+
+ if (consecutive_count < required_blocks)
+ continue;
+
+ /* Check DMA contiguity only if more than one block is needed */
+ if (required_blocks > 1) {
+ hd = &p->blk_array[*start_block];
+ tl = &p->blk_array[*start_block + required_blocks - 1];
+
+ if (((uintptr_t)tl->blk_dma - (uintptr_t)hd->blk_dma) !=
+ ((uintptr_t)tl->blk - (uintptr_t)hd->blk)) {
+ /* Not contiguous, skip this start and try again */
+ if (++skip_attempts > MAX_SKIP_ATTEMPTS)
+ return -WD_ENOMEM;
+
+ i = *start_block; // will be incremented by loop
+ consecutive_count = 0;
+ continue;
+ }
+ }
+
+ /* Found and DMA is contiguous */
+ return WD_SUCCESS;
+ }
+
+ return -WD_ENOMEM;
+}
+
+void *wd_mem_alloc(void *pool, size_t size)
+{
+ unsigned int required_blocks;
+ unsigned int start_block = 0;
+ struct wd_blk_hd *hd = NULL;
+ struct wd_blkpool *p = pool;
+ unsigned int j;
+ int ret;
+
+ if (unlikely(!p || !size)) {
+ WD_ERR("blk alloc pool is null!\n");
+ return NULL;
+ }
+
+ if (!p->act_blk_sz) {
+ WD_ERR("blk pool is error!\n");
+ return NULL;
+ }
+
+ /* Calculate the number of blocks required. */
+ required_blocks = (size + p->act_blk_sz - 1) / p->act_blk_sz;
+ if (required_blocks > p->free_blk_num) {
+ p->alloc_failures++;
+ WD_ERR("Not enough free blocks.\n");
+ return NULL;
+ }
+
+ pthread_spin_lock(&p->pool_lock);
+ /* Find contiguous free blocks. */
+ ret = wd_find_contiguous_blocks(p, required_blocks, &start_block);
+ if (ret != 0) {
+ p->alloc_failures++;
+ pthread_spin_unlock(&p->pool_lock);
+ WD_ERR("Failed to find contiguous blocks.\n");
+ return NULL;
+ }
+
+ /* Mark all required blocks as used */
+ for (j = start_block; j < start_block + required_blocks; j++) {
+ p->blk_array[j].blk_tag = TAG_USED;
+ bitmap_set_bit(p->free_bitmap, j);
+ }
+
+ p->free_blk_num -= required_blocks;
+ hd = &p->blk_array[start_block];
+ hd->blk_num = required_blocks;
+ pthread_spin_unlock(&p->pool_lock);
+
+ return hd->blk;
+}
+
+void *wd_mem_map(void *pool, void *buf, size_t sz)
+{
+ struct wd_blkpool *p = pool;
+ struct wd_blk_hd *hd;
+ unsigned long offset;
+ unsigned long blk_sz;
+ unsigned long blk_idx;
+
+ if (unlikely(!pool || !buf)) {
+ WD_ERR("blk map err, pool is NULL!\n");
+ return NULL;
+ }
+
+ if (!sz || (uintptr_t)buf < (uintptr_t)p->act_start) {
+ WD_ERR("map buf addr is error.\n");
+ return NULL;
+ }
+ /* Calculate the block index. */
+ offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start);
+ blk_sz = p->act_hd_sz + p->act_blk_sz;
+ blk_idx = offset / blk_sz;
+
+ /* Check if the index is valid. */
+ if (blk_idx >= p->total_blocks) {
+ WD_ERR("Invalid block index<%lu> in map.\n", blk_idx);
+ return NULL;
+ }
+
+ hd = &p->blk_array[blk_idx];
+ if (unlikely(hd->blk_tag != TAG_USED ||
+ (uintptr_t)buf < (uintptr_t)hd->blk)) {
+ WD_ERR("dma map fail!\n");
+ return NULL;
+ }
+
+ return (void *)((uintptr_t)hd->blk_dma + ((uintptr_t)buf -
+ (uintptr_t)hd->blk));
+}
+
+void wd_mem_unmap(void *pool, void *buf_dma, void *buf, size_t sz)
+{
+ /* do nothing at no-iommu mode */
+}
+
+int wd_get_free_num(void *pool, __u32 *free_num)
+{
+ struct wd_blkpool *p = pool;
+
+ if (!p || !free_num) {
+ WD_ERR("get_free_blk_num err, parameter err!\n");
+ return -WD_EINVAL;
+ }
+
+ *free_num = __atomic_load_n(&p->free_blk_num, __ATOMIC_RELAXED);
+
+ return WD_SUCCESS;
+}
+
+int wd_get_fail_num(void *pool, __u32 *fail_num)
+{
+ struct wd_blkpool *p = pool;
+
+ if (!p || !fail_num) {
+ WD_ERR("get_blk_alloc_failure err, pool is NULL!\n");
+ return -WD_EINVAL;
+ }
+
+ *fail_num = __atomic_load_n(&p->alloc_failures, __ATOMIC_RELAXED);
+
+ return WD_SUCCESS;
+}
+
+__u32 wd_get_bufsize(void *pool)
+{
+ struct wd_blkpool *p = pool;
+
+ if (!p) {
+ WD_ERR("get dev id is null!\n");
+ return 0;
+ }
+
+ return p->act_blk_sz;
+}
+
+__u32 wd_get_dev_id(void *pool)
+{
+ struct wd_blkpool *p = pool;
+
+ if (!p) {
+ WD_ERR("failed to get dev id!\n");
+ return 0;
+ }
+
+ return p->dev_id;
+}
+
diff --git a/wd_util.c b/wd_util.c
index e8a2934..d0d83eb 100644
--- a/wd_util.c
+++ b/wd_util.c
@@ -13,6 +13,8 @@
#include <ctype.h>
#include "wd_sched.h"
#include "wd_util.h"
+#include "wd_alg.h"
+#include "wd_bmm.h"
#define WD_ASYNC_DEF_POLL_NUM 1
#define WD_ASYNC_DEF_QUEUE_DEPTH 1024
@@ -100,11 +102,6 @@ struct acc_alg_item {
const char *algtype;
};
-struct wd_ce_ctx {
- char *drv_name;
- void *priv;
-};
-
static struct acc_alg_item alg_options[] = {
{"zlib", "zlib"},
{"gzip", "gzip"},
@@ -172,6 +169,93 @@ static struct acc_alg_item alg_options[] = {
{"", ""}
};
+static void *wd_internal_alloc(void *usr, size_t size)
+{
+ if (size != 0)
+ return malloc(size);
+ else
+ return NULL;
+}
+
+static void wd_internal_free(void *usr, void *va)
+{
+ if (va != NULL)
+ free(va);
+}
+
+static __u32 wd_mem_bufsize(void *usr)
+{
+ /* Malloc memory min size is 1 Byte */
+ return 1;
+}
+
+int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type)
+{
+ int ret;
+
+ ret = wd_is_sva(h_ctx);
+ if (ret == UACCE_DEV_SVA || ret == -WD_HW_EACCESS) {
+ /*
+ * In software queue scenario, all memory is handled as virtual memory
+ * and processed in the same way as SVA mode
+ */
+ mm_ops->sva_mode = true;
+ } else if (!ret) {
+ mm_ops->sva_mode = false;
+ } else {
+ WD_ERR("failed to check ctx!\n");
+ return ret;
+ }
+
+ /*
+ * Under SVA mode, there is no need to consider the memory type;
+ * directly proceed with virtual memory handling
+ */
+ if (mm_ops->sva_mode) {
+ mm_ops->alloc = (void *)wd_internal_alloc;
+ mm_ops->free = (void *)wd_internal_free;
+ mm_ops->iova_map = NULL;
+ mm_ops->iova_unmap = NULL;
+ mm_ops->get_bufsize = (void *)wd_mem_bufsize;
+ mm_ops->usr = NULL;
+ return 0;
+ }
+
+ switch (mem_type) {
+ case UADK_MEM_AUTO:
+ /*
+ * The memory pool needs to be allocated according to
+ * the block size when it is first executed in the UADK
+ */
+ mm_ops->usr = NULL;
+ WD_ERR("automatic under No-SVA mode is not supported!\n");
+ return -WD_EINVAL;
+ case UADK_MEM_USER:
+ if (!mm_ops->alloc || !mm_ops->free || !mm_ops->iova_map ||
+ !mm_ops->iova_unmap || !mm_ops->usr) { // The user create a memory pool
+ WD_ERR("failed to check memory ops, some ops function is NULL!\n");
+ return -WD_EINVAL;
+ }
+ break;
+ case UADK_MEM_PROXY:
+ if (!mm_ops->usr) {
+ WD_ERR("failed to check memory pool!\n");
+ return -WD_EINVAL;
+ }
+ mm_ops->alloc = (void *)wd_mem_alloc;
+ mm_ops->free = (void *)wd_mem_free;
+ mm_ops->iova_map = (void *)wd_mem_map;
+ mm_ops->iova_unmap = (void *)wd_mem_unmap;
+ mm_ops->get_bufsize = (void *)wd_get_bufsize;
+ break;
+ default:
+ WD_ERR("failed to check memory type!\n");
+ return -WD_EINVAL;
+ }
+
+ return 0;
+}
+
static void clone_ctx_to_internal(struct wd_ctx *ctx,
struct wd_ctx_internal *ctx_in)
{
@@ -257,6 +341,12 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in,
WD_ERR("failed to init ctxs lock!\n");
goto err_out;
}
+
+ ret = wd_insert_ctx_list(cfg->ctxs[i].ctx, in->alg_name);
+ if (ret) {
+ WD_ERR("failed to add ctx to mem list!\n");
+ goto err_out;
+ }
}
in->ctxs = ctxs;
@@ -318,6 +408,7 @@ void wd_clear_ctx_config(struct wd_ctx_config_internal *in)
in->ctxs = NULL;
}
+ wd_remove_ctx_list();
wd_shm_delete(in);
}
@@ -2485,7 +2576,7 @@ static int wd_init_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *l
/* If the ctx set number is 0, the initialization is skipped. */
if (!ctx_set_num)
- return 0;
+ return -WD_ENOPROC;
dev = wd_find_dev_by_numa(list, numa_id);
if (WD_IS_ERR(dev))
@@ -2573,7 +2664,9 @@ static int wd_init_ctx_and_sched(struct wd_init_attrs *attrs, struct bitmask *bm
for (j = 0; j < op_type_num; j++) {
ctx_nums = ctx_params->ctx_set_num[j];
ret = wd_init_ctx_set(attrs, list, idx, i, j);
- if (ret)
+ if (ret == -WD_ENOPROC)
+ continue;
+ else if (ret)
goto free_ctxs;
ret = wd_instance_sched_set(attrs->sched, ctx_nums, idx, i, j);
if (ret)
--
2.33.0
1
29
您好！
sig-AccLib 邀请您参加 2025-11-12 11:00 召开的Zoom会议
会议主题：【2025/11/12 AccLIb SIG 双周例会 11:00 - 12:00】
会议内容：
会议链接：linaro-org.zoom.us/j/91879279131
会议纪要：etherpad.openeuler.org/p/sig-AccLib-meet
会议链接：https://us06web.zoom.us/j/83926110482?pwd=aEbqlYoZb2CHsaTYMRJs0YzjeUf7TY.1
会议纪要：https://etherpad.openeuler.org/p/sig-AccLib-meetings
更多资讯尽在：https://www.openeuler.org/zh/
Hello!
sig-AccLib invites you to attend the Zoom conference will be held at 2025-11-12 11:00,
The subject of the conference is 【2025/11/12 AccLIb SIG 双周例会 11:00 - 12:00】
Summary:
会议链接：linaro-org.zoom.us/j/91879279131
会议纪要：etherpad.openeuler.org/p/sig-AccLib-meet
You can join the meeting at https://us06web.zoom.us/j/83926110482?pwd=aEbqlYoZb2CHsaTYMRJs0YzjeUf7TY.1
Add topics at https://etherpad.openeuler.org/p/sig-AccLib-meetings
More information: https://www.openeuler.org/en/
1
0