*** BLURB HERE ***
Chenghai Huang (1): uadk - reduce the print rating of specific logs
Weili Qian (4): Revert "uadk/v1: replace wd_spinlock to pthread_spin_lock" Revert "uadk/v1: check queue status before sending doorbells" uadk/v1: not check hardware status when BD not processed uadk/v1: check queue status before sending doorbells
Wenkai Lin (1): uadk/v1: replace wd_spinlock to pthread_spin_lock
drv/hisi_comp.c | 11 +++++++++-- v1/drv/hisi_zip_udrv.c | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 6 deletions(-)
From: Chenghai Huang huangchenghai2@huawei.com
Reduce the frequency of printing error logs when the status is 0xe for avoid a large amount of printed information.
Signed-off-by: Chenghai Huang huangchenghai2@huawei.com --- drv/hisi_comp.c | 11 +++++++++-- v1/drv/hisi_zip_udrv.c | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 6 deletions(-)
diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index b4c216fa..07d87548 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -29,6 +29,9 @@
#define ZSTD_MAX_SIZE (1 << 17)
+/* Error status 0xe indicates that dest_avail_out insufficient */ +#define ERR_DSTLEN_OUT 0xe + #define swab32(x) \ ((((x) & 0x000000ff) << 24) | \ (((x) & 0x0000ff00) << 8) | \ @@ -1039,8 +1042,12 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe,
if (unlikely(status != 0 && status != HZ_NEGACOMPRESS && status != HZ_CRC_ERR && status != HZ_DECOMP_END)) { - WD_ERR("bad request(ctx_st = 0x%x, status = 0x%x, algorithm type = %u)!\n", - ctx_st, status, type); + if (status == ERR_DSTLEN_OUT) + WD_DEBUG("bad request(ctx_st=0x%x, status=0x%x, algorithm type=%u)!\n", + ctx_st, status, type); + else + WD_ERR("bad request(ctx_st=0x%x, status=0x%x, algorithm type=%u)!\n", + ctx_st, status, type); recv_msg->req.status = WD_IN_EPARA; }
diff --git a/v1/drv/hisi_zip_udrv.c b/v1/drv/hisi_zip_udrv.c index 01d76a39..8497ebc5 100644 --- a/v1/drv/hisi_zip_udrv.c +++ b/v1/drv/hisi_zip_udrv.c @@ -26,6 +26,7 @@ #include <sys/ioctl.h> #include <sys/epoll.h> #include <sys/eventfd.h> +#include <sys/wait.h> #include <sys/types.h> #include "v1/wd_util.h" #include "v1/wd_comp.h" @@ -57,6 +58,10 @@ #define ZSTD_FREQ_DATA_SIZE 784 #define REPCODE_SIZE 12
+/* Error status 0xe indicates that dest_avail_out insufficient */ +#define ERR_DSTLEN_OUT 0xe +#define PRINT_TIME_INTERVAL 21600 + #define CTX_PRIV1_OFFSET 4 #define CTX_PRIV2_OFFSET 8 #define CTX_REPCODE1_OFFSET 12 @@ -95,6 +100,35 @@ struct zip_fill_sqe_ops { void (*fill_sqe_hw_info)(void *ssqe, struct wcrypto_comp_msg *msg); };
+static unsigned int g_err_print_enable = 1; + +static void zip_err_print_alarm_end(int sig) +{ + if (sig == SIGALRM) { + g_err_print_enable = 1; + alarm(0); + } +} + +static void zip_err_print_time_start(void) +{ + g_err_print_enable = 0; + signal(SIGALRM, zip_err_print_alarm_end); + alarm(PRINT_TIME_INTERVAL); +} + +static void zip_err_bd_print(__u16 ctx_st, __u32 status, __u32 type) +{ + if (status != ERR_DSTLEN_OUT) { + WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", + ctx_st, status, type); + } else if (g_err_print_enable == 1) { + WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", + ctx_st, status, type); + zip_err_print_time_start(); + } +} + static int fill_zip_comp_alg_v1(struct hisi_zip_sqe *sqe, struct wcrypto_comp_msg *msg) { @@ -274,8 +308,7 @@ int qm_parse_zip_sqe(void *hw_msg, const struct qm_queue_info *info,
if (status != 0 && status != HW_NEGACOMPRESS && status != HW_CRC_ERR && status != HW_DECOMP_END) { - WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", - ctx_st, status, type); + zip_err_bd_print(ctx_st, status, type); recv_msg->status = WD_IN_EPARA; } else { recv_msg->status = 0; @@ -707,8 +740,7 @@ int qm_parse_zip_sqe_v3(void *hw_msg, const struct qm_queue_info *info, return 0;
if (status != 0 && status != HW_NEGACOMPRESS && status != HW_DECOMP_END) { - WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", - ctx_st, status, type); + zip_err_bd_print(ctx_st, status, type); recv_msg->status = WD_IN_EPARA; } else { recv_msg->status = 0;
From: Weili Qian qianweili@huawei.com
This reverts commit f7c2a7a3e5116dc0ce4af539070f2ed93bb18af8. --- v1/drv/hisi_qm_udrv.c | 51 +++++++++++++----------------------------- v1/drv/hisi_qm_udrv.h | 4 ++-- v1/drv/hisi_rng_udrv.c | 25 ++++++--------------- v1/drv/hisi_rng_udrv.h | 2 +- 4 files changed, 26 insertions(+), 56 deletions(-)
diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c index 1d4f1d87..175a5c4a 100644 --- a/v1/drv/hisi_qm_udrv.c +++ b/v1/drv/hisi_qm_udrv.c @@ -20,7 +20,6 @@ #include <sys/mman.h> #include <string.h> #include <stdint.h> -#include <pthread.h> #include <sys/ioctl.h> #include <sys/epoll.h> #include <sys/eventfd.h> @@ -459,11 +458,6 @@ static int qm_init_queue_info(struct wd_queue *q) struct hisi_qp_ctx qp_ctx = {0}; int ret;
- if (!info->sqe_size) { - WD_ERR("invalid: sqe size is 0!\n"); - return -WD_EINVAL; - } - info->sq_tail_index = 0; info->cq_head_index = 0; info->cqc_phase = 1; @@ -508,6 +502,11 @@ static int qm_set_queue_info(struct wd_queue *q) ret = qm_set_queue_regions(q); if (ret) return -WD_EINVAL; + if (!info->sqe_size) { + WD_ERR("sqe size =%d err!\n", info->sqe_size); + ret = -WD_EINVAL; + goto err_with_regions; + } info->cq_base = (void *)((uintptr_t)info->sq_base + info->sqe_size * info->sq_depth);
@@ -535,24 +534,8 @@ static int qm_set_queue_info(struct wd_queue *q) goto err_with_regions; }
- ret = pthread_spin_init(&info->sd_lock, PTHREAD_PROCESS_PRIVATE); - if (ret) { - WD_ERR("failed to init qinfo sd_lock!\n"); - goto free_cache; - } - - ret = pthread_spin_init(&info->rc_lock, PTHREAD_PROCESS_PRIVATE); - if (ret) { - WD_ERR("failed to init qinfo rc_lock!\n"); - goto uninit_lock; - } - return 0;
-uninit_lock: - pthread_spin_destroy(&info->sd_lock); -free_cache: - free(info->req_cache); err_with_regions: qm_unset_queue_regions(q); return ret; @@ -593,10 +576,8 @@ void qm_uninit_queue(struct wd_queue *q) struct q_info *qinfo = q->qinfo; struct qm_queue_info *info = qinfo->priv;
- pthread_spin_destroy(&info->rc_lock); - pthread_spin_destroy(&info->sd_lock); - free(info->req_cache); qm_unset_queue_regions(q); + free(info->req_cache); free(qinfo->priv); qinfo->priv = NULL; } @@ -624,10 +605,10 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) int ret; __u32 i;
- pthread_spin_lock(&info->sd_lock); + wd_spinlock(&info->sd_lock); if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > info->sq_depth - num - 1)) { - pthread_spin_unlock(&info->sd_lock); + wd_unspinlock(&info->sd_lock); WD_ERR("queue is full!\n"); return -WD_EBUSY; } @@ -636,7 +617,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) ret = info->sqe_fill[qinfo->atype](req[i], qinfo->priv, info->sq_tail_index); if (unlikely(ret != WD_SUCCESS)) { - pthread_spin_unlock(&info->sd_lock); + wd_unspinlock(&info->sd_lock); WD_ERR("sqe fill error, ret %d!\n", ret); return -WD_EINVAL; } @@ -648,7 +629,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) }
ret = qm_tx_update(info, num); - pthread_spin_unlock(&info->sd_lock); + wd_unspinlock(&info->sd_lock);
return ret; } @@ -681,9 +662,9 @@ static int check_ds_rx_base(struct qm_queue_info *info, return 0;
if (before) { - pthread_spin_lock(&info->rc_lock); + wd_spinlock(&info->rc_lock); qm_rx_from_cache(info, resp, num); - pthread_spin_unlock(&info->rc_lock); + wd_unspinlock(&info->rc_lock); WD_ERR("wd queue hw error happened before qm receive!\n"); } else { WD_ERR("wd queue hw error happened after qm receive!\n"); @@ -724,7 +705,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) if (unlikely(ret)) return ret;
- pthread_spin_lock(&info->rc_lock); + wd_spinlock(&info->rc_lock); for (i = 0; i < num; i++) { cqe = info->cq_base + info->cq_head_index * sizeof(struct cqe); if (info->cqc_phase != CQE_PHASE(cqe)) @@ -733,7 +714,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) mb(); /* make sure the data is all in memory before read */ sq_head = CQE_SQ_HEAD_INDEX(cqe); if (unlikely(sq_head >= info->sq_depth)) { - pthread_spin_unlock(&info->rc_lock); + wd_unspinlock(&info->rc_lock); WD_ERR("CQE_SQ_HEAD_INDEX(%u) error\n", sq_head); return -WD_EIO; } @@ -745,7 +726,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) if (!ret) { break; } else if (ret < 0) { - pthread_spin_unlock(&info->rc_lock); + wd_unspinlock(&info->rc_lock); WD_ERR("recv sqe error %u\n", sq_head); return ret; } @@ -766,7 +747,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) ret = i; }
- pthread_spin_unlock(&info->rc_lock); + wd_unspinlock(&info->rc_lock);
return ret; } diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h index 06ac66a0..4d54cf6a 100644 --- a/v1/drv/hisi_qm_udrv.h +++ b/v1/drv/hisi_qm_udrv.h @@ -166,8 +166,8 @@ struct qm_queue_info { qm_sqe_parse sqe_parse[WCRYPTO_MAX_ALG]; hisi_qm_sqe_fill_priv sqe_fill_priv; hisi_qm_sqe_parse_priv sqe_parse_priv; - pthread_spinlock_t sd_lock; - pthread_spinlock_t rc_lock; + struct wd_lock sd_lock; + struct wd_lock rc_lock; struct wd_queue *q; int (*sgl_info)(struct hw_sgl_info *info); int (*sgl_init)(void *pool, struct wd_sgl *sgl); diff --git a/v1/drv/hisi_rng_udrv.c b/v1/drv/hisi_rng_udrv.c index 605ef275..86a20cb9 100644 --- a/v1/drv/hisi_rng_udrv.c +++ b/v1/drv/hisi_rng_udrv.c @@ -17,7 +17,6 @@ #include <stdlib.h> #include <unistd.h> #include <stdio.h> -#include <pthread.h> #include <sys/mman.h> #include <string.h> #include <stdint.h> @@ -35,7 +34,6 @@ int rng_init_queue(struct wd_queue *q) { struct q_info *qinfo = q->qinfo; struct rng_queue_info *info; - int ret;
info = calloc(1, sizeof(*info)); if (!info) { @@ -43,20 +41,12 @@ int rng_init_queue(struct wd_queue *q) return -ENOMEM; }
- ret = pthread_spin_init(&info->lock, PTHREAD_PROCESS_PRIVATE); - if (ret) { - free(info); - WD_ERR("failed to init rng qinfo lock!\n"); - return ret; - } - qinfo->priv = info; info->mmio_base = wd_drv_mmap_qfr(q, WD_UACCE_QFRT_MMIO, 0); if (info->mmio_base == MAP_FAILED) { info->mmio_base = NULL; + free(qinfo->priv); qinfo->priv = NULL; - pthread_spin_destroy(&info->lock); - free(info); WD_ERR("mmap trng mmio fail\n"); return -ENOMEM; } @@ -73,7 +63,6 @@ void rng_uninit_queue(struct wd_queue *q)
free(qinfo->priv); qinfo->priv = NULL; - pthread_spin_destroy(&info->lock); }
int rng_send(struct wd_queue *q, void **req, __u32 num) @@ -81,14 +70,14 @@ int rng_send(struct wd_queue *q, void **req, __u32 num) struct q_info *qinfo = q->qinfo; struct rng_queue_info *info = qinfo->priv;
- pthread_spin_lock(&info->lock); + wd_spinlock(&info->lock); if (!info->req_cache[info->send_idx]) { info->req_cache[info->send_idx] = req[0]; info->send_idx++; - pthread_spin_unlock(&info->lock); + wd_unspinlock(&info->lock); return 0; } - pthread_spin_unlock(&info->lock); + wd_unspinlock(&info->lock);
WD_ERR("queue is full!\n"); return -WD_EBUSY; @@ -139,16 +128,16 @@ int rng_recv(struct wd_queue *q, void **resp, __u32 num) struct wcrypto_cb_tag *tag; __u32 currsize = 0;
- pthread_spin_lock(&info->lock); + wd_spinlock(&info->lock); msg = info->req_cache[info->recv_idx]; if (!msg) { - pthread_spin_unlock(&info->lock); + wd_unspinlock(&info->lock); return 0; }
info->req_cache[info->recv_idx] = NULL; info->recv_idx++; - pthread_spin_unlock(&info->lock); + wd_unspinlock(&info->lock);
tag = (void *)(uintptr_t)msg->usr_tag; if (usr && tag->ctx_id != usr) diff --git a/v1/drv/hisi_rng_udrv.h b/v1/drv/hisi_rng_udrv.h index 3efa10e9..56814a42 100644 --- a/v1/drv/hisi_rng_udrv.h +++ b/v1/drv/hisi_rng_udrv.h @@ -29,7 +29,7 @@ struct rng_queue_info { void *req_cache[TRNG_Q_DEPTH]; __u8 send_idx; __u8 recv_idx; - pthread_spinlock_t lock; + struct wd_lock lock; };
int rng_init_queue(struct wd_queue *q);
From: Weili Qian qianweili@huawei.com
This reverts commit afc1082d85c6cdac7d3eaaac8130ffb5fa39b138. --- v1/drv/hisi_hpre_udrv.c | 10 +++---- v1/drv/hisi_qm_udrv.c | 60 ++++++++++++++++------------------------- v1/drv/hisi_qm_udrv.h | 4 +-- v1/wd_util.h | 8 +++++- 4 files changed, 36 insertions(+), 46 deletions(-)
diff --git a/v1/drv/hisi_hpre_udrv.c b/v1/drv/hisi_hpre_udrv.c index 05518abc..5832e9a4 100644 --- a/v1/drv/hisi_hpre_udrv.c +++ b/v1/drv/hisi_hpre_udrv.c @@ -1930,10 +1930,10 @@ static int fill_sm2_enc_sqe(void *msg, struct qm_queue_info *info, __u16 idx) goto fail_fill_sqe; }
+ /* make sure the request is all in memory before doorbell */ + mb(); info->sq_tail_index = i; - ret = qm_tx_update(info, 1); - if (unlikely(ret)) - goto fail_fill_sqe; + qm_tx_update(info, 1);
return ret;
@@ -2083,9 +2083,7 @@ static int parse_first_sqe(void *hw_msg, struct qm_queue_info *info, __u16 idx, WD_ERR("first BD error = %u\n", msg->result);
info->cq_head_index = i; - ret = qm_rx_update(info, 1); - if (unlikely(ret)) - return ret; + qm_rx_update(info, 1);
return 1; } diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c index 175a5c4a..ea4b1991 100644 --- a/v1/drv/hisi_qm_udrv.c +++ b/v1/drv/hisi_qm_udrv.c @@ -582,20 +582,10 @@ void qm_uninit_queue(struct wd_queue *q) qinfo->priv = NULL; }
-int qm_tx_update(struct qm_queue_info *info, __u32 num) +void qm_tx_update(struct qm_queue_info *info, __u32 num) { - if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { - WD_ERR("wd queue hw error happened before qm send!\n"); - return -WD_HW_EACCESS; - } - - /* make sure the request is all in memory before doorbell */ - mb(); - info->db(info, DOORBELL_CMD_SQ, info->sq_tail_index, 0); __atomic_add_fetch(&info->used, num, __ATOMIC_RELAXED); - - return WD_SUCCESS; }
int qm_send(struct wd_queue *q, void **req, __u32 num) @@ -605,6 +595,11 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) int ret; __u32 i;
+ if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { + WD_ERR("wd queue hw error happened before qm send!\n"); + return -WD_HW_EACCESS; + } + wd_spinlock(&info->sd_lock); if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > info->sq_depth - num - 1)) { @@ -628,10 +623,19 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) info->sq_tail_index++; }
- ret = qm_tx_update(info, num); + /* make sure the request is all in memory before doorbell */ + mb(); + qm_tx_update(info, num); wd_unspinlock(&info->sd_lock);
- return ret; + return WD_SUCCESS; +} + +void qm_rx_update(struct qm_queue_info *info, __u32 num) +{ + /* set c_flag to enable interrupt when use poll */ + info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); + __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); }
void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num) @@ -673,24 +677,6 @@ static int check_ds_rx_base(struct qm_queue_info *info, return -WD_HW_EACCESS; }
-int qm_rx_update(struct qm_queue_info *info, __u32 num) -{ - int ret; - - ret = check_ds_rx_base(info, NULL, 0, 0); - if (unlikely(ret)) - return ret; - - /* make sure queue status check is complete. */ - rmb(); - - /* set c_flag to enable interrupt when use poll */ - info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); - __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); - - return WD_SUCCESS; -} - int qm_recv(struct wd_queue *q, void **resp, __u32 num) { struct q_info *qinfo = q->qinfo; @@ -741,15 +727,15 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) } }
- if (i) { - ret = qm_rx_update(info, i); - if (!ret) - ret = i; - } + if (i) + qm_rx_update(info, i);
wd_unspinlock(&info->rc_lock); + ret = check_ds_rx_base(info, resp, num, 0); + if (unlikely(ret)) + return ret;
- return ret; + return i; }
static int hw_type_check(struct wd_queue *q, const char *hw_type) diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h index 4d54cf6a..542d20df 100644 --- a/v1/drv/hisi_qm_udrv.h +++ b/v1/drv/hisi_qm_udrv.h @@ -191,8 +191,8 @@ int qm_init_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); int qm_uninit_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); int qm_merge_hwsgl(struct wd_queue *q, void *pool, struct wd_sgl *dst_sgl, struct wd_sgl *src_sgl); -int qm_tx_update(struct qm_queue_info *info, __u32 num); -int qm_rx_update(struct qm_queue_info *info, __u32 num); +void qm_tx_update(struct qm_queue_info *info, __u32 num); +void qm_rx_update(struct qm_queue_info *info, __u32 num); void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num);
#define HISI_QM_API_VER_BASE "hisi_qm_v1" diff --git a/v1/wd_util.h b/v1/wd_util.h index 21137d2d..9180fc11 100644 --- a/v1/wd_util.h +++ b/v1/wd_util.h @@ -383,11 +383,17 @@ struct wcrypto_ecc_out { static inline void wd_reg_write(void *reg_addr, uint32_t value) { *((uint32_t *)reg_addr) = value; + wmb(); /* load fence */ }
static inline uint32_t wd_reg_read(void *reg_addr) { - return *((uint32_t *)reg_addr); + uint32_t temp; + + temp = *((uint32_t *)reg_addr); + rmb(); /* load fence */ + + return temp; }
void wd_spinlock(struct wd_lock *lock);
From: Weili Qian qianweili@huawei.com
If the CQE status is not updated during packet receiving, the hardware has not processed the packet. In this case, 0 can be returned without checking the hardware status. If a fault occurs, the hardware status will be checked when the user attempts to receive packets next time.
Signed-off-by: Weili Qian qianweili@huawei.com --- v1/drv/hisi_qm_udrv.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c index ea4b1991..731492b5 100644 --- a/v1/drv/hisi_qm_udrv.c +++ b/v1/drv/hisi_qm_udrv.c @@ -727,8 +727,12 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) } }
- if (i) + if (i) { qm_rx_update(info, i); + } else { + wd_unspinlock(&info->rc_lock); + return 0; + }
wd_unspinlock(&info->rc_lock); ret = check_ds_rx_base(info, resp, num, 0);
From: Weili Qian qianweili@huawei.com
When the device needs to be reset, the queue status is set to disable before resetting. The user process checks the queue status before sending the doorbell. If the queue is disable, the user process returns failure.
Currently, the task execution order in user mode is as follows: 1. check the queue status. 2. fill in or parse the BD. 3. send the doorbell to the hardware.
To reduce the possibility of sending doorbells during reset, the task execution order is modified as follows: 1. fill in or parse the BD. 2. check the queue status. 3. send the doorbell to the hardware.
In addition, a rmb() is added before the doorbell is sent to ensure that the queue status check is complete.
rmb() and wmb() can be replaced by mb() in hisi_qm_send(). Therefore, the barrier on the wd_reg_read() and wd_reg_write() can be deleted.
Signed-off-by: Weili Qian qianweili@huawei.com Signed-off-by: Qi Tao taoqi10@huawei.com --- v1/drv/hisi_hpre_udrv.c | 10 ++++--- v1/drv/hisi_qm_udrv.c | 60 ++++++++++++++++++++++++----------------- v1/drv/hisi_qm_udrv.h | 4 +-- v1/wd_util.h | 8 +----- 4 files changed, 44 insertions(+), 38 deletions(-)
diff --git a/v1/drv/hisi_hpre_udrv.c b/v1/drv/hisi_hpre_udrv.c index 5832e9a4..05518abc 100644 --- a/v1/drv/hisi_hpre_udrv.c +++ b/v1/drv/hisi_hpre_udrv.c @@ -1930,10 +1930,10 @@ static int fill_sm2_enc_sqe(void *msg, struct qm_queue_info *info, __u16 idx) goto fail_fill_sqe; }
- /* make sure the request is all in memory before doorbell */ - mb(); info->sq_tail_index = i; - qm_tx_update(info, 1); + ret = qm_tx_update(info, 1); + if (unlikely(ret)) + goto fail_fill_sqe;
return ret;
@@ -2083,7 +2083,9 @@ static int parse_first_sqe(void *hw_msg, struct qm_queue_info *info, __u16 idx, WD_ERR("first BD error = %u\n", msg->result);
info->cq_head_index = i; - qm_rx_update(info, 1); + ret = qm_rx_update(info, 1); + if (unlikely(ret)) + return ret;
return 1; } diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c index 731492b5..175a5c4a 100644 --- a/v1/drv/hisi_qm_udrv.c +++ b/v1/drv/hisi_qm_udrv.c @@ -582,10 +582,20 @@ void qm_uninit_queue(struct wd_queue *q) qinfo->priv = NULL; }
-void qm_tx_update(struct qm_queue_info *info, __u32 num) +int qm_tx_update(struct qm_queue_info *info, __u32 num) { + if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { + WD_ERR("wd queue hw error happened before qm send!\n"); + return -WD_HW_EACCESS; + } + + /* make sure the request is all in memory before doorbell */ + mb(); + info->db(info, DOORBELL_CMD_SQ, info->sq_tail_index, 0); __atomic_add_fetch(&info->used, num, __ATOMIC_RELAXED); + + return WD_SUCCESS; }
int qm_send(struct wd_queue *q, void **req, __u32 num) @@ -595,11 +605,6 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) int ret; __u32 i;
- if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { - WD_ERR("wd queue hw error happened before qm send!\n"); - return -WD_HW_EACCESS; - } - wd_spinlock(&info->sd_lock); if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > info->sq_depth - num - 1)) { @@ -623,19 +628,10 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) info->sq_tail_index++; }
- /* make sure the request is all in memory before doorbell */ - mb(); - qm_tx_update(info, num); + ret = qm_tx_update(info, num); wd_unspinlock(&info->sd_lock);
- return WD_SUCCESS; -} - -void qm_rx_update(struct qm_queue_info *info, __u32 num) -{ - /* set c_flag to enable interrupt when use poll */ - info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); - __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); + return ret; }
void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num) @@ -677,6 +673,24 @@ static int check_ds_rx_base(struct qm_queue_info *info, return -WD_HW_EACCESS; }
+int qm_rx_update(struct qm_queue_info *info, __u32 num) +{ + int ret; + + ret = check_ds_rx_base(info, NULL, 0, 0); + if (unlikely(ret)) + return ret; + + /* make sure queue status check is complete. */ + rmb(); + + /* set c_flag to enable interrupt when use poll */ + info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); + __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); + + return WD_SUCCESS; +} + int qm_recv(struct wd_queue *q, void **resp, __u32 num) { struct q_info *qinfo = q->qinfo; @@ -728,18 +742,14 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) }
if (i) { - qm_rx_update(info, i); - } else { - wd_unspinlock(&info->rc_lock); - return 0; + ret = qm_rx_update(info, i); + if (!ret) + ret = i; }
wd_unspinlock(&info->rc_lock); - ret = check_ds_rx_base(info, resp, num, 0); - if (unlikely(ret)) - return ret;
- return i; + return ret; }
static int hw_type_check(struct wd_queue *q, const char *hw_type) diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h index 542d20df..4d54cf6a 100644 --- a/v1/drv/hisi_qm_udrv.h +++ b/v1/drv/hisi_qm_udrv.h @@ -191,8 +191,8 @@ int qm_init_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); int qm_uninit_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); int qm_merge_hwsgl(struct wd_queue *q, void *pool, struct wd_sgl *dst_sgl, struct wd_sgl *src_sgl); -void qm_tx_update(struct qm_queue_info *info, __u32 num); -void qm_rx_update(struct qm_queue_info *info, __u32 num); +int qm_tx_update(struct qm_queue_info *info, __u32 num); +int qm_rx_update(struct qm_queue_info *info, __u32 num); void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num);
#define HISI_QM_API_VER_BASE "hisi_qm_v1" diff --git a/v1/wd_util.h b/v1/wd_util.h index 9180fc11..21137d2d 100644 --- a/v1/wd_util.h +++ b/v1/wd_util.h @@ -383,17 +383,11 @@ struct wcrypto_ecc_out { static inline void wd_reg_write(void *reg_addr, uint32_t value) { *((uint32_t *)reg_addr) = value; - wmb(); /* load fence */ }
static inline uint32_t wd_reg_read(void *reg_addr) { - uint32_t temp; - - temp = *((uint32_t *)reg_addr); - rmb(); /* load fence */ - - return temp; + return *((uint32_t *)reg_addr); }
void wd_spinlock(struct wd_lock *lock);
From: Wenkai Lin linwenkai6@hisilicon.com
Due to memory differences, using wd_spinlock may cause synchronization problems, it is better to use the standard pthread spin lock of glibc.
Signed-off-by: Wenkai Lin linwenkai6@hisilicon.com Signed-off-by: Qi Tao taoqi10@huawei.com --- v1/drv/hisi_qm_udrv.c | 51 +++++++++++++++++++++++++++++------------- v1/drv/hisi_qm_udrv.h | 4 ++-- v1/drv/hisi_rng_udrv.c | 25 +++++++++++++++------ v1/drv/hisi_rng_udrv.h | 2 +- 4 files changed, 56 insertions(+), 26 deletions(-)
diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c index 175a5c4a..1d4f1d87 100644 --- a/v1/drv/hisi_qm_udrv.c +++ b/v1/drv/hisi_qm_udrv.c @@ -20,6 +20,7 @@ #include <sys/mman.h> #include <string.h> #include <stdint.h> +#include <pthread.h> #include <sys/ioctl.h> #include <sys/epoll.h> #include <sys/eventfd.h> @@ -458,6 +459,11 @@ static int qm_init_queue_info(struct wd_queue *q) struct hisi_qp_ctx qp_ctx = {0}; int ret;
+ if (!info->sqe_size) { + WD_ERR("invalid: sqe size is 0!\n"); + return -WD_EINVAL; + } + info->sq_tail_index = 0; info->cq_head_index = 0; info->cqc_phase = 1; @@ -502,11 +508,6 @@ static int qm_set_queue_info(struct wd_queue *q) ret = qm_set_queue_regions(q); if (ret) return -WD_EINVAL; - if (!info->sqe_size) { - WD_ERR("sqe size =%d err!\n", info->sqe_size); - ret = -WD_EINVAL; - goto err_with_regions; - } info->cq_base = (void *)((uintptr_t)info->sq_base + info->sqe_size * info->sq_depth);
@@ -534,8 +535,24 @@ static int qm_set_queue_info(struct wd_queue *q) goto err_with_regions; }
+ ret = pthread_spin_init(&info->sd_lock, PTHREAD_PROCESS_PRIVATE); + if (ret) { + WD_ERR("failed to init qinfo sd_lock!\n"); + goto free_cache; + } + + ret = pthread_spin_init(&info->rc_lock, PTHREAD_PROCESS_PRIVATE); + if (ret) { + WD_ERR("failed to init qinfo rc_lock!\n"); + goto uninit_lock; + } + return 0;
+uninit_lock: + pthread_spin_destroy(&info->sd_lock); +free_cache: + free(info->req_cache); err_with_regions: qm_unset_queue_regions(q); return ret; @@ -576,8 +593,10 @@ void qm_uninit_queue(struct wd_queue *q) struct q_info *qinfo = q->qinfo; struct qm_queue_info *info = qinfo->priv;
- qm_unset_queue_regions(q); + pthread_spin_destroy(&info->rc_lock); + pthread_spin_destroy(&info->sd_lock); free(info->req_cache); + qm_unset_queue_regions(q); free(qinfo->priv); qinfo->priv = NULL; } @@ -605,10 +624,10 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) int ret; __u32 i;
- wd_spinlock(&info->sd_lock); + pthread_spin_lock(&info->sd_lock); if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > info->sq_depth - num - 1)) { - wd_unspinlock(&info->sd_lock); + pthread_spin_unlock(&info->sd_lock); WD_ERR("queue is full!\n"); return -WD_EBUSY; } @@ -617,7 +636,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) ret = info->sqe_fill[qinfo->atype](req[i], qinfo->priv, info->sq_tail_index); if (unlikely(ret != WD_SUCCESS)) { - wd_unspinlock(&info->sd_lock); + pthread_spin_unlock(&info->sd_lock); WD_ERR("sqe fill error, ret %d!\n", ret); return -WD_EINVAL; } @@ -629,7 +648,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) }
ret = qm_tx_update(info, num); - wd_unspinlock(&info->sd_lock); + pthread_spin_unlock(&info->sd_lock);
return ret; } @@ -662,9 +681,9 @@ static int check_ds_rx_base(struct qm_queue_info *info, return 0;
if (before) { - wd_spinlock(&info->rc_lock); + pthread_spin_lock(&info->rc_lock); qm_rx_from_cache(info, resp, num); - wd_unspinlock(&info->rc_lock); + pthread_spin_unlock(&info->rc_lock); WD_ERR("wd queue hw error happened before qm receive!\n"); } else { WD_ERR("wd queue hw error happened after qm receive!\n"); @@ -705,7 +724,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) if (unlikely(ret)) return ret;
- wd_spinlock(&info->rc_lock); + pthread_spin_lock(&info->rc_lock); for (i = 0; i < num; i++) { cqe = info->cq_base + info->cq_head_index * sizeof(struct cqe); if (info->cqc_phase != CQE_PHASE(cqe)) @@ -714,7 +733,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) mb(); /* make sure the data is all in memory before read */ sq_head = CQE_SQ_HEAD_INDEX(cqe); if (unlikely(sq_head >= info->sq_depth)) { - wd_unspinlock(&info->rc_lock); + pthread_spin_unlock(&info->rc_lock); WD_ERR("CQE_SQ_HEAD_INDEX(%u) error\n", sq_head); return -WD_EIO; } @@ -726,7 +745,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) if (!ret) { break; } else if (ret < 0) { - wd_unspinlock(&info->rc_lock); + pthread_spin_unlock(&info->rc_lock); WD_ERR("recv sqe error %u\n", sq_head); return ret; } @@ -747,7 +766,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) ret = i; }
- wd_unspinlock(&info->rc_lock); + pthread_spin_unlock(&info->rc_lock);
return ret; } diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h index 4d54cf6a..06ac66a0 100644 --- a/v1/drv/hisi_qm_udrv.h +++ b/v1/drv/hisi_qm_udrv.h @@ -166,8 +166,8 @@ struct qm_queue_info { qm_sqe_parse sqe_parse[WCRYPTO_MAX_ALG]; hisi_qm_sqe_fill_priv sqe_fill_priv; hisi_qm_sqe_parse_priv sqe_parse_priv; - struct wd_lock sd_lock; - struct wd_lock rc_lock; + pthread_spinlock_t sd_lock; + pthread_spinlock_t rc_lock; struct wd_queue *q; int (*sgl_info)(struct hw_sgl_info *info); int (*sgl_init)(void *pool, struct wd_sgl *sgl); diff --git a/v1/drv/hisi_rng_udrv.c b/v1/drv/hisi_rng_udrv.c index 86a20cb9..605ef275 100644 --- a/v1/drv/hisi_rng_udrv.c +++ b/v1/drv/hisi_rng_udrv.c @@ -17,6 +17,7 @@ #include <stdlib.h> #include <unistd.h> #include <stdio.h> +#include <pthread.h> #include <sys/mman.h> #include <string.h> #include <stdint.h> @@ -34,6 +35,7 @@ int rng_init_queue(struct wd_queue *q) { struct q_info *qinfo = q->qinfo; struct rng_queue_info *info; + int ret;
info = calloc(1, sizeof(*info)); if (!info) { @@ -41,12 +43,20 @@ int rng_init_queue(struct wd_queue *q) return -ENOMEM; }
+ ret = pthread_spin_init(&info->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) { + free(info); + WD_ERR("failed to init rng qinfo lock!\n"); + return ret; + } + qinfo->priv = info; info->mmio_base = wd_drv_mmap_qfr(q, WD_UACCE_QFRT_MMIO, 0); if (info->mmio_base == MAP_FAILED) { info->mmio_base = NULL; - free(qinfo->priv); qinfo->priv = NULL; + pthread_spin_destroy(&info->lock); + free(info); WD_ERR("mmap trng mmio fail\n"); return -ENOMEM; } @@ -63,6 +73,7 @@ void rng_uninit_queue(struct wd_queue *q)
free(qinfo->priv); qinfo->priv = NULL; + pthread_spin_destroy(&info->lock); }
int rng_send(struct wd_queue *q, void **req, __u32 num) @@ -70,14 +81,14 @@ int rng_send(struct wd_queue *q, void **req, __u32 num) struct q_info *qinfo = q->qinfo; struct rng_queue_info *info = qinfo->priv;
- wd_spinlock(&info->lock); + pthread_spin_lock(&info->lock); if (!info->req_cache[info->send_idx]) { info->req_cache[info->send_idx] = req[0]; info->send_idx++; - wd_unspinlock(&info->lock); + pthread_spin_unlock(&info->lock); return 0; } - wd_unspinlock(&info->lock); + pthread_spin_unlock(&info->lock);
WD_ERR("queue is full!\n"); return -WD_EBUSY; @@ -128,16 +139,16 @@ int rng_recv(struct wd_queue *q, void **resp, __u32 num) struct wcrypto_cb_tag *tag; __u32 currsize = 0;
- wd_spinlock(&info->lock); + pthread_spin_lock(&info->lock); msg = info->req_cache[info->recv_idx]; if (!msg) { - wd_unspinlock(&info->lock); + pthread_spin_unlock(&info->lock); return 0; }
info->req_cache[info->recv_idx] = NULL; info->recv_idx++; - wd_unspinlock(&info->lock); + pthread_spin_unlock(&info->lock);
tag = (void *)(uintptr_t)msg->usr_tag; if (usr && tag->ctx_id != usr) diff --git a/v1/drv/hisi_rng_udrv.h b/v1/drv/hisi_rng_udrv.h index 56814a42..3efa10e9 100644 --- a/v1/drv/hisi_rng_udrv.h +++ b/v1/drv/hisi_rng_udrv.h @@ -29,7 +29,7 @@ struct rng_queue_info { void *req_cache[TRNG_Q_DEPTH]; __u8 send_idx; __u8 recv_idx; - struct wd_lock lock; + pthread_spinlock_t lock; };
int rng_init_queue(struct wd_queue *q);
-
Qi Tao