From: zzry 1245464216@qq.com
Drop QP locks when associated to a PAD holding a TD. --- providers/hns/hns_roce_u.h | 2 +- providers/hns/hns_roce_u_hw_v2.c | 26 ++++++++++------- providers/hns/hns_roce_u_verbs.c | 49 +++++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 16 deletions(-)
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 338f162..f414da9 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -299,7 +299,7 @@ struct hns_roce_srq {
struct hns_roce_wq { unsigned long *wrid; - pthread_spinlock_t lock; + struct hns_roce_spinlock hr_lock; unsigned int wqe_cnt; int max_post; unsigned int head; diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index daef17a..8b4af93 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -1270,7 +1270,7 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, return ret; }
- pthread_spin_lock(&qp->sq.lock); + hns_roce_spin_lock(&qp->sq.hr_lock);
sge_info.start_idx = qp->next_sge; /* start index of extend sge */
@@ -1331,7 +1331,7 @@ out: *(qp->sdb) = qp->sq.head & 0xffff; }
- pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock);
if (ibvqp->state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; @@ -1420,7 +1420,7 @@ static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, return ret; }
- pthread_spin_lock(&qp->rq.lock); + hns_roce_spin_lock(&qp->rq.hr_lock);
max_sge = qp->rq.max_gs - qp->rq.rsv_sge; for (nreq = 0; wr; ++nreq, wr = wr->next) { @@ -1454,7 +1454,7 @@ out: hns_roce_update_rq_db(ctx, ibvqp->qp_num, qp->rq.head); }
- pthread_spin_unlock(&qp->rq.lock); + hns_roce_spin_unlock(&qp->rq.hr_lock);
if (ibvqp->state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; @@ -1544,8 +1544,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, bool flag = false; /* modify qp to error */
if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) { - pthread_spin_lock(&hr_qp->sq.lock); - pthread_spin_lock(&hr_qp->rq.lock); + hns_roce_spin_lock(&hr_qp->sq.hr_lock); + hns_roce_spin_lock(&hr_qp->rq.hr_lock); flag = true; }
@@ -1554,8 +1554,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, if (flag) { if (!ret) qp->state = IBV_QPS_ERR; - pthread_spin_unlock(&hr_qp->rq.lock); - pthread_spin_unlock(&hr_qp->sq.lock); + hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + hns_roce_spin_unlock(&hr_qp->rq.hr_lock); }
if (ret) @@ -1628,6 +1628,7 @@ static void hns_roce_unlock_cqs(struct ibv_qp *qp) static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) { struct hns_roce_context *ctx = to_hr_ctx(ibqp->context); + struct hns_roce_pad *pad = to_hr_pad(ibqp->pd); struct hns_roce_qp *qp = to_hr_qp(ibqp); int ret;
@@ -1654,6 +1655,9 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp)
hns_roce_free_qp_buf(qp, ctx);
+ if (pad) + atomic_fetch_sub(&pad->pd.refcount, 1); + free(qp);
return ret; @@ -2543,7 +2547,7 @@ static void wr_start(struct ibv_qp_ex *ibv_qp) return; }
- pthread_spin_lock(&qp->sq.lock); + hns_roce_spin_lock(&qp->sq.hr_lock); qp->sge_info.start_idx = qp->next_sge; qp->rb_sq_head = qp->sq.head; qp->err = 0; @@ -2576,7 +2580,7 @@ static int wr_complete(struct ibv_qp_ex *ibv_qp) }
out: - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); if (ibv_qp->qp_base.state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; hns_roce_u_v2_modify_qp(&ibv_qp->qp_base, &attr, IBV_QP_STATE); @@ -2591,7 +2595,7 @@ static void wr_abort(struct ibv_qp_ex *ibv_qp)
qp->sq.head = qp->rb_sq_head;
- pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); }
enum { diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 89114f3..eb52ff9 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -1009,6 +1009,41 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, return verify_qp_create_cap(ctx, attr); }
+static int hns_roce_qp_spinlock_init(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp) +{ + bool sq_need_lock; + bool rq_need_lock; + int ret; + + sq_need_lock = hns_roce_whether_need_lock(attr->pd); + if (!sq_need_lock) + verbs_info(&ctx->ibv_ctx, "configure sq as no lock.\n"); + + rq_need_lock = hns_roce_whether_need_lock(attr->pd); + if (!rq_need_lock) + verbs_info(&ctx->ibv_ctx, "configure rq as no lock.\n"); + + ret = hns_roce_spinlock_init(&qp->sq.hr_lock, sq_need_lock); + if (ret) + return ret; + + ret = hns_roce_spinlock_init(&qp->rq.hr_lock, rq_need_lock); + if (ret) { + hns_roce_spinlock_destroy(&qp->sq.hr_lock); + return ret; + } + + return 0; +} + +void hns_roce_qp_spinlock_destroy(struct hns_roce_qp *qp) +{ + hns_roce_spinlock_destroy(&qp->rq.hr_lock); + hns_roce_spinlock_destroy(&qp->sq.hr_lock); +} + static int alloc_recv_rinl_buf(uint32_t max_sge, struct hns_roce_rinl_buf *rinl_buf) { @@ -1379,10 +1414,6 @@ static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr_ex *attr, { int ret;
- if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) || - pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE)) - return -ENOMEM; - ret = qp_alloc_wqe(&attr->cap, qp, ctx); if (ret) return ret; @@ -1409,6 +1440,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, struct ibv_qp_init_attr_ex *attr) { struct hns_roce_context *context = to_hr_ctx(ibv_ctx); + struct hns_roce_pad *pad = to_hr_pad(attr->pd); struct hns_roce_qp *qp; uint64_t dwqe_mmap_key; int ret; @@ -1425,6 +1457,13 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
hns_roce_set_qp_params(attr, qp, context);
+ if (pad) + atomic_fetch_add(&pad->pd.refcount, 1); + + ret = hns_roce_qp_spinlock_init(context, attr, qp); + if (ret) + goto err_spinlock; + ret = hns_roce_alloc_qp_buf(attr, qp, context); if (ret) goto err_buf; @@ -1458,6 +1497,8 @@ err_ops: err_cmd: hns_roce_free_qp_buf(qp, context); err_buf: + hns_roce_qp_spinlock_destroy(qp); +err_spinlock: free(qp); err: if (ret < 0)