driver inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9FIHP
----------------------------------------------------------------------
During the reset, issuing the cmdq command to destroy resources such as qp, cq, and mr may fail, and the destruction failure information is printed. When a large number of resources fail to be destroyed, printk() causes multiple threads to compete for a global log buffer spinlock. As a result, the CPU performance deteriorates and the CPU is stuck. To fix it, use a function like printk_limit_rate() to limit the print frequency of printk().
Fixes: 9a4435375cd1 ("IB/hns: Add driver files for hns RoCE driver") Fixes: c7bcb13442e1 ("RDMA/hns: Add SRQ support for hip08 kernel mode") Fixes: 626903e9355b ("RDMA/hns: Add support for reporting wc as software mode") Fixes: 6eef524201de ("RDMA/hns: Replace not intuitive function/macro names") Fixes: 6f5f556d3795 ("RDMA/hns: Use the reserved loopback QPs to free MR before destroying MPT") Fixes: 357f34294686 ("RDMA/hns: Simplify the state judgment code of qp") Fixes: 926a01dc000d ("RDMA/hns: Add QP operations support for hip08 SoC") Fixes: 70d469ff011b ("RDMA/hns: Add method to detach WQE buffer") Fixes: 9c9edf689a60 ("RDMA/hns: Remove redundant DFX file and DFX ops structure") Signed-off-by: Chengchang Tang tangchengchang@huawei.com Signed-off-by: Juan Zhou zhoujuan51@h-partners.com --- drivers/infiniband/hw/hns/hns_roce_cq.c | 5 ++- drivers/infiniband/hw/hns/hns_roce_hem.c | 5 ++- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 50 +++++++++++++--------- drivers/infiniband/hw/hns/hns_roce_mr.c | 2 +- drivers/infiniband/hw/hns/hns_roce_srq.c | 5 ++- 5 files changed, 39 insertions(+), 28 deletions(-)
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 7250d0643..d1d93fc66 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -179,8 +179,9 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_CQC, hr_cq->cqn); if (ret) - dev_err(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret, - hr_cq->cqn); + dev_err_ratelimited(dev, + "DESTROY_CQ failed (%d) for CQN %06lx\n", + ret, hr_cq->cqn);
xa_erase(&cq_table->array, hr_cq->cqn);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index a4b3f1916..9ac68efc3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -672,8 +672,9 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev,
ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); if (ret) - dev_warn(dev, "failed to clear HEM base address, ret = %d.\n", - ret); + dev_warn_ratelimited(dev, + "failed to clear HEM base address, ret = %d.\n", + ret);
hns_roce_free_hem(hr_dev, table->hem[i]); table->hem[i] = NULL; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index a3494257b..e854e559f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -443,19 +443,21 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, if (unlikely(hr_qp->state == IB_QPS_RESET || hr_qp->state == IB_QPS_INIT || hr_qp->state == IB_QPS_RTR)) { - ibdev_err(ibdev, "failed to post WQE, QP state %u!\n", - hr_qp->state); + ibdev_err_ratelimited(ibdev, + "failed to post WQE, QP state %u!\n", + hr_qp->state); return -EINVAL; } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) { - ibdev_err(ibdev, "failed to post WQE, dev state %d!\n", - hr_dev->state); + ibdev_err_ratelimited(ibdev, + "failed to post WQE, dev state %d!\n", + hr_dev->state); return -EIO; }
if (check_dca_attach_enable(hr_qp)) { ret = dca_attach_qp_buf(hr_dev, hr_qp); if (unlikely(ret)) { - ibdev_err(ibdev, + ibdev_err_ratelimited(ibdev, "failed to attach DCA for QP-%lu send!\n", hr_qp->qpn); return ret; @@ -3613,8 +3615,9 @@ static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)
ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr); if (ret) { - ibdev_err(ibdev, "failed to post wqe for free mr, ret = %d.\n", - ret); + ibdev_err_ratelimited(ibdev, + "failed to post wqe for free mr, ret = %d.\n", + ret); return ret; }
@@ -3653,7 +3656,7 @@ static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev)
ret = free_mr_post_send_lp_wqe(hr_qp); if (ret) { - ibdev_err(ibdev, + ibdev_err_ratelimited(ibdev, "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n", hr_qp->qpn, ret); break; @@ -3666,14 +3669,14 @@ static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) while (cqe_cnt) { npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc); if (npolled < 0) { - ibdev_err(ibdev, + ibdev_err_ratelimited(ibdev, "failed to poll cqe for free mr, remain %d cqe.\n", cqe_cnt); goto out; }
if (time_after(jiffies, end)) { - ibdev_err(ibdev, + ibdev_err_ratelimited(ibdev, "failed to poll cqe for free mr and timeout, remain %d cqe.\n", cqe_cnt); goto out; @@ -5261,7 +5264,8 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, int ret = 0;
if (!check_qp_state(cur_state, new_state)) { - ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n"); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "Illegal state for QP!\n"); return -EINVAL; }
@@ -5525,7 +5529,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, /* SW pass context to HW */ ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp); if (ret) { - ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, "failed to modify QP, ret = %d.\n", ret); goto out; }
@@ -5710,7 +5714,9 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context); if (ret) { - ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, + "failed to query QPC, ret = %d.\n", + ret); ret = -EINVAL; goto out; } @@ -5718,7 +5724,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, state = hr_reg_read(&context, QPC_QP_ST); tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); if (tmp_qp_state == -1) { - ibdev_err(ibdev, "Illegal ib_qp_state\n"); + ibdev_err_ratelimited(ibdev, "Illegal ib_qp_state\n"); ret = -EINVAL; goto out; } @@ -5800,7 +5806,9 @@ static bool hns_roce_v2_chk_dca_buf_inactive(struct hns_roce_dev *hr_dev,
ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context); if (ret) { - ibdev_err(ibdev, "failed to query DCA QPC, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, + "failed to query DCA QPC, ret = %d.\n", + ret); return false; }
@@ -5844,7 +5852,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET, udata); if (ret) - ibdev_err(ibdev, + ibdev_err_ratelimited(ibdev, "failed to modify QP to RST, ret = %d.\n", ret); } @@ -5882,7 +5890,7 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); if (ret) - ibdev_err(&hr_dev->ib_dev, + ibdev_err_ratelimited(&hr_dev->ib_dev, "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n", hr_qp->qpn, ret);
@@ -6178,7 +6186,7 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn); hns_roce_free_cmd_mailbox(hr_dev, mailbox); if (ret) - ibdev_err(&hr_dev->ib_dev, + ibdev_err_ratelimited(&hr_dev->ib_dev, "failed to process cmd when modifying CQ, ret = %d.\n", ret);
@@ -6204,9 +6212,9 @@ static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn, ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_CQC, cqn); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd when querying CQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd when querying CQ, ret = %d.\n", + ret); goto err_mailbox; }
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 15382fb89..00b81e09c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -138,7 +138,7 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1)); if (ret) - ibdev_warn(ibdev, "failed to destroy mpt, ret = %d.\n", + ibdev_warn_ratelimited(ibdev, "failed to destroy mpt, ret = %d.\n", ret); }
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 31f100211..293c76632 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -150,8 +150,9 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ, srq->srqn); if (ret) - dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", - ret, srq->srqn); + dev_err_ratelimited(hr_dev->dev, + "DESTROY_SRQ failed (%d) for SRQN %06lx\n", + ret, srq->srqn);
xa_erase(&srq_table->xa, srq->srqn);