- High-performance-network - mailweb.openeuler.org

[PATCH] libhns: Add error logs to help diagnosis
by Chengchang Tang 03 Jan '25

03 Jan '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> Add error logs to help diagnosis. Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...hns-Add-error-logs-to-help-diagnosis.patch | 242 ++++++++++++++++++ rdma-core.spec | 9 +- 2 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 0042-libhns-Add-error-logs-to-help-diagnosis.patch diff --git a/0042-libhns-Add-error-logs-to-help-diagnosis.patch b/0042-libhns-Add-error-logs-to-help-diagnosis.patch new file mode 100644 index 0000000..9d880ea --- /dev/null +++ b/0042-libhns-Add-error-logs-to-help-diagnosis.patch @@ -0,0 +1,242 @@ +From 60c45b5f7c2cd0c2e7139d472406f071f327bb91 Mon Sep 17 00:00:00 2001 +From: Junxian Huang <huangjunxian6(a)hisilicon.com> +Date: Fri, 27 Dec 2024 14:02:29 +0800 +Subject: [PATCH] libhns: Add error logs to help diagnosis + +mainline inclusion +from mainline-master +commit 7849f1b17f89b8baa0065adaf9cd04204698ea82 +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBFGPH +CVE: NA + +Reference: https://github.com/linux-rdma/rdma-core/pull/1533/commits/7849f1b17f89b8baa… + +---------------------------------------------------------------------- + +Add error logs to help diagnosis. + +Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> +--- + providers/hns/hns_roce_u.c | 4 +- + providers/hns/hns_roce_u_hw_v2.c | 3 ++ + providers/hns/hns_roce_u_verbs.c | 79 ++++++++++++++++++++++++++------ + 3 files changed, 70 insertions(+), 16 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index e219b9e..ec995e7 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -424,8 +424,10 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + + context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, cmd_fd, 0); +- if (context->uar == MAP_FAILED) ++ if (context->uar == MAP_FAILED) { ++ verbs_err(&context->ibv_ctx, "failed to mmap uar page.\n"); + goto err_set_attr; ++ } + + if (init_dca_context(context, cmd_fd, + &resp, ctx_attr, hr_dev->page_size)) +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index c746e03..0628646 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -3057,6 +3057,9 @@ static int fill_send_wr_ops(const struct ibv_qp_init_attr_ex *attr, + fill_send_wr_ops_ud(qp_ex); + break; + default: ++ verbs_err(verbs_get_ctx(qp_ex->qp_base.context), ++ "QP type %d not supported for qp_ex send ops.\n", ++ attr->qp_type); + return -EOPNOTSUPP; + } + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index c733b21..e9acfab 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -422,8 +422,11 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + { + struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain); + +- if (!attr->cqe || attr->cqe > context->max_cqe) ++ if (!attr->cqe || attr->cqe > context->max_cqe) { ++ verbs_err(&context->ibv_ctx, "unsupported cq depth %u.\n", ++ attr->cqe); + return EINVAL; ++ } + + if (!check_comp_mask(attr->comp_mask, CREATE_CQ_SUPPORTED_COMP_MASK)) { + verbs_err(&context->ibv_ctx, "unsupported cq comps 0x%x\n", +@@ -431,8 +434,11 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + return EOPNOTSUPP; + } + +- if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS)) ++ if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS)) { ++ verbs_err(&context->ibv_ctx, "unsupported wc flags 0x%llx.\n", ++ attr->wc_flags); + return EOPNOTSUPP; ++ } + + if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { + if (!pad) { +@@ -504,8 +510,11 @@ static int exec_cq_create_cmd(struct ibv_context *context, + ret = ibv_cmd_create_cq_ex(context, attr, &cq->verbs_cq, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex), 0); +- if (ret) ++ if (ret) { ++ verbs_err(verbs_get_ctx(context), ++ "failed to exec create cq cmd, ret = %d.\n", ret); + return ret; ++ } + + cq->cqn = resp_drv->cqn; + cq->flags = resp_drv->cap_flags; +@@ -724,13 +733,20 @@ static int verify_srq_create_attr(struct hns_roce_context *context, + struct ibv_srq_init_attr_ex *attr) + { + if (attr->srq_type != IBV_SRQT_BASIC && +- attr->srq_type != IBV_SRQT_XRC) ++ attr->srq_type != IBV_SRQT_XRC) { ++ verbs_err(&context->ibv_ctx, ++ "unsupported srq type, type = %d.\n", attr->srq_type); + return -EINVAL; ++ } + + if (!attr->attr.max_sge || + attr->attr.max_wr > context->max_srq_wr || +- attr->attr.max_sge > context->max_srq_sge) ++ attr->attr.max_sge > context->max_srq_sge) { ++ verbs_err(&context->ibv_ctx, ++ "invalid srq attr size, max_wr = %u, max_sge = %u.\n", ++ attr->attr.max_wr, attr->attr.max_sge); + return -EINVAL; ++ } + + attr->attr.max_wr = max_t(uint32_t, attr->attr.max_wr, + HNS_ROCE_MIN_SRQ_WQE_NUM); +@@ -862,8 +878,12 @@ static int exec_srq_create_cmd(struct ibv_context *context, + ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, init_attr, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex)); +- if (ret) ++ if (ret) { ++ verbs_err(verbs_get_ctx(context), ++ "failed to exec create srq cmd, ret = %d.\n", ++ ret); + return ret; ++ } + + srq->srqn = resp_ex.srqn; + srq->cap_flags = resp_ex.cap_flags; +@@ -1086,9 +1106,12 @@ static int check_qp_create_mask(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr) + { + struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device); ++ int ret = 0; + +- if (!check_comp_mask(attr->comp_mask, CREATE_QP_SUP_COMP_MASK)) +- return EOPNOTSUPP; ++ if (!check_comp_mask(attr->comp_mask, CREATE_QP_SUP_COMP_MASK)) { ++ ret = EOPNOTSUPP; ++ goto out; ++ } + + if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS && + !check_comp_mask(attr->send_ops_flags, SEND_OPS_FLAG_MASK)) +@@ -1102,17 +1125,21 @@ static int check_qp_create_mask(struct hns_roce_context *ctx, + case IBV_QPT_RC: + case IBV_QPT_XRC_SEND: + if (!(attr->comp_mask & IBV_QP_INIT_ATTR_PD)) +- return EINVAL; ++ ret = EINVAL; + break; + case IBV_QPT_XRC_RECV: + if (!(attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) +- return EINVAL; ++ ret = EINVAL; + break; + default: + return EOPNOTSUPP; + } + +- return 0; ++out: ++ if (ret) ++ verbs_err(&ctx->ibv_ctx, "invalid comp_mask 0x%x.\n", ++ attr->comp_mask); ++ return ret; + } + + static int hns_roce_qp_has_rq(struct ibv_qp_init_attr_ex *attr) +@@ -1137,8 +1164,13 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, + if (cap->max_send_wr > ctx->max_qp_wr || + cap->max_recv_wr > ctx->max_qp_wr || + cap->max_send_sge > ctx->max_sge || +- cap->max_recv_sge > ctx->max_sge) ++ cap->max_recv_sge > ctx->max_sge) { ++ verbs_err(&ctx->ibv_ctx, ++ "invalid qp cap size, max_send/recv_wr = {%u, %u}, max_send/recv_sge = {%u, %u}.\n", ++ cap->max_send_wr, cap->max_recv_wr, ++ cap->max_send_sge, cap->max_recv_sge); + return -EINVAL; ++ } + + has_rq = hns_roce_qp_has_rq(attr); + if (!has_rq) { +@@ -1147,12 +1179,20 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, + } + + min_wqe_num = HNS_ROCE_V2_MIN_WQE_NUM; +- if (cap->max_send_wr < min_wqe_num) ++ if (cap->max_send_wr < min_wqe_num) { ++ verbs_debug(&ctx->ibv_ctx, ++ "change sq depth from %u to minimum %u.\n", ++ cap->max_send_wr, min_wqe_num); + cap->max_send_wr = min_wqe_num; ++ } + + if (cap->max_recv_wr) { +- if (cap->max_recv_wr < min_wqe_num) ++ if (cap->max_recv_wr < min_wqe_num) { ++ verbs_debug(&ctx->ibv_ctx, ++ "change rq depth from %u to minimum %u.\n", ++ cap->max_recv_wr, min_wqe_num); + cap->max_recv_wr = min_wqe_num; ++ } + + if (!cap->max_recv_sge) + return -EINVAL; +@@ -1646,6 +1686,11 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, + ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex)); ++ if (ret) { ++ verbs_err(&ctx->ibv_ctx, ++ "failed to exec create qp cmd, ret = %d.\n", ret); ++ return ret; ++ } + + qp->flags = resp_ex.drv_payload.cap_flags; + *dwqe_mmap_key = resp_ex.drv_payload.dwqe_mmap_key; +@@ -1707,8 +1752,12 @@ static int mmap_dwqe(struct ibv_context *ibv_ctx, struct hns_roce_qp *qp, + { + qp->dwqe_page = mmap(NULL, HNS_ROCE_DWQE_PAGE_SIZE, PROT_WRITE, + MAP_SHARED, ibv_ctx->cmd_fd, dwqe_mmap_key); +- if (qp->dwqe_page == MAP_FAILED) ++ if (qp->dwqe_page == MAP_FAILED) { ++ verbs_err(verbs_get_ctx(ibv_ctx), ++ "failed to mmap direct wqe page, QPN = %u.\n", ++ qp->verbs_qp.qp.qp_num); + return -EINVAL; ++ } + + return 0; + } +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 121f2f5..db8e113 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 17 +Release: 18 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -47,6 +47,7 @@ patch38: 0038-libhns-Fix-reference-to-uninitialized-cq-pointer.patch patch39: 0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch patch40: 0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch patch41: 0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch +patch42: 0042-libhns-Add-error-logs-to-help-diagnosis.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -626,6 +627,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Fri Jan 3 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-18 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Add error logs to help diagnosis + * Thu Nov 28 2024 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-17 - Type: bugfix - ID: NA -- 2.33.0

1 0

[PATCH] libhns: Fixed two bugs in libhns
by Chengchang Tang 03 Jan '25

03 Jan '25

From: Xinghai Cen <cenxinghai(a)h-partners.com> Fixed two bugs in libhns: libhns: Fix bypassed vendor check in hnsdv_query_device() libhns: Fix coredump during QP destruction when send_cq： == recv_cq Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...sed-vendor-check-in-hnsdv_query_devi.patch | 40 ++++++++++++++ ...ump-during-QP-destruction-when-send_.patch | 53 +++++++++++++++++++ rdma-core.spec | 10 +++- 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 0089-libhns-Fix-bypassed-vendor-check-in-hnsdv_query_devi.patch create mode 100644 0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch diff --git a/0089-libhns-Fix-bypassed-vendor-check-in-hnsdv_query_devi.patch b/0089-libhns-Fix-bypassed-vendor-check-in-hnsdv_query_devi.patch new file mode 100644 index 0000000..14f2edc --- /dev/null +++ b/0089-libhns-Fix-bypassed-vendor-check-in-hnsdv_query_devi.patch @@ -0,0 +1,40 @@ +From 485cddd47c83d6f229450b28d55d8e07f60ddcc0 Mon Sep 17 00:00:00 2001 +From: Yuyu Li <liyuyu6(a)huawei.com> +Date: Thu, 21 Nov 2024 21:37:15 +0800 +Subject: [PATCH] libhns: Fix bypassed vendor check in hnsdv_query_device() + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBF87T + +-------------------------------------------------------------------------- + +The device vendor check is actually bypassed currently due +to the wrong if-condition. It should be a '||' statement. + +Fixes: 19e1eabc154f ("libhns: Add input parameter check for hnsdv_query_device()") +Signed-off-by: Yuyu Li <liyuyu6(a)huawei.com> +--- + providers/hns/hns_roce_u_verbs.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 090efbf..a6afce2 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -128,10 +128,10 @@ int hnsdv_query_device(struct ibv_context *context, + struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; + struct hns_roce_device *hr_dev; + +- if (!ctx || !attrs_out) ++ if (!ctx || !context->device || !attrs_out) + return EINVAL; + +- if (!context->device && !is_hns_dev(context->device)) { ++ if (!is_hns_dev(context->device)) { + verbs_err(verbs_get_ctx(context), "not a HNS RoCE device!\n"); + return EOPNOTSUPP; + } +-- +2.33.0 + diff --git a/0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch b/0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch new file mode 100644 index 0000000..4909bd3 --- /dev/null +++ b/0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch @@ -0,0 +1,53 @@ +From ad5055f9b32ab0915803575385985fb10a29434a Mon Sep 17 00:00:00 2001 +From: Yuyu Li <liyuyu6(a)huawei.com> +Date: Mon, 25 Nov 2024 15:42:16 +0800 +Subject: [PATCH] libhns: Fix coredump during QP destruction when send_cq + == recv_cq + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IBF87T + +-------------------------------------------------------------------------- + +If the specified send CQ and recv CQ are both +the same CQ, the QP node in SCQ is not deleted. +which causes a segfault to occur when recreating +the QP. Here fix it. + +coredump info: +0x0000ffff8fbc37d4 in list_add_before_ +0x0000ffff8fbc381c in list_add_tail_ +0x0000ffff8fbc9d9c in add_qp_to_cq_list +0x0000ffff8fbca008 in create_qp +0x0000ffff8fbca110 in hns_roce_u_create_qp +0x0000ffff8feae39c in __ibv_create_qp_1_1 +0x0000000000401420 in test_ctrl_path + +Fixes: 5bebdb5ba77b ("libhns: Support reporting wc as software mode") +Signed-off-by: Yuyu Li <liyuyu6(a)huawei.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 8f071e1..48a7566 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -2033,9 +2033,10 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + list_del(&qp->rcq_node); + } + +- if (ibqp->send_cq && ibqp->send_cq != ibqp->recv_cq) { +- __hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), ibqp->qp_num, +- NULL); ++ if (ibqp->send_cq) { ++ if (ibqp->send_cq != ibqp->recv_cq) ++ __hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), ibqp->qp_num, ++ NULL); + list_del(&qp->scq_node); + } + +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 229810a..5e7be99 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 28 +Release: 29 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -94,6 +94,8 @@ patch85: 0085-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch patch86: 0086-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch patch87: 0087-libhns-Fix-out-of-order-issue-of-requester-when-sett.patch patch88: 0088-libhns-Fix-reference-to-uninitialized-cq-pointer.patch +patch89: 0089-libhns-Fix-bypassed-vendor-check-in-hnsdv_query_devi.patch +patch90: 0090-libhns-Fix-coredump-during-QP-destruction-when-send_.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -341,6 +343,12 @@ fi %{_mandir}/* %changelog +* Thu Jan 2 2025 Xinghai Cen <cenxinghai(a)h-partners.com> - 41.0-29 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Fixed two bugs in libhns + * Thu Nov 21 2024 Wentao Hu <huwentao19(a)h-partners.com> - 41.0-28 - Type: bugfix - ID: NA -- 2.33.0

1 0

[PATCH v1.1 rdma-core 0/3] Misc patches
by Junxian Huang 27 Dec '24

27 Dec '24

Junxian Huang (3): MAINTAINERS: Update hns maintainers README: Update the name of hns kernel module libhns: Add error logs to help diagnosis MAINTAINERS | 6 +-- README.md | 2 +- providers/hns/hns_roce_u.c | 5 +- providers/hns/hns_roce_u_hw_v2.c | 3 ++ providers/hns/hns_roce_u_verbs.c | 79 ++++++++++++++++++++++++++------ 5 files changed, 74 insertions(+), 21 deletions(-) -- 2.33.0

1 3

[PATCH] libhns: Fix coredump during QP destruction when send_cq == recv_cq
by Chengchang Tang 28 Nov '24

28 Nov '24

From: Xinghai Cen <cenxinghai(a)h-partners.com> driver inclusion category: feature bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IB7JZL ------------------------------------------------------------------ If the specified send CQ and recv CQ are both the same CQ, the QP node in SCQ is not deleted. which causes a segfault to occur when recreating the QP. Here fix it. Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...ump-during-QP-destruction-when-send_.patch | 54 +++++++++++++++++++ rdma-core.spec | 9 +++- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch diff --git a/0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch b/0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch new file mode 100644 index 0000000..b3ee84f --- /dev/null +++ b/0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch @@ -0,0 +1,54 @@ +From 263479c6fb4712528ccae276960ec94fd77afc51 Mon Sep 17 00:00:00 2001 +From: Yuyu Li <liyuyu6(a)huawei.com> +Date: Mon, 25 Nov 2024 16:13:48 +0800 +Subject: [PATCH] libhns: Fix coredump during QP destruction when send_cq == + recv_cq + +driver inclusion +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IB7JZL + +------------------------------------------------------------------ + +If the specified send CQ and recv CQ are both +the same CQ, the QP node in SCQ is not deleted. +which causes a segfault to occur when recreating +the QP. Here fix it. + +coredump info: +0x0000ffff8fbc37d4 in list_add_before_ +0x0000ffff8fbc381c in list_add_tail_ +0x0000ffff8fbc9d9c in add_qp_to_cq_list +0x0000ffff8fbca008 in create_qp +0x0000ffff8fbca110 in hns_roce_u_create_qp +0x0000ffff8feae39c in __ibv_create_qp_1_1 +0x0000000000401420 in test_ctrl_path + +Fixes: 95e05809d2d2 ("[BigDipperV3R9,NeZha][ROCE] libhns: Support reporting wc as software mode") +Signed-off-by: Yuyu Li <liyuyu6(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index e4232ea..c746e03 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -2006,9 +2006,10 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + list_del(&qp->rcq_node); + } + +- if (ibqp->send_cq && ibqp->send_cq != ibqp->recv_cq) { +- __hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), ibqp->qp_num, +- NULL); ++ if (ibqp->send_cq) { ++ if (ibqp->send_cq != ibqp->recv_cq) ++ __hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), ibqp->qp_num, ++ NULL); + list_del(&qp->scq_node); + } + +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index fb2c277..121f2f5 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 16 +Release: 17 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -46,6 +46,7 @@ patch37: 0037-libhns-Fix-out-of-order-issue-of-requester-when-sett.patch patch38: 0038-libhns-Fix-reference-to-uninitialized-cq-pointer.patch patch39: 0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch patch40: 0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch +patch41: 0041-libhns-Fix-coredump-during-QP-destruction-when-send_.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -625,6 +626,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Thu Nov 28 2024 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-17 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Fix coredump during QP destruction when send_cq == recv_cq + * Mon Nov 25 2024 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-16 - Type: bugfix - ID: NA -- 2.33.0

1 0

[PATCH OLK-6.6] RDMA/hns: Fix bonding failure due to wrong reset_state
by Chengchang Tang 22 Nov '24

22 Nov '24

From: Junxian Huang <huangjunxian6(a)hisilicon.com> driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB3K00 ---------------------------------------------------------------------- When roce driver is removed during reset, the reset flow of roce may not be fully completed. This may lead to the reset_state of roce handler stored in nic driver remaining in a middle state, such as HNS_ROCE_STATE_RST_DOWN or HNS_ROCE_STATE_RST_UNINIT. The reset_state won't be cleared even if roce driver is re-inited. This cause that roce bonding which currently relies on reset_state fails in this case. Replace the reset detection for bonding with nic APIs (.ae_dev_resetting() and .get_hw_reset_stat()), just like the reset detection elsewhere in roce driver. Fixes: 26d71e7c13cf ("RDMA/hns: Fix the concurrency error between bond and reset.") Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> --- drivers/infiniband/hw/hns/hns_roce_bond.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 7adae8990acd..f76667335189 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -543,6 +543,7 @@ static void hns_roce_do_bond(struct hns_roce_bond_group *bond_grp) bool is_bond_slave_in_reset(struct hns_roce_bond_group *bond_grp) { + const struct hnae3_ae_ops *ops; struct hnae3_handle *handle; struct net_device *net_dev; int i; @@ -550,9 +551,11 @@ bool is_bond_slave_in_reset(struct hns_roce_bond_group *bond_grp) for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { net_dev = bond_grp->bond_func_info[i].net_dev; handle = bond_grp->bond_func_info[i].handle; - if (net_dev && handle && - handle->rinfo.reset_state != HNS_ROCE_STATE_NON_RST && - handle->rinfo.reset_state != HNS_ROCE_STATE_RST_INITED) + if (!net_dev || !handle) + continue; + ops = handle->ae_algo->ops; + if (ops->ae_dev_resetting(handle) || + ops->get_hw_reset_stat(handle)) return true; } -- 2.33.0

1 0

[PATCH] libhns: Fixes several bugs for libhns
by Chengchang Tang 22 Nov '24

22 Nov '24

From: Xinghai Cen <cenxinghai(a)h-partners.com> Fixes several bugs for libhns: libhns: Fix memory leakage when DCA is enabled libhns: Fix the exception branch of wr_start() is not locked Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- ...xception-branch-of-wr_start-is-not-l.patch | 42 +++++++++++++++++++ ...x-memory-leakage-when-DCA-is-enabled.patch | 41 ++++++++++++++++++ rdma-core.spec | 10 ++++- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch create mode 100644 0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch diff --git a/0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch b/0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch new file mode 100644 index 0000000..a5fc588 --- /dev/null +++ b/0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch @@ -0,0 +1,42 @@ +From 7acbe6f79bbfe32f207800173e5b2b0c13ef23d9 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Wed, 12 Jun 2024 17:11:13 +0800 +Subject: [PATCH 39/40] libhns: Fix the exception branch of wr_start() is not + locked + +driver inclusion +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IB66RT + +------------------------------------------------------------------ + +The provider should provide locking to ensure that ibv_wr_start() +and ibv_wr_complete()/abort() form a per-QP critical section +where no other threads can enter. + +The exception branch of wr_start() is not locked, fix it here. +Because check_qp_send () does not require lock protection, +hns_roce_spin_lock () is placed after check_qp_send (). + +Fixes: 36446a56eea5 ("libhns: Extended QP supports the new post send mechanism") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_hw_v2.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 465ef1e..e4232ea 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -2930,6 +2930,7 @@ static void wr_start(struct ibv_qp_ex *ibv_qp) + + ret = check_qp_send(qp, ctx); + if (ret) { ++ hns_roce_spin_lock(&qp->sq.hr_lock); + qp->err = ret; + return; + } +-- +2.33.0 + diff --git a/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch b/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch new file mode 100644 index 0000000..6f5a9c6 --- /dev/null +++ b/0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch @@ -0,0 +1,41 @@ +From 1f5569fa17fc65391b504ca8118c3089e598dbc1 Mon Sep 17 00:00:00 2001 +From: wenglianfa <wenglianfa(a)huawei.com> +Date: Thu, 25 Jul 2024 11:06:01 +0800 +Subject: [PATCH 40/40] libhns: Fix memory leakage when DCA is enabled + +driver inclusion +category: feature +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/IB66RT + +------------------------------------------------------------------ + +After DCA is enabled and a QP is created, the memory block +applied for DCA is not free when the QP is destroyed. Here +fix it. + +Fixes: 41e39ab792c8 ("[BigDipperV3R9,NeZha][ROCE] libhns: Add support for attaching QP's WQE buffer") +Signed-off-by: wenglianfa <wenglianfa(a)huawei.com> +Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> +--- + providers/hns/hns_roce_u_verbs.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index e30880c..154e800 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1357,7 +1357,10 @@ static void qp_free_wqe(struct hns_roce_qp *qp) + + if (qp->rq.wqe_cnt) + free(qp->rq.wrid); +- hns_roce_free_buf(&qp->buf); ++ if (qp->dca_wqe.bufs) ++ free(qp->dca_wqe.bufs); ++ else ++ hns_roce_free_buf(&qp->buf); + } + + static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr, +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index fd956d5..2f4ebe5 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 50.0 -Release: 15 +Release: 16 Summary: RDMA core userspace libraries and daemons License: GPL-2.0-only OR BSD-2-Clause AND BSD-3-Clause Url: https://github.com/linux-rdma/rdma-core @@ -44,6 +44,8 @@ patch35: 0035-Fix-the-stride-calculation-for-MSN-PSN-area.patch patch36: 0036-add-ZTE-Dinghai-rdma-driver.patch patch37: 0037-libhns-Fix-out-of-order-issue-of-requester-when-sett.patch patch38: 0038-libhns-Fix-reference-to-uninitialized-cq-pointer.patch +patch39: 0039-libhns-Fix-the-exception-branch-of-wr_start-is-not-l.patch +patch40: 0040-libhns-Fix-memory-leakage-when-DCA-is-enabled.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) systemd systemd-devel @@ -623,6 +625,12 @@ fi %doc %{_docdir}/%{name}-%{version}/70-persistent-ipoib.rules %changelog +* Thu Nov 21 2024 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-16 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Fixes several bugs for libhns + * Fri Nov 15 2024 Xinghai Cen <cenxinghai(a)h-partners.com> - 50.0-15 - Type: bugfix - ID: NA -- 2.33.0

1 0

[PATCH OLK-6.6 00/13] Some patches for RDMA/hns
by Chengchang Tang 20 Nov '24

20 Nov '24

From: Xinghai Cen <cenxinghai(a)h-partners.com> Some patches for RDMA/hns Chengchang Tang (6): RDMA/hns: Fix HW UAF when destroy context timeout RDMA/hns: Fix integer overflow in calc_loading_percent() RDMA/hns: Fix possible RAS when DCA is not attached RDMA/hns: Fix a meaningless loop in active_dca_pages_proc() RDMA/hns: Fix list_*_careful() not being used in pairs RDMA/hns: Use one CQ bank per context Junxian Huang (5): RDMA/hns: Fix wrong output of sysfs scc pram when configuration failed RDMA/hns: Fix concurrency between sysfs store and FW configuration of scc params RDMA/hns: Fix mixed use of u32 and __le32 in sysfs RDMA/hns: Fix dereference of noderef expression RDMA/hns: Fix "Should it be static?" warnings wenglianfa (2): RDMA/hns: Fix the modification of max_send_sge RDMA/hns: Fix RoCEE hang when multiple QP banks use EXT_SGE EXT_SGE drivers/infiniband/hw/hns/hns_roce_bond.c | 8 ++- drivers/infiniband/hw/hns/hns_roce_cq.c | 73 +++++++++++++++++-- drivers/infiniband/hw/hns/hns_roce_dca.c | 60 ++++++++++++++-- drivers/infiniband/hw/hns/hns_roce_dca.h | 2 + drivers/infiniband/hw/hns/hns_roce_debugfs.c | 10 ++- drivers/infiniband/hw/hns/hns_roce_device.h | 17 ++++- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 26 +++++-- drivers/infiniband/hw/hns/hns_roce_main.c | 20 ++++++ drivers/infiniband/hw/hns/hns_roce_mr.c | 6 +- drivers/infiniband/hw/hns/hns_roce_qp.c | 74 ++++++++++++++++---- drivers/infiniband/hw/hns/hns_roce_srq.c | 6 +- drivers/infiniband/hw/hns/hns_roce_sysfs.c | 41 +++++++---- 12 files changed, 288 insertions(+), 55 deletions(-) -- 2.33.0

1 13

[PATCH OLK-6.6] RDMA/hns: Fix different dgids mapping to the same dip_idx
by Chengchang Tang 20 Nov '24

20 Nov '24

From: Feng Fang <fangfeng4(a)huawei.com> maillist inclusion commit faa62440a5772b40bb7d78bf9e29556a82ecf153 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IB4OOG Reference: https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git/commit/?id=fa… --------------------------------------------------------------------- DIP algorithm requires a one-to-one mapping between dgid and dip_idx. Currently a queue 'spare_idx' is used to store QPN of QPs that use DIP algorithm. For a new dgid, use a QPN from spare_idx as dip_idx. This method lacks a mechanism for deduplicating QPN, which may result in different dgids sharing the same dip_idx and break the one-to-one mapping requirement. This patch replaces spare_idx with xarray and introduces a refcnt of a dip_idx to indicate the number of QPs that using this dip_idx. The state machine for dip_idx management is implemented as: * The entry at an index in xarray is empty -- This indicates that the corresponding dip_idx hasn't been created. * The entry at an index in xarray is not empty but with 0 refcnt -- This indicates that the corresponding dip_idx has been created but not used as dip_idx yet. * The entry at an index in xarray is not empty and with non-0 refcnt -- This indicates that the corresponding dip_idx is being used by refcnt number of DIP QPs. Fixes: eb653eda1e91 ("RDMA/hns: Bugfix for incorrect association between dip_idx and dgid") Fixes: f91696f2f053 ("RDMA/hns: Support congestion control type selection according to the FW") Signed-off-by: Feng Fang <fangfeng4(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> Link: https://patch.msgid.link/20241112055553.3681129-1-huangjunxian6@hisilicon.c… Signed-off-by: Leon Romanovsky <leon(a)kernel.org> Signed-off-by: Xinghai Cen <cenxinghai(a)h-partners.com> --- drivers/infiniband/hw/hns/hns_roce_device.h | 11 +-- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 96 +++++++++++++++------ drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 2 +- drivers/infiniband/hw/hns/hns_roce_main.c | 2 - drivers/infiniband/hw/hns/hns_roce_qp.c | 8 +- 5 files changed, 75 insertions(+), 44 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index fdc1fe5e6a81..e3303cc3584a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -570,12 +570,6 @@ struct hns_roce_bank { u32 next; /* Next ID to allocate. */ }; -struct hns_roce_idx_table { - u32 *spare_idx; - u32 head; - u32 tail; -}; - struct hns_roce_qp_table { struct hns_roce_hem_table qp_table; struct hns_roce_hem_table irrl_table; @@ -584,7 +578,7 @@ struct hns_roce_qp_table { struct mutex scc_mutex; struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM]; struct mutex bank_mutex; - struct hns_roce_idx_table idx_table; + struct xarray dip_xa; }; struct hns_roce_cq_table { @@ -742,6 +736,7 @@ struct hns_roce_qp { bool delayed_destroy_flag; struct hns_roce_mtr_node *mtr_node; spinlock_t flush_lock; + struct hns_roce_dip *dip; }; struct hns_roce_ib_iboe { @@ -1102,8 +1097,6 @@ struct hns_roce_dev { enum hns_roce_device_state state; struct list_head qp_list; /* list of all qps on this dev */ spinlock_t qp_list_lock; /* protect qp_list */ - struct list_head dip_list; /* list of all dest ips on this dev */ - spinlock_t dip_list_lock; /* protect dip_list */ struct list_head pgdir_list; struct mutex pgdir_mutex; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 3f254ac48b42..88bd75c5743e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2720,20 +2720,19 @@ static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev) free_link_table_buf(hr_dev, &priv->ext_llm); } -static void free_dip_list(struct hns_roce_dev *hr_dev) +static void free_dip_entry(struct hns_roce_dev *hr_dev) { struct hns_roce_dip *hr_dip; - struct hns_roce_dip *tmp; - unsigned long flags; + unsigned long idx; - spin_lock_irqsave(&hr_dev->dip_list_lock, flags); + xa_lock(&hr_dev->qp_table.dip_xa); - list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) { - list_del(&hr_dip->node); + xa_for_each(&hr_dev->qp_table.dip_xa, idx, hr_dip) { + __xa_erase(&hr_dev->qp_table.dip_xa, hr_dip->dip_idx); kfree(hr_dip); } - spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); + xa_unlock(&hr_dev->qp_table.dip_xa); } static int hns_roce_v2_get_reset_page(struct hns_roce_dev *hr_dev) @@ -3182,7 +3181,7 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) hns_roce_v2_put_reset_page(hr_dev); if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09) - free_dip_list(hr_dev); + free_dip_entry(hr_dev); } static inline void mbox_desc_init(struct hns_roce_post_mbox *mb, @@ -5042,26 +5041,49 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask, return 0; } +static int alloc_dip_entry(struct xarray *dip_xa, u32 qpn) +{ + struct hns_roce_dip *hr_dip; + int ret; + + hr_dip = xa_load(dip_xa, qpn); + if (hr_dip) + return 0; + + hr_dip = kzalloc(sizeof(*hr_dip), GFP_KERNEL); + if (!hr_dip) + return -ENOMEM; + + ret = xa_err(xa_store(dip_xa, qpn, hr_dip, GFP_KERNEL)); + if (ret) + kfree(hr_dip); + + return ret; +} + static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, u32 *dip_idx) { const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - u32 *spare_idx = hr_dev->qp_table.idx_table.spare_idx; - u32 *head = &hr_dev->qp_table.idx_table.head; - u32 *tail = &hr_dev->qp_table.idx_table.tail; + struct xarray *dip_xa = &hr_dev->qp_table.dip_xa; + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct hns_roce_dip *hr_dip; - unsigned long flags; + unsigned long idx; int ret = 0; - spin_lock_irqsave(&hr_dev->dip_list_lock, flags); + ret = alloc_dip_entry(dip_xa, ibqp->qp_num); + if (ret) + return ret; - spare_idx[*tail] = ibqp->qp_num; - *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1); + xa_lock(dip_xa); - list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { - if (!memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { + xa_for_each(dip_xa, idx, hr_dip) { + if (hr_dip->qp_cnt && + !memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { *dip_idx = hr_dip->dip_idx; + hr_dip->qp_cnt++; + hr_qp->dip = hr_dip; goto out; } } @@ -5069,19 +5091,24 @@ static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, /* If no dgid is found, a new dip and a mapping between dgid and * dip_idx will be created. */ - hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC); - if (!hr_dip) { - ret = -ENOMEM; - goto out; + xa_for_each(dip_xa, idx, hr_dip) { + if (hr_dip->qp_cnt) + continue; + + *dip_idx = idx; + memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); + hr_dip->dip_idx = idx; + hr_dip->qp_cnt++; + hr_qp->dip = hr_dip; + break; } - memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); - hr_dip->dip_idx = *dip_idx = spare_idx[*head]; - *head = (*head == hr_dev->caps.num_qps - 1) ? 0 : (*head + 1); - list_add_tail(&hr_dip->node, &hr_dev->dip_list); + /* This should never happen. */ + if (WARN_ON_ONCE(!hr_qp->dip)) + ret = -ENOSPC; out: - spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); + xa_unlock(dip_xa); return ret; } @@ -6005,6 +6032,20 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, return ret; } +static void put_dip_ctx_idx(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct hns_roce_dip *hr_dip = hr_qp->dip; + + xa_lock(&hr_dev->qp_table.dip_xa); + + hr_dip->qp_cnt--; + if (!hr_dip->qp_cnt) + memset(hr_dip->dgid, 0, GID_LEN_V2); + + xa_unlock(&hr_dev->qp_table.dip_xa); +} + int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); @@ -6018,6 +6059,9 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) spin_unlock_irqrestore(&hr_qp->flush_lock, flags); flush_work(&hr_qp->flush_work.work); + if (hr_qp->cong_type == CONG_TYPE_DIP) + put_dip_ctx_idx(hr_dev, hr_qp); + ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); if (ret) ibdev_err_ratelimited(&hr_dev->ib_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 201902dff611..f43444be0308 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -1366,7 +1366,7 @@ struct hns_roce_v2_priv { struct hns_roce_dip { u8 dgid[GID_LEN_V2]; u32 dip_idx; - struct list_head node; /* all dips are on a list */ + u32 qp_cnt; }; struct fmea_ram_ecc { diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d488c3d5986f..520112bd43d5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -1250,8 +1250,6 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) INIT_LIST_HEAD(&hr_dev->qp_list); spin_lock_init(&hr_dev->qp_list_lock); - INIT_LIST_HEAD(&hr_dev->dip_list); - spin_lock_init(&hr_dev->dip_list_lock); INIT_LIST_HEAD(&hr_dev->uctx_list); mutex_init(&hr_dev->uctx_list_mutex); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 5ed2647567aa..77ec0c8678d3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1701,14 +1701,10 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) unsigned int reserved_from_bot; unsigned int i; - qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps, - sizeof(u32), GFP_KERNEL); - if (!qp_table->idx_table.spare_idx) - return -ENOMEM; - mutex_init(&qp_table->scc_mutex); mutex_init(&qp_table->bank_mutex); xa_init(&hr_dev->qp_table_xa); + xa_init(&qp_table->dip_xa); reserved_from_bot = hr_dev->caps.reserved_qps; @@ -1733,7 +1729,7 @@ void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) ida_destroy(&hr_dev->qp_table.bank[i].ida); + xa_destroy(&hr_dev->qp_table.dip_xa); mutex_destroy(&hr_dev->qp_table.bank_mutex); mutex_destroy(&hr_dev->qp_table.scc_mutex); - kfree(hr_dev->qp_table.idx_table.spare_idx); } -- 2.33.0

1 0

[PATCH OLK-6.6] RDMA/hns: Fix different dgids mapping to the same dip_idx
by Chengchang Tang 19 Nov '24

19 Nov '24

From: Feng Fang <fangfeng4(a)huawei.com> mainline inclusion from mainline-v6.12-rc2 commit faa62440a5772b40bb7d78bf9e29556a82ecf153 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IB4OOG CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git/commit/?id=fa… --------------------------------------------------------------------- DIP algorithm requires a one-to-one mapping between dgid and dip_idx. Currently a queue 'spare_idx' is used to store QPN of QPs that use DIP algorithm. For a new dgid, use a QPN from spare_idx as dip_idx. This method lacks a mechanism for deduplicating QPN, which may result in different dgids sharing the same dip_idx and break the one-to-one mapping requirement. This patch replaces spare_idx with xarray and introduces a refcnt of a dip_idx to indicate the number of QPs that using this dip_idx. The state machine for dip_idx management is implemented as: * The entry at an index in xarray is empty -- This indicates that the corresponding dip_idx hasn't been created. * The entry at an index in xarray is not empty but with 0 refcnt -- This indicates that the corresponding dip_idx has been created but not used as dip_idx yet. * The entry at an index in xarray is not empty and with non-0 refcnt -- This indicates that the corresponding dip_idx is being used by refcnt number of DIP QPs. Fixes: eb653eda1e91 ("RDMA/hns: Bugfix for incorrect association between dip_idx and dgid") Fixes: f91696f2f053 ("RDMA/hns: Support congestion control type selection according to the FW") Signed-off-by: Feng Fang <fangfeng4(a)huawei.com> Signed-off-by: Junxian Huang <huangjunxian6(a)hisilicon.com> Link: https://patch.msgid.link/20241112055553.3681129-1-huangjunxian6@hisilicon.c… Signed-off-by: Leon Romanovsky <leon(a)kernel.org> --- drivers/infiniband/hw/hns/hns_roce_device.h | 11 +-- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 96 +++++++++++++++------ drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 2 +- drivers/infiniband/hw/hns/hns_roce_main.c | 2 - drivers/infiniband/hw/hns/hns_roce_qp.c | 8 +- 5 files changed, 75 insertions(+), 44 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index fdc1fe5e6a81..e3303cc3584a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -570,12 +570,6 @@ struct hns_roce_bank { u32 next; /* Next ID to allocate. */ }; -struct hns_roce_idx_table { - u32 *spare_idx; - u32 head; - u32 tail; -}; - struct hns_roce_qp_table { struct hns_roce_hem_table qp_table; struct hns_roce_hem_table irrl_table; @@ -584,7 +578,7 @@ struct hns_roce_qp_table { struct mutex scc_mutex; struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM]; struct mutex bank_mutex; - struct hns_roce_idx_table idx_table; + struct xarray dip_xa; }; struct hns_roce_cq_table { @@ -742,6 +736,7 @@ struct hns_roce_qp { bool delayed_destroy_flag; struct hns_roce_mtr_node *mtr_node; spinlock_t flush_lock; + struct hns_roce_dip *dip; }; struct hns_roce_ib_iboe { @@ -1102,8 +1097,6 @@ struct hns_roce_dev { enum hns_roce_device_state state; struct list_head qp_list; /* list of all qps on this dev */ spinlock_t qp_list_lock; /* protect qp_list */ - struct list_head dip_list; /* list of all dest ips on this dev */ - spinlock_t dip_list_lock; /* protect dip_list */ struct list_head pgdir_list; struct mutex pgdir_mutex; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 3f254ac48b42..88bd75c5743e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2720,20 +2720,19 @@ static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev) free_link_table_buf(hr_dev, &priv->ext_llm); } -static void free_dip_list(struct hns_roce_dev *hr_dev) +static void free_dip_entry(struct hns_roce_dev *hr_dev) { struct hns_roce_dip *hr_dip; - struct hns_roce_dip *tmp; - unsigned long flags; + unsigned long idx; - spin_lock_irqsave(&hr_dev->dip_list_lock, flags); + xa_lock(&hr_dev->qp_table.dip_xa); - list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) { - list_del(&hr_dip->node); + xa_for_each(&hr_dev->qp_table.dip_xa, idx, hr_dip) { + __xa_erase(&hr_dev->qp_table.dip_xa, hr_dip->dip_idx); kfree(hr_dip); } - spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); + xa_unlock(&hr_dev->qp_table.dip_xa); } static int hns_roce_v2_get_reset_page(struct hns_roce_dev *hr_dev) @@ -3182,7 +3181,7 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) hns_roce_v2_put_reset_page(hr_dev); if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09) - free_dip_list(hr_dev); + free_dip_entry(hr_dev); } static inline void mbox_desc_init(struct hns_roce_post_mbox *mb, @@ -5042,26 +5041,49 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask, return 0; } +static int alloc_dip_entry(struct xarray *dip_xa, u32 qpn) +{ + struct hns_roce_dip *hr_dip; + int ret; + + hr_dip = xa_load(dip_xa, qpn); + if (hr_dip) + return 0; + + hr_dip = kzalloc(sizeof(*hr_dip), GFP_KERNEL); + if (!hr_dip) + return -ENOMEM; + + ret = xa_err(xa_store(dip_xa, qpn, hr_dip, GFP_KERNEL)); + if (ret) + kfree(hr_dip); + + return ret; +} + static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, u32 *dip_idx) { const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - u32 *spare_idx = hr_dev->qp_table.idx_table.spare_idx; - u32 *head = &hr_dev->qp_table.idx_table.head; - u32 *tail = &hr_dev->qp_table.idx_table.tail; + struct xarray *dip_xa = &hr_dev->qp_table.dip_xa; + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct hns_roce_dip *hr_dip; - unsigned long flags; + unsigned long idx; int ret = 0; - spin_lock_irqsave(&hr_dev->dip_list_lock, flags); + ret = alloc_dip_entry(dip_xa, ibqp->qp_num); + if (ret) + return ret; - spare_idx[*tail] = ibqp->qp_num; - *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1); + xa_lock(dip_xa); - list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { - if (!memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { + xa_for_each(dip_xa, idx, hr_dip) { + if (hr_dip->qp_cnt && + !memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { *dip_idx = hr_dip->dip_idx; + hr_dip->qp_cnt++; + hr_qp->dip = hr_dip; goto out; } } @@ -5069,19 +5091,24 @@ static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, /* If no dgid is found, a new dip and a mapping between dgid and * dip_idx will be created. */ - hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC); - if (!hr_dip) { - ret = -ENOMEM; - goto out; + xa_for_each(dip_xa, idx, hr_dip) { + if (hr_dip->qp_cnt) + continue; + + *dip_idx = idx; + memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); + hr_dip->dip_idx = idx; + hr_dip->qp_cnt++; + hr_qp->dip = hr_dip; + break; } - memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); - hr_dip->dip_idx = *dip_idx = spare_idx[*head]; - *head = (*head == hr_dev->caps.num_qps - 1) ? 0 : (*head + 1); - list_add_tail(&hr_dip->node, &hr_dev->dip_list); + /* This should never happen. */ + if (WARN_ON_ONCE(!hr_qp->dip)) + ret = -ENOSPC; out: - spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); + xa_unlock(dip_xa); return ret; } @@ -6005,6 +6032,20 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, return ret; } +static void put_dip_ctx_idx(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct hns_roce_dip *hr_dip = hr_qp->dip; + + xa_lock(&hr_dev->qp_table.dip_xa); + + hr_dip->qp_cnt--; + if (!hr_dip->qp_cnt) + memset(hr_dip->dgid, 0, GID_LEN_V2); + + xa_unlock(&hr_dev->qp_table.dip_xa); +} + int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); @@ -6018,6 +6059,9 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) spin_unlock_irqrestore(&hr_qp->flush_lock, flags); flush_work(&hr_qp->flush_work.work); + if (hr_qp->cong_type == CONG_TYPE_DIP) + put_dip_ctx_idx(hr_dev, hr_qp); + ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); if (ret) ibdev_err_ratelimited(&hr_dev->ib_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 201902dff611..f43444be0308 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -1366,7 +1366,7 @@ struct hns_roce_v2_priv { struct hns_roce_dip { u8 dgid[GID_LEN_V2]; u32 dip_idx; - struct list_head node; /* all dips are on a list */ + u32 qp_cnt; }; struct fmea_ram_ecc { diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d488c3d5986f..520112bd43d5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -1250,8 +1250,6 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) INIT_LIST_HEAD(&hr_dev->qp_list); spin_lock_init(&hr_dev->qp_list_lock); - INIT_LIST_HEAD(&hr_dev->dip_list); - spin_lock_init(&hr_dev->dip_list_lock); INIT_LIST_HEAD(&hr_dev->uctx_list); mutex_init(&hr_dev->uctx_list_mutex); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 5ed2647567aa..77ec0c8678d3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1701,14 +1701,10 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) unsigned int reserved_from_bot; unsigned int i; - qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps, - sizeof(u32), GFP_KERNEL); - if (!qp_table->idx_table.spare_idx) - return -ENOMEM; - mutex_init(&qp_table->scc_mutex); mutex_init(&qp_table->bank_mutex); xa_init(&hr_dev->qp_table_xa); + xa_init(&qp_table->dip_xa); reserved_from_bot = hr_dev->caps.reserved_qps; @@ -1733,7 +1729,7 @@ void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) ida_destroy(&hr_dev->qp_table.bank[i].ida); + xa_destroy(&hr_dev->qp_table.dip_xa); mutex_destroy(&hr_dev->qp_table.bank_mutex); mutex_destroy(&hr_dev->qp_table.scc_mutex); - kfree(hr_dev->qp_table.idx_table.spare_idx); } -- 2.33.0

1 0

高性能网络sig会议
by openEuler conference 18 Nov '24

18 Nov '24

您好！ sig-high-performance-network 邀请您参加 2024-11-18 14:30 召开的WeLink会议(自动录制) 会议主题：高性能网络sig会议会议链接：https://meeting.huaweicloud.com:36443/#/j/982559975 会议纪要：https://etherpad.openeuler.org/p/sig-high-performance-network-meetings 更多资讯尽在：https://www.openeuler.org/zh/ Hello! sig-high-performance-network invites you to attend the WeLink conference(auto recording) will be held at 2024-11-18 14:30, The subject of the conference is 高性能网络sig会议, You can join the meeting at https://meeting.huaweicloud.com:36443/#/j/982559975. Add topics at https://etherpad.openeuler.org/p/sig-high-performance-network-meetings. More information: https://www.openeuler.org/en/

1 0

2025

2024

2023

2022

2021

2020

High-performance-network