driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9H643 CVE: NA
---------------------------------
The RDMA driver supports the following features: Supports Huawei SP600 series NICs; Supports RoCEv2; Supports RoCE XRC, UD, UC, and RC modes; Supports RoCE UC, RC, and UD local switching; Supports RoCE MR, PD, CQ, QoS, QP, and SRQ management; Supports RoCE congestion control; Supports RoCE Bond; Supports RoCE FLR; Supports RoCE entry specifications; Supports RoCE error detection and reporting;
Signed-off-by: Shuai Wu wushuai51@huawei.com --- MAINTAINERS | 7 + arch/arm64/configs/openeuler_defconfig | 1 + arch/x86/configs/openeuler_defconfig | 1 + drivers/infiniband/Kconfig | 1 + drivers/infiniband/hw/Makefile | 1 + drivers/infiniband/hw/hiroce3/Kconfig | 14 + drivers/infiniband/hw/hiroce3/Makefile | 98 + .../infiniband/hw/hiroce3/bond/roce_bond.h | 147 + .../hw/hiroce3/bond/roce_bond_common.c | 938 +++ drivers/infiniband/hw/hiroce3/cq/roce_cq.h | 250 + .../infiniband/hw/hiroce3/cq/roce_cq_common.c | 195 + .../infiniband/hw/hiroce3/cq/roce_cq_cqe.c | 744 +++ .../infiniband/hw/hiroce3/cq/roce_cq_create.c | 629 ++ .../infiniband/hw/hiroce3/cq/roce_cq_ctrl.c | 930 +++ .../hw/hiroce3/cq/roce_cq_destroy.c | 247 + drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c | 122 + drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h | 177 + .../infiniband/hw/hiroce3/dfx/roce_dfx_cap.c | 688 +++ .../infiniband/hw/hiroce3/dfx/roce_dfx_cap.h | 181 + .../hw/hiroce3/dfx/roce_dfx_query.c | 643 ++ .../hiroce3/extension/roce_cdev_extension.c | 11 + .../hiroce3/extension/roce_event_extension.c | 23 + .../hiroce3/extension/roce_main_extension.c | 188 + .../hw/hiroce3/extension/roce_mr_extension.c | 31 + .../hiroce3/extension/roce_netdev_extension.c | 121 + .../hw/hiroce3/extension/roce_qp_extension.c | 241 + .../extension/roce_qp_post_send_extension.c | 11 + .../hw/hiroce3/extension/roce_srq_extension.c | 27 + .../hw/hiroce3/host/hmm/hmm_buddy.c | 161 + .../hw/hiroce3/host/hmm/hmm_buddy.h | 32 + .../infiniband/hw/hiroce3/host/hmm/hmm_comp.c | 164 + .../infiniband/hw/hiroce3/host/hmm/hmm_comp.h | 228 + .../hw/hiroce3/host/hmm/hmm_comp_init.c | 123 + .../hw/hiroce3/host/hmm/hmm_comp_mtt.c | 494 ++ .../hw/hiroce3/host/hmm/hmm_comp_mw_mr.c | 220 + .../hw/hiroce3/host/hmm/hmm_comp_res.c | 58 + .../infiniband/hw/hiroce3/host/hmm/hmm_em.c | 348 ++ .../infiniband/hw/hiroce3/host/hmm/hmm_em.h | 47 + .../infiniband/hw/hiroce3/host/hmm/hmm_mr.c | 429 ++ .../infiniband/hw/hiroce3/host/hmm/hmm_mr.h | 32 + .../infiniband/hw/hiroce3/host/hmm/hmm_umem.c | 273 + .../infiniband/hw/hiroce3/host/hmm/hmm_umem.h | 124 + .../hw/hiroce3/include/hinic3_hmm.h | 93 + .../hw/hiroce3/include/hinic3_rdma.h | 202 + .../hw/hiroce3/include/nic/nic_mpu_cmd.h | 181 + .../hw/hiroce3/include/nic/nic_npu_cmd.h | 29 + .../hw/hiroce3/include/nic/nic_npu_cmd_defs.h | 134 + .../infiniband/hw/hiroce3/include/node_id.h | 52 + .../include/rdma/rdma_context_format.h | 5181 +++++++++++++++++ .../include/rdma/rdma_ext_ctx_format.h | 379 ++ .../hw/hiroce3/include/rdma/roce_ccf_format.h | 722 +++ .../hiroce3/include/rdma/roce_compile_macro.h | 66 + .../hw/hiroce3/include/rdma/roce_ctx_api.h | 258 + .../hw/hiroce3/include/rdma/roce_dif_format.h | 492 ++ .../hw/hiroce3/include/rdma/roce_err_type.h | 122 + .../hiroce3/include/rdma/roce_hmm_context.h | 210 + .../hw/hiroce3/include/rdma/roce_mpu_common.h | 234 + .../hw/hiroce3/include/rdma/roce_pub.h | 259 + .../hw/hiroce3/include/rdma/roce_pub_cmd.h | 261 + .../hw/hiroce3/include/rdma/roce_ulp.h | 178 + .../hw/hiroce3/include/rdma/roce_vbs_format.h | 206 + .../hw/hiroce3/include/rdma/roce_verbs_attr.h | 413 ++ .../include/rdma/roce_verbs_attr_qpc_chip.h | 360 ++ .../hw/hiroce3/include/rdma/roce_verbs_cmd.h | 248 + .../hiroce3/include/rdma/roce_verbs_cq_attr.h | 196 + .../include/rdma/roce_verbs_ext_attr.h | 48 + .../hiroce3/include/rdma/roce_verbs_format.h | 132 + .../include/rdma/roce_verbs_gid_attr.h | 111 + .../hiroce3/include/rdma/roce_verbs_mr_attr.h | 330 ++ .../hw/hiroce3/include/rdma/roce_verbs_pub.h | 225 + .../include/rdma/roce_verbs_srq_attr.h | 264 + .../include/rdma/roce_verbs_ulp_format.h | 91 + .../hw/hiroce3/include/rdma/roce_wqe_format.h | 930 +++ .../hw/hiroce3/include/rdma/roce_xqe_format.h | 722 +++ .../hw/hiroce3/include/roce_cdev_extension.h | 13 + .../hw/hiroce3/include/roce_event_extension.h | 13 + .../hw/hiroce3/include/roce_main_extension.h | 78 + .../hw/hiroce3/include/roce_mr_extension.h | 17 + .../hiroce3/include/roce_netdev_extension.h | 19 + .../hw/hiroce3/include/roce_qp_extension.h | 64 + .../include/roce_qp_post_send_extension.h | 13 + .../hw/hiroce3/include/roce_srq_extension.h | 14 + drivers/infiniband/hw/hiroce3/mr/roce_mr.c | 949 +++ drivers/infiniband/hw/hiroce3/mr/roce_mr.h | 97 + drivers/infiniband/hw/hiroce3/qp/roce_post.h | 167 + drivers/infiniband/hw/hiroce3/qp/roce_qp.h | 244 + .../infiniband/hw/hiroce3/qp/roce_qp_create.c | 1239 ++++ .../hw/hiroce3/qp/roce_qp_destroy.c | 260 + .../infiniband/hw/hiroce3/qp/roce_qp_exp.h | 80 + .../infiniband/hw/hiroce3/qp/roce_qp_modify.c | 2243 +++++++ .../hw/hiroce3/qp/roce_qp_post_recv.c | 223 + .../hw/hiroce3/qp/roce_qp_post_send.c | 1315 +++++ .../infiniband/hw/hiroce3/qp/roce_qp_query.c | 393 ++ .../infiniband/hw/hiroce3/rdma/rdma_bitmap.c | 129 + .../infiniband/hw/hiroce3/rdma/rdma_bitmap.h | 36 + .../infiniband/hw/hiroce3/rdma/rdma_comp.c | 22 + .../infiniband/hw/hiroce3/rdma/rdma_comp.h | 131 + .../hw/hiroce3/rdma/rdma_comp_gid.c | 281 + .../hw/hiroce3/rdma/rdma_comp_init.c | 366 ++ .../hw/hiroce3/rdma/rdma_comp_mw_mr.c | 242 + .../infiniband/hw/hiroce3/rdma/rdma_comp_pd.c | 50 + .../hw/hiroce3/rdma/rdma_comp_res.c | 245 + drivers/infiniband/hw/hiroce3/roce.h | 574 ++ drivers/infiniband/hw/hiroce3/roce_cdev.c | 1259 ++++ drivers/infiniband/hw/hiroce3/roce_cmd.c | 722 +++ drivers/infiniband/hw/hiroce3/roce_cmd.h | 74 + drivers/infiniband/hw/hiroce3/roce_compat.h | 60 + drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c | 52 + drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h | 17 + drivers/infiniband/hw/hiroce3/roce_db.c | 88 + drivers/infiniband/hw/hiroce3/roce_db.h | 29 + drivers/infiniband/hw/hiroce3/roce_event.c | 566 ++ drivers/infiniband/hw/hiroce3/roce_event.h | 36 + drivers/infiniband/hw/hiroce3/roce_k_ioctl.h | 89 + drivers/infiniband/hw/hiroce3/roce_main.c | 1609 +++++ drivers/infiniband/hw/hiroce3/roce_mix.c | 1194 ++++ drivers/infiniband/hw/hiroce3/roce_mix.h | 205 + drivers/infiniband/hw/hiroce3/roce_netdev.c | 786 +++ drivers/infiniband/hw/hiroce3/roce_netdev.h | 59 + drivers/infiniband/hw/hiroce3/roce_netlink.c | 352 ++ drivers/infiniband/hw/hiroce3/roce_netlink.h | 164 + drivers/infiniband/hw/hiroce3/roce_pd.c | 66 + drivers/infiniband/hw/hiroce3/roce_pd.h | 24 + drivers/infiniband/hw/hiroce3/roce_sysfs.c | 1800 ++++++ drivers/infiniband/hw/hiroce3/roce_sysfs.h | 108 + drivers/infiniband/hw/hiroce3/roce_user.h | 65 + drivers/infiniband/hw/hiroce3/roce_xrc.c | 128 + drivers/infiniband/hw/hiroce3/roce_xrc.h | 23 + drivers/infiniband/hw/hiroce3/srq/roce_srq.h | 201 + .../infiniband/hw/hiroce3/srq/roce_srq_comm.c | 93 + .../hw/hiroce3/srq/roce_srq_create.c | 635 ++ .../infiniband/hw/hiroce3/srq/roce_srq_ctrl.c | 570 ++ 132 files changed, 43320 insertions(+) create mode 100644 drivers/infiniband/hw/hiroce3/Kconfig create mode 100644 drivers/infiniband/hw/hiroce3/Makefile create mode 100644 drivers/infiniband/hw/hiroce3/bond/roce_bond.h create mode 100644 drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq.h create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h create mode 100644 drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c create mode 100644 drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h create mode 100644 drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h create mode 100644 drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h create mode 100644 drivers/infiniband/hw/hiroce3/include/node_id.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_event_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_main_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h create mode 100644 drivers/infiniband/hw/hiroce3/mr/roce_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/mr/roce_mr.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_post.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c create mode 100644 drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c create mode 100644 drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce3/roce.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_cdev.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cmd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_compat.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_db.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_db.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_event.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_event.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_k_ioctl.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_main.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_mix.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_mix.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_netdev.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_netdev.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_pd.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_pd.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_user.h create mode 100644 drivers/infiniband/hw/hiroce3/roce_xrc.c create mode 100644 drivers/infiniband/hw/hiroce3/roce_xrc.h create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq.h create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c create mode 100644 drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c
diff --git a/MAINTAINERS b/MAINTAINERS index 3747bb15a..a6eb60eec 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8176,6 +8176,13 @@ F: drivers/net/ethernet/huawei/hinic3/cqm/ F: drivers/net/ethernet/huawei/hinic3/hw/ F: drivers/net/ethernet/huawei/hinic3/include/
+HUAWEI PVRDMA DRIVER +M: Chengbo Gu guchengbo@huawei.com +R: Xiaoping zheng zhengxiaoping5@huawei.com +L: linux-rdma@vger.kernel.org +S: Supported +F: drivers/infiniband/hw/hiroce3/ + HUGETLB FILESYSTEM M: Mike Kravetz mike.kravetz@oracle.com L: linux-mm@kvack.org diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index af9890a17..19936ef53 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -5644,6 +5644,7 @@ CONFIG_INFINIBAND_HNS_HIP08=y CONFIG_INFINIBAND_BNXT_RE=m CONFIG_INFINIBAND_QEDR=m CONFIG_INFINIBAND_XSC=m +CONFIG_HIROCE3=m CONFIG_RDMA_RXE=m # CONFIG_RDMA_SIW is not set CONFIG_INFINIBAND_IPOIB=m diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 39ec74f1e..b43404447 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -6280,6 +6280,7 @@ CONFIG_INFINIBAND_HFI1=m CONFIG_INFINIBAND_QEDR=m CONFIG_INFINIBAND_XSC=m CONFIG_INFINIBAND_RDMAVT=m +CONFIG_HIROCE3=m CONFIG_RDMA_RXE=m # CONFIG_RDMA_SIW is not set CONFIG_INFINIBAND_IPOIB=m diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index ec00b4f72..194e82483 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -92,6 +92,7 @@ source "drivers/infiniband/hw/bnxt_re/Kconfig" source "drivers/infiniband/hw/hfi1/Kconfig" source "drivers/infiniband/hw/qedr/Kconfig" source "drivers/infiniband/hw/xsc/Kconfig" +source "drivers/infiniband/hw/hiroce3/Kconfig" source "drivers/infiniband/sw/rdmavt/Kconfig" source "drivers/infiniband/sw/rxe/Kconfig" source "drivers/infiniband/sw/siw/Kconfig" diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile index 478dc420a..af9b9b87c 100644 --- a/drivers/infiniband/hw/Makefile +++ b/drivers/infiniband/hw/Makefile @@ -14,3 +14,4 @@ obj-$(CONFIG_INFINIBAND_HNS) += hns/ obj-$(CONFIG_INFINIBAND_QEDR) += qedr/ obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re/ obj-$(CONFIG_INFINIBAND_XSC) += xsc/ +obj-$(CONFIG_HIROCE3) += hiroce3/ diff --git a/drivers/infiniband/hw/hiroce3/Kconfig b/drivers/infiniband/hw/hiroce3/Kconfig new file mode 100644 index 000000000..9e351bf10 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Huawei driver configuration +# + +config HIROCE3 + tristate "Huawei Intelligent Network Interface Card RDMA Driver" + depends on HINIC3 && PCI_MSI && NUMA && PCI_IOV && DCB && (X86 || ARM64) + help + This driver supports HiROCE PCIE Ethernet cards. + To compile this driver as part of the kernel, choose Y here. + If unsure, choose N. + The default is N. + diff --git a/drivers/infiniband/hw/hiroce3/Makefile b/drivers/infiniband/hw/hiroce3/Makefile new file mode 100644 index 000000000..379bcb8aa --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/Makefile @@ -0,0 +1,98 @@ +EXPORT_SYMBOL := true + +OFED_VERSION := OFED_MLNX_5_8 +KBUILD_EXTRA_SYMBOLS += $(srctree)/drivers/net/ethernet/huawei/hinic3/Module.symvers + +ccflags-y += -DHW_CONVERT_ENDIAN +ccflags-y += -DROCE_SERVICE +ccflags-y += -D__ROCE_DFX__ +ccflags-y += -DROCE_CC_EN +ccflags-y += -D$(OFED_VERSION) +ccflags-y += -DROCE_COMPUTE +ccflags-y += -DROCE_BONDING_EN +ccflags-y += -DROCE_STANDARD +ccflags-y += -DOFED_MLNX_5_8 + +# Set CFLAGS from default file directories +ccflags-y += -I$(srctree)/include/linux +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3 +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/host/hmm +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/bond +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/cq +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/dfx +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/extension +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include/nic +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include/rdma +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/include/mag +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/mr +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/qp +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/rdma +ccflags-y += -I$(srctree)/drivers/infiniband/hw/hiroce3/srq +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3 +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/hw +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/bond +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/cfg_mgmt +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/mpu +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/bond +ccflags-y += -I$(srctree)/drivers/net/ethernet/huawei/hinic3/include/cqm + +obj-$(CONFIG_HIROCE3) += hiroce3.o + +hiroce3-y := cq/roce_cq_common.o \ + cq/roce_cq_cqe.o \ + cq/roce_cq_create.o \ + cq/roce_cq_ctrl.o \ + cq/roce_cq_destroy.o \ + extension/roce_event_extension.o \ + extension/roce_main_extension.o \ + extension/roce_mr_extension.o \ + extension/roce_netdev_extension.o \ + extension/roce_qp_extension.o \ + extension/roce_qp_post_send_extension.o \ + extension/roce_srq_extension.o \ + extension/roce_cdev_extension.o \ + roce_db.o \ + roce_main.o \ + roce_netlink.o \ + roce_event.o \ + roce_netdev.o \ + roce_mix.o \ + mr/roce_mr.o \ + roce_pd.o \ + qp/roce_qp_create.o \ + qp/roce_qp_destroy.o \ + qp/roce_qp_modify.o \ + qp/roce_qp_post_recv.o \ + qp/roce_qp_post_send.o \ + qp/roce_qp_query.o \ + roce_xrc.o \ + dfx/roce_dfx.o \ + dfx/roce_dfx_query.o \ + dfx/roce_dfx_cap.o \ + roce_cdev.o \ + roce_sysfs.o \ + roce_cmd.o \ + roce_cqm_cmd.o \ + bond/roce_bond_common.o \ + rdma/rdma_bitmap.o \ + rdma/rdma_comp.o \ + rdma/rdma_comp_res.o \ + rdma/rdma_comp_gid.o \ + rdma/rdma_comp_init.o \ + rdma/rdma_comp_pd.o \ + rdma/rdma_comp_mw_mr.o \ + srq/roce_srq_comm.o \ + srq/roce_srq_create.o \ + srq/roce_srq_ctrl.o \ + host/hmm/hmm_buddy.o \ + host/hmm/hmm_comp.o \ + host/hmm/hmm_comp_init.o \ + host/hmm/hmm_comp_mtt.o \ + host/hmm/hmm_comp_mw_mr.o \ + host/hmm/hmm_comp_res.o \ + host/hmm/hmm_em.o \ + host/hmm/hmm_mr.o \ + host/hmm/hmm_umem.o + diff --git a/drivers/infiniband/hw/hiroce3/bond/roce_bond.h b/drivers/infiniband/hw/hiroce3/bond/roce_bond.h new file mode 100644 index 000000000..5406893bf --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/bond/roce_bond.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_BOND_H +#define ROCE_BOND_H + +#include <net/bonding.h> +#include <linux/netdevice.h> + +#include "hinic3_bond.h" + +#include "roce.h" +#include "roce_qp.h" +#include "roce_cmd.h" +#include "roce_verbs_attr.h" + +#define ROCE_BOND_MAX_GROUPS 2 +#define ROCE_BOND_2_100G_MAX_GROUPS 1 +#define ROCE_BOND_4_25G_MAX_GROUPS 2 + +/* Adding two roce at a time consumes one bit of the array. + * Currently, a maximum of 33 nodes are supported, and at least 17. + * To support roce hot swap, the code logic needs to be reconstructed. + */ +#define ROCE_BOND_HCA_NUM 17 + +#define ROCE_BOND_MAX_PORTS 4 +#define ROCE_BOND_MAX_FUNCS 4 + +#define ROCE_BOND_NO_VLAN_ID 0 +#define ROCE_BOND_RSVD_VLAN_ID 4095 + +#define ROCE_BOND_PORT_ENABLE 1 +#define ROCE_BOND_PORT_DISABLE 0 + +#define ROCE_BOND_ADD_MAC_TBL 1 +#define ROCE_BOND_DEL_MAC_TBL 0 + +#define ROCE_BOND_FWD_ID_TBL_ALL_BITS 32 +#define ROCE_BOND_FWD_ID_TBL_PER_BITS 3 +#define ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK 0x7 + +#define ROCE_BOND_MAX_ACX_QP_NUM 32 +#define ROCE_BOND_ACX_QP_ENABLE 1 +#define ROCE_BOND_ACX_QP_DISABLE 0 + +#define ROCE_BOND_WAIT_ACTIVE_500MS 500 +enum { + ROCE_BOND_FUNC_OWN_FLAG = (1 << 0) +}; + +enum { + ROCE_BOND_FLAG = (1 << 0) +}; + +enum { + ROCE_BOND_WANT_TWO_SLAVES = 2, /* 2 slaves per bond_dev */ + ROCE_BOND_WANT_THREE_SLAVES = 3, /* 3 slaves per bond_dev */ + ROCE_BOND_WANT_FOUR_SLAVES = 4 /* 4 slaves per bond_dev */ +}; + +enum { + ROCE_BOND_WANT_TWO_SLAVES_MASK = 0x3, + ROCE_BOND_WANT_THREE_SLAVES_MASK = 0x7, + ROCE_BOND_WANT_FOUR_SLAVES_MASK = 0xf +}; + +enum { + ROCE_BOND_2_PORT_NUM = 2, + ROCE_BOND_4_PORT_NUM = 4 +}; + +enum { + ROCE_BOND_25G_PORT_SPEED = 25, + ROCE_BOND_100G_PORT_SPEED = 100 +}; + +enum { + ROCE_BOND_2_FUNC_NUM = 2, + ROCE_BOND_4_FUNC_NUM = 4 +}; + +enum { + ROCE_BOND_INVALID_HCA = -1, + ROCE_BOND_2_100G_HCA = 0, + ROCE_BOND_4_25G_HCA = 1, + ROCE_BOND_2_25G_HCA = 2 +}; + +#define SDI_BOND_SUPPORT_ROCE_FUNC_BIT 1 +#define SDI_BOND_SUPPORT_ROCE_FUNC_CNT 1 +#define SDI_BOND_SLAVES_FUNC_NUM 2 + +struct roce3_bond_slave { + struct net_device *netdev; + struct hinic3_lld_dev *lld_dev; + struct hinic3_lld_dev *ppf_dev; + struct netdev_lag_lower_state_info netdev_state; + u32 update_cnt; + u16 func_id; + u8 er_id; + bool is_ppf; +}; + +typedef void (*roce3_bond_service_func)(const char *bond_name, struct bond_attr *attr); + +struct roce3_bond_device { + char name[IFNAMSIZ]; + struct list_head entry; + struct roce3_bond_slave slaves[ROCE_BOND_MAX_FUNCS]; + struct mutex slave_lock; + u32 slave_cnt; + atomic_t next_port; + struct work_struct detach_work; + struct roce3_device *attached_rdev; + struct bond_attr attr; +}; + +bool roce3_bond_is_active(struct roce3_device *rdev); +struct net_device *roce3_bond_get_netdev(struct roce3_device *rdev); + +int roce3_add_bond_real_slave_mac(struct roce3_device *rdev, u8 *mac); +int roce3_add_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id); +void roce3_del_bond_real_slave_mac(struct roce3_device *rdev); +void roce3_del_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id); + +int roce3_bond_is_eth_port_of_netdev(struct roce3_device *rdev, + struct net_device *event_ndev); + +void roce3_bond_rr_set_flow(struct roce3_device *rdev, + struct roce3_qp *rqp, struct tag_roce_verbs_qp_attr *qp_attr); + +int roce3_bond_event_cfg_rdev(struct hinic3_lld_dev *lld_dev, + void *uld_dev, struct roce3_device **rdev); +int roce3_bonded_port_event_report(struct roce3_device *rdev, + const struct hinic3_event_info *event); +void roce3_handle_bonded_port_state_event(struct roce3_device *rdev); + +bool roce3_get_bond_ipsurx_en(void); +void roce3_set_bond_ipsurx_en(bool ipsurx_en); + +int roce3_bond_attach(struct roce3_device *rdev); +int roce3_bond_init(void); +void roce3_bond_pre_exit(void); +void roce3_bond_exit(void); + +#endif diff --git a/drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c b/drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c new file mode 100644 index 000000000..8b9db6ca9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/bond/roce_bond_common.c @@ -0,0 +1,938 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> + +#include <net/bonding.h> + +#include "bond_common_defs.h" + +#include "hinic3_hw.h" +#include "hinic3_srv_nic.h" + +#include "roce_bond.h" +#include "roce_cmd.h" +#include "roce_netdev.h" + +static bool g_roce3_bond_ipsurx_en = true; + +static LIST_HEAD(g_roce3_bond_list); +static DEFINE_MUTEX(g_roce3_bond_mutex); + +static struct workqueue_struct *g_bond_wq; + +struct roce3_detach_work { + u16 bond_id; + struct work_struct work; +}; + +struct roce3_bond_work { + char name[IFNAMSIZ]; + struct work_struct work; +}; + +bool roce3_get_bond_ipsurx_en(void) +{ + return g_roce3_bond_ipsurx_en; +} + +void roce3_set_bond_ipsurx_en(bool ipsurx_en) +{ + g_roce3_bond_ipsurx_en = ipsurx_en; +} + +static enum netdev_lag_tx_type roce3_get_tx_type_by_bond_mode(u16 bond_mode) +{ + switch (bond_mode) { + case BOND_MODE_8023AD: + case BOND_MODE_XOR: + return NETDEV_LAG_TX_TYPE_HASH; + case BOND_MODE_ACTIVEBACKUP: + return NETDEV_LAG_TX_TYPE_ACTIVEBACKUP; + default: + return NETDEV_LAG_TX_TYPE_UNKNOWN; + } +} + +static bool roce3_bond_mode_is_supported(u16 bond_mode) +{ + enum netdev_lag_tx_type tx_type = roce3_get_tx_type_by_bond_mode(bond_mode); + + if ((tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) && (tx_type != NETDEV_LAG_TX_TYPE_HASH)) { + pr_err("[ROCE, ERR] %s: Failed to support bond mode(%d)\n", __func__, tx_type); + return false; + } + return true; +} + +static bool is_hinic3_netdev(struct net_device *netdev) +{ + return (hinic3_get_lld_dev_by_netdev(netdev) != NULL); +} + +static bool roce3_can_do_bond(struct bonding *bond) +{ + bool ret = false; + int slave_cnt = 0; + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic3_lld_dev *lld_dev = NULL; + struct hinic3_lld_dev *ppf_dev = NULL; + + if (!bond || !roce3_bond_mode_is_supported(bond->params.mode)) + return ret; + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + lld_dev = hinic3_get_lld_dev_by_netdev(slave->dev); + if (!lld_dev) + goto out; + + if (!hinic3_support_roce(lld_dev->hwdev, NULL)) + goto out; + + if (!ppf_dev) { + ppf_dev = hinic3_get_ppf_lld_dev(lld_dev); + if (!ppf_dev) + goto out; + } + + if (hinic3_get_ppf_lld_dev(lld_dev) != ppf_dev) + goto out; + + slave_cnt++; + pr_info("%s:can do bond? slave_cnt(%d), slave_name(%s)", + __func__, slave_cnt, slave->dev->name); + } + + ret = (slave_cnt == ROCE_BOND_2_FUNC_NUM); +out: + rcu_read_unlock(); + return ret; +} + +struct net_device *roce3_bond_get_netdev(struct roce3_device *rdev) +{ + int i; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct net_device *ret_dev = NULL; + struct slave *slave = NULL; + + mutex_lock(&g_roce3_bond_mutex); + if (!bond_dev) { + mutex_unlock(&g_roce3_bond_mutex); + return ret_dev; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + rcu_read_lock(); + slave = bond_slave_get_rcu(bond_dev->slaves[i].netdev); + rcu_read_unlock(); + if (!slave) + continue; + + if (bond_is_active_slave(slave)) { + if (netif_running(bond_dev->slaves[i].netdev) && + netif_carrier_ok(bond_dev->slaves[i].netdev)) { + ret_dev = bond_dev->slaves[i].netdev; + } else if (netif_running(bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt].netdev) && + netif_carrier_ok(bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt].netdev)) { + ret_dev = bond_dev->slaves[(i + 1) % bond_dev->slave_cnt].netdev; + } else { + ret_dev = bond_dev->slaves[i].netdev; + } + dev_hold(ret_dev); + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce3_bond_mutex); + return ret_dev; + } + } + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce3_bond_mutex); + return ret_dev; +} + +void roce3_bond_rr_set_flow(struct roce3_device *rdev, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + u32 bond_tx_hash; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!bond_dev) + return; + + bond_tx_hash = (u32)atomic_add_return(1, &bond_dev->next_port); + rqp->tx_hash_value = bond_tx_hash; + + qp_attr->path_info.dw0.bs.bond_tx_hash_value = (u16)bond_tx_hash; +} + +static int roce3_bond_modify_mac_tbl_for_sdi(struct roce3_device *rdev, u8 *mac, + roce3_modify_mac_tbl modify_mac_tbl) +{ + u16 func_id; + int ret; + + for (func_id = 0; func_id < SDI_BOND_SLAVES_FUNC_NUM; func_id++) { + if (func_id == rdev->glb_func_id) + continue; + + ret = modify_mac_tbl(rdev->hwdev, mac, ROCE_BOND_RSVD_VLAN_ID, + rdev->glb_func_id, func_id); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to modify mac table, ret(%d)\n", + __func__, ret); + return ret; + } + } + + return 0; +} + +int roce3_add_bond_real_slave_mac(struct roce3_device *rdev, u8 *mac) +{ + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct roce3_bond_slave *slave = NULL; + int ret; + int i; + + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: Failed to find bond_dev\n", __func__); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + ret = roce3_add_mac_tbl_mac_entry(rdev->hwdev, mac, ROCE_BOND_RSVD_VLAN_ID, + rdev->glb_func_id, slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to add mac_vlan entry, ret(%d)\n", + __func__, ret); + return ret; + } + + if (slave->func_id != rdev->glb_func_id) { + /* + * The IPSU MAC table is used for fast forwarding. Even + * if the addition fails, the forwarding information + * can still be obtained by checking the MAC table later, + * without judging the execution result. + */ + (void)roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, mac, 0, + rdev->glb_func_id, slave->er_id); + } + } + mutex_unlock(&bond_dev->slave_lock); + + if (rdev->sdi_bond_name != NULL) { + ret = roce3_bond_modify_mac_tbl_for_sdi(rdev, mac, roce3_add_mac_tbl_mac_entry); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to modify mac table of sdi, ret(%d)\n", + __func__, ret); + return ret; + } + } + + return 0; +} + +int roce3_add_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id) +{ + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct roce3_bond_slave *slave = NULL; + int ret; + int i; + + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: Failed to find bond_dev\n", __func__); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) + continue; + + ret = roce3_add_mac_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + slave->func_id, slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to add mac_vlan entry, ret(%d)\n", + __func__, ret); + return ret; + } + + /* + * The IPSU MAC table is used for fast forwarding. Even + * if the addition fails, the forwarding information + * can still be obtained by checking the MAC table later, + * without judging the execution result. + */ + (void)roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + rdev->glb_func_id, slave->er_id); + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +void roce3_del_bond_real_slave_mac(struct roce3_device *rdev) +{ + int i; + struct roce3_bond_slave *slave = NULL; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!bond_dev) + return; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id != rdev->glb_func_id) { + (void)roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, rdev->mac, 0, + rdev->glb_func_id, slave->er_id); + } + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, rdev->mac, ROCE_BOND_RSVD_VLAN_ID, + rdev->glb_func_id, slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); + + if (rdev->sdi_bond_name != NULL) { + (void)roce3_bond_modify_mac_tbl_for_sdi(rdev, rdev->mac, + roce3_del_mac_tbl_mac_entry); + } +} + +void roce3_del_bond_vlan_slave_mac(struct roce3_device *rdev, u8 *mac, u16 vlan_id) +{ + int i; + struct roce3_bond_slave *slave = NULL; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!bond_dev) + return; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) + continue; + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + slave->func_id, slave->er_id); + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, mac, vlan_id, + rdev->glb_func_id, slave->er_id); + } + mutex_unlock(&bond_dev->slave_lock); +} + +bool roce3_bond_is_active(struct roce3_device *rdev) +{ + return (rdev->bond_dev != NULL); +} + +int roce3_bond_event_cfg_rdev(struct hinic3_lld_dev *lld_dev, void *uld_dev, + struct roce3_device **rdev) +{ + int i; + struct roce3_bond_device *bond_dev = NULL; + + if (lld_dev == NULL) { + pr_err("[ROCE, ERR] %s: Lld_dev is null\n", __func__); + return -EINVAL; + } + + if (uld_dev != NULL) { + *rdev = (struct roce3_device *)uld_dev; + return 0; + } + + mutex_lock(&g_roce3_bond_mutex); + list_for_each_entry(bond_dev, &g_roce3_bond_list, entry) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->slaves[i].lld_dev == lld_dev) { + *rdev = bond_dev->attached_rdev; + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + } + mutex_unlock(&bond_dev->slave_lock); + } + +out: + mutex_unlock(&g_roce3_bond_mutex); + return *rdev ? 0 : -EINVAL; +} + +int roce3_bonded_port_event_report(struct roce3_device *rdev, const struct hinic3_event_info *event) +{ + u32 type = HINIC3_SRV_EVENT_TYPE(event->service, event->type); + + if ((type != HINIC3_SRV_EVENT_TYPE(EVENT_SRV_NIC, EVENT_NIC_LINK_UP)) && + (type != HINIC3_SRV_EVENT_TYPE(EVENT_SRV_NIC, EVENT_NIC_LINK_DOWN))) { + pr_err("[ROCE] %s: event_service(%d), type(%d)\n", + __func__, event->service, event->type); + return -ERANGE; + } + + return 0; +} + +int roce3_bond_is_eth_port_of_netdev(struct roce3_device *rdev, struct net_device *event_ndev) +{ + struct roce3_bond_device *bond_dev = rdev->bond_dev; + struct net_device *tmp_ndev = NULL; + int ret; + int i; + /* judge current net device */ + tmp_ndev = rdev->ndev; + ret = roce3_is_eth_port_of_netdev(tmp_ndev, event_ndev); + if (ret != 0) + return 1; + + if (!bond_dev) + return 0; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (roce3_is_eth_port_of_netdev(bond_dev->slaves[i].netdev, event_ndev)) { + mutex_unlock(&bond_dev->slave_lock); + return 1; + } + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +struct roce3_bond_device *roce3_get_bond_dev(const char *bond_name) +{ + struct roce3_bond_device *bdev = NULL; + + list_for_each_entry(bdev, &g_roce3_bond_list, entry) { + if (!strcmp(bdev->name, bond_name)) + return bdev; + } + + return NULL; +} + +struct roce3_bond_device *roce3_get_bond_dev_by_name(const char *bond_name) +{ + struct roce3_bond_device *bdev = NULL; + + mutex_lock(&g_roce3_bond_mutex); + bdev = roce3_get_bond_dev(bond_name); + mutex_unlock(&g_roce3_bond_mutex); + return bdev; +} + +void roce3_bond_init_slave(struct roce3_bond_slave *slave, struct bond_tracker *tracker, int index, + struct bond_attr *attr) +{ + void *hwdev; + + slave->func_id = index; + slave->netdev = tracker->ndev[index]; + + dev_hold(slave->netdev); + pr_info("[ROCE, INFO] %s: dev_hold: name(%s),tracker_cnt(%d)\n", + __func__, slave->netdev->name, tracker->cnt); + slave->lld_dev = hinic3_get_lld_dev_by_netdev(slave->netdev); + slave->ppf_dev = hinic3_get_ppf_lld_dev(slave->lld_dev); + hwdev = slave->lld_dev->hwdev; + slave->is_ppf = hinic3_func_type(hwdev) == TYPE_PPF; + slave->er_id = hinic3_er_id(hwdev); + slave->netdev_state.link_up = tracker->netdev_state[index].link_up; + slave->netdev_state.tx_enabled = tracker->netdev_state[index].tx_enabled; + + if (slave->is_ppf) + attr->first_roce_func = slave->func_id; + else + hinic3_detach_service(slave->lld_dev, SERVICE_T_ROCE); +} + +bool roce3_bond_before_active_check(struct bond_tracker *tracker, struct bond_attr *attr) +{ + int i; + struct hinic3_lld_dev *lld_dev = NULL; + struct hinic3_lld_dev *ppf_dev = NULL; + + if (!roce3_bond_mode_is_supported(attr->bond_mode)) + return false; + + for (i = 0; i < ROCE_BOND_2_FUNC_NUM; i++) { + lld_dev = hinic3_get_lld_dev_by_netdev(tracker->ndev[i]); + if (!lld_dev) { + pr_err("[ROCE, ERR] %s: get lld dev err\n", __func__); + return false; + } + + if (!hinic3_support_roce(lld_dev->hwdev, NULL)) { + pr_err("[ROCE, ERR] %s: Not support roce\n", __func__); + return false; + } + + if (!ppf_dev) { + ppf_dev = hinic3_get_ppf_lld_dev(lld_dev); + if (!ppf_dev) { + pr_err("[ROCE, ERR] %s: get ppf dev err\n", __func__); + return false; + } + } + + if (hinic3_get_ppf_lld_dev(lld_dev) != ppf_dev) + return false; + } + + return true; +} + +void roce3_detach_nic_bond_work(struct work_struct *work) +{ + struct roce3_detach_work *detach_work = container_of(work, struct roce3_detach_work, work); + + hinic3_bond_detach(detach_work->bond_id, HINIC3_BOND_USER_ROCE); + + kfree(detach_work); +} + +static void roce3_attach_bond_work(struct work_struct *_work) +{ + u16 bond_id; + struct roce3_bond_work *work = container_of(_work, struct roce3_bond_work, work); + + pr_info("roce_attach: %s: work_name(%s)\n", __func__, work->name); + hinic3_bond_attach(work->name, HINIC3_BOND_USER_ROCE, &bond_id); + + kfree(work); +} +void roce3_deatch_bond(u16 bond_id) +{ + struct roce3_detach_work *detach_work = NULL; + + detach_work = kmalloc(sizeof(*detach_work), GFP_KERNEL); + if (!detach_work) + return; + + detach_work->bond_id = bond_id; + INIT_WORK(&detach_work->work, roce3_detach_nic_bond_work); + queue_work(g_bond_wq, &detach_work->work); +} + +bool roce3_bond_tracker_get(const char *bond_name, struct bond_tracker *tracker) +{ + int ret = 0; + + ret = hinic3_get_bond_tracker_by_name(bond_name, tracker); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: get bond tracker failed name(%s), ret(%d)\n", + __func__, bond_name, ret); + return false; + } + if (!tracker->is_bonded) { + pr_err("[ROCE, ERR] %s: tracker is NOT bond (%s)\n", __func__, bond_name); + return false; + } + if (tracker->cnt == ROCE_BOND_2_FUNC_NUM) + return true; + + pr_err("[ROCE, ERR] %s: get tracker cnt fail, cnt(%d) name(%s)\n", + __func__, tracker->cnt, bond_name); + return false; +} + +void roce3_before_bond_active(const char *bond_name, struct bond_attr *attr) +{ + struct roce3_bond_device *bond_dev = NULL; + struct roce3_bond_slave *slave = NULL; + struct bond_tracker tracker; + int i; + + if (!roce3_bond_tracker_get(bond_name, &tracker)) { + pr_err("[ROCE, ERR] %s: get bond tracker failed\n", __func__); + goto err; + } + + if (!roce3_bond_before_active_check(&tracker, attr)) { + pr_err("[ROCE, ERR] %s: active check failed\n", __func__); + goto err; + } + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (bond_dev) { + pr_info("[ROCE, INFO] %s: Find exist bond device\n", __func__); + return; + } + + bond_dev = kzalloc(sizeof(*bond_dev), GFP_KERNEL); + if (!bond_dev) + goto err; + + strscpy(bond_dev->name, bond_name, sizeof(bond_dev->name)); + + bond_dev->attr = *attr; + bond_dev->slave_cnt = tracker.cnt; + mutex_init(&bond_dev->slave_lock); + + for (i = 0; i < ROCE_BOND_2_FUNC_NUM; i++) { + slave = &bond_dev->slaves[i]; + roce3_bond_init_slave(slave, &tracker, i, attr); + } + + hinic3_detach_service(bond_dev->slaves[0].ppf_dev, SERVICE_T_ROCE); + mutex_lock(&g_roce3_bond_mutex); + list_add_tail(&bond_dev->entry, &g_roce3_bond_list); + mutex_unlock(&g_roce3_bond_mutex); + return; +err: + roce3_deatch_bond(attr->bond_id); +} + +void roce3_after_bond_active(const char *bond_name, struct bond_attr *attr) +{ + int ret; + struct roce3_bond_device *bond_dev = NULL; + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + return; + } + + ret = hinic3_attach_service(bond_dev->slaves[0].ppf_dev, SERVICE_T_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to attach roce device, ret(%d), bond name(%s)\n", + __func__, ret, bond_name); + } +} + +void roce3_after_bond_modify(const char *bond_name, struct bond_attr *attr) +{ + struct roce3_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + int i; + int j; + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + return; + } + + if (hinic3_get_bond_tracker_by_name(bond_name, &tracker) != 0) { + pr_err("[ROCE, ERR] %s: get bond tracker failed\n", __func__); + return; + } + + bond_dev->attr = *attr; + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < BOND_PORT_MAX_NUM; i++) { + for (j = 0; j < bond_dev->slave_cnt; j++) { + if (bond_dev->slaves[j].netdev != tracker.ndev[i]) + continue; + + bond_dev->slaves[j].netdev_state.link_up = tracker.netdev_state[i].link_up; + bond_dev->slaves[j].netdev_state.tx_enabled = + tracker.netdev_state[i].tx_enabled; + break; + } + } + mutex_unlock(&bond_dev->slave_lock); +} + +void roce3_before_bond_deactive(const char *bond_name, struct bond_attr *attr) +{ +} + +void roce3_after_bond_deactive(const char *bond_name, struct bond_attr *attr) +{ +} + +void roce3_bond_destroy(const char *bond_name) +{ + int ret; + int i; + struct roce3_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce3_bond_mutex); + bond_dev = roce3_get_bond_dev(bond_name); + if (bond_dev) + list_del(&bond_dev->entry); + + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + mutex_unlock(&g_roce3_bond_mutex); + return; + } + if (bond_dev->attached_rdev != NULL) + bond_dev->attached_rdev->bond_dev = NULL; + + mutex_unlock(&g_roce3_bond_mutex); + + hinic3_detach_service(bond_dev->slaves[0].ppf_dev, SERVICE_T_ROCE); + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + ret = hinic3_attach_service(bond_dev->slaves[i].lld_dev, SERVICE_T_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to attach roce device, ret(%d)\n", + __func__, ret); + } + dev_put(bond_dev->slaves[i].netdev); + pr_info("[ROCE, INFO] %s: dev_put: name(%s),slave_cnt(%d), slave_name(%s)\n", + __func__, bond_name, bond_dev->slave_cnt, bond_dev->slaves[i].netdev->name); + } + bond_dev->slave_cnt = 0; + mutex_unlock(&bond_dev->slave_lock); + + hinic3_bond_detach(bond_dev->attr.bond_id, HINIC3_BOND_USER_ROCE); + kfree(bond_dev); +} + +void roce3_before_bond_modify(const char *bond_name, struct bond_attr *attr) +{ + struct roce3_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + int i; + + bond_dev = roce3_get_bond_dev_by_name(bond_name); + if (!bond_dev) { + pr_err("[ROCE, ERR] %s: not find bond device by name(%s)\n", __func__, bond_name); + return; + } + + if (hinic3_get_bond_tracker_by_name(bond_name, &tracker) != 0) { + pr_err("[ROCE, ERR] %s: get bond tracker failed\n", __func__); + return; + } + + if (tracker.cnt == bond_dev->slave_cnt) { + bond_dev->attr = *attr; + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->slaves[i].is_ppf) { + attr->first_roce_func = bond_dev->slaves[i].func_id; + break; + } + } + return; + } + + if (tracker.cnt > bond_dev->slave_cnt) { + pr_err("[ROCE, ERR] %s: Add slave is not support, bond name(%s)\n", + __func__, bond_name); + return; + } + + if (tracker.cnt < ROCE_BOND_2_FUNC_NUM) { + roce3_bond_destroy(bond_dev->name); + return; + } +} + +static roce3_bond_service_func g_roce3_bond_proc[] = { + roce3_before_bond_active, + roce3_after_bond_active, + roce3_before_bond_modify, + roce3_after_bond_modify, + roce3_before_bond_deactive, + roce3_after_bond_deactive, +}; + +void roce3_bond_service_proc(const char *bond_name, void *bond_attr, enum bond_service_proc_pos pos) +{ + struct bond_attr *attr = (struct bond_attr *)bond_attr; + + if (bond_name == NULL) { + pr_err("[ROCE, ERR] %s: Bond_name is NULL\n", __func__); + return; + } + + if (pos >= BOND_POS_MAX) { + pr_err("[ROCE, ERR] %s: The pos is out of the range of proc_func\n", __func__); + return; + } + + if (g_roce3_bond_proc[pos] != NULL) + g_roce3_bond_proc[pos](bond_name, attr); +} + +int roce3_bond_attach(struct roce3_device *rdev) +{ + int i; + int ret = 0; + struct roce3_bond_device *bond_dev; + + mutex_lock(&g_roce3_bond_mutex); + list_for_each_entry(bond_dev, &g_roce3_bond_list, entry) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (rdev->ndev != bond_dev->slaves[i].netdev) + continue; + + if (bond_dev->attached_rdev == NULL) { + bond_dev->attached_rdev = rdev; + rdev->bond_dev = bond_dev; + } else { + ret = -EEXIST; + } + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + mutex_unlock(&bond_dev->slave_lock); + } +out: + mutex_unlock(&g_roce3_bond_mutex); + return ret; +} + +static void roce3_detach_bond_work(struct work_struct *_work) +{ + struct roce3_bond_work *work = container_of(_work, struct roce3_bond_work, work); + + roce3_bond_destroy(work->name); + + kfree(work); +} + +void roce3_queue_bond_work(struct net_device *upper_netdev, work_func_t func) +{ + struct roce3_bond_work *work; + struct bonding *bond = netdev_priv(upper_netdev); + + if (!bond) { + pr_info("%s: (name:%s) has no bond dev.\n", __func__, upper_netdev->name); + return; + } + + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return; + + strscpy(work->name, upper_netdev->name, sizeof(work->name)); + INIT_WORK(&work->work, func); + queue_work(g_bond_wq, &work->work); +} + +int roce3_bond_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) +{ + struct net_device *net_dev = NULL; + struct netdev_notifier_changeupper_info *info = NULL; + struct net_device *upper_netdev = NULL; + + info = (struct netdev_notifier_changeupper_info *)ptr; + net_dev = netdev_notifier_info_to_dev(ptr); + if (net_eq(dev_net(net_dev), &init_net) == 0) + return NOTIFY_DONE; + + if (event != NETDEV_CHANGEUPPER) + return NOTIFY_DONE; + + if (!is_hinic3_netdev(net_dev)) + return NOTIFY_DONE; + + upper_netdev = info->upper_dev; + if (upper_netdev == NULL) + return NOTIFY_DONE; + + if (!netif_is_lag_master(upper_netdev)) + return NOTIFY_DONE; + + if (!roce3_can_do_bond(netdev_priv(upper_netdev))) { + roce3_queue_bond_work(upper_netdev, roce3_detach_bond_work); + return NOTIFY_DONE; + } + + roce3_queue_bond_work(upper_netdev, roce3_attach_bond_work); + return NOTIFY_DONE; +} + +static struct notifier_block nb_netdevice = { + .notifier_call = roce3_bond_netdev_event +}; + +int roce3_bond_init(void) +{ + int ret; + struct net_device *upper_netdev; + + g_bond_wq = alloc_ordered_workqueue("roce3-bond-wq", 0); + if (!g_bond_wq) { + pr_err("[ROCE, ERR] %s: Failed to alloc workqueue\n", __func__); + return -ENOMEM; + } + + ret = register_netdevice_notifier(&nb_netdevice); + if (ret) { + pr_err("[ROCE, ERR] %s: Failed to register netdevice notifier(%d)\n", + __func__, ret); + goto nb_err; + } + + ret = hinic3_bond_register_service_func(HINIC3_BOND_USER_ROCE, roce3_bond_service_proc); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to register bond(%d)\n", __func__, ret); + goto err; + } + + rtnl_lock(); + for_each_netdev(&init_net, upper_netdev) { + if (netif_is_bond_master(upper_netdev) && + roce3_can_do_bond(netdev_priv(upper_netdev))) { + roce3_queue_bond_work(upper_netdev, roce3_attach_bond_work); + } + } + rtnl_unlock(); + + return 0; +err: + unregister_netdevice_notifier(&nb_netdevice); +nb_err: + destroy_workqueue(g_bond_wq); + return ret; +} + +void roce3_bond_pre_exit(void) +{ + int ret; + + ret = hinic3_bond_unregister_service_func(HINIC3_BOND_USER_ROCE); + if (ret != 0) + pr_err("[ROCE, ERR] %s: Failed to unregister service func(%d)\n", __func__, ret); + + unregister_netdevice_notifier(&nb_netdevice); + destroy_workqueue(g_bond_wq); +} + +void roce3_bond_exit(void) +{ + struct roce3_bond_device *bond_dev = NULL; + int i; + + while (!list_empty(&g_roce3_bond_list)) { + bond_dev = list_first_entry(&g_roce3_bond_list, struct roce3_bond_device, entry); + list_del(&bond_dev->entry); + for (i = 0; i < bond_dev->slave_cnt; i++) { + pr_info("[ROCE, INFO] %s: EXIT dev_put: bond_name(%s),slave_cnt(%d), slave_name(%s)\n", + __func__, bond_dev->name, bond_dev->slave_cnt, + bond_dev->slaves[i].netdev->name); + dev_put(bond_dev->slaves[i].netdev); + } + hinic3_bond_detach(bond_dev->attr.bond_id, HINIC3_BOND_USER_ROCE); + kfree(bond_dev); + } +} + +#endif diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq.h b/drivers/infiniband/hw/hiroce3/cq/roce_cq.h new file mode 100644 index 000000000..63909fac0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq.h @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CQ_H +#define ROCE_CQ_H + +#include <rdma/ib_verbs.h> +#include <linux/slab.h> + +#include "hinic3_hw.h" +#include "hinic3_rdma.h" + +#include "rdma_context_format.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_srq.h" +#include "roce_xrc.h" +#include "roce_user.h" +#include "hinic3_hmm.h" + +#define ROCE_CQN_INVLD 0xFFFFFFFF + +/* DB type of ARM_CQ */ +enum { + ROCE_CQ_DB_REQ_NOT_SOL = 1, + ROCE_CQ_DB_REQ_NOT = 2 +}; + +#define CQ_GPA_SIG_LEN 3 +#define CQ_DW_TO_BYTE 4 + +/* the type of err cqe + * IB compliant completion with error syndrome: + * 0x01 - Local Length Error () + * 0x02 - Local QP Operation Error + * 0x04 - Local Protection Error + * 0x05 - Work Request Flushed Error + * 0x06 - Memory Window Bind Error + * 0x10 - Bad Response Error + * 0x11 - Local Access Error + * 0x12 - Remote Invalid Request Error + * 0x13 - Remote Access Error + * 0x14 - Remote Operation Error + * 0x15 - Transport Retry Counter Exceeded + * 0x16 - RNR Retry Counter Exceeded + * 0x22 - Remote Aborted Error + * other - reserved + * Syndrome is defined according to the InfiniBand Architecture Specification, + * Volume 1. For a detailed explanation of the syndromes, refer to the Software + * Transport Interface and Software Transport Verbs chapters of the IB specifica-tion. + */ +enum roce3_cqe_syndrome { + ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, + ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, + ROCE_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, + ROCE_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, + + ROCE_CQE_SYNDROME_MW_BIND_ERR = 0x06, + ROCE_CQE_SYNDROME_BAD_RESP_ERR = 0x10, + + ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, + ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, + ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, + ROCE_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, + ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, + + ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, + + ROCE_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, + ROCE_CQE_SYNDROME_MAX = 0x23 +}; + +/* + * 1.Same type of operation as SQ WQE + * 8'h00-Send + * 8'h01-Send with Invalidate + * 8'h02-Send with Immediate Data + * 8'h03-reserved + * 8'h04-RDMA Write + * 8'h05-RDMA Write with Immediate Data + * 8'h06-reserved + * 8'h07-reserved + * 8'h08-RDMA Read + * 8'h09-reserved + * 8'h0a-reserved + * 8'h0b-reserved + * 8'h0c-Atomic Compare & Swap + * 8'h0d-Atomic Fetch & Add + * 8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation) + * 8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation) + * 8'h10-Fast Register PMR + * 8'h11-Local Invalidate + * 8'h12-Bind Memory Window Type1/2 + * 8'h13-Local opreation(Extended for further local opreation) + * other-Reserved + * 2.Receive + * 00000 - RDMA Write with Immediate + * 00001 - Send + * 00010 - Send with Immediate + * 00011 - Send & Invalidate + * 3.The following are general + * 11110 Error coding + * 10110 Resize coding + */ +enum roce3_cqe_send_opcode { + ROCE_OPCODE_SEND = 0x0, + ROCE_OPCODE_SEND_WITH_INV = 0x1, + ROCE_OPCODE_SEND_WITH_IMM = 0x2, + /* 0x3 reserved */ + + ROCE_OPCODE_RDMA_WRITE = 0x4, + ROCE_OPCODE_RDMA_WRITE_WITH_IMM = 0x5, + /* 0x6 and 0x7 reserved */ + + ROCE_OPCODE_RDMA_READ = 0x8, + /* 0x9~0xb reserved */ + + ROCE_OPCODE_ATOMIC_COMP_AND_SWP = 0xc, + ROCE_OPCODE_ATOMIC_FETCH_AND_ADD = 0xd, + ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP = 0xe, + ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD = 0xf, + + ROCE_OPCODE_FAST_REG_PMR = 0x10, + ROCE_OPCODE_LOCAL_INV = 0x11, + ROCE_OPCODE_BIND_TYPE2_MW = 0x12, + ROCE_OPCODE_REG_SIG_MR = 0x13, + + ROCE_OPCODE_RESIZE_CQE = 0x16, + ROCE_OPCODE_ERR = 0x1e, + ROCE_OPCODE_CQE_UNUSED = 0x1f /* Be used in new CQ buf when reszie cq */ +}; + +enum roce3_cqe_recv_opcode { + ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM = 0x0, + ROCE_RECV_OPCODE_SEND = 0x1, + ROCE_RECV_OPCODE_SEND_WITH_IMM = 0x2, + ROCE_RECV_OPCODE_SEND_WITH_INV = 0x3 +}; + +/* Define the state type of the CQ */ +enum cq_state { + ROCE_CQ_STATE_INVALID = 0x0, + ROCE_CQ_STATE_ERR = 0x1, + ROCE_CQ_STATE_OVERFLOW = 0x2, + ROCE_CQ_STATE_VALID = 0xf, + ROCE_CQ_STATE_MEM_INIT = 0xa /* Initial value of Host Memory */ +}; + +#define ROCE_CQ_TIME_OUT_CHECK_VALUE 0xe +#define ROCE_CQ_STATE_CHECK_VALUE 0x0 + +#define ROCE_CQE_RQ_INLINE 1 +#define ROCE_CQE_RQ_NORMAL 0 +#define ROCE_CQE_SEND_COMP 1 +#define ROCE_CQE_RECV_COMP 0 + +#define ATOMIC_DATA_LEN 8 /* Specified as 8B by protocol */ + +#define ROCE_CQE_INVALID_VALUE 0xff +#define ROCE_CQ_RESIZE_POLL_TIMES 100 + +/* roce Commands, bufs and data structures related to cq */ +struct roce3_cq_query_outbuf { + struct roce_cq_context cqc; +}; + +struct roce3_cq_buf { + struct tag_cqm_buf *buf; /* pointer to describe the buf structure b cqm */ + /* the mtt struct used by kernel mode and user mode to discribe buf */ + struct rdma_mtt mtt; + int entry_size; /* the size of cqe */ + int buf_size; /* the size of cq_buf */ +}; + +struct roce3_cq_resize_buf { + struct roce3_cq_buf buf; /* the size of buf that was resized */ + int cqe; /* the number of resized cqe */ +}; + +struct roce3_cq { + struct ib_cq ibcq; + struct tag_cqm_queue *cqm_cq; /* Save the handle obtained from cqm */ + /* The address information of the software DB that stores the CI user mode/kernel mode */ + struct roce3_db db; + __be32 *set_ci_db; /* Kernel-mode software DB */ + u32 cons_index; /* consumer pointer */ + + u32 arm_sn; /* the serial number of arm */ + /* Used to determine whether the arm command has been sent, to avoid repeated sending */ + int arm_flag; + u32 cqn; + unsigned int vector; /* associated to the eq used */ + struct roce3_cq_buf buf; /* pointer describing the buf structure of cqm */ + struct roce3_cq_resize_buf *resize_buf; /* resize buf struction */ + + spinlock_t lock; /* Need to lock when operating cq */ + struct mutex resize_mutex; /* resize the mutex of cq */ + struct ib_umem *umem; /* record the information mapped by User-mode buf */ + /* record the information mapped by buf which the User-mode resized */ + struct ib_umem *resize_umem; + + struct list_head send_qp_list; /* send queue qp */ + struct list_head recv_qp_list; /* receive queue qp */ + + int reset_notify_added; + struct list_head reset_notify; + + void (*reset_flow_comp)(struct roce3_cq *cq); +}; + +/* cross ibcroce3_e_cq */ +static inline struct roce3_cq *to_roce3_cq(const struct ib_cq *ibcq) +{ + return container_of(ibcq, struct roce3_cq, ibcq); +} + +/* obroce3_roce_cq crossing cqm */ +static inline struct roce3_cq *cqmobj_to_roce3_cq(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_cq; + + cqm_cq = container_of(object, struct tag_cqm_queue, object); + return (struct roce3_cq *)cqm_cq->priv; +} + +/* Used when destroy QP */ +void roce3_cq_clean(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq); +void roce3_cq_async_event(struct roce3_device *rdev, struct roce3_cq *cq, int type); +void roce3_cq_clean_process(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq); +void roce3_cq_put_umem(struct roce3_device *rdev, struct roce3_cq_buf *buf, struct ib_umem **umem); + +int roce3_cq_get_umem(struct roce3_device *rdev, struct ib_udata *udata, struct roce3_cq_buf *buf, + struct ib_umem **umem, u64 buf_addr, int cqe); +int roce3_create_cq_common(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce3_cq *rcq, u32 index); + +void *roce3_get_sw_cqe(struct roce3_cq *cq, unsigned int n); +void *roce3_get_cqe_from_buf(struct roce3_cq_buf *buf, unsigned int n); +void roce3_cq_set_ci(struct roce3_cq *cq); +void roce3_cq_buf_init(struct roce3_cq_buf *buf); +void *roce3_get_cqe(struct roce3_cq *cq, unsigned int n); +void roce_reset_flow_comp(struct roce3_cq *rcq); + +void roce3_lock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __acquires(&roce3_send_cq->lock) __acquires(&roce3_recv_cq->lock); +void roce3_unlock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __releases(&roce3_send_cq->lock) __releases(&roce3_recv_cq->lock); + +#endif // ROCE_CQ_H diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c new file mode 100644 index 000000000..ece655612 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_common.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_user.h" +#include "roce_cq.h" +#include "roce_pub_cmd.h" +#include "hinic3_hmm.h" + +/* + **************************************************************************** + Prototype : roce3_cq_async_event + Description : roce3_cq_async_event + Input : struct roce3_device *rdev + struct roce3_cq *cq + int type + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_async_event(struct roce3_device *rdev, struct roce3_cq *cq, int type) +{ + struct ib_cq *ibcq = &cq->ibcq; + struct ib_event event; + + memset(&event, 0, sizeof(event)); + if (type != ROCE_EVENT_TYPE_CQ_ERROR) { + dev_warn_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Unexpected event type(0x%x) on CQ(%06x), func_id(%d)\n", + __func__, type, cq->cqn, rdev->glb_func_id); + return; + } + + if (ibcq->event_handler) { + event.device = ibcq->device; + event.event = IB_EVENT_CQ_ERR; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } +} + +/* + **************************************************************************** + Prototype : roce3_cq_buf_init + Description : roce3_cq_buf_init + Input : struct roce3_cq_buf *buf + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_buf_init(struct roce3_cq_buf *buf) +{ + /* optype was initialized into ROCE_OPCODE_CQE_UNUSED(0x1f) */ + memset(buf->buf->direct.va, ROCE_CQE_INVALID_VALUE, + (unsigned long)((unsigned int)buf->buf_size)); +} + +/* + **************************************************************************** + Prototype : roce3_cq_put_umem + Description : Release umem and corresponding mtt, corresponding to put + Input : struct roce3_device *rdev + struct roce3_cq_buf *buf + struct ib_umem **umem + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_put_umem(struct roce3_device *rdev, struct roce3_cq_buf *buf, struct ib_umem **umem) +{ + /* free MTT of Buffer */ + hmm_rdma_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + + /* release umem */ + ib_umem_release(*umem); +} + +static void *roce3_cq_buf_offset(struct tag_cqm_buf *buf, unsigned int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce3_get_cqe_from_buf(struct roce3_cq_buf *buf, unsigned int n) +{ + return roce3_cq_buf_offset(buf->buf, (n * buf->entry_size)); +} + +void *roce3_get_cqe(struct roce3_cq *cq, unsigned int n) +{ + return roce3_get_cqe_from_buf(&cq->buf, n); +} + +/* + **************************************************************************** + Prototype : roce3_get_sw_cqe + Description : roce3_get_sw_cqe + Input : struct roce3_cq *cq + unsigned int n + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void *roce3_get_sw_cqe(struct roce3_cq *cq, unsigned int n) +{ + struct roce_cqe *cqe = (struct roce_cqe *)roce3_get_cqe( + cq, (n & ((unsigned int)cq->ibcq.cqe))); + struct roce_cqe tmp_cqe; + unsigned int tmp_val; + + tmp_cqe.dw0.value = roce3_convert_cpu32(cqe->dw0.value); + tmp_cqe.dw1.value = roce3_convert_cpu32(cqe->dw1.value); + + /* + * Add judgment condition: the optype of CQE cannot be UNUSED, + * UNUSED means that it has been initialized in resize cq + */ + tmp_val = ((n & ((unsigned int)cq->ibcq.cqe + 1)) == 0) ? 1 : 0; + if ((ROCE_LIKELY(tmp_cqe.dw1.bs.op_type != ROCE_OPCODE_CQE_UNUSED)) && + ((tmp_cqe.dw0.bs.owner ^ tmp_val) != 0)) + return cqe; + + return NULL; +} + +int roce3_cq_get_umem(struct roce3_device *rdev, struct ib_udata *udata, struct roce3_cq_buf *buf, + struct ib_umem **umem, u64 buf_addr, int cqe) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + int cqe_size = (int)rdev->rdma_cap.cqe_size; + + *umem = ib_umem_get(&rdev->ib_dev, buf_addr, + (unsigned long)(cqe * cqe_size), IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(*umem)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get ib_umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + return (int)PTR_ERR(*umem); + } + + npages = (u32)ib_umem_num_pages(*umem); + page_shift = PAGE_SHIFT; + + buf->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &buf->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_buf; + } + + ret = roce3_umem_write_mtt(rdev, &buf->mtt, *umem); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to write mtt for umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_mtt; + } + + return 0; + +err_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + +err_buf: + ib_umem_release(*umem); + + return ret; +} + +void roce_reset_flow_comp(struct roce3_cq *rcq) +{ + struct ib_cq *ibcq = &rcq->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c new file mode 100644 index 000000000..2db465377 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_cqe.c @@ -0,0 +1,744 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#include <linux/slab.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" + +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : roce3_next_cqe_sw + Description : roce3_next_cqe_sw + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static struct roce_cqe *roce3_next_cqe_sw(struct roce3_cq *cq) +{ + return (struct roce_cqe *)roce3_get_sw_cqe(cq, cq->cons_index); +} + +/*lint -e26*/ +static int g_error_cqe_to_wc[ROCE_CQE_SYNDROME_MAX] = { + [ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR] = IB_WC_LOC_LEN_ERR, + [ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR] = IB_WC_LOC_QP_OP_ERR, + [ROCE_CQE_SYNDROME_LOCAL_PROT_ERR] = IB_WC_LOC_PROT_ERR, + [ROCE_CQE_SYNDROME_WR_FLUSH_ERR] = IB_WC_WR_FLUSH_ERR, + + [ROCE_CQE_SYNDROME_MW_BIND_ERR] = IB_WC_MW_BIND_ERR, + [ROCE_CQE_SYNDROME_BAD_RESP_ERR] = IB_WC_BAD_RESP_ERR, + + [ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR] = IB_WC_LOC_ACCESS_ERR, + [ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR] = IB_WC_REM_INV_REQ_ERR, + [ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR] = IB_WC_REM_ACCESS_ERR, + [ROCE_CQE_SYNDROME_REMOTE_OP_ERR] = IB_WC_REM_OP_ERR, + [ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR] = IB_WC_RETRY_EXC_ERR, + + [ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR] = IB_WC_RNR_RETRY_EXC_ERR, + + [ROCE_CQE_SYNDROME_REMOTE_ABORTED_ERR] = IB_WC_REM_ABORT_ERR, +}; +/*lint +e26*/ + +/* + **************************************************************************** + Prototype : roce3_handle_error_cqe + Description : roce3_handle_error_cqe + Input : struct roce_err_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_error_cqe(struct roce_err_cqe *cqe, struct ib_wc *wc) +{ + if (cqe->dw7.bs.syndrome >= ROCE_CQE_SYNDROME_MAX) { + wc->status = IB_WC_GENERAL_ERR; + } else if (cqe->dw7.bs.syndrome > 0) { + wc->status = g_error_cqe_to_wc[cqe->dw7.bs.syndrome]; + if (wc->status == 0) + wc->status = IB_WC_GENERAL_ERR; + } + + wc->vendor_err = cqe->dw7.bs.vendor_err; +} + +static void roce3_get_local_opcode_from_cqe(const struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case ROCE_OPCODE_REG_SIG_MR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + pr_warn("[ROCE] %s: Unknown cqe optype\n", __func__); + break; + } +} + +static void roce3_get_opcode_type_part_1(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + + case ROCE_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_SEND: + wc->opcode = IB_WC_SEND; + break; + + case ROCE_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + break; + default: + roce3_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +static void roce3_get_opcode_type_part_2(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = cqe->byte_cnt; + break; + + case ROCE_OPCODE_ATOMIC_COMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + case ROCE_OPCODE_FAST_REG_PMR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + roce3_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +/* + **************************************************************************** + Prototype : roce3_get_opcode_from_scqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_get_opcode_from_scqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->wc_flags = 0; + + if (cqe->dw1.bs.op_type < ROCE_OPCODE_RDMA_READ) + roce3_get_opcode_type_part_1(cqe, wc); + else + roce3_get_opcode_type_part_2(cqe, wc); +} + +/* + **************************************************************************** + Prototype : roce3_get_opcode_from_rcqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_get_opcode_from_rcqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->byte_len = cqe->byte_cnt; + + switch (cqe->dw1.bs.op_type) { + case ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + /* The driver does not perform big or small endian + * conversion for immediate data, but the incoming CQE + * has been converted to the CPU endian, so it needs to be converted back + */ + wc->ex.imm_data = roce3_convert_be32(cqe->imm_invalid_rkey); + break; + + case ROCE_RECV_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = cqe->imm_invalid_rkey; + break; + + case ROCE_RECV_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + + case ROCE_RECV_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + /* + * The driver does not perform big or small endian conversion + * for immediate data, but the incoming CQE + * has been converted to the CPU endian, so it needs to be converted back + */ + wc->ex.imm_data = roce3_convert_be32(cqe->imm_invalid_rkey); + break; + + default: + pr_warn("[ROCE] %s: Not supported\n", __func__); + break; + } +} + +/* + **************************************************************************** + Prototype : roce3_cq_poll_and_resize + Description : roce3_cq_poll_and_resize + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_cq_poll_and_resize(struct roce3_device *rdev, struct roce3_cq *cq) +{ + if (ROCE_LIKELY(cq->resize_buf != NULL)) { + /* Release the original Buffer of CQ */ + hiudk_cqm_object_resize_free_old(rdev->hwdev, &cq->cqm_cq->object); + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + } + + --cq->cons_index; +} + +/* + **************************************************************************** + Prototype : roce3_cq_get_cur_qp + Description : roce3_cq_get_cur_qp + Input : struct roce3_cq *cq + struct roce3_qp **cur_qp + struct roce_cqe *cqe + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_get_cur_qp(struct roce3_cq *cq, struct roce3_qp **cur_qp, struct roce_cqe *cqe) +{ + struct roce3_device *rdev = NULL; + struct tag_cqm_object *cqm_obj_qp = NULL; + + if ((!*cur_qp) || (cqe->dw0.bs.qpn != (u32)(*cur_qp)->qpn)) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + rdev = to_roce3_dev(cq->ibcq.device); + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, + cqe->dw0.bs.qpn, false); + if (cqm_obj_qp == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: CQ(%06x) with entry for unknown QPN(%06x), func_id(%d)\n", + __func__, cq->cqn, cqe->dw0.bs.qpn, rdev->glb_func_id); + return -EINVAL; + } + + *cur_qp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_get_xrc_srq + Description : roce3_cq_get_xrc_srq + Input : struct roce3_cq *cq + struct roce3_srq **srq + struct roce_cqe *cqe + u32 qp_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_get_xrc_srq(struct roce3_cq *cq, struct roce3_srq **srq, + struct roce_cqe *cqe, u32 qp_type) +{ + u32 srq_num = 0; + struct roce3_device *rdev = NULL; + struct tag_cqm_object *cqm_obj_srq = NULL; + + if (qp_type == IB_QPT_XRC_TGT) { + srq_num = cqe->dw6.bs.srqn_rqpn; + + /* SRQ is also in the radix tree */ + rdev = to_roce3_dev(cq->ibcq.device); + cqm_obj_srq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srq_num, false); + if (cqm_obj_srq == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: CQ(%06x) with entry for unknown SRQN(%06x), func_id(%d)\n", + __func__, cq->cqn, srq_num, rdev->glb_func_id); + return -EINVAL; + } + + /* Get roce3__srq structure through cqm object */ + *srq = cqmobj_to_roce3_srq(cqm_obj_srq); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_srq); + } + + return 0; +} + +static int roce3_poll_recv_cqe(struct roce3_qp *cur_qp, struct roce3_srq *srq, + struct roce_cqe trans_cqe, struct ib_wc *wc) +{ + u16 wqe_ctr = 0; + unsigned int tail = 0; + struct roce3_srq *srq_tmp; + + if (cur_qp->ibqp.srq) { + srq_tmp = to_roce3_srq(cur_qp->ibqp.srq); + + wqe_ctr = (u16)trans_cqe.dw1.bs.wqebb_cnt; + /* Determine whether to overflow */ + if (wqe_ctr > srq_tmp->max_depth) { + pr_err("[ROCE, ERR] %s: Get wqe index(0x%x) from cqe\n", __func__, wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq_tmp->wrid[wqe_ctr]; + roce3_free_srq_wqe(srq_tmp, wqe_ctr); + } else if (srq) { + wqe_ctr = (u16)trans_cqe.dw1.bs.wqebb_cnt; + /* Determine whether to overflow */ + if (wqe_ctr > srq->max_depth) { + pr_err("[ROCE, ERR] %s: Get wqe index(0x%x) from cqe\n", __func__, wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq->wrid[wqe_ctr]; + roce3_free_srq_wqe(srq, wqe_ctr); + } else { + struct roce3_wq *wq = &(cur_qp->rq); + + tail = (u32)(wq->tail & ((unsigned int)wq->wqebb_cnt - 1)); + wc->wr_id = wq->wrid[tail]; + ++wq->tail; + } + + return 0; +} + +static void roce3_poll_move_tail(const struct roce3_qp *cur_qp, struct roce3_wq *wq, + struct roce_cqe *trans_cqe) +{ + u16 wqe_ctr = 0; + + if (cur_qp->sq_signal_bits == 0) { + wqe_ctr = trans_cqe->dw7.bs.wqe_cnt; + wq->tail += (u16)(wqe_ctr - (u16)wq->tail); + } +} + +static void roce3_poll_one_get_av_info(const struct roce3_qp *cur_qp, struct ib_wc *wc, + struct roce_cqe trans_cqe, struct roce_cqe *cqe) +{ + if ((cur_qp->qp_type == IB_QPT_UD) || (cur_qp->qp_type == IB_QPT_GSI)) { + wc->sl = trans_cqe.dw4.bs.vlan_pri; + wc->src_qp = trans_cqe.dw6.bs.srqn_rqpn; + wc->network_hdr_type = trans_cqe.dw6.bs.stp; + /* Take the first Byte data, shift 8bit */ + wc->smac[0] = (u8)(trans_cqe.dw4.bs.smac_h >> 8); + wc->smac[1] = (u8)(trans_cqe.dw4.bs.smac_h & 0xff); + memcpy(&wc->smac[2], &cqe->smac_l, sizeof(cqe->smac_l)); // 2 : smac array idx + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_SMAC); + if (trans_cqe.dw6.bs.vlan_pre != 0) { + wc->vlan_id = trans_cqe.dw4.bs.vlan_id; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_VLAN); + } + } +} + +static int roce3_poll_one_get_qp_and_srq(struct roce3_cq *cq, struct roce3_qp **cur_qp, + struct ib_wc *wc, struct roce_cqe *trans_cqe, struct roce3_srq **srq) +{ + int ret = 0; + + ret = roce3_cq_get_cur_qp(cq, cur_qp, trans_cqe); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get current qp\n", __func__); + return ret; + } + + wc->qp = &((*cur_qp)->ibqp); + + ret = roce3_cq_get_xrc_srq(cq, srq, trans_cqe, wc->qp->qp_type); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get xrc srq\n", __func__); + return ret; + } + + return ret; +} + +static void roce3_bytes_trans(u32 *addr, int dw_num) +{ + u32 *dw = addr; + int i = 0; + + for (i = 0; i < dw_num; i++) { + *dw = roce3_convert_cpu32(*dw); + dw++; + } +} + +static int roce3_get_and_trans_cqe(struct roce3_device *rdev, struct roce3_cq *cq, + struct roce_cqe *trans_cqe, int *is_send, struct roce_cqe **cqe) +{ + do { + *cqe = roce3_next_cqe_sw(cq); + if (*cqe == NULL) + return -EAGAIN; + + /* Later, it is necessary to perform big and small end conversion on CQE. + * In order to avoid turning back after processing, copy a copy + */ + memcpy((void *)trans_cqe, (void *)*cqe, sizeof(struct roce_cqe)); + ++cq->cons_index; + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + /* Convert the CQE into the CPU endian corresponding to the running environment */ + roce3_bytes_trans((u32 *)(void *)trans_cqe, + (int)(sizeof(struct roce_cqe) / CQ_DW_TO_BYTE)); + + *is_send = trans_cqe->dw1.bs.s_r; + /* Resize CQ operation in progress */ + if (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == ROCE_OPCODE_RESIZE_CQE)) + roce3_cq_poll_and_resize(rdev, cq); + } while (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == ROCE_OPCODE_RESIZE_CQE)); + + return 0; +} + +static void roce3_poll_send_cqe(struct roce3_qp *cur_qp, struct roce_cqe *trans_cqe, + struct ib_wc *wc) +{ + struct roce3_wq *wq = &(cur_qp->sq); + /* QP of type IB_SIGNAL_REQ_WR, tail needs to skip several WRs that do not generate CQE */ + roce3_poll_move_tail(cur_qp, wq, trans_cqe); + + wc->wr_id = wq->wrid[wq->tail & ((unsigned int)wq->wqebb_cnt - 1)]; + ++wq->tail; + + if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce3_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe, wc); + return; + } + + roce3_get_opcode_from_scqe(trans_cqe, wc); +} + +/* + **************************************************************************** + Prototype : roce3_poll_one + Description : roce3_poll_one + Input : struct roce3_cq *cq + struct roce3_qp **cur_qp + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_poll_one(struct roce3_device *rdev, struct roce3_cq *cq, + struct roce3_qp **cur_qp, struct ib_wc *wc) +{ + int ret = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe trans_cqe; + struct roce3_srq *srq = NULL; + int is_send = 0; + bool need_poll = true; + + while (need_poll) { + ret = roce3_get_and_trans_cqe(rdev, cq, &trans_cqe, &is_send, &cqe); + if (ret != 0) + return -EAGAIN; + + ret = roce3_poll_one_get_qp_and_srq(cq, cur_qp, wc, &trans_cqe, &srq); + if (ret != 0) + return ret; + + /* + * Poll cq exception handling, generally cq judges + * user state information, xrc judges wrid + */ + if ((*cur_qp)->umem || ((*cur_qp)->ibqp.xrcd && !(srq->wrid))) { + pr_err("[ROCE, ERR] %s: qp(%u) create in user space , but poll cq in kernel. NOT PERMIT!\n", + __func__, (*cur_qp)->qpn); + return -EACCES; + } + + if (trans_cqe.dw1.bs.fake != 0) { + pr_info("[ROCE] %s: Fake cqe go repoll.\n", __func__); + continue; + } + + wc->status = IB_WC_SUCCESS; + if (is_send != 0) { + roce3_poll_send_cqe(*cur_qp, &trans_cqe, wc); + return 0; + } + ret = roce3_poll_recv_cqe(*cur_qp, srq, trans_cqe, wc); + if (ret != 0) + return ret; + + if (trans_cqe.dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce3_handle_error_cqe((struct roce_err_cqe *)(void *)&trans_cqe, wc); + return 0; + } + + roce3_get_opcode_from_rcqe(&trans_cqe, wc); + wc->wc_flags = (int)((unsigned int)wc->wc_flags | IB_WC_GRH); + wc->pkey_index = 0; + wc->vlan_id = 0xffff; + wc->sl = 0; + + /* avoid cm_req_handler()->ib_lid_be16() trigger call trace */ + wc->slid = 0; + + roce3_poll_one_get_av_info((*cur_qp), wc, trans_cqe, cqe); + + /* Kernel mode does not support receiving inline */ + if (trans_cqe.dw1.bs.inline_r == ROCE_CQE_RQ_INLINE) { + pr_err("[ROCE, ERR] %s: Receive inline not supported in kernel space\n", + __func__); + return -EINVAL; + } + need_poll = false; + } + + return 0; +} + +static int sw_send_comp(struct roce3_qp *rqp, int num_entries, int *npolled, struct ib_wc *wc) +{ + struct roce3_wq *wq = &rqp->sq; + unsigned int cur = wq->head - wq->tail; + unsigned int i; + + if (cur == 0) + goto out; + + for (i = 0; (i < cur) && (*npolled < num_entries); i++) { + wc[*npolled].wr_id = wq->wrid[wq->tail & (wq->wqebb_cnt - 1)]; + wc[*npolled].status = IB_WC_WR_FLUSH_ERR; + wc[*npolled].vendor_err = ROCE_CQE_SYNDROME_WR_FLUSH_ERR; + wc[*npolled].qp = &rqp->ibqp; + wq->tail++; + (*npolled)++; + } + +out: + return (*npolled >= num_entries); +} + +static int sw_recv_comp(struct roce3_qp *rqp, int num_entries, int *npolled, struct ib_wc *wc) +{ + struct roce3_wq *wq = &rqp->rq; + unsigned int cur = wq->head - wq->tail; + unsigned int i; + + if (cur == 0) + goto out; + + for (i = 0; (i < cur) && (*npolled < num_entries); i++) { + wc[*npolled].wr_id = wq->wrid[wq->tail & (wq->wqebb_cnt - 1)]; + wc[*npolled].status = IB_WC_WR_FLUSH_ERR; + wc[*npolled].vendor_err = ROCE_CQE_SYNDROME_WR_FLUSH_ERR; + wc[*npolled].qp = &rqp->ibqp; + wq->tail++; + (*npolled)++; + } + +out: + return (*npolled >= num_entries); +} + +static int roce_poll_sw_comp(const struct roce3_cq *rcq, int num_entries, struct ib_wc *wc) +{ + struct roce3_qp *rqp = NULL; + int npolled = 0; + + list_for_each_entry(rqp, &rcq->send_qp_list, cq_send_list) { + if (sw_send_comp(rqp, num_entries, &npolled, wc) != 0) + return npolled; + } + + list_for_each_entry(rqp, &rcq->recv_qp_list, cq_recv_list) { + if (sw_recv_comp(rqp, num_entries, &npolled, wc) != 0) + return npolled; + } + + return npolled; +} + +/* the format of cq_ci DB:the bits of ci less than 24bit */ +void roce3_cq_set_ci(struct roce3_cq *cq) +{ + *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); +} + +int roce3_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct roce3_device *rdev = NULL; + struct roce3_cq *rcq = to_roce3_cq(ibcq); + struct roce3_qp *cur_qp = NULL; + unsigned long flags = 0; + int npolled = 0; + int ret = 0; + + spin_lock_irqsave(&rcq->lock, flags); + + rdev = to_roce3_dev(ibcq->device); + if (roce3_hca_is_present(rdev) == 0) { + npolled = roce_poll_sw_comp(rcq, num_entries, wc); + goto poll_out; + } + + for (npolled = 0; npolled < num_entries; ++npolled) { + ret = roce3_poll_one(rdev, rcq, &cur_qp, wc + npolled); + if (ret != 0) + break; + } + + roce3_cq_set_ci(rcq); + +poll_out: + spin_unlock_irqrestore(&rcq->lock, flags); + + if ((ret == 0) || (ret == -EAGAIN)) + return npolled; + + return ret; +} + +static void roce3_cq_arm(struct roce3_cq *cq, u32 cmd, void __iomem *uar_page) +{ + struct roce_db_cq_arm db_value; + + memset(&db_value, 0, sizeof(db_value)); + + db_value.dw0.bs.type = 3; + db_value.dw0.bs.cos = 0; /* arm_cq don't need cos */ + db_value.dw0.bs.cp = 1; /* 1 for cp */ + db_value.dw0.bs.non_filter = 1; + db_value.dw0.bs.cqc_type = 0; + db_value.dw0.bs.cqn = cq->cqn; + db_value.dw0.value = roce3_convert_be32(db_value.dw0.value); + + db_value.dw1.bs.cmd_sn = cq->arm_sn; + db_value.dw1.bs.cmd = cmd; + db_value.dw1.bs.ci = cq->cons_index; + db_value.dw1.value = roce3_convert_be32(db_value.dw1.value); + + /* + * Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + wmb(); + + roce3_write64((u32 *)(void *)&db_value, uar_page); +} + +int roce3_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + unsigned long lock_flags = 0; + struct roce3_cq *cq = to_roce3_cq(ibcq); + + spin_lock_irqsave(&cq->lock, lock_flags); + if (cq->arm_flag != 0) { + spin_unlock_irqrestore(&cq->lock, lock_flags); + return 0; + } + + cq->arm_flag = 1; + spin_unlock_irqrestore(&cq->lock, lock_flags); + + /* Only 64-bit is supported, 64-bit writes are atomic, no need to lock */ + roce3_cq_arm(to_roce3_cq(ibcq), + (((unsigned int)flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) ? + ROCE_CQ_DB_REQ_NOT_SOL : ROCE_CQ_DB_REQ_NOT, + to_roce3_dev(ibcq->device)->kernel_db_map); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c new file mode 100644 index 000000000..775f6a0bd --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_create.c @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_cq.h" +#include "roce_pub_cmd.h" +#include "roce_cqm_cmd.h" +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : roce_cq_attr_assign + Description : Assign value to CQC + Input : struct roce_cq_attr *cq_attr + struct roce3_cq *rcq + int eqn + int page_shift + Output : None + + 1.Date : 2019/10/13 + Modification : Created function + +**************************************************************************** +*/ +static void roce_cq_attr_assign(struct roce_verbs_cq_attr *cq_attr, struct roce3_cq *rcq, + int eqn, int page_shift) +{ + /* Assign value to CQ */ + /* dw0 */ + cq_attr->dw0.bs.size = (u32)ROCE_ILOG2((unsigned int)(rcq->ibcq.cqe + 1)); + cq_attr->dw0.bs.page_size = (u32)page_shift; + /* entry_size=(2^cq_cqe_size)*16B. */ + cq_attr->dw0.bs.cqe_size = + (u32)ROCE_ILOG2((unsigned int)(rcq->buf.entry_size / 16)); + cq_attr->dw0.bs.mtt_page_size = rcq->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + cq_attr->dw0.bs.tss_timer_num = 7; /* 7 : The maximum number of timers supported by cq */ + cq_attr->dw0.bs.arm_timer_en = 0; + cq_attr->dw0.bs.timer_mode = 0; + cq_attr->dw0.bs.cnt_adjust_en = 1; + cq_attr->dw0.bs.cnt_clear_en = 1; + cq_attr->dw0.bs.ci_on_chip = 0; + cq_attr->dw0.value = cpu_to_be32(cq_attr->dw0.value); + + /* dw1 */ + cq_attr->dw1.bs.dma_attr_idx = 0; + cq_attr->dw1.bs.so_ro = 0; + cq_attr->dw1.bs.state = ROCE_CQ_STATE_VALID; + cq_attr->dw1.value = cpu_to_be32(cq_attr->dw1.value); + + /* dw2 */ + cq_attr->dw2.bs.idle_max_count = 0; + cq_attr->dw2.bs.cqecnt_lth = 6; /* update ci threshold: 2^6=64 */ + cq_attr->dw2.bs.cqecnt_rctl_en = 0; + cq_attr->dw2.bs.ceqe_en = 1; + cq_attr->dw2.bs.arm_ceqe_en = 1; + cq_attr->dw2.bs.ceqn = (u8)eqn; + cq_attr->dw2.value = cpu_to_be32(cq_attr->dw2.value); + + /* dw3 */ + /* The timeout mechanism is disabled by default when CQ is created */ + cq_attr->dw3.bs.timeout = 0; + /* The overtime mechanism is disabled by default when CQ is created */ + cq_attr->dw3.bs.max_cnt = 0; + cq_attr->dw3.value = cpu_to_be32(cq_attr->dw3.value); + + /* dw4 - dw5 */ + cq_attr->cqc_l0mtt_gpa = rcq->buf.mtt.mtt_paddr; + cq_attr->cqc_l0mtt_gpa = cpu_to_be64(cq_attr->cqc_l0mtt_gpa); + + /* dw6 - dw7 */ + cq_attr->ci_record_gpa_at_hop_num = + cpu_to_be64((rcq->db.dma & (~0x3uLL)) | rcq->buf.mtt.mtt_layers); +} + +static int roce3_cq_fill_create_inbuf(struct roce3_device *rdev, struct roce3_cq *rcq, + int vector, int page_shift, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + int eqn = 0; + struct roce_verbs_cq_attr *cq_attr = NULL; + struct tag_roce_uni_cmd_creat_cq *cq_sw2hw_inbuf = NULL; + + cq_sw2hw_inbuf = (struct tag_roce_uni_cmd_creat_cq *)cqm_cmd_inbuf->buf; + cq_sw2hw_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_attr = &cq_sw2hw_inbuf->cq_attr; + + /* Get the EQN of the CEQ based on the Vector index */ + eqn = hinic3_vector_to_eqn(rdev->hwdev, SERVICE_T_ROCE, vector); + if (eqn < 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get eqn from hinic vector, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + roce_cq_attr_assign(cq_attr, rcq, eqn, page_shift); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_sw2hw + Description : Send the cqc configuration command + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int vector + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_sw2hw(struct roce3_device *rdev, struct roce3_cq *rcq, + int vector, int page_shift) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_creat_cq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + ret = roce3_cq_fill_create_inbuf(rdev, rcq, vector, page_shift, cqm_cmd_inbuf); + if (ret != 0) { + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; + } + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_SW2HW_CQ, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send SW2HW_CQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(SW2HW_CQ), CQN(0x%x), func_id(%u)\n", + __func__, rcq->cqn, rdev->glb_func_id); + + /* + * CMDq times out or CMDq does not work, update the + * device status, notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + ret = -1; + goto err_send_cmd; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + rcq->cons_index = 0; + rcq->arm_sn = 1; + rcq->arm_flag = 0; + rcq->vector = (u32)vector; + + return 0; + +err_send_cmd: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_create_cq_check + Description : roce3_create_cq_check + Input : struct ib_device *ibdev + int entries + int vector + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_cq_check(struct ib_device *ibdev, int entries, int vector, + const struct ib_ucontext *ibcontext, const struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + + if (ibdev == NULL) { + pr_err("[ROCE, ERR] %s: Ibdev is null\n", __func__); + return -EINVAL; + } + + if ((ibcontext != NULL) && (udata == NULL)) { + pr_err("[ROCE, ERR] %s: Udata is null ptr, but ibcontext is not null ptr\n", + __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Cqe number invalid, entries(%d), func_id(%d)\n", + __func__, entries, rdev->glb_func_id); + return -EINVAL; + } + + if (vector > (int)rdev->rdma_cap.num_comp_vectors) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Vector over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +static int roce3_check_cq_create_flags(u32 flags) +{ + /* + * It returns non-zero value for unsupported CQ + * create flags, otherwise it returns zero. + */ + return (int)(flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN | + IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)); /*lint !e40*/ +} + +static int roce3_cq_cqc_cfg(struct roce3_device *rdev, struct roce3_cq *rcq, int vector, + struct ib_udata *udata) +{ + int ret = 0; + int page_shift = 0; + + /* configurate CQC */ + ret = roce3_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to handle cq sw2hw, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + /* User mode requires outgoing CQN */ + if (ib_copy_to_udata(udata, &rcq->cqn, sizeof(u32)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EFAULT; + return ret; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_create_user_cq + Description : roce3_create_user_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + int vector + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_user_cq(struct roce3_device *rdev, struct roce3_cq *rcq, int entries, + int vector, struct ib_ucontext *ibcontext, struct ib_udata *udata, u32 index) +{ + int ret = 0; + struct roce3_create_cq_cmd ucmd = { 0 }; + + rcq->cqm_cq = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SCQ, 0, rcq, false, index); + if (rcq->cqm_cq == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma queue from cqm object, func_id(%d), index(%d)\n", + __func__, rdev->glb_func_id, index); + return -ENOMEM; + } + + /* record CQN */ + rcq->cqn = rcq->cqm_cq->index; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EFAULT; + goto err_free_cqm_cq; + } + + /* + * Entries has been decreased by one when it was input by + * User Mode, and it is the expected value exactly + * after increased by one by Kernel Mode + */ +#if defined(OFED_MLNX_5_8) + ret = roce3_cq_get_umem(rdev, udata, &rcq->buf, &rcq->umem, ucmd.buf_addr, entries); +#else + ret = roce3_cq_get_umem(rdev, ibcontext, &rcq->buf, &rcq->umem, ucmd.buf_addr, entries); +#endif + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_cqm_cq; + } + + ret = roce3_db_map_user(to_roce3_ucontext(ibcontext), + (unsigned long)ucmd.db_addr, &rcq->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to map kernel_mem to user_mem, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + ret = roce3_cq_cqc_cfg(rdev, rcq, vector, udata); + if (ret != 0) + goto err_unmap_db; + + return 0; + +err_unmap_db: + roce3_db_unmap_user(to_roce3_ucontext(ibcontext), &rcq->db); + +err_free_mtt: + roce3_cq_put_umem(rdev, &rcq->buf, &rcq->umem); + rcq->umem = NULL; + +err_free_cqm_cq: + hiudk_cqm_object_delete(rdev->hwdev, &rcq->cqm_cq->object); + + return ret; +} + +static void roce3_fill_rcq_info(struct roce3_cq *rcq) +{ + rcq->cqn = rcq->cqm_cq->index; + rcq->buf.buf = &rcq->cqm_cq->q_room_buf_1; + + /* Initialize buf to unused */ + roce3_cq_buf_init(&rcq->buf); + + /* Software DB assignment */ + rcq->set_ci_db = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr->doorbell_record); + *rcq->set_ci_db = 0; + rcq->db.db_record = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr->doorbell_record); + rcq->db.dma = rcq->cqm_cq->q_header_paddr; +} + +/* + **************************************************************************** + Prototype : roce3_create_kernel_cq + Description : roce3_create_kernel_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + int vector + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_kernel_cq(struct roce3_device *rdev, struct roce3_cq *rcq, + int entries, int vector, u32 index) +{ + int ret = 0; + int page_shift = 0; + + rcq->buf.buf_size = entries * (int)rdev->rdma_cap.cqe_size; + rcq->cqm_cq = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SCQ, (u32)rcq->buf.buf_size, rcq, true, index); + if (rcq->cqm_cq == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma_queue from cqm_object, func_id(%d), index(%d)\n", + __func__, rdev->glb_func_id, index); + return (-ENOMEM); + } + + /* Buffer is obtained from room1 when the queue is just created */ + if (rcq->cqm_cq->current_q_room != CQM_RDMA_Q_ROOM_1) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Not use CQM_RDMA_Q_ROOM_1, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EINVAL; + goto err_free_cqm_cq; + } + + roce3_fill_rcq_info(rcq); + + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size); + + /* allocate MTT */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, rcq->buf.buf->buf_number, (u32)page_shift, + &rcq->buf.mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_cqm_cq; + } + + /* Write the PA of the CQ Buffer to the MTT */ + ret = roce3_buf_write_mtt(rdev, &rcq->buf.mtt, rcq->buf.buf); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to write rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + /*lint -e834*/ + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size) - PAGE_SHIFT_4K; + /*lint +e834*/ + + /* configurate CQC */ + ret = roce3_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to configure CQC, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + return 0; + +err_free_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_free_cqm_cq: + hiudk_cqm_object_delete(rdev->hwdev, &rcq->cqm_cq->object); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_do_create_cq + Description : OFED_3_12 + Input : struct ib_device *ibdev + struct ib_ucontext *ibcontext + struct ib_ucontext *ibcontext + struct ib_udata *udata + struct roce3_cq *rcq + u32 index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + 2.Date : 2015/8/10 + Modification : modify function + 3.Date : 2017/11/10 + Modification : modify function + +**************************************************************************** +*/ +static int roce3_do_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, struct ib_udata *udata, struct roce3_cq *rcq, u32 index) +{ + struct roce3_device *rdev = NULL; + int ret = 0; + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + + /* The CQE queue should reserve a special CQE for resize cq */ + entries++; + entries = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)entries) & 0xffffffff); /*lint !e587*/ + + rdev = to_roce3_dev(ibdev); + /* Chip Constraints: Minimum queue depth needs to be page-aligned */ + if ((u32)((u32)entries * rdev->rdma_cap.cqe_size) < PAGE_SIZE) + entries = (PAGE_SIZE >> (u32)ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + + /* Check if max spec is exceeded */ + if (entries > ((int)rdev->rdma_cap.max_cqes + 1)) { + ret = (-EINVAL); + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Over range after align, entries(%d), max_cqes(%d), func_id(%d)\n", + __func__, entries, rdev->rdma_cap.max_cqes + 1, rdev->glb_func_id); + return ret; + } + + rcq->ibcq.cqe = entries - 1; + mutex_init(&rcq->resize_mutex); + /*lint -e708*/ + spin_lock_init(&rcq->lock); + /*lint +e708*/ + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + rcq->buf.entry_size = (int)rdev->rdma_cap.cqe_size; + + INIT_LIST_HEAD(&rcq->send_qp_list); + INIT_LIST_HEAD(&rcq->recv_qp_list); + if (ibcontext) { + ret = roce3_create_user_cq(rdev, rcq, entries, vector, ibcontext, udata, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to create user_cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } else { + ret = roce3_create_kernel_cq(rdev, rcq, entries, vector, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to create kernel_cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + rcq->reset_flow_comp = roce_reset_flow_comp; + + return 0; +} + +#if defined(OFED_MLNX_5_8) +int roce3_create_cq_common(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce3_cq *rcq, u32 index) +{ + int ret; + struct roce3_device *rdev = to_roce3_dev(ibdev); + struct roce3_ucontext *context = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (roce3_check_cq_create_flags(attr->flags) != 0) { + pr_err("[ROCE, ERR] %s: Not support the cq_create flag(%x)\n", + __func__, attr->flags); + return -EOPNOTSUPP; + } + + ret = roce3_create_cq_check(ibdev, entries, vector, &context->ibucontext, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check cq information\n", __func__); + return ret; + } + + return roce3_do_create_cq(ibdev, attr, &context->ibucontext, udata, rcq, index); +} + +int roce3_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) +{ + struct roce3_cq *rcq = to_roce3_cq(ibcq); + + return roce3_create_cq_common(ibcq->device, attr, udata, rcq, ROCE_CQN_INVLD); +} +#else +struct ib_cq *roce3_create_cq_common(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, struct ib_udata *udata, u32 index) +{ + int ret = 0; + struct roce3_cq *rcq = NULL; + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + struct roce3_device *rdev = to_roce3_dev(ibdev); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return ERR_PTR((long)-EPERM); + } + + if (roce3_check_cq_create_flags(attr->flags) != 0) { + pr_err("[ROCE, ERR] %s: Not support the cq_create flag(%x)\n", + __func__, attr->flags); + return ERR_PTR((long)(-EOPNOTSUPP)); + } + + ret = roce3_create_cq_check(ibdev, entries, vector, ibcontext, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check cq information\n", __func__); + goto err_out; + } + + rcq = kzalloc(sizeof(*rcq), GFP_KERNEL); + if (rcq == NULL) { + ret = -ENOMEM; + goto err_out; + } + + ret = roce3_do_create_cq(ibdev, attr, ibcontext, udata, rcq, index); + if (ret != 0) + goto err_free_cq; + + return &rcq->ibcq; + +err_free_cq: + kfree(rcq); + +err_out: + return (struct ib_cq *)ERR_PTR((long)ret); +} + +/* + **************************************************************************** + Prototype : roce3_create_cq + Description : OFED_3_12 + Input : struct ib_device *ibdev + struct ib_cq_init_attr *attr(flags for exp) + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + 2.Date : 2015/8/10 + Modification : modify function + 3.Date : 2017/11/10 + Modification : modify function + 4.Date : 2021/1/7 + Modification : modified function + +**************************************************************************** +*/ +struct ib_cq *roce3_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, struct ib_udata *udata) +{ + return roce3_create_cq_common(ibdev, attr, ibcontext, udata, ROCE_CQN_INVLD); +} +#endif + diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c new file mode 100644 index 000000000..19489bcec --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_ctrl.c @@ -0,0 +1,930 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : roce3_cq_modify + Description : Send the command of cq_modify + Input : struct roce3_device *rdev + struct roce3_cq *cq + u32 cnt + u32 period + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_modify(struct roce3_device *rdev, struct roce3_cq *cq, u32 cnt, u32 period) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_modify_cq *cq_modify_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_modify_cq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_modify_inbuf = (struct tag_roce_cmd_modify_cq *)cqm_cmd_inbuf->buf; + cq_modify_inbuf->com.index = cpu_to_be32(cq->cqn); + cq_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_modify_inbuf->max_cnt = cpu_to_be32(cnt); + cq_modify_inbuf->timeout = cpu_to_be32(period); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_MODIFY_CQ, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to send MODIFY_CQ, cqn(0x%x), ret(%d), func_id(%u)\n", + __func__, cq->cqn, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(MODIFY_CQ), cqn(0x%x), func_id(%u)\n", + __func__, cq->cqn, rdev->glb_func_id); + + /* + * CMDq times out or CMDq does not work, update the device status, + * notify the PCIe module to reset the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + /* CMDq may return a positive number, so its return value cannot be used directly */ + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_modify_cq + Description : OFED_3_12 + Input : struct ib_cq *ibcq + u16 cq_count + u16 cq_period + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period) +{ + int ret = 0; + struct roce3_cq *cq = NULL; + struct roce3_device *rdev = NULL; + + if (ibcq == NULL) { + pr_err("[ROCE, ERR] %s: Ibcq is null\n", __func__); + return -EINVAL; + } + + cq = to_roce3_cq(ibcq); + rdev = to_roce3_dev(ibcq->device); + + ret = roce3_cq_modify(rdev, cq, cq_count, cq_period); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to modify cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_alloc_resize_buf + Description : Kernel mode applies for resize_buf, and its corresponding MTT + When there is an error in execution, this function + ensures that all intermediate resources are released; + when the execution is successful, this function + can confirm that all resources are acquired. + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_alloc_resize_buf(struct roce3_device *rdev, struct roce3_cq *rcq, int entries) +{ + int ret = 0; + int page_shift = 0; + + if (rcq->resize_buf) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Resize buffer is busy, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EBUSY; + } + + /* Apply for resize_buf and assign it */ + rcq->resize_buf = kmalloc(sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) + return -ENOMEM; + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + ret = hiudk_cqm_object_resize_alloc_new(rdev->hwdev, &rcq->cqm_cq->object, + (u32)rcq->resize_buf->buf.buf_size); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize cq buffer, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_resize_buf_alloc; + } + + /* If the application is successful, record the buffer information of the new buffer */ + rcq->resize_buf->cqe = entries - 1; + rcq->resize_buf->buf.buf = &rcq->cqm_cq->q_room_buf_2; + + /* Initialize new buf to unused state */ + roce3_cq_buf_init(&rcq->resize_buf->buf); + + page_shift = (int)ROCE_ILOG2(rcq->resize_buf->buf.buf->buf_size); + + /* Apply MTT for resize_buf */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, rcq->resize_buf->buf.buf->buf_number, + (u32)page_shift, &rcq->resize_buf->buf.mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_mtt_alloc; + } + + /* configurate MTT, write PA of resize_buf to MTT */ + ret = roce3_buf_write_mtt(rdev, &rcq->resize_buf->buf.mtt, rcq->resize_buf->buf.buf); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to write rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_mtt_write; + } + + return 0; + +err_mtt_write: + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_mtt_alloc: + /* free resize_buf */ + hiudk_cqm_object_resize_free_new(rdev->hwdev, &rcq->cqm_cq->object); + +err_resize_buf_alloc: + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_cq_free_resize_buf + Description : Kernel Mode releases resize_buf and its corresponding MTT + Input : struct roce3_device *rdev + struct roce3_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_cq_free_resize_buf(struct roce3_device *rdev, struct roce3_cq *rcq) +{ + /* Release the MTT of resize_buf first */ + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + /* free resize_buf */ + hiudk_cqm_object_resize_free_new(rdev->hwdev, &rcq->cqm_cq->object); + + /* free the resize_buf pointer itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* + **************************************************************************** + Prototype : roce3_cq_alloc_resize_umem + Description : User mode applies for umem of + resize_buf,and MTT corresponding to user mode buf + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_alloc_resize_umem(struct roce3_device *rdev, struct roce3_cq *rcq, int entries, + struct ib_udata *udata) +{ + int ret = 0; + struct roce3_resize_cq_cmd ucmd = { 0 }; + + /* + * If a resize is being executed, it is not allowed + * to execute another resize at the same time + */ + if (rcq->resize_umem) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Resize_umem is busy, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EBUSY; + } + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EFAULT; + } + + rcq->resize_buf = kmalloc(sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) + return -ENOMEM; + + ret = roce3_cq_get_umem(rdev, udata, &rcq->resize_buf->buf, + &rcq->resize_umem, ucmd.buf_addr, entries); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; + } + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + rcq->resize_buf->cqe = entries - 1; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_free_resize_umem + Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf + Input : struct roce3_device *rdev + struct roce3_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_cq_free_resize_umem(struct roce3_device *rdev, struct roce3_cq *rcq) +{ + /* Free MTT and umem of resize_buf */ + roce3_cq_put_umem(rdev, &rcq->resize_buf->buf, &rcq->resize_umem); + rcq->resize_umem = NULL; + + /* free resize_buf itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* + **************************************************************************** + Prototype : roce3_cq_get_outstanding_cqes + Description : roce3_cq_get_outstanding_cqes + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_get_outstanding_cqes(struct roce3_cq *cq) +{ + u32 i = 0; + + i = cq->cons_index; + while (roce3_get_sw_cqe(cq, i)) + ++i; + + return (int)(i - cq->cons_index); +} + +static int roce3_cq_get_next_cqe(struct roce_cqe **cqe, struct roce3_cq *cq, + unsigned int *i, u32 *times, const struct roce_cqe *start_cqe) +{ + int ret = 0; + + do { + *cqe = (struct roce_cqe *)roce3_get_sw_cqe(cq, ++(*i)); + if (*cqe == NULL) { + ROCE_MDELAY(MS_DELAY); + --(*times); + } + } while ((*cqe == NULL) && (*times != 0)); + + if ((*cqe == start_cqe) || (*cqe == NULL)) { + pr_err("[ROCE, ERR] %s: Failed to get resize CQE, CQN(0x%x)\n", __func__, cq->cqn); + return -ENOMEM; + } + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_cq_resize_copy_cqes + Description : roce3_cq_resize_copy_cqes + Input : struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_resize_copy_cqes(struct roce3_cq *cq) +{ + struct roce_cqe *cqe = NULL; + struct roce_cqe *new_cqe = NULL; + struct roce_cqe *start_cqe = NULL; + u32 times; + unsigned int i = 0; + int ret = 0; + + i = cq->cons_index; + + times = ROCE_CQ_RESIZE_POLL_TIMES; + do { + cqe = (struct roce_cqe *)roce3_get_sw_cqe(cq, i); + if (cqe == NULL) { + ROCE_MDELAY(MS_DELAY); + --times; + } + } while ((cqe == NULL) && (times != 0)); + + if (cqe == NULL) { + pr_err("[ROCE, ERR] %s: Failed to get resize CQE, CQN(0x%x)\n", __func__, cq->cqn); + return -ENOMEM; + } + + start_cqe = cqe; + + /* + * r_cqe:resize_cqe + * CI + * +--------+-----+-----+-----+----------+ + * old_buf: | | CQE | CQE |R_CQE| | + * +--------+-----+-----+-----+----------+ + * | | + * | | + * +--------+-----+-----+-----+----------+ + * new_buf: | | CQE | CQE | | | + * +--------+-----+-----+-----+----------+ + */ + /* Convert the fields needed by CQE to little endian */ + cqe->dw0.value = roce3_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_cpu32(cqe->dw1.value); + while (cqe->dw1.bs.op_type != ROCE_OPCODE_RESIZE_CQE) { + /* + * The resized PI can be inherited, and the index of + * the original CQE remains unchanged in resize_buf + */ + new_cqe = + (struct roce_cqe *)roce3_get_cqe_from_buf(&cq->resize_buf->buf, + (i & ((unsigned int)cq->resize_buf->cqe))); + + memcpy((void *)new_cqe, (void *)cqe, sizeof(struct roce_cqe)); + + /* If the CQE has been wrapped in resize_buf, + * the corresponding Owner bit needs to be modified + * to be owned by the software. + * The rule is: when the owner bit is owned by CQE software, + * the O bit is opposite to the high bit of ci; + * when the owner bit is owned by hardware, + * the O bit is the same as the high bit of CI. + */ + new_cqe->dw0.bs.owner = + ((i & ((unsigned int)cq->resize_buf->cqe + 1)) != 0) ? 1 : 0; + + /* After processing, turn the DW0/DW1 of CQE back to big endian */ + cqe->dw0.value = roce3_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_be32(cqe->dw1.value); + + /* Convert DW0/DW1 of the CQE in the new queue back to big endian */ + new_cqe->dw0.value = roce3_convert_be32(new_cqe->dw0.value); + new_cqe->dw1.value = roce3_convert_be32(new_cqe->dw1.value); + + /* Get the next CQE */ + ret = roce3_cq_get_next_cqe(&cqe, cq, &i, ×, start_cqe); + if (ret != 0) + return ret; + + /* Convert DW0/DW1 of CQE to little endian */ + cqe->dw0.value = roce3_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_cpu32(cqe->dw1.value); + } + + return 0; +} + +static void roce3_cq_fill_resize_inbuf(struct roce3_device *rdev, struct roce3_cq *rcq, + int page_shift, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct rdma_service_cap *rdma_cap = NULL; + u32 cq_size = 0; + u32 mtt_layer = 0; + u64 mtt_paddr = 0; + u32 mtt_page_size = 0; + struct tag_roce_cmd_resize_cq *cq_resize_inbuf = NULL; + + rdma_cap = &rdev->rdma_cap; + cq_size = (u32)rcq->resize_buf->cqe + 1; + cq_size = (u32)ROCE_ILOG2(cq_size); + mtt_layer = rcq->resize_buf->buf.mtt.mtt_layers; + mtt_paddr = rcq->resize_buf->buf.mtt.mtt_paddr; + mtt_page_size = rcq->resize_buf->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + + cq_resize_inbuf = (struct tag_roce_cmd_resize_cq *)cqm_cmd_inbuf->buf; + cq_resize_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_resize_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_resize_inbuf->page_size = cpu_to_be32((u32)page_shift); + cq_resize_inbuf->log_cq_size = cpu_to_be32(cq_size); + cq_resize_inbuf->mtt_layer_num = cpu_to_be32(mtt_layer); + cq_resize_inbuf->mtt_base_addr = mtt_paddr; + cq_resize_inbuf->mtt_base_addr = cpu_to_be64(cq_resize_inbuf->mtt_base_addr); + cq_resize_inbuf->mtt_page_size = cpu_to_be32(mtt_page_size); + + cq_resize_inbuf->mtt_info.mtt_flags = 0; + cq_resize_inbuf->mtt_info.mtt_num = 0; + cq_resize_inbuf->mtt_info.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + cq_resize_inbuf->mtt_info.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + cq_resize_inbuf->mtt_info.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); +} + +/* + **************************************************************************** + Prototype : roce3_cq_resize + Description : roce3_cq_resize + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_cq_resize(struct roce3_device *rdev, struct roce3_cq *rcq, int page_shift) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_resize_cq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce3_cq_fill_resize_inbuf(rdev, rcq, page_shift, cqm_cmd_inbuf); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_RESIZE_CQ, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send RESIZE_CQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(RESIZE_CQ), cqn(0x%x), func_id(%u)\n", + __func__, rcq->cqn, rdev->glb_func_id); + + /* + * When CMDq times out or CMDq cannot work, update the + * device status and notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_user_cq_resize + Description : roce3_user_cq_resize + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_user_cq_resize(struct roce3_device *rdev, struct roce3_cq *rcq, + int entries, struct ib_udata *udata) +{ + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Over range after align, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + ret = roce3_cq_alloc_resize_umem(rdev, rcq, entries, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc resize_umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + /* + * Send the cq_resize command to configure CQC. + * After the configuration is successful, the new CQE is written to + * resize_buf, and the old buffer may still retain the old CQE. + */ + ret = roce3_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cq_free_resize_umem(rdev, rcq); + return ret; + } + + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_resize_user_cq + Description : roce3_resize_user_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_resize_user_cq(struct roce3_device *rdev, struct roce3_cq *rcq, + int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_resize_cq_cmd ucmd = { 0 }; + int cqe_entries = entries; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EFAULT; + } + + if (ucmd.stage == 1) + goto release_flow; + + mutex_lock(&rcq->resize_mutex); + + cqe_entries++; + cqe_entries = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); /*lint !e587*/ + // Minimum queue depth needs to be aligned by page + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) + cqe_entries = (int)(PAGE_SIZE >> (unsigned int)ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + + /* + * The new depth of CQ cannot be the same as the old + * depth. A special CQE space is reserved when CQ is created, + * so the value of ibcq->cqe is 1 smaller than the actual value + */ + if (cqe_entries == (rcq->ibcq.cqe + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: No need to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = 0; + goto out; + } + + ret = roce3_user_cq_resize(rdev, rcq, cqe_entries, udata); + if (ret != 0) + goto out; + + mutex_unlock(&rcq->resize_mutex); + + return 0; + +release_flow: + /* free old MTT */ + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + rcq->buf = rcq->resize_buf->buf; + ib_umem_release(rcq->umem); + rcq->umem = rcq->resize_umem; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_kernel_cq_resize + Description : roce3_kernel_cq_resize + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_kernel_cq_resize(struct roce3_device *rdev, struct roce3_cq *rcq, int entries) +{ + int outst_cqe = 0; + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Over range after align, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + /* The number of resized CQEs cannot be smaller than the number of outstanding CQEs */ + outst_cqe = roce3_cq_get_outstanding_cqes(rcq); + if (entries < (outst_cqe + 1)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't resize, because smaller than the number of outstanding CQES, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + ret = roce3_cq_alloc_resize_buf(rdev, rcq, entries); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc resize buffer, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + /*lint -e834*/ + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_2.buf_size) - PAGE_SHIFT_4K; + /*lint +e834*/ + + /* + * Send the cq_resize command to configure CQC. After + * the configuration is successful, the new CQE is written to + * resize_buf, and the old buffer may still retain the old CQE. + */ + ret = roce3_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cq_free_resize_buf(rdev, rcq); + return ret; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_resize_kernel_cq + Description : roce3_resize_kernel_cq + Input : struct roce3_device *rdev + struct roce3_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_resize_kernel_cq(struct roce3_device *rdev, struct roce3_cq *rcq, int entries) +{ + int ret = 0; + int tmp_cqe = 0; + int cqe_entries = entries; + + mutex_lock(&rcq->resize_mutex); + + cqe_entries++; + cqe_entries = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); /*lint !e587*/ + /* Minimum queue depth needs to be aligned by page */ + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) + cqe_entries = (PAGE_SIZE >> (unsigned int)ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + + /* + * The new depth of CQ cannot be the same as the + * old depth. A special CQE space is reserved when CQ is created, + * so the value of ibcq->cqe is 1 smaller than the actual value + */ + if (cqe_entries == (rcq->ibcq.cqe + 1)) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: No need to resize cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + mutex_unlock(&rcq->resize_mutex); + return 0; + } + + ret = roce3_kernel_cq_resize(rdev, rcq, cqe_entries); + if (ret != 0) + goto out; + + /* free old MTT */ + hmm_rdma_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + /* + * When copying CQE from the old buffer to resize_buf, + * the user may be polling cqe from the old buf, so + * it needs to be locked.If CQE of the old buffer has + * been polled in the polling process, polling process will + * free old buffer and switch to new buffer. + */ + spin_lock_irq(&rcq->lock); + + /* If the new buf has been switched to, nothing needs to be done */ + if (rcq->resize_buf) { + ret = roce3_cq_resize_copy_cqes(rcq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Have not polled resize cqe(fuc:%d, cqn:%u, ret:%d)\n", + __func__, rdev->glb_func_id, rcq->cqn, ret); + spin_unlock_irq(&rcq->lock); + mutex_unlock(&rcq->resize_mutex); + return ret; + } + tmp_cqe = rcq->ibcq.cqe; + rcq->buf = rcq->resize_buf->buf; + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + } + + spin_unlock_irq(&rcq->lock); + + /* + * Non-0 means that the above resize_buf non-empty + * branch has been entered, the old buffer needs to be released + */ + if (tmp_cqe != 0) + hiudk_cqm_object_resize_free_old(rdev->hwdev, &rcq->cqm_cq->object); + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +static int roce3_resize_cq_check(struct ib_cq *ibcq, int entries, const struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + + if (ibcq == NULL) { + pr_err("[ROCE, ERR] %s: Ibcq is null\n", __func__); + return -EINVAL; + } + + if ((ibcq->uobject != NULL) && (udata == NULL)) { + pr_err("[ROCE, ERR] %s: Udata is null, but uobject is not null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibcq->device); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: resize CQEs invalid. entries(%d), func_id(%d)\n", + __func__, entries, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_resize_cq + Description : roce3_resize_cq + Input : struct ib_cq *ibcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_cq *rcq = NULL; + struct roce3_device *rdev = NULL; + + ret = roce3_resize_cq_check(ibcq, entries, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: failed to check resize_cq\n", __func__); + return ret; + } + + rcq = to_roce3_cq(ibcq); + rdev = to_roce3_dev(ibcq->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -1; + } + + if (ibcq->uobject) { + ret = roce3_resize_user_cq(rdev, rcq, entries, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize user cq, func_id(%u) ret(%d)\n", + __func__, (u32)rdev->glb_func_id, ret); + return ret; + } + } else { + ret = roce3_resize_kernel_cq(rdev, rcq, entries); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to resize kernel cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +void roce3_lock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __acquires(&roce3_send_cq->lock) __acquires(&roce3_recv_cq->lock) +{ + if (roce3_send_cq == roce3_recv_cq) { + spin_lock_irq(&roce3_send_cq->lock); + __acquire(&roce3_recv_cq->lock); + } else if (roce3_send_cq->cqn < roce3_recv_cq->cqn) { + spin_lock_irq(&roce3_send_cq->lock); + spin_lock_nested(&roce3_recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&roce3_recv_cq->lock); + spin_lock_nested(&roce3_send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +void roce3_unlock_cqs(struct roce3_cq *roce3_send_cq, struct roce3_cq *roce3_recv_cq) + __releases(&roce3_send_cq->lock) __releases(&roce3_recv_cq->lock) +{ + if (roce3_send_cq == roce3_recv_cq) { + __release(&roce3_recv_cq->lock); + spin_unlock_irq(&roce3_send_cq->lock); + } else if (roce3_send_cq->cqn < roce3_recv_cq->cqn) { + spin_unlock(&roce3_recv_cq->lock); + spin_unlock_irq(&roce3_send_cq->lock); + } else { + spin_unlock(&roce3_send_cq->lock); + spin_unlock_irq(&roce3_recv_cq->lock); + } +} diff --git a/drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c b/drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c new file mode 100644 index 000000000..c9e555ce0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/cq/roce_cq_destroy.c @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_pub_cmd.h" +#include "hinic3_hmm.h" +#include "roce_main_extension.h" + +static int roce3_check_cqc_data_state(struct roce3_device *rdev, const struct roce3_cq *cq, + struct roce_cq_context *cqc_data, u32 check_state) +{ + int read_back_flag = 0; + int times = rdev->try_times; + struct roce_cq_context check_cqc_data; + + while ((times--) != 0) { + if (roce3_hca_is_present(rdev) == 0) + return 0; + + check_cqc_data.dw2.value = be32_to_cpu(cqc_data->dw2.value); + if (check_cqc_data.dw2.bs.state == check_state) { + read_back_flag = 1; + break; + } + ROCE_UDELAY(US_PERF_DELAY); + } + + if (read_back_flag == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read back after try %d times, CQ state(0x%x), func_id(%u)\n", + __func__, rdev->try_times, cqc_data->dw2.value, (u32)rdev->glb_func_id); + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Cqn(0x%x), timer_dw(0x%x), func_id(%u)\n", + __func__, cq->cqn, cqc_data->dw2.value, (u32)rdev->glb_func_id); + return -1; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_cq_hw2sw + Description : roce3_cq_hw2sw + Input : struct roce3_device *rdev + struct roce3_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_cq_hw2sw(struct roce3_device *rdev, struct roce3_cq *cq) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct roce_cq_context *cqc_data = NULL; + struct tag_roce_cmd_cq_hw2sw *cq_hw2sw_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_cq_hw2sw), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_hw2sw_inbuf = (struct tag_roce_cmd_cq_hw2sw *)cqm_cmd_inbuf->buf; + cq_hw2sw_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + + cqc_data = (struct roce_cq_context *)((void *)cq->cqm_cq->q_ctx_vaddr); + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_HW2SW_CQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send HW2SW_CQ command, func_id(%u), cqn(0x%x), (ret:%d)\n", + __func__, (u32)rdev->glb_func_id, cq->cqn, ret); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(HW2SW_CQ), cqn(0x%x), func_id(%u)\n", + __func__, cq->cqn, rdev->glb_func_id); + + /* + * When CMDq times out or CMDq cannot work, update + * the device status and notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + /* CMDq may return a positive number, so its return value cannot be used directly */ + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return roce3_check_cqc_data_state(rdev, cq, cqc_data, ROCE_CQ_TIME_OUT_CHECK_VALUE); +} + +static int roce3_cq_cache_out(struct roce3_device *rdev, struct roce3_cq *cq) +{ + int ret = 0; + struct rdma_service_cap *rdma_cap = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_cq_cache_invalidate *cq_cache_invld_inbuf = NULL; + + /* Send the cache out command */ + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_cq_cache_invalidate), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + dev_dbg(rdev->hwdev_hdl, "[ROCE, INFO] %s: func_id(%d) cqn(%d)\n", + __func__, rdev->glb_func_id, cq->cqn); + rdma_cap = &rdev->rdma_cap; + cq_cache_invld_inbuf = (struct tag_roce_cmd_cq_cache_invalidate *)cqm_cmd_inbuf->buf; + cq_cache_invld_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); //lint !e778 + cq_cache_invld_inbuf->mtt_info.mtt_flags = 0; + cq_cache_invld_inbuf->mtt_info.mtt_num = 0; + cq_cache_invld_inbuf->mtt_info.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + cq_cache_invld_inbuf->mtt_info.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + cq_cache_invld_inbuf->mtt_info.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_MISC_CQ_CACHE_INVLD, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send CQ_CACHE_INVLD command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA is present(CQ_CACHE_INVLD), cqn(0x%x), func_id(%u)\n", + __func__, cq->cqn, rdev->glb_func_id); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + /* + * When CMDq times out or CMDq cannot work, update + * the device status and notify the PCIe module to reset + * the device through OFED + */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return -1; + } + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +static void destroy_cq_user(struct roce3_cq *cq, struct ib_udata *udata) +{ + struct roce3_ucontext *context = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + + roce3_db_unmap_user(context, &cq->db); + ib_umem_release(cq->umem); +} + +int roce3_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + struct roce3_cq *cq = NULL; + struct roce_cq_context *cqc_data = NULL; + + if (ibcq == NULL) { + pr_err("[ROCE, ERR] %s: Ibcq is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibcq->device); + cq = to_roce3_cq(ibcq); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return ok), func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_roce_cq_free; + } + + /* Modify CQC to be owned by the software */ + ret = roce3_cq_hw2sw(rdev, cq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to modify CQC, ret(%d), func_id(%u), cqn(0x%x)\n", + __func__, ret, rdev->glb_func_id, cq->cqn); + return ret; + } + + /* Send the cache out command */ + ret = roce3_cq_cache_out(rdev, cq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to modify cqc, ret(%d), func_id(%u)\n", + __func__, ret, rdev->glb_func_id); + return ret; + } + + cqc_data = (struct roce_cq_context *)((void *)cq->cqm_cq->q_ctx_vaddr); + ret = roce3_check_cqc_data_state(rdev, cq, cqc_data, ROCE_CQ_STATE_CHECK_VALUE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: check cqc data state fail, func_id(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + +err_roce_cq_free: + /* Release the MTT corresponding to cq_buf */ + hmm_rdma_mtt_free(rdev->hwdev, &cq->buf.mtt, SERVICE_T_ROCE); + + /* + * Call the CQM interface to release CQN and CQC. + * Since there is no cq_buf and software DB in user mode, + * it does not need to be released; since kernel mode + * has both, it needs to be released + */ + hiudk_cqm_object_delete(rdev->hwdev, &cq->cqm_cq->object); + + /* + * If it is user mode, you also need to cancel + * the mapping of the corresponding software DB + */ + if (ibcq->uobject) + destroy_cq_user(cq, udata); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c new file mode 100644 index 000000000..ab9bee809 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> + +#include "hinic3_mt.h" + +#include "roce.h" +#include "roce_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_dfx.h" + + +void roce3_dfx_clean_up(struct roce3_device *rdev) +{ +#ifdef ROCE_PKT_CAP_EN + (void)roce3_dfx_stop_cap_pkt(rdev, NULL, NULL); +#endif +} + +int roce3_get_drv_version(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct drv_version_info *ver_info = buf_out; + int rc; + + if (!buf_out) { + pr_err("Buf_out is NULL.\n"); + return -EINVAL; + } + + if (*out_size != sizeof(*ver_info)) { + pr_err("Unexpect out buf size from user :%u, expect: %lu\n", + *out_size, sizeof(*ver_info)); + return -EINVAL; + } + + rc = snprintf(ver_info->ver, sizeof(ver_info->ver), "%s %s", + HIROCE3_DRV_VERSION, "2024-04-16_15:14:30"); + if (rc == -1) { + pr_err("Snprintf roce version err\n"); + return -EFAULT; + } + + return 0; +} + +static int roce3_dfx_enable_bw_ctrl(struct roce3_device *rdev, struct roce3_bw_ctrl_inbuf *inbuf, + struct roce3_bw_ctrl_outbuf *outbuf) +{ + if (rdev->hw_info.hca_type == ROCE3_2_100G_HCA) { + inbuf->ctrl_param.cir = ROCE3_100G_CIR; + inbuf->ctrl_param.pir = ROCE3_100G_PIR; + inbuf->ctrl_param.cnp = ROCE3_100G_CNP; + } else { + inbuf->ctrl_param.cir = ROCE3_25G_CIR; + inbuf->ctrl_param.pir = ROCE3_25G_PIR; + inbuf->ctrl_param.cnp = ROCE3_25G_CNP; + } + return roce3_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_EN, inbuf); +} + +static int roce3_dfx_disable_bw_ctrl(struct roce3_device *rdev, struct roce3_bw_ctrl_inbuf *inbuf, + struct roce3_bw_ctrl_outbuf *outbuf) +{ + inbuf->ctrl_param.cir = 0; + inbuf->ctrl_param.pir = 0; + inbuf->ctrl_param.cnp = 0; + return roce3_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_DIS, inbuf); +} + +static int roce3_dfx_change_bw_ctrl_param(struct roce3_device *rdev, + struct roce3_bw_ctrl_inbuf *inbuf, struct roce3_bw_ctrl_outbuf *outbuf) +{ + return roce3_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_RESET, inbuf); +} + +static int roce3_dfx_query_bw_ctrl_param(struct roce3_device *rdev, + struct roce3_bw_ctrl_inbuf *inbuf, struct roce3_bw_ctrl_outbuf *outbuf) +{ + return roce3_query_bw_ctrl_state(rdev, &outbuf->bw_ctrl_param); +} + +typedef int (*roce3_adm_dfx_bw_ctrl_func_t)(struct roce3_device *rdev, + struct roce3_bw_ctrl_inbuf *inbuf, struct roce3_bw_ctrl_outbuf *outbuf); + +/*lint -e26*/ +static roce3_adm_dfx_bw_ctrl_func_t g_roce3_adm_dfx_bw_ctrl_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [ROCE_CMD_ENABLE_BW_CTRL] = roce3_dfx_enable_bw_ctrl, + [ROCE_CMD_DISABLE_BW_CTRL] = roce3_dfx_disable_bw_ctrl, + [ROCE_CMD_CHANGE_BW_CTRL_PARAM] = roce3_dfx_change_bw_ctrl_param, + [ROCE_CMD_QUERY_BW_CTRL_PARAM] = roce3_dfx_query_bw_ctrl_param, +}; +/*lint +e26*/ + +int roce3_adm_dfx_bw_ctrl(struct roce3_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct roce3_bw_ctrl_inbuf *inbuf = (struct roce3_bw_ctrl_inbuf *)buf_in; + struct roce3_bw_ctrl_outbuf *outbuf = (struct roce3_bw_ctrl_outbuf *)buf_out; + roce3_adm_dfx_bw_ctrl_func_t roce3_adm_dfx_bw_ctrl_func; + + memset(buf_out, 0, sizeof(struct roce3_bw_ctrl_outbuf)); + *out_size = sizeof(struct roce3_bw_ctrl_outbuf); + + if (inbuf->cmd_type >= COMMON_CMD_VM_COMPAT_TEST) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + roce3_adm_dfx_bw_ctrl_func = g_roce3_adm_dfx_bw_ctrl_funcs[inbuf->cmd_type]; + if (roce3_adm_dfx_bw_ctrl_func == NULL) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + return roce3_adm_dfx_bw_ctrl_func(rdev, inbuf, outbuf); +} +#endif /* __ROCE_DFX__ */ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h new file mode 100644 index 000000000..b61f6fa97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DFX_H +#define ROCE_DFX_H + +#include <linux/types.h> + +#include "hinic3_rdma.h" + +#include "roce_sysfs.h" +#include "roce.h" +#include "roce_verbs_cmd.h" +#include "rdma_context_format.h" + +#ifdef ROCE_PKT_CAP_EN +#include "roce_dfx_cap.h" +#endif + +#define MR_KEY_2_INDEX_SHIFT 8 + +#define ROCE_IO_DFX_CFG_VADDR_ID 0 +#define ROCE_IO_DFX_CFG_PADDR_ID 1 +#define ROCE_IO_DFX_CFG_ADDR_NUM 2 + +struct roce3_mpt_query_outbuf { + struct roce_mpt_context mpt_entry; +}; + +#define roce3_dfx_print pr_info + +struct roce3_dfx_query_inbuf { + u32 cmd_type; + /*lint -e658*/ + union { + u32 qpn; + u32 cqn; + u32 srqn; + u32 mpt_key; + u32 gid_index; + struct { + u32 qpn; + u32 cqn; + } query_pi_ci; + }; + /*lint +e658*/ +}; + +struct roce3_dfx_pi_ci { + u32 qpc_sq_pi_on_chip; + u32 qpc_sq_pi; + u32 qpc_sq_load_pi; + u32 qpc_rq_pi_on_chip; + u32 qpc_rq_load_pi; + u32 qpc_rq_pi; + u32 qpc_rc_pi; + u32 qpc_sq_ci; + u32 qpc_sq_wqe_prefetch_ci; + u32 qpc_sq_mtt_prefetch_wqe_ci; + u32 qpc_sqa_ci; + u32 qpc_sqa_wqe_prefetch_ci; + u32 qpc_rq_ci; + u32 qpc_rq_wqe_prefetch_ci; + u32 qpc_rq_mtt_prefetch_wqe_ci; + u32 qpc_rq_base_ci; + u32 qpc_rc_ci; + u32 qpc_rc_prefetch_ci; + u32 cq_ci_on_chip; + u32 cq_ci; + u32 cq_load_ci; + u64 cq_ci_record_gpa_at_hop_num; + u32 cq_last_solicited_pi; + u32 cq_pi; + u32 cq_last_notified_pi; +}; + +struct roce3_dfx_qp_count { + u32 qp_alloced; + u32 qp_deleted; + u32 qp_alive; +}; + +union roce3_dfx_query_outbuf { + struct roce_qp_context qp_ctx; + struct roce_cq_context cq_ctx; + struct roce_srq_context srq_ctx; + struct roce_mpt_context mpt; + struct rdma_gid_entry gid_entry; + struct roce3_dfx_pi_ci pi_ci; + struct roce3_dfx_qp_count qp_count; + u32 algo_type; +}; + +enum roce3_bw_ctrl_cmd_e { + ROCE_BW_CTRL_DIS, + ROCE_BW_CTRL_EN, + ROCE_BW_CTRL_RESET +}; +struct rdma_gid_query_outbuf { + struct rdma_gid_entry gid_entry; +}; + +struct roce3_bw_ctrl_inbuf { + u32 cmd_type; + struct { + u32 cir; + u32 pir; + u32 cnp; + } ctrl_param; +}; + +struct roce3_bw_ctrl_param { + u8 color_type; + u16 ptype; + u8 hw_wred_mode; + + u32 cir; + u32 pir; + u32 cbs; + u32 xbs; + u32 cnp; + u32 enable; +}; + +struct roce3_bw_ctrl_outbuf { + struct roce3_bw_ctrl_param bw_ctrl_param; +}; + +enum roce3_dfx_io_cmd_type { + ROCE_IO_CTRL_DIS, + ROCE_IO_CTRL_EN +}; + +struct roce3_dfx_io_alarm { + enum roce3_dfx_io_cmd_type en_flag; + u16 pf_id; + u16 rsvd; + u16 io_latency_thd; + u16 exec_time; + u32 exp_qpn; + void *rcd_uaddr; + struct timespec64 start; + struct mutex io_alarm_mutex; +}; + +struct roce3_dfx_io_inbuf { + u32 cmd_type; + struct roce3_dfx_io_alarm io_alarm; +}; + +struct roce3_dfx_io_outbuf { + struct roce3_dfx_io_alarm io_alarm; +}; + +int roce3_get_drv_version(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce3_adm_dfx_query(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce3_adm_dfx_bw_ctrl(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +void roce3_dfx_clean_up(struct roce3_device *rdev); + +void *global_roce3_io_alarm_va_get(void); + +void global_roce3_io_alarm_va_set(u64 va); + +void global_roce3_io_alarm_pa_set(dma_addr_t pa); + +dma_addr_t global_roce3_io_alarm_pa_get(void); + +int roce3_dfx_cmd_query_qp(struct roce3_device *rdev, u32 qpn, struct roce_qp_context *qp_ctx); + +int roce3_dfx_cmd_query_cq(struct roce3_device *rdev, u32 cqn, struct roce_cq_context *cq_ctx); + +int roce3_dfx_cmd_query_srq(struct roce3_device *rdev, u32 srqn, struct roce_srq_context *srq_ctx); + +#endif // __ROCE_DFX_H__ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c new file mode 100644 index 000000000..cdc75962f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef ROCE_PKT_CAP_EN + +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> +#include <linux/etherdevice.h> +#include <rdma/ib_verbs.h> + +#include "hinic3_hw.h" + +#include "roce.h" +#include "roce_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_dfx.h" +#include "roce_dfx_cap.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +static struct roce3_cap_block_num_attr g_nattr[8] = { + { .block_num_idx = 0, .shift = 4, .num = 16 }, + { .block_num_idx = 1, .shift = 5, .num = 32 }, + { .block_num_idx = 2, .shift = 6, .num = 64 }, + { .block_num_idx = 3, .shift = 7, .num = 128 }, + { .block_num_idx = 4, .shift = 8, .num = 256 }, + { .block_num_idx = 5, .shift = 9, .num = 512 }, + { .block_num_idx = 6, .shift = 10, .num = 1024 }, + { .block_num_idx = 7, .shift = 11, .num = 2048 }, +}; + +struct roce3_pkt_cap_info g_roce3_cap_info = { + .cap_status = ROCE_CAPTURE_STOP, + .cap_mode = 1, + .poll_ci = 0, + .rdev = NULL, + .func_id = 0, + // Configurable, currently using 4, when modifying, + // the size of cfg bl needs to be modified synchronously + .block_num_idx = 4, + .mode = 0, + .qp_list_cnt = 0, + .qp_list_head = LIST_HEAD_INIT(g_roce3_cap_info.qp_list_head), +}; + +static void roce3_stop_thread(struct sdk_thread_info *thread_info) +{ + if (thread_info->thread_obj) { + (void)kthread_stop(thread_info->thread_obj); + thread_info->thread_obj = NULL; + } +} + +static int roce3_linux_thread_func(void *thread) +{ + struct sdk_thread_info *info = (struct sdk_thread_info *)thread; + + while (!kthread_should_stop()) { + info->thread_fn(info->data); + + // Warning: kernel thread should reschedule + schedule(); + } + + return 0; +} +#ifdef ROCE_BONDING_EN +static struct net_device *roce3_get_roce_bond_ndev(struct roce3_device *rdev) +{ + struct bonding *bond = NULL; + struct slave *slave = NULL; + struct net_device *netdev = rdev->ndev; + + if (netif_is_lag_master(netdev)) { + bond = netdev_priv(netdev); + } else if (netif_is_bond_slave(netdev)) { + rcu_read_lock(); + slave = bond_slave_get_rcu(netdev); + rcu_read_unlock(); + if (slave) + bond = bond_get_bond_by_slave(slave); + } + + if ((bond == NULL) || (bond->dev == NULL)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Bond/bond->dev is NULL\n", __func__); + return NULL; + } + + return bond->dev; +} +#endif +static void roce3_netif_skb_info(struct roce3_device *rdev, const char *pkt, u8 len) +{ + struct sk_buff *skb = NULL; + unsigned char *packet = NULL; + struct net_device *netdev = rdev->ndev; +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + netdev = roce3_get_roce_bond_ndev(rdev); + if (netdev == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get roce_bond ndev\n", + __func__); + return; + } + } +#endif + /* alloc skb and set skb->dev */ + skb = netdev_alloc_skb_ip_align(netdev, CAP_NUM_PER_BLOCK + 1); + if (unlikely(skb == NULL)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc skb\n", __func__); + return; + } + + skb_reserve(skb, CAP_COUNTER_TWO); + + /* put space for pkt */ + packet = (unsigned char *)skb_put(skb, len); + if (packet == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get packet\n", __func__); + kfree_skb(skb); + return; + } + + /* copy pkt hdr */ + memcpy(packet, pkt, len); + + prefetchw(skb->data); + + /* resolve protocol and type */ + skb->protocol = eth_type_trans(skb, netdev); + + /* force to host */ + skb->pkt_type = 0; + + netif_receive_skb(skb); +} + +static int roce3_set_cap_func_disable(struct roce3_device *rdev) +{ + int ret; + + ret = hinic3_set_func_capture_en(rdev->hwdev, rdev->glb_func_id, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set capture disable, ret(%d)\n", + __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_create_thread(struct sdk_thread_info *thread_info) +{ + thread_info->thread_obj = kthread_run(roce3_linux_thread_func, + thread_info, thread_info->name); + if (!thread_info->thread_obj) { + pr_err("[ROCE, ERR] %s: Failed to create thread\n", __func__); + return (-EFAULT); + } + + return 0; +} + +static int roce3_set_cap_func_en(struct roce3_device *rdev) +{ + int ret; + + ret = hinic3_set_func_capture_en(rdev->hwdev, rdev->glb_func_id, 1); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set capture func enable, ret(%d)\n", + __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_pkt_cap_poll_check(struct roce3_device *rdev) +{ + if (rdev == NULL) { + pr_err("[ROCE] %s: Rdev is null\n", __func__); + return (-EINVAL); + } + + if (rdev->hwdev == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Rdev->hwdev is null\n", __func__); + return (-EINVAL); + } + + if (g_roce3_cap_info.cap_status == ROCE_CAPTURE_STOP) { + /* Don't add log here */ + return (-EINVAL); + } + + return 0; +} + +static int roce3_cap_hdr_vld(const roce3_cap_hdr_u *cap_hdr) +{ + return (cap_hdr->value != 0); +} + +static void roce3_fill_cap_cfg(struct roce3_dfx_cap_cfg_tbl *cfg_info, + u32 state, u32 ci_index, u8 mode) +{ + cfg_info->ci_index = ci_index; + + cfg_info->dw1.bs.cap_block_num_shift = + (u8)g_nattr[g_roce3_cap_info.block_num_idx].shift; // 8 + cfg_info->dw1.bs.cap_mode = (u8)g_roce3_cap_info.cap_mode; + cfg_info->dw1.bs.qp_mode = mode; + + cfg_info->dw2.bs.state = state; + cfg_info->dw2.bs.cap_func = g_roce3_cap_info.func_id; + + cfg_info->maxnum = g_nattr[g_roce3_cap_info.block_num_idx].num * CAP_NUM_PER_BLOCK; +} + +static void roce3_pkt_cap_poll_hdr(struct roce3_pkt_cap_info *cap_info, + union roce3_cap_hdr **cap_hdr, union roce3_cap_hdr *hdr_value) +{ + u32 lt_index; + u32 lt_offset; + u8 sel; + u32 block_num_per_entry = g_nattr[g_roce3_cap_info.block_num_idx].num; + u32 block_num_shift = g_nattr[g_roce3_cap_info.block_num_idx].shift; + + /* resolve index and offset */ + lt_index = (((cap_info->poll_ci) / block_num_per_entry) % CAP_PA_TBL_ENTRY_NUM); + lt_offset = ((cap_info->poll_ci) % (block_num_per_entry >> 1)); + sel = ((cap_info->poll_ci >> (block_num_shift - 1)) & 0x1); + + *cap_hdr = (roce3_cap_hdr_u *)(cap_info->que_addr[sel][0][lt_index] + + (lt_offset * CAP_PKT_ITEM_SIZE)); + + hdr_value->value = (*cap_hdr)->value; + /* First, execute big endian and small endian switch operation for ctrl information */ + hdr_value->value = ntohl(hdr_value->value); +} + +static void roce3_pkt_cap_poll(void *data) +{ + int ret; + u32 count = 0; + struct roce3_dfx_cap_cfg_tbl cfg_info = { 0 }; + union roce3_cap_hdr hdr_value = { { 0 } }; + struct roce3_pkt_cap_info *cap_info = (struct roce3_pkt_cap_info *)data; + struct roce3_device *rdev = cap_info->rdev; + union roce3_cap_hdr *cap_hdr = NULL; + + if (roce3_pkt_cap_poll_check(rdev) != 0) + return; + + roce3_pkt_cap_poll_hdr(cap_info, &cap_hdr, &hdr_value); + + if (roce3_cap_hdr_vld(&hdr_value) == 0) + return; + + do { + if (g_roce3_cap_info.cap_status == ROCE_CAPTURE_STOP) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Cap is stopped\n", __func__); + return; + } + + roce3_netif_skb_info(rdev, ((u8 *)cap_hdr + CAP_HDR_OFFSET + + hdr_value.cap_ctrl_info.pad), CAP_SKB_LEN); + + /* reset to 0 */ + cap_hdr->value = 0; + cap_info->poll_ci++; + + if (g_roce3_cap_info.cap_mode != 0) { + count++; + if (count >= (FILL_CAP_CFG_PKT_NUM)) { + roce3_fill_cap_cfg(&cfg_info, 0, cap_info->poll_ci, 0); + + (void)roce3_set_cap_cfg(rdev->hwdev, CAP_PA_TBL_ENTRY_NUM, + (u32 *)&cfg_info); + + count = 0; + } + } + + roce3_pkt_cap_poll_hdr(cap_info, &cap_hdr, &hdr_value); + + if (unlikely(kthread_should_stop())) + break; + + schedule(); + } while (roce3_cap_hdr_vld(&hdr_value) != 0); + + roce3_fill_cap_cfg(&cfg_info, 0, cap_info->poll_ci, 0); + + ret = roce3_set_cap_cfg(rdev->hwdev, CAP_PA_TBL_ENTRY_NUM, (u32 *)&cfg_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap cfg, ret(%d)\n", + __func__, ret); + return; + } +} + +static int roce3_dfx_dma_alloc_addr(struct roce3_device *rdev, int cap_index, int que_index, + struct roce3_dfx_cap_tbl *pa_info) +{ + u64 v_addr; + u64 p_addr = 0; + + v_addr = dma_alloc_coherent(&rdev->pdev->dev, + (unsigned long)(CAP_PKT_ITEM_SIZE * + ((g_nattr[g_roce3_cap_info.block_num_idx].num) >> 1)), &p_addr, GFP_KERNEL); + if (v_addr == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to alloc v_addr for que_index %d\n", + __func__, que_index); + return -ENOMEM; + } + + g_roce3_cap_info.que_addr[que_index][0][cap_index] = v_addr; + g_roce3_cap_info.que_addr[que_index][1][cap_index] = p_addr; + + pa_info->pa[que_index].wr_init_pc_h32 = + (u32)((p_addr >> CAP_ADDR_COMBIN_SHIFT) & 0xffffffff); + pa_info->pa[que_index].wr_init_pc_l32 = (u32)(p_addr & 0xffffffff); + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: v_addr que_index(%d), cap_index(%d), PA:0x%x %x\n", + __func__, que_index, cap_index, + pa_info->pa[que_index].wr_init_pc_h32, pa_info->pa[que_index].wr_init_pc_l32); + + return 0; +} + +static void roce3_dfx_dma_free_addr(struct roce3_device *rdev, int cap_index, int que_index) +{ + u64 v_addr; + u64 p_addr; + + v_addr = g_roce3_cap_info.que_addr[que_index][0][cap_index]; + p_addr = g_roce3_cap_info.que_addr[que_index][1][cap_index]; + dma_free_coherent(&rdev->pdev->dev, + (unsigned long)(CAP_PKT_ITEM_SIZE * + ((g_nattr[g_roce3_cap_info.block_num_idx].num) >> 1)), (void *)v_addr, + (dma_addr_t)p_addr); +} + +static int roce3_dfx_alloc_mem_for_one_cap(struct roce3_device *rdev, int cap_index) +{ + struct roce3_dfx_cap_tbl pa_info; + int ret; + + memset(&pa_info, 0x0, sizeof(pa_info)); + + ret = roce3_dfx_dma_alloc_addr(rdev, cap_index, 0, &pa_info); + if (ret != 0) + return ret; + + ret = roce3_dfx_dma_alloc_addr(rdev, cap_index, 1, &pa_info); + if (ret != 0) + goto err_dma_mem_alloc_free_1; + + ret = roce3_set_cap_cfg(rdev->hwdev, (u16)cap_index, (u32 *)&pa_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap cfg, ret(%d)\n", + __func__, ret); + goto err_dma_mem_alloc_free; + } + + return 0; + +err_dma_mem_alloc_free: + roce3_dfx_dma_free_addr(rdev, cap_index, 1); + +err_dma_mem_alloc_free_1: + roce3_dfx_dma_free_addr(rdev, cap_index, 0); + return ret; +} + +static void roce3_dfx_free_mem_for_one_cap(struct roce3_device *rdev, int cap_index) +{ + roce3_dfx_dma_free_addr(rdev, cap_index, 1); + roce3_dfx_dma_free_addr(rdev, cap_index, 0); +} + +static int roce3_dfx_alloc_cap(struct roce3_device *rdev) +{ + u32 counter, times; + int ret; + int i; + + /* alloc lt memory resource */ + for (i = 0; i < CAP_PA_TBL_ENTRY_NUM; i++) { + ret = roce3_dfx_alloc_mem_for_one_cap(rdev, i); + if (ret != 0) + goto err_dma_mem_alloc_free; + } + + for (times = 0; times < CLEAR_CAP_TRYING_TIMES; times++) { + if (times == CLEAR_CAP_TRYING_TIMES - 1) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear pi\n", __func__); + goto err_dma_mem_alloc_free; + } + + /* clear pi */ + ret = roce3_clear_cap_counter(rdev->hwdev, ROCE_CAP_COUNTER_INDEX, &counter); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to read counter(lt index %x), ret(%d)\n", + __func__, ROCE_CAP_COUNTER_INDEX, ret); + goto err_dma_mem_alloc_free; + } + + if (counter == 0) + break; + + msleep(CLEAR_SLEEP_TIME); + } + + return 0; + +err_dma_mem_alloc_free: + for (i--; i >= 0; i--) + roce3_dfx_free_mem_for_one_cap(rdev, i); + + return ret; +} + +static void roce3_dfx_free_cap(struct roce3_device *rdev) +{ + int i; + + for (i = 0; i < CAP_PA_TBL_ENTRY_NUM; i++) + roce3_dfx_free_mem_for_one_cap(rdev, i); +} + +static void roce3_dfx_init_g_cap_info(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf) +{ + u32 mode = inbuf->mode; + + g_roce3_cap_info.func_id = rdev->glb_func_id; + g_roce3_cap_info.task.data = &g_roce3_cap_info; + g_roce3_cap_info.task.thread_fn = roce3_pkt_cap_poll; + g_roce3_cap_info.task.name = "capture_thread"; + g_roce3_cap_info.rdev = rdev; + g_roce3_cap_info.poll_ci = 0; + g_roce3_cap_info.block_num_per_entry = g_nattr[g_roce3_cap_info.block_num_idx].num; + g_roce3_cap_info.maxnum = + (g_nattr[g_roce3_cap_info.block_num_idx].num) * CAP_PA_TBL_ENTRY_NUM; + g_roce3_cap_info.mode = mode; + g_roce3_cap_info.qp_list_cnt = 0; +} + +static int roce3_dfx_start_cap_pkt(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf, union roce3_dfx_capture_outbuf *outbuf) +{ + struct roce3_dfx_cap_cfg_tbl cfg_info; + int ret; + + mutex_lock(&cap_mutex); + if (g_roce3_cap_info.cap_status == ROCE_CAPTURE_START) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Capture is running\n", __func__); + mutex_unlock(&cap_mutex); + return (-EBUSY); + } + + ret = roce3_dfx_alloc_cap(rdev); + if (ret != 0) { + mutex_unlock(&cap_mutex); + return ret; + } + + roce3_dfx_init_g_cap_info(rdev, inbuf); + + memset(&cfg_info, 0, sizeof(cfg_info)); + roce3_fill_cap_cfg(&cfg_info, 0, 0, 0); + + ret = roce3_set_cap_cfg(rdev->hwdev, CAP_PA_TBL_ENTRY_NUM, (u32 *)&cfg_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap cfg, ret(%d)\n", + __func__, ret); + goto err_alloc_cap; + } + + dev_err(rdev->hwdev_hdl, "[ROCE] %s: roce create thread\n", __func__); + ret = roce3_create_thread(&(g_roce3_cap_info.task)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to create thread, ret(%d)\n", + __func__, ret); + goto err_alloc_cap; + } + + ret = roce3_set_cap_func_en(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap enable, ret(%d)\n", + __func__, ret); + goto err_thread_release; + } + + g_roce3_cap_info.cap_status = ROCE_CAPTURE_START; + INIT_LIST_HEAD(&g_roce3_cap_info.qp_list_head); + mutex_unlock(&cap_mutex); + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: Start to capture pkt, func(%u)\n", + __func__, g_roce3_cap_info.func_id); + return 0; + +err_thread_release: + roce3_stop_thread(&(g_roce3_cap_info.task)); + msleep(CAP_STOP_SLEEP_TIME); + +err_alloc_cap: + roce3_dfx_free_cap(rdev); + + g_roce3_cap_info.cap_status = ROCE_CAPTURE_STOP; + mutex_unlock(&cap_mutex); + + return ret; +} + +static int roce3_clear_cap_counter_encap(struct roce3_device *rdev, u16 lt_index) +{ + int ret; + u32 counter = 0; + u32 times; + + for (times = 0; times < CLEAR_CAP_TRYING_TIMES; times++) { + if (times == CLEAR_CAP_TRYING_TIMES - 1) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear pi\n", __func__); + return -EFAULT; + } + + ret = roce3_clear_cap_counter(rdev->hwdev, lt_index, &counter); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear cap counter(lt index 1), ret(%d)\n", + __func__, ret); + return -EFAULT; + } + + if (counter == 0) + break; + + msleep(CLEAR_SLEEP_TIME); + } + + return 0; +} + +int roce3_dfx_stop_cap_pkt(struct roce3_device *rdev, const struct roce3_dfx_capture_inbuf *inbuf, + union roce3_dfx_capture_outbuf *outbuf) +{ + int ret = 0; + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: start to stop cap,\n", __func__); + + mutex_lock(&cap_mutex); + if (g_roce3_cap_info.cap_status != ROCE_CAPTURE_START) { // 1:RUNNING + mutex_unlock(&cap_mutex); + return 0; + } + + if (g_roce3_cap_info.func_id != rdev->glb_func_id) { + mutex_unlock(&cap_mutex); + return 0; + } + + roce3_stop_thread(&(g_roce3_cap_info.task)); + + /* stop cap */ + ret = roce3_set_cap_func_disable(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to set cap enable, ret(%d)\n", + __func__, ret); + ret = -EFAULT; + goto err; + } + + msleep(CAP_STOP_SLEEP_TIME); + + /* release lt mem resource */ + roce3_dfx_free_cap(rdev); + + /* lt index_1 */ + ret = roce3_clear_cap_counter_encap(rdev, ROCE_CAP_COUNTER_INDEX); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to clear pi(lt index %x), ret(%d)\n", + __func__, ROCE_CAP_COUNTER_INDEX, ret); + goto err; + } + + g_roce3_cap_info.cap_status = ROCE_CAPTURE_STOP; + g_roce3_cap_info.poll_ci = 0; + g_roce3_cap_info.func_id = 0; + g_roce3_cap_info.rdev = NULL; + dev_info(rdev->hwdev_hdl, "[ROCE] %s: Stop capture pkt, func(%d)\n", + __func__, rdev->glb_func_id); + +err: + mutex_unlock(&cap_mutex); + return ret; +} + +static int roce3_dfx_query_cap_pkt(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf, union roce3_dfx_capture_outbuf *outbuf) +{ + int ret; + u32 pi = 0; + u32 total = 0; + struct roce3_dfx_cap_cfg_tbl cfg_info; + struct roce3_dfx_capture_info *capture_info = &outbuf->capture_info; + + memset(&cfg_info, 0, sizeof(cfg_info)); + ret = roce3_get_cap_cfg(rdev->hwdev, CAP_NUM_PER_BLOCK, (u32 *)&cfg_info); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to read table, ret(%d)\n", + __func__, ret); + return -EFAULT; + } + + ret = roce3_read_cap_counter(rdev->hwdev, CAP_COUNTER_TWO, &total); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to read counter(lt index 2), ret(%d)\n", + __func__, ret); + return -EFAULT; + } + + ret = roce3_read_cap_counter(rdev->hwdev, ROCE_CAP_COUNTER_INDEX, &pi); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read counter(lt index %x), ret(%d)\n", + __func__, ROCE_CAP_COUNTER_INDEX, ret); + return -EFAULT; + } + + capture_info->cap_status = g_roce3_cap_info.cap_status; + capture_info->cap_mode = cfg_info.dw1.bs.cap_mode; + capture_info->qp_mode = cfg_info.dw1.bs.qp_mode; + capture_info->cap_block_num_shift = cfg_info.dw1.bs.cap_block_num_shift; + capture_info->cap_func = cfg_info.dw2.bs.cap_func; + capture_info->cap_state = cfg_info.dw2.bs.state; + capture_info->cap_max_num = cfg_info.maxnum; + capture_info->cap_ci = g_roce3_cap_info.poll_ci; + capture_info->cap_pi = pi; + capture_info->cap_total = total; + + return 0; +} + +typedef int (*roce3_adm_dfx_capture_func_t)(struct roce3_device *rdev, + const struct roce3_dfx_capture_inbuf *inbuf, union roce3_dfx_capture_outbuf *outbuf); + +/*lint -e26*/ +static roce3_adm_dfx_capture_func_t g_roce3_adm_dfx_capture_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [ROCE_CMD_START_CAP_PACKET] = roce3_dfx_start_cap_pkt, + [ROCE_CMD_STOP_CAP_PACKET] = roce3_dfx_stop_cap_pkt, + [ROCE_CMD_QUERY_CAP_INFO] = roce3_dfx_query_cap_pkt, + [ROCE_CMD_ENABLE_QP_CAP_PACKET] = NULL, + [ROCE_CMD_DISABLE_QP_CAP_PACKET] = NULL, + [ROCE_CMD_QUERY_QP_CAP_INFO] = NULL, +}; +/*lint +e26*/ + +int roce3_adm_dfx_capture(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret; + const struct roce3_dfx_capture_inbuf *inbuf = (struct roce3_dfx_capture_inbuf *)buf_in; + union roce3_dfx_capture_outbuf *outbuf = (union roce3_dfx_capture_outbuf *)buf_out; + roce3_adm_dfx_capture_func_t roce3_adm_dfx_capture_func; + + memset(buf_out, 0, sizeof(union roce3_dfx_capture_outbuf)); + *out_size = sizeof(union roce3_dfx_capture_outbuf); + + if (inbuf->cmd_type >= COMMON_CMD_VM_COMPAT_TEST) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + roce3_adm_dfx_capture_func = g_roce3_adm_dfx_capture_funcs[inbuf->cmd_type]; + if (roce3_adm_dfx_capture_func == NULL) { + dev_err(rdev->hwdev_hdl, "Not support this type(%d)\n", inbuf->cmd_type); + return -EINVAL; + } + + return roce3_adm_dfx_capture_func(rdev, inbuf, outbuf); +} + +#endif /* ROCE_PKT_CAP_EN */ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h new file mode 100644 index 000000000..acd3c407c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_cap.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DFX_CAP_H +#define ROCE_DFX_CAP_H + +#include <linux/types.h> + +#include "hinic3_rdma.h" + +#include "roce_sysfs.h" +#include "roce.h" + +#define CAP_PKT_ITEM_SIZE 128 + +#define CAP_NUM_PER_BLOCK 255 + +#define CAP_SKB_LEN 120 + +#define CAP_HDR_OFFSET 4 + +#define FILL_CAP_CFG_PKT_NUM 8192 + +#define CAP_LOOP 255 + +#define CLEAR_MAX_TRY_TIME 10 + +#define CLEAR_SLEEP_TIME 2 + +#define CAP_STOP_SLEEP_TIME 50 + +#define CAP_COUNTER_TWO 2 + +#define CAP_PA_TBL_ENTRY_NUM 255 + +#define CAP_ADDR_COMBIN_SHIFT 32 + +#define CLEAR_CAP_TRYING_TIMES 10 + +#define ROCE_CAP_COUNTER_INDEX 0x39ff + +union roce3_cap_hdr { + struct { + u32 rsvd : 16; + u32 pad : 8; + u32 rsvd0 : 2; + u32 col_num : 4; + u32 tx_rx : 1; + u32 vld : 1; + } cap_ctrl_info; + u32 value; +}; + +struct roce3_dfx_cap_pa_entry { + u32 wr_init_pc_h32; + u32 wr_init_pc_l32; +}; + +struct roce3_dfx_cap_tbl { + struct roce3_dfx_cap_pa_entry pa[2]; +}; + +struct roce3_dfx_cap_cfg_tbl { + u32 ci_index; + + union { + struct { + /* + * Driver configurate bit offset by the number of per + * pa block, for example, 7 means 128, 8 means 256 + */ + u8 cap_block_num_shift; + u8 cap_mode; /* Packet Capture mode */ + u8 qp_mode; /* according to qp Packet Capture */ + u8 rsvd; /* reserved */ + } bs; + u32 value; + } dw1; + + union { + struct { + u32 state : 8; /* Packet Capture rate control */ + u32 rsvd : 8; + u32 cap_func : 16; /* Packet Capture function */ + } bs; + u32 value; + } dw2; + + u32 maxnum; +}; + +struct roce3_qp_cap_pkt_list { + u32 xid; + struct list_head list; +}; + +struct roce3_cap_block_num_attr { + u32 block_num_idx; + u32 shift; + u32 num; +}; + +struct sdk_thread_info { + struct task_struct *thread_obj; + char *name; + void (*thread_fn)(void *unused); + void *thread_event; + void *data; +}; + +struct roce3_pkt_cap_info { + struct roce3_device *rdev; + u32 func_id; + u32 poll_ci; + u32 cap_mode; /* 0 : drop mode 1: overwrite mode */ + u32 mode; /* 0 : func mode 1: qp mode */ + u32 cap_status; /* 0 : stopped 1: running */ + u32 block_num_idx; + u32 block_num_per_entry; + u32 maxnum; + struct sdk_thread_info task; + char thread_name[20]; + u64 que_addr[2][2][256]; + struct list_head qp_list_head; /* Based on qp Packet Capture linked list */ + u32 qp_list_cnt; +}; + +#define ROCE_DFX_MAX_CAPTURE_QP_NUM 511 + +struct roce3_qp_cap_pkt { + __be32 xid; +}; + +struct roce3_dfx_capture_inbuf { + u32 cmd_type; + u32 mode; + u32 qpn; +}; + +struct roce3_dfx_capture_info { + u32 cap_status; + u32 cap_mode; + u32 qp_mode; + u32 cap_block_num_shift; + u32 cap_func; + u32 cap_state; + u32 cap_max_num; + u32 cap_pi; + u32 cap_ci; + u32 cap_total; +}; + +struct roce3_dfx_qp_capture_info { + u32 qp_num; + u32 qpn[ROCE_DFX_MAX_CAPTURE_QP_NUM]; +}; + +union roce3_dfx_capture_outbuf { + struct roce3_dfx_capture_info capture_info; + struct roce3_dfx_qp_capture_info qp_capture_info; +}; + +enum roce3_dfx_capture_mode { + ROCE_CAPTURE_MODE_CAP_FUNC = 0, + ROCE_CAPTURE_MODE_CAP_QP, +}; + +enum roce3_dfx_capture_state { + ROCE_CAPTURE_START = 0, + ROCE_CAPTURE_STOP, +}; + +extern int hinic3_set_func_capture_en(void *hwdev, u16 func_id, bool cap_en); + +int roce3_adm_dfx_capture(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce3_dfx_stop_cap_pkt(struct roce3_device *rdev, const struct roce3_dfx_capture_inbuf *inbuf, + union roce3_dfx_capture_outbuf *outbuf); +static DEFINE_MUTEX(cap_mutex); + +#endif /* ROCE_DFX_CAP_H */ diff --git a/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c new file mode 100644 index 000000000..1058ba0ac --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/dfx/roce_dfx_query.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/kthread.h> + +#include <rdma/ib_verbs.h> +#include "roce_compat.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_cq.h" +#include "hinic3_hw.h" +#include "roce_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_cqm_cmd.h" + +int roce3_dfx_cmd_query_qp(struct roce3_device *rdev, u32 qpn, struct roce_qp_context *qp_ctx) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_qp_query *qp_query_inbuf = NULL; + struct roce3_qp_query_outbuf *qp_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_qp_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_qp_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + qp_query_inbuf = (struct tag_roce_cmd_qp_query *)cqm_cmd_inbuf->buf; + qp_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + qp_query_inbuf->com.index = cpu_to_be32(qpn); + + if ((rdev->cfg_info.lb_en != 0) && (rdev->cfg_info.lb_mode == ROCE_LB_MODE_1)) { + u8 cos = qpn & 0x3; + + roce3_dfx_print("%s: lb_mode1 func_id(%d) qpn:%u cos:%u\n", + __func__, rdev->glb_func_id, qpn, cos); + ret = cqm_lb_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_QP, + cos, cqm_cmd_inbuf, cqm_cmd_outbuf, + NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + } else { + roce3_dfx_print("%s: Not lb_mode1 func_id(%d) qpn:%u\n", + __func__, rdev->glb_func_id, qpn); + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + } + + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_QP"); + ret = -1; + goto out; + } + + qp_query_outbuf = (struct roce3_qp_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(qp_ctx, &qp_query_outbuf->qpc, sizeof(struct roce_qp_context)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +static int roce3_dfx_get_dev_algo(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + u32 cc_algo; + struct roce3_ecn_ctx ecn_ctx; + u32 *algo_type = &outbuf->algo_type; + + if (rdev == NULL) { + pr_err("[ROCE, ERR] %s: Failed to get roce device.\n", __func__); + return -EINVAL; + } + ecn_ctx = rdev->ecn_ctx; + cc_algo = ecn_ctx.cc_algo; + *algo_type = cc_algo; + + return 0; +} + +static int roce3_dfx_get_qp_count(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct roce3_dfx_qp_count *qp_count = &outbuf->qp_count; + + qp_count->qp_alloced = rdev->qp_cnt.alloc_qp_cnt; + qp_count->qp_deleted = rdev->qp_cnt.del_qp_cnt; + qp_count->qp_alive = rdev->qp_cnt.alloc_qp_cnt - rdev->qp_cnt.del_qp_cnt; + + return 0; +} + +/* + * QPC is statically allocated, so it is not necessary to judge + * whether it exists when querying the cache content + */ +static int roce3_dfx_query_cache_qpc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + u32 qpn = inbuf->qpn; + struct roce_qp_context *qp_ctx = &outbuf->qp_ctx; + + memset(qp_ctx, 0, sizeof(struct roce_qp_context)); + ret = roce3_dfx_cmd_query_qp(rdev, qpn, qp_ctx); + if (ret != 0) { + roce3_dfx_print("Failed to query QPC from cache!"); + roce3_dfx_print("******************From Cache: qpn(%#x) ********************", qpn); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY QPC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: qpn(%#x) ********************", qpn); + + return ret; + } + + return 0; +} + +static int roce3_dfx_query_host_qpc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj_qp = NULL; + struct roce3_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + struct roce_qp_context *qp_ctx = &outbuf->qp_ctx; + + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_qp == NULL) { + roce3_dfx_print("Failed to get cqm_obj_qp."); + roce3_dfx_print("******************From Host: qpn(%#x) ********************", qpn); + roce3_dfx_print(">>>>>>>>>>> QUERY QPC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: qpn(%#x) ********************", qpn); + + return -EINVAL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + + memcpy(qp_ctx, rqp->qpc_info->vaddr, sizeof(struct roce_qp_context)); + + return 0; +} + +int roce3_dfx_cmd_query_cq(struct roce3_device *rdev, u32 cqn, struct roce_cq_context *cq_ctx) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_roce_cq_query *cq_query_inbuf = NULL; + struct roce3_cq_query_outbuf *cq_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_roce_cq_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_cq_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + cq_query_inbuf = (struct tag_roce_cmd_roce_cq_query *)cqm_cmd_inbuf->buf; + cq_query_inbuf->com.index = cpu_to_be32(cqn); + cq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_CQ, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_CQ, ret=%d", ret); + ret = -1; + goto out; + } + + cq_query_outbuf = (struct roce3_cq_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(cq_ctx, &cq_query_outbuf->cqc, sizeof(struct roce_cq_context)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + * CQC is dynamic allocation. When querying cache content, + * you need to determine whether it exists or not. + */ +static int roce3_dfx_query_cache_cqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + struct tag_cqm_object *cqm_obj_cq = NULL; + u32 cqn = inbuf->cqn; + struct roce_cq_context *cq_ctx = &outbuf->cq_ctx; + + cqm_obj_cq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_cq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_cq."); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + roce3_dfx_print(">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + + return -EINVAL; + } + + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_cq); + + memset(cq_ctx, 0, sizeof(struct roce_cq_context)); + ret = roce3_dfx_cmd_query_cq(rdev, cqn, cq_ctx); + if (ret != 0) { + roce3_dfx_print("Failed to query cq from cache!"); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY CQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: cqn(%#x) ********************", cqn); + + return -EINVAL; + } + + return 0; +} + +static int roce3_dfx_query_host_cqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj_cq = NULL; + struct roce3_cq *rcq = NULL; + u32 cqn = inbuf->cqn; + struct roce_cq_context *cq_ctx = &outbuf->cq_ctx; + + cqm_obj_cq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_cq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_cq."); + roce3_dfx_print("******************From Host: cqn(%#x) ********************", cqn); + roce3_dfx_print(">>>>>>>>>>> QUERY CQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: cqn(%#x) ********************", cqn); + + return -EINVAL; + } + + rcq = cqmobj_to_roce3_cq(cqm_obj_cq); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_cq); + + memcpy(cq_ctx, rcq->cqm_cq->q_ctx_vaddr, sizeof(struct roce_cq_context)); + + return 0; +} + +int roce3_dfx_cmd_query_srq(struct roce3_device *rdev, u32 srqn, struct roce_srq_context *srq_ctx) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_srq_query *srq_query_inbuf = NULL; + struct roce3_srq_query_outbuf *srq_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_srq_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + srq_query_inbuf = (struct tag_roce_cmd_srq_query *)cqm_cmd_inbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32(srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_SRQ, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_SRQ, ret=%d", ret); + ret = -1; + goto out; + } + + srq_query_outbuf = (struct roce3_srq_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(srq_ctx, srq_query_outbuf, + sizeof(struct roce3_srq_query_outbuf)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + * SRQC is dynamic allocation. When querying the cache content, + * you need to determine whether it exists or not. + */ +static int roce3_dfx_query_cache_srqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + struct tag_cqm_object *cqm_obj_srq = NULL; + u32 srqn = inbuf->srqn; + struct roce_srq_context *srq_ctx = &outbuf->srq_ctx; + + cqm_obj_srq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_srq"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + roce3_dfx_print(">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_srq); + + memset(srq_ctx, 0, sizeof(struct roce_srq_context)); + ret = roce3_dfx_cmd_query_srq(rdev, srqn, srq_ctx); + if (ret != 0) { + roce3_dfx_print("Failed to query srq from cache"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY SRQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + return 0; +} + +static int roce3_dfx_query_host_srqc(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj_srq = NULL; + struct roce3_srq *rsrq = NULL; + u32 srqn = inbuf->srqn; + struct roce_srq_context *srq_ctx = &outbuf->srq_ctx; + + cqm_obj_srq = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + roce3_dfx_print("Failed to get cqm_obj_srq."); + roce3_dfx_print("******************From Host: srqn(%#x) ********************", + srqn); + roce3_dfx_print(">>>>>>>>>>> QUERY SRQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + rsrq = cqmobj_to_roce3_srq(cqm_obj_srq); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_srq); + + memcpy(srq_ctx, rsrq->cqm_srq->q_ctx_vaddr, sizeof(struct roce_srq_context)); + + return 0; +} + +static int roce3_dfx_cmd_query_mpt(struct roce3_device *rdev, u32 key, + struct roce_mpt_context *mpt_entry) +{ + int ret; + u32 mpt_index; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_mpt_query *mpt_query_inbuf = NULL; + struct roce3_mpt_query_outbuf *mpt_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_mpt_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_mpt_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + mpt_query_inbuf = (struct tag_roce_cmd_mpt_query *)cqm_cmd_inbuf->buf; + mpt_index = (key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + mpt_query_inbuf->com.index = cpu_to_be32(mpt_index); + mpt_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_MPT, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_MPT"); + ret = -1; + goto out; + } + + mpt_query_outbuf = (struct roce3_mpt_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(mpt_entry, &mpt_query_outbuf->mpt_entry, + sizeof(struct roce_mpt_context)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +static int roce3_dfx_cmd_query_gid(struct roce3_device *rdev, u32 port, u32 gid_index, + struct rdma_gid_entry *gid_entry) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_qurey_gid *gid_query_inbuf = NULL; + struct rdma_gid_query_outbuf *gid_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_qurey_gid), &cqm_cmd_outbuf, + (u16)sizeof(struct rdma_gid_query_outbuf)); + if (ret != 0) { + roce3_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + gid_query_inbuf = (struct tag_roce_qurey_gid *)cqm_cmd_inbuf->buf; + gid_query_inbuf->port = cpu_to_be32(port); + gid_query_inbuf->com.index = cpu_to_be32(gid_index); + gid_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_GID, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + roce3_dfx_print("Failed to send cmd QUERY_GID"); + ret = -1; + goto out; + } + + gid_query_outbuf = (struct rdma_gid_query_outbuf *)cqm_cmd_outbuf->buf; + memcpy(gid_entry, &gid_query_outbuf->gid_entry, sizeof(struct rdma_gid_entry)); +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + * MPT is statically allocated, so it is not necessary to + * judge whether it exists when querying the cache content + */ +static int roce3_dfx_query_cache_mpt(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + memset(mpt_entry, 0, sizeof(struct roce_mpt_context)); + ret = roce3_dfx_cmd_query_mpt(rdev, mpt_key, mpt_entry); + if (ret != 0) { + roce3_dfx_print("Failed to query mpt from cache!"); + roce3_dfx_print("******************From Cache: mpt_key(%#x) ********************", + mpt_key); + roce3_dfx_print(">>>>>>>>>>>>>>>> QUERY MPT FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce3_dfx_print("******************From Cache: mpt_key(%#x) ********************", + mpt_key); + + return ret; + } + + return 0; +} + +static int roce3_dfx_query_host_mpt(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + u32 mpt_index = 0; + struct tag_cqm_object *cqm_obj_mpt = NULL; + struct tag_cqm_qpc_mpt *cqm_mpt = NULL; + struct rdma_mpt *rmpt = NULL; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + mpt_index = (mpt_key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + cqm_obj_mpt = cqm_object_get(rdev->hwdev, CQM_OBJECT_MPT, mpt_index, false); + if (cqm_obj_mpt == NULL) { + roce3_dfx_print("Failed to get cqm_obj_mpt."); + roce3_dfx_print("******************From Host: mpt_key(%#x) ********************", + mpt_key); + roce3_dfx_print(">>>>>>>>>>> QUERY MPT FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce3_dfx_print("******************From Host: mpt_key(%#x) ********************", + mpt_key); + + return -EINVAL; + } + + cqm_mpt = container_of(cqm_obj_mpt, struct tag_cqm_qpc_mpt, object); + rmpt = (struct rdma_mpt *)cqm_mpt->priv; + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_mpt); + + memcpy(mpt_entry, rmpt->vaddr, sizeof(struct roce_mpt_context)); + + return 0; +} + +static int roce3_dfx_query_cache_gid(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + u32 gid_index = inbuf->gid_index; + struct rdma_gid_entry *gid_entry = &outbuf->gid_entry; + + memset(gid_entry, 0, sizeof(struct rdma_gid_entry)); + ret = roce3_dfx_cmd_query_gid(rdev, 0, gid_index, gid_entry); + if (ret != 0) { + roce3_dfx_print("***************port(1), gid_index(%d) *****************", + gid_index); + roce3_dfx_print("Failed to query gid from cache!"); + roce3_dfx_print("***************port(1), gid_index(%d) *****************", + gid_index); + + return -EINVAL; + } + + return 0; +} + +static void roce3_dfx_query_pi_ci_b32_to_cpu(struct roce_qp_context *qp_ctx) +{ + qp_ctx->chip_seg.sqc.dw0.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw0.value); + qp_ctx->chip_seg.sqc.dw2.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw2.value); + qp_ctx->chip_seg.sqc.dw3.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw3.value); + qp_ctx->chip_seg.sqc.dw7.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw7.value); + qp_ctx->chip_seg.sqc.dw14.value = be32_to_cpu(qp_ctx->chip_seg.sqc.dw14.value); + qp_ctx->chip_seg.rqc.dw0.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw0.value); + qp_ctx->chip_seg.rqc.dw3.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw3.value); + qp_ctx->chip_seg.rqc.dw7.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw7.value); + qp_ctx->chip_seg.rqc.dw14.value = be32_to_cpu(qp_ctx->chip_seg.rqc.dw14.value); + qp_ctx->chip_seg.sqac.dw3.value = be32_to_cpu(qp_ctx->chip_seg.sqac.dw3.value); + qp_ctx->chip_seg.sqac.dw7.value = be32_to_cpu(qp_ctx->chip_seg.sqac.dw7.value); + qp_ctx->chip_seg.rcc.dw5.value = be32_to_cpu(qp_ctx->chip_seg.rcc.dw5.value); + qp_ctx->chip_seg.rcc.dw6.value = be32_to_cpu(qp_ctx->chip_seg.rcc.dw6.value); + qp_ctx->chip_seg.rcc.dw7.value = be32_to_cpu(qp_ctx->chip_seg.rcc.dw7.value); + qp_ctx->chip_seg.qpcc.dw4.value = be32_to_cpu(qp_ctx->chip_seg.qpcc.dw4.value); +} + +static void roce3_dfx_query_pi_ci_set(struct roce_qp_context qp_ctx, struct roce3_dfx_pi_ci *pi_ci) +{ + pi_ci->qpc_sq_pi_on_chip = qp_ctx.chip_seg.sqc.dw0.bs.sq_pi_on_chip; + pi_ci->qpc_sq_pi = qp_ctx.chip_seg.sqc.dw2.bs.sq_pi; + pi_ci->qpc_sq_load_pi = qp_ctx.chip_seg.sqc.dw3.bs.sq_load_pi; + pi_ci->qpc_rq_pi_on_chip = qp_ctx.chip_seg.rqc.dw0.bs.rq_pi_on_chip; + pi_ci->qpc_rq_load_pi = qp_ctx.chip_seg.rqc.dw3.bs.rq_load_pi; + pi_ci->qpc_rq_pi = qp_ctx.chip_seg.rqc.dw7.bs.rq_pi; + pi_ci->qpc_rc_pi = qp_ctx.chip_seg.rcc.dw5.bs.rc_pi; + pi_ci->qpc_sq_ci = qp_ctx.chip_seg.sqc.dw3.bs.sq_ci; + pi_ci->qpc_sq_wqe_prefetch_ci = qp_ctx.chip_seg.sqc.dw7.bs.sq_wqe_prefetch_ci; + pi_ci->qpc_sq_mtt_prefetch_wqe_ci = qp_ctx.chip_seg.sqc.dw14.bs.sq_mtt_prefetch_wqe_ci; + pi_ci->qpc_sqa_ci = qp_ctx.chip_seg.sqac.dw3.bs.sqa_ci; + pi_ci->qpc_sqa_wqe_prefetch_ci = qp_ctx.chip_seg.sqac.dw7.bs.sqa_wqe_prefetch_ci; + pi_ci->qpc_rq_ci = qp_ctx.chip_seg.rqc.dw3.bs.rq_ci; + pi_ci->qpc_rq_wqe_prefetch_ci = qp_ctx.chip_seg.rqc.dw7.bs.rq_wqe_prefetch_ci; + pi_ci->qpc_rq_mtt_prefetch_wqe_ci = qp_ctx.chip_seg.rqc.dw14.bs.rq_mtt_prefetch_wqe_ci; + pi_ci->qpc_rq_base_ci = qp_ctx.chip_seg.qpcc.dw4.bs.rq_base_ci; + pi_ci->qpc_rc_ci = qp_ctx.chip_seg.rcc.dw6.bs.rc_ci; + pi_ci->qpc_rc_prefetch_ci = qp_ctx.chip_seg.rcc.dw7.bs.rc_prefetch_ci; +} + +static int roce3_dfx_query_pi_ci(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf) +{ + int ret; + struct roce_qp_context qp_ctx; + struct roce_cq_context cq_ctx; + u32 qpn = inbuf->query_pi_ci.qpn; + u32 cqn = inbuf->query_pi_ci.cqn; + struct roce3_dfx_pi_ci *pi_ci = &outbuf->pi_ci; + + memset(pi_ci, 0, sizeof(struct roce3_dfx_pi_ci)); + + memset(&qp_ctx, 0, sizeof(qp_ctx)); + ret = roce3_dfx_cmd_query_qp(rdev, qpn, &qp_ctx); + if (ret != 0) + return ret; + + roce3_dfx_query_pi_ci_b32_to_cpu(&qp_ctx); + roce3_dfx_query_pi_ci_set(qp_ctx, pi_ci); + + memset(&cq_ctx, 0, sizeof(cq_ctx)); + ret = roce3_dfx_cmd_query_cq(rdev, cqn, &cq_ctx); + if (ret != 0) + return ret; + + cq_ctx.dw0.value = be32_to_cpu(cq_ctx.dw0.value); + cq_ctx.dw1.value = be32_to_cpu(cq_ctx.dw1.value); + cq_ctx.dw2.value = be32_to_cpu(cq_ctx.dw2.value); + cq_ctx.dw3.value = be32_to_cpu(cq_ctx.dw3.value); + cq_ctx.dw9.value = be32_to_cpu(cq_ctx.dw9.value); + cq_ctx.ci_record_gpa_at_hop_num = be32_to_cpu(cq_ctx.ci_record_gpa_at_hop_num); + + pi_ci->cq_ci_on_chip = cq_ctx.dw0.bs.ci_on_chip; + pi_ci->cq_ci = cq_ctx.dw1.bs.ci; + pi_ci->cq_load_ci = cq_ctx.dw3.bs.load_ci; + pi_ci->cq_ci_record_gpa_at_hop_num = cq_ctx.ci_record_gpa_at_hop_num; + pi_ci->cq_last_solicited_pi = cq_ctx.dw0.bs.last_solicited_pi; + pi_ci->cq_pi = cq_ctx.dw2.bs.pi; + pi_ci->cq_last_notified_pi = cq_ctx.dw9.bs.last_notified_pi; + + return 0; +} + +typedef int (*roce3_adm_dfx_query_t)(const struct roce3_dfx_query_inbuf *inbuf, + struct roce3_device *rdev, union roce3_dfx_query_outbuf *outbuf); + +static roce3_adm_dfx_query_t roce3_adm_dfx_query_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [ROCE_CMD_GET_QPC_FROM_CACHE] = roce3_dfx_query_cache_qpc, + [ROCE_CMD_GET_QPC_FROM_HOST] = roce3_dfx_query_host_qpc, + [ROCE_CMD_GET_CQC_FROM_CACHE] = roce3_dfx_query_cache_cqc, + [ROCE_CMD_GET_CQC_FROM_HOST] = roce3_dfx_query_host_cqc, + [ROCE_CMD_GET_SRQC_FROM_CACHE] = roce3_dfx_query_cache_srqc, + [ROCE_CMD_GET_SRQC_FROM_HOST] = roce3_dfx_query_host_srqc, + [ROCE_CMD_GET_MPT_FROM_CACHE] = roce3_dfx_query_cache_mpt, + [ROCE_CMD_GET_MPT_FROM_HOST] = roce3_dfx_query_host_mpt, + [ROCE_CMD_GET_GID_FROM_CACHE] = roce3_dfx_query_cache_gid, + [ROCE_CMD_GET_QPC_CQC_PI_CI] = roce3_dfx_query_pi_ci, + [ROCE_CMD_GET_QP_COUNT] = roce3_dfx_get_qp_count, + [ROCE_CMD_GET_DEV_ALGO] = roce3_dfx_get_dev_algo, +}; + +int roce3_adm_dfx_query(struct roce3_device *rdev, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct roce3_dfx_query_inbuf *inbuf = (struct roce3_dfx_query_inbuf *)buf_in; + union roce3_dfx_query_outbuf *outbuf = (union roce3_dfx_query_outbuf *)buf_out; + roce3_adm_dfx_query_t roce3_adm_dfx_query_func; + + memset(buf_out, 0, sizeof(union roce3_dfx_query_outbuf)); + *out_size = (u32)sizeof(union roce3_dfx_query_outbuf); + + if (inbuf->cmd_type >= COMMON_CMD_VM_COMPAT_TEST) { + roce3_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + roce3_adm_dfx_query_func = roce3_adm_dfx_query_funcs[inbuf->cmd_type]; + if (roce3_adm_dfx_query_func == NULL) { + roce3_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + return roce3_adm_dfx_query_func(inbuf, rdev, outbuf); +} + +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c new file mode 100644 index 000000000..c27cf826e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_cdev_extension.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#ifndef PANGEA_NOF + +#include "roce_cdev_extension.h" + +long ioctl_non_bonding_extend(unsigned int cmd, struct roce3_device *rdev, unsigned long arg) +{ + return NOT_SUPOORT_TYPE; +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c new file mode 100644 index 000000000..cd881052d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_event_extension.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_event_extension.h" + +#ifndef PANGEA_NOF +void roce3_event_report_extend(const struct roce3_device *rdev, int event_str_index) +{ + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [non ofed event type] Invalid extend event type error. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ +} + +int roce3_async_event_handle_extend(u8 event_type, u8 *val, struct roce3_device *rdev) +{ + int event_str_index = 0; + + event_str_index = to_unknown_event_str_index(event_type, val); + + return event_str_index; +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c new file mode 100644 index 000000000..d3885af3f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_main_extension.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_main_extension.h" + +#ifndef PANGEA_NOF + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" + +static struct mutex g_rdev_mutex; +#endif + +void roce3_service_init_pre(void) +{ +#ifdef ROCE_BONDING_EN + mutex_init(&g_rdev_mutex); +#endif + +} + +void roce3_service_init_ext(void) +{ +} + +void roce3_lock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_lock(&g_rdev_mutex); +#endif + +} + +void roce3_unlock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_unlock(&g_rdev_mutex); +#endif +} + +int roce3_get_rdev_by_uld(struct hinic3_lld_dev *lld_dev, void *uld_dev, struct roce3_device **rdev, + struct hinic3_event_info *event) +{ +#ifdef ROCE_BONDING_EN + int ret; + + ret = roce3_bond_event_cfg_rdev(lld_dev, uld_dev, rdev); + if (ret != 0) { + pr_err("[ROCE] %s: Cfg bond rdev failed(%d)\n", __func__, ret); + return ret; + } + ret = roce3_bonded_port_event_report(*rdev, event); + if (ret != 0) { + pr_err("[ROCE] %s: Report bond event failed(%d)\n", __func__, ret); + return ret; + } +#else + if ((lld_dev == NULL) || (uld_dev == NULL)) { + pr_err("[ROCE] %s: Input params is null\n", __func__); + return -ENODEV; + } + *rdev = (struct roce3_device *)uld_dev; +#endif + return 0; +} +#endif /* !PANGEA_NOF */ + +#ifdef ROCE_STANDARD +void roce3_init_dev_ext_handlers(struct roce3_device *rdev) +{ +} +#endif /* ROCE_STANDARD */ + +#ifndef PANGEA_NOF +void roce3_remove_clean_res_ext(struct roce3_device *rdev) +{ +#ifdef __ROCE_DFX__ + roce3_dfx_clean_up(rdev); +#endif +} +#endif /* PANGEA_NOF */ + +#ifndef PANGEA_NOF +int roce3_board_cfg_check(struct roce3_device *rdev) +{ + int ret = 0; + int port_num = 0; + int port_speed = 0; + + port_num = rdev->board_info.port_num; + port_speed = rdev->board_info.port_speed; + if ((port_num == ROCE3_2_PORT_NUM) && (port_speed == ROCE3_100G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE3_2_100G_HCA; + } else if ((port_num == ROCE3_4_PORT_NUM) && (port_speed == ROCE3_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE3_4_25G_HCA; + } else if ((port_num == ROCE3_2_PORT_NUM) && (port_speed == ROCE3_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE3_2_25G_HCA; + } else { + pr_err("[ROCE] %s: Invalid fw cfg\n", __func__); + ret = (-EINVAL); + } + + return ret; +} + +int roce3_mmap_ext(struct roce3_device *rdev, struct roce3_ucontext *ucontext, + struct vm_area_struct *vma) +{ + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Not support, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; +} + +int roce3_dfx_mem_alloc(struct roce3_device *rdev) +{ + return 0; +} + +void roce3_dfx_mem_free(struct roce3_device *rdev) +{ +} + +void *roce3_ucontext_alloc_ext(void) +{ + return kzalloc(sizeof(struct roce3_ucontext), GFP_KERNEL); +} + +void *roce3_resp_alloc_ext(void) +{ + return kzalloc(sizeof(struct roce3_alloc_ucontext_resp), GFP_KERNEL); +} + +void roce3_resp_set_ext(struct roce3_device *rdev, struct roce3_alloc_ucontext_resp *resp) +{ +} + +void roce3_ucontext_set_ext(struct roce3_device *rdev, struct roce3_ucontext *context) +{ +} + +void *roce3_rdev_alloc_ext(void) +{ + return (void *)ib_alloc_device(roce3_device, ib_dev); +} + +void roce3_rdev_set_ext(struct roce3_device *rdev) +{ +} + +int ib_copy_to_udata_ext(struct ib_udata *udata, struct roce3_alloc_ucontext_resp *resp) +{ + return ib_copy_to_udata(udata, resp, sizeof(struct roce3_alloc_ucontext_resp)); +} + +int roce3_set_comm_event(const struct roce3_device *rdev, const struct hinic3_event_info *event) +{ + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Comm event unsupported, func_id(%d), event(%d)\n", + __func__, rdev->glb_func_id, event->type); + return -1; +} + +bool roce3_hca_is_present(const struct roce3_device *rdev) +{ + return true; +} +#endif /* PANGEA_NOF */ + +#if !defined(NOF_AA) +int roce3_init_dev_ext(struct roce3_device *rdev) +{ + return 0; +} + +#if defined(EULER_2_10_OFED_4_19) || defined(KY10_OFED_4_19) +void roce3_rdma_cap_ext(struct rdma_service_cap *rdma_cap) +{ + rdma_cap->max_sq_desc_sz = RDMA_MAX_SQ_DESC_SZ_COMPUTE; + rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes = ROCE_MAX_WQES_COMPUTE; + rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz = + ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE; +} +#else +void roce3_rdma_cap_ext(struct rdma_service_cap *rdma_cap) +{ +} +#endif + +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c new file mode 100644 index 000000000..c27dbda09 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_mr_extension.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include "roce_mr.h" +#include "roce_main_extension.h" +#include "roce_mr_extension.h" + +#ifdef ROCE_STANDARD +int roce3_check_alloc_mr_type(enum ib_mr_type mr_type) +{ + int ret = 0; + + if (mr_type != IB_MR_TYPE_MEM_REG) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: mr_type is invalid. mr_type:%u\n", __func__, mr_type); + } + + return ret; +} + +enum rdma_mr_type roce3_get_mrtype(enum ib_mr_type ib_mr_type) +{ + switch (ib_mr_type) { + case IB_MR_TYPE_MEM_REG: + return RDMA_DMA_MR; + + default: + return RDMA_DMA_MR; + } +} +#endif + diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c new file mode 100644 index 000000000..99d868613 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_netdev_extension.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_netdev_extension.h" +#include "roce_mpu_common.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifndef PANGEA_NOF +int roce3_add_real_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + int ret; + u32 vlan_id = 0; + + /* + * no need to configure IPSURX vf table for vroce here, + * this action has been done in vroce driver + */ + roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + ret = roce3_add_bond_real_slave_mac(rdev, (u8 *)netdev->dev_addr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add bond ipsu mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, 0, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + return ret; + } + } +#endif + + memcpy(rdev->mac, netdev->dev_addr, sizeof(rdev->mac)); + + return 0; +} + +int roce3_add_vlan_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + int ret = 0; + u32 vlan_id = 0; + + vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID(((u32)rdma_vlan_dev_vlan_id(netdev))); + dev_info(rdev->hwdev_hdl, "[ROCE] %s: enter add vlan, vlan_id(0x%x), func_id(%d)\n", + __func__, vlan_id, rdev->glb_func_id); + ret = roce3_add_mac_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, + vlan_id, rdev->glb_func_id, rdev->glb_func_id); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set vlan mac, vlan_id(0x%x), func_id(%d)\n", + __func__, vlan_id, rdev->glb_func_id); + return ret; + } + + roce3_add_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + ret = roce3_add_bond_vlan_slave_mac(rdev, (u8 *)netdev->dev_addr, (u16)vlan_id); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set bond, vlan_id(0x%x), func_id(%d)\n", + __func__, vlan_id, rdev->glb_func_id); + goto err_add_bond_vlan_slave_mac; + } + } +#endif + + return ret; + +#ifdef ROCE_BONDING_EN +err_add_bond_vlan_slave_mac: + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, (u8 *)netdev->dev_addr, + vlan_id, rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + + (void)hinic3_del_mac(rdev->hwdev, (u8 *)netdev->dev_addr, (u16)vlan_id, + rdev->glb_func_id, HINIC3_CHANNEL_ROCE); + + return ret; +#endif +} + +void roce3_del_real_device_mac(struct roce3_device *rdev) +{ + u32 vlan_id = 0; +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + roce3_del_bond_real_slave_mac(rdev); +#endif + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, rdev->mac, vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); +} + +void roce3_del_vlan_device_mac(struct roce3_device *rdev, struct roce3_vlan_dev_list *old_list) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + roce3_del_bond_vlan_slave_mac(rdev, old_list->mac, (u16)old_list->vlan_id); +#endif + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, old_list->mac, old_list->vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, old_list->mac, old_list->vlan_id, + rdev->glb_func_id, rdev->glb_func_id); +} + +void roce3_event_up_extend(struct roce3_device *rdev) +{ + if (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) == 0) + roce3_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ACTIVE); +} + +#endif /* PANGEA_NOF */ diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c new file mode 100644 index 000000000..50170d779 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_qp_extension.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_qp_extension.h" +#include "cfg_mgmt_mpu_cmd_defs.h" + +#ifdef ROCE_EXTEND +#include "roce_ctx_api.h" +#include "roce_pd.h" +#endif + +int to_roce3_qp_type(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return ROCE_QP_ST_RC; + + case IB_QPT_UC: + return ROCE_QP_ST_UC; + + case IB_QPT_UD: + return ROCE_QP_ST_UD; + + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + return ROCE_QP_ST_XRC; + + case IB_QPT_GSI: + return ROCE_QP_ST_UD; + + default: + return -1; + } +} + +bool roce3_check_qp_modify_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, + enum ib_qp_type type, enum ib_qp_attr_mask mask, enum rdma_link_layer ll) +{ + return ib_modify_qp_is_ok(cur_state, next_state, type, mask); +} + +#ifndef PANGEA_NOF +int roce3_create_qp_pre_ext(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr) +{ + return 0; +} + +int roce3_create_qp_user_pre_ext(struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, u32 *qpn) +{ + *qpn = ROCE_QP_INVLID_QP_NUM; + + return 0; +} + +int roce3_create_qp_user_post_ext(struct ib_pd *ibpd, struct roce3_device *rdev, + struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr) +{ + return 0; +} + +int roce3_qp_modify_cmd_ext(struct tag_cqm_cmd_buf *cqm_cmd_inbuf, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar) +{ + return 0; +} + +bool roce3_need_qpn_lb1_consistent_srqn(const struct roce3_qp *rqp, const struct roce3_device *rdev, + const struct ib_qp_init_attr *init_attr) +{ + if (init_attr->srq == NULL) + return false; + + if ((rdev->cfg_info.scence_id == SCENES_ID_CLOUD) || + (rdev->cfg_info.scence_id == SCENES_ID_COMPUTE_ROCE) || + (rdev->cfg_info.scence_id == SCENES_ID_COMPUTE_STANDARD)) + return true; + + return false; +} + +int roce3_is_qp_normal(struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr) +{ + return 1; +} + +#ifdef ROCE_EXTEND +static struct roce3_qp *roce3_cdev_lookup_and_check_rqp(struct roce3_device *rdev, u32 qpn) +{ + struct tag_cqm_object *cqm_obj_qp = NULL; + struct roce3_qp *rqp = NULL; + + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_qp == NULL) { + pr_err("[ROCE, ERR] %s: Can't find rqp according to qpn(0x%x), func_id(%d)\n", + __func__, qpn, rdev->glb_func_id); + return NULL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + + if (rqp->qpn >= QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: qpn[%u] more than sqpc num offset(%d) for sqpc.\n", + __func__, rqp->qpn, QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + return NULL; + } + + return rqp; +} + +static int roce3_alloc_sqpc(struct roce3_device *rdev, struct roce3_qp *rqp, + struct roce3_vbs_qp *vbs_rqp) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + + /* Call the CQM interface to allocate QPNs and QPCs */ + /* For SQPC, do not need to allocate QPN, QPN is directly spcified in rsvd segment */ + qpc_info = cqm_object_qpc_mpt_create(rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, + NULL, rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, false); + if (qpc_info == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Failed to create qpc by cqm object, func_id(%d), qpn(%u)\n", + __func__, rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + return -ENOMEM; + } + + if (qpc_info->xid != (rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC)) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Create qpc error, func_id(%d), expect qpn(%d), actual qpn(%d)\n", + __func__, rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, qpc_info->xid); + hiudk_cqm_object_delete(rdev->hwdev, &(qpc_info->object)); + return -EFAULT; + } + + vbs_rqp->vbs_sqpc_info = qpc_info; + rqp->vbs_qp_ptr = (void *)vbs_rqp; + + return 0; +} + +long roce3_set_qp_ext_attr(struct roce3_device *rdev, void *buf) +{ + int ret; + struct roce3_qp *rqp = NULL; + struct roce3_set_qp_ext_attr_cmd cmd; + struct roce_qp_context *context = NULL; + + ret = (int)copy_from_user(&cmd, buf, sizeof(cmd)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data from user\n", + __func__); + return (long)ret; + } + + rqp = roce3_cdev_lookup_and_check_rqp(rdev, cmd.qpn); + if (rqp == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + context = (struct roce_qp_context *)((void *)rqp->qpc_info->vaddr); + if (((cmd.attr_mask) & ROCE_QP_VBS_FLAG) != 0) { /* IBV_QP_VBS_OSD_FLAG */ + context->sw_seg.ucode_seg.common.dw0.bs.ulp_type = ROCE_ULP_VBS; + } + + return (long)ret; +} + +long roce3_vbs_create_sqpc(struct roce3_device *rdev, void *buf) +{ + struct roce3_modify_qp_vbs_cmd cmd; + struct roce3_vbs_qp *vbs_rqp = NULL; + struct roce3_qp *rqp = NULL; + struct roce3_pd *pd = NULL; + int ret; + + ret = (int)copy_from_user(&cmd, buf, sizeof(cmd)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data from user\n", + __func__); + return (long)ret; + } + + rqp = roce3_cdev_lookup_and_check_rqp(rdev, cmd.qpn); + if (rqp == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + vbs_rqp = kzalloc(sizeof(*vbs_rqp), GFP_KERNEL); + if (vbs_rqp == NULL) + return -ENOMEM; + + pd = roce3_get_pd(rqp); + ret = roce3_db_map_user(to_roce3_ucontext(pd->ibpd.uobject->context), + cmd.ci_record_addr, &vbs_rqp->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Failed to map db page to user, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto free_rqp; + } + + ret = roce3_alloc_sqpc(rdev, rqp, vbs_rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE_VBS, ERR] %s: Failed to alloc sqpc, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto free_rqp; + } + + return 0; + +free_rqp: + kfree(vbs_rqp); + + return (long)ret; +} +#endif + +#endif /* !PANGEA_NOF */ + +#ifndef ROCE_CHIP_TEST +void roce3_set_qp_dif_attr(struct roce3_qp *rqp, const struct ib_qp_init_attr *init_attr, + const struct roce3_device *rdev) +{ + if (((unsigned int)init_attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) != 0) { + rqp->signature_en = true; + dev_info(rdev->hwdev_hdl, "[ROCE] %s: func(%d) qp(%u) roce3_create_qp signature_en.\n", + __func__, rdev->glb_func_id, rqp->qpn); + } +} +#endif + +#ifndef ROCE_VBS_EN +int roce3_qp_modify_pre_extend(struct roce3_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + return 0; +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c new file mode 100644 index 000000000..25b599020 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_qp_post_send_extension.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_qp_post_send_extension.h" + +int roce3_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + return roce3_post_send_standard(ibqp, (const struct ib_send_wr *)wr, + (const struct ib_send_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c b/drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c new file mode 100644 index 000000000..b701f014c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/extension/roce_srq_extension.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_srq_extension.h" + +#ifndef ROCE_CHIP_TEST +void roce3_srq_container_init(struct ib_srq_init_attr *init_attr, struct roce3_srq *rsrq, + struct roce3_device *rdev) +{ + rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC); + + if (rsrq->xrc_en != 0) + rsrq->container_flag = rdev->cfg_info.srq_container_en; + + rsrq->container_mode = (rsrq->xrc_en != 0) ? + rdev->cfg_info.xrc_srq_container_mode : rdev->cfg_info.srq_container_mode; + rsrq->container_warn_th = roce3_calculate_cont_th(init_attr->attr.srq_limit); + rsrq->rqe_cnt_th = rdev->cfg_info.warn_th; + rsrq->container_size = roce3_get_container_sz(rsrq->container_mode); +} +#endif + +#ifndef PANGEA_NOF +void roce3_create_user_srq_update_ext(u32 *cqn, u32 srqn) +{ +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c new file mode 100644 index 000000000..89b321ebe --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include "hinic3_hw.h" +#include "hmm_comp.h" +#include "hmm_buddy.h" + + +u32 hmm_buddy_alloc(struct hmm_buddy *buddy, u32 order) +{ + u32 first_index = 0; + u32 cur_order = 0; + u32 cur_bit_num = 0; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return HMM_INVALID_INDEX; + } + if (order > buddy->max_order) { + pr_err("%s: Order(%d) is bigger than max order(%d)\n", + __func__, order, buddy->max_order); + return HMM_INVALID_INDEX; + } + spin_lock(&buddy->lock); + + for (cur_order = order; cur_order <= buddy->max_order; ++cur_order) { + if (buddy->num_free[cur_order] != 0) { + cur_bit_num = 1U << (buddy->max_order - cur_order); + first_index = (u32)find_first_bit(buddy->bits[cur_order], + (unsigned long)cur_bit_num); + if (first_index < cur_bit_num) + goto found; + } + } + spin_unlock(&buddy->lock); + pr_err("%s: Get a invalid index\n", __func__); + return HMM_INVALID_INDEX; + +found: + clear_bit((int)first_index, buddy->bits[cur_order]); + --buddy->num_free[cur_order]; + + while (cur_order > order) { + --cur_order; + first_index <<= 1; + set_bit(first_index ^ 1, buddy->bits[cur_order]); + ++buddy->num_free[cur_order]; + } + first_index <<= order; + spin_unlock(&buddy->lock); + return first_index; +} + +void hmm_buddy_free(struct hmm_buddy *buddy, u32 first_index, u32 order) +{ + u32 tmp_first_index = first_index; + u32 tmp_order = order; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return; + } + if (tmp_order > buddy->max_order) { + pr_err("%s: Order(%d) is bigger than max order(%d)\n", + __func__, tmp_order, buddy->max_order); + return; + } + tmp_first_index >>= tmp_order; + spin_lock(&buddy->lock); + while (test_bit((int)(tmp_first_index ^ 1), buddy->bits[tmp_order]) != 0) { + clear_bit((int)(tmp_first_index ^ 1), buddy->bits[tmp_order]); + --buddy->num_free[tmp_order]; + tmp_first_index >>= 1; + ++tmp_order; + } + set_bit(tmp_first_index, buddy->bits[tmp_order]); + ++buddy->num_free[tmp_order]; + spin_unlock(&buddy->lock); +} + +static void hmm_buddy_alloc_bitmap_fail(struct hmm_buddy *buddy, u32 i) +{ + u32 j = 0; + + for (j = 0; j < i; j++) { + if (is_vmalloc_addr(buddy->bits[j])) + vfree(buddy->bits[j]); + else + kfree(buddy->bits[j]); + buddy->bits[j] = NULL; + } + kfree(buddy->bits); + buddy->bits = NULL; +} + +int hmm_buddy_init(struct hmm_buddy *buddy, u32 max_order) +{ + u32 i = 0; + u32 bit_num = 0; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return -EINVAL; + } + buddy->max_order = max_order; + /*lint -e708*/ + spin_lock_init(&buddy->lock); + /*lint +e708*/ + buddy->num_free = kcalloc( + (unsigned long)(buddy->max_order + 1UL), sizeof(int), GFP_KERNEL); + if (buddy->num_free == NULL) + return -ENOMEM; + buddy->bits = kcalloc( + (unsigned long)(buddy->max_order + 1UL), sizeof(long *), GFP_KERNEL); + if (buddy->bits == NULL) + goto alloc_bits_fail; + + for (i = 0; i <= buddy->max_order; i++) { + bit_num = (u32)BITS_TO_LONGS(1UL << (buddy->max_order - i)); + buddy->bits[i] = kcalloc( + (unsigned long)bit_num, sizeof(long), GFP_KERNEL | __GFP_NOWARN); + if (buddy->bits[i] == NULL) { + buddy->bits[i] = vzalloc((unsigned long)bit_num * sizeof(long)); + if (buddy->bits[i] == NULL) + goto alloc_bitmap_fail; + } + } + set_bit(0, buddy->bits[buddy->max_order]); + buddy->num_free[buddy->max_order] = 1; + return 0; + +alloc_bitmap_fail: + hmm_buddy_alloc_bitmap_fail(buddy, i); +alloc_bits_fail: + kfree(buddy->num_free); + buddy->num_free = NULL; + return -ENOMEM; +} + +void hmm_buddy_cleanup(struct hmm_buddy *buddy) +{ + u32 i; + + if (buddy == NULL) { + pr_err("%s: Buddy is null\n", __func__); + return; + } + for (i = 0; i <= buddy->max_order; i++) { + if (is_vmalloc_addr(buddy->bits[i])) + vfree(buddy->bits[i]); + else + kfree(buddy->bits[i]); + buddy->bits[i] = NULL; + } + kfree(buddy->bits); + buddy->bits = NULL; + kfree(buddy->num_free); + buddy->num_free = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h new file mode 100644 index 000000000..303de57ba --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_buddy.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_BUDDY_H +#define HMM_BUDDY_H + +#include <linux/spinlock.h> +#include <linux/mm.h> + +#if defined(__i386__) +#include <linux/highmem.h> +#endif + +#ifndef HMM_INVALID_INDEX +#define HMM_INVALID_INDEX 0xFFFFFFFF +#endif + +struct hmm_buddy { + unsigned long **bits; /* 指向多级bitmap的内存 */ + unsigned int *num_free; /* 指示各级bitmap中可用的索引个数 */ + u32 max_order; /* 指bitmap的级数 */ + spinlock_t lock; /* buddy的自旋锁 */ +}; + +u32 hmm_buddy_alloc(struct hmm_buddy *buddy, u32 order); +void hmm_buddy_free(struct hmm_buddy *buddy, u32 first_index, u32 order); + +int hmm_buddy_init(struct hmm_buddy *buddy, u32 max_order); +void hmm_buddy_cleanup(struct hmm_buddy *buddy); + + +#endif // HMM_BUDDY_H diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c new file mode 100644 index 000000000..c66ac63c5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> +#include "hinic3_hw.h" +#include "hmm_comp.h" + + +struct hmm_comp_priv *get_hmm_comp_priv(void *hwdev, u32 service_type) +{ + return (struct hmm_comp_priv *)hinic3_get_service_adapter(hwdev, + (enum hinic3_service_type)service_type); +} + +static void assemble_mpt_hw2sw(struct rdma_mpt_hw2sw_inbuf **mpt_hw2sw_inbuf, + struct hmm_comp_priv *comp_priv, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + *mpt_hw2sw_inbuf = (struct rdma_mpt_hw2sw_inbuf *)cqm_cmd_inbuf->buf; + memset(*mpt_hw2sw_inbuf, 0, sizeof(struct rdma_mpt_hw2sw_inbuf)); + + (*mpt_hw2sw_inbuf)->dmtt_flags = 0; /* 默认按VF踢除cache */ + (*mpt_hw2sw_inbuf)->dmtt_num = 0; + (*mpt_hw2sw_inbuf)->dmtt_cache_line_start = cpu_to_be32(comp_priv->rdma_cap.dmtt_cl_start); + (*mpt_hw2sw_inbuf)->dmtt_cache_line_end = cpu_to_be32(comp_priv->rdma_cap.dmtt_cl_end); + (*mpt_hw2sw_inbuf)->dmtt_cache_line_size = cpu_to_be32(comp_priv->rdma_cap.dmtt_cl_sz); +} + +int hmm_enable_roce_mpt(void *hwdev, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u16 channel) +{ + int ret; + + ret = cqm_send_cmd_box(hwdev, HINIC3_MOD_ROCE, RDMA_ROCE_CMD_SW2HW_MPT, + cqm_cmd_inbuf, NULL, NULL, RDMA_CMD_TIME_OUT_A, channel); + if (ret != 0) { + if (hinic3_get_heartbeat_status(hwdev) != PCIE_LINK_DOWN) { + pr_err("%s: Send cmd rdma_roce_cmd_sw2hw_mpt failed, ret(%d)\n", + __func__, ret); + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + return -RDMA_CMDQ_TIMEOUT; + + return -RDMA_CMDQ_ERR; + } + pr_err("%s: Card not present, return err\n", __func__); + return -RDMA_CMDQ_ERR; + } + + return 0; +} + +static int hmm_mpt_read_back_test(struct hmm_comp_priv *comp_priv, struct rdma_mpt *mpt) +{ + int retry; + struct rdma_mpt_entry *mpt_entry = NULL; + struct rdma_mpt_entry check_mpt_entry; + + /* 获取Host MPT内容 */ + mpt_entry = (struct rdma_mpt_entry *)mpt->vaddr; + for (retry = 0; retry < RDMA_MAX_RETRY; retry++) { + if (hinic3_get_heartbeat_status(comp_priv->hwdev) == PCIE_LINK_DOWN) { + pr_err("%s: Card not present, return ok\n", __func__); + return 0; + } + /* + * Confirm that the chip operation is complete by comparing the MPT State field. + * If the readback status is correct, the loop exits. Otherwise, + * the loop continues to read and compare data after a forcible + * delay until the loop ends. + */ + check_mpt_entry.roce_mpt_ctx.dw2.value = + be32_to_cpu(mpt_entry->roce_mpt_ctx.dw2.value); + if (check_mpt_entry.roce_mpt_ctx.dw2.bs.status == RDMA_MPT_STATUS_INVALID) + return 0; + + /*lint -e160 -e506*/ + mdelay(RDMA_MS_DELAY); + } + + pr_err("%s: RoCE mpt state read times(%d), mpt_index(0x%x), state_dw(0x%x)\n", + __func__, retry, mpt->mpt_index, mpt_entry->roce_mpt_ctx.dw2.value); + return -RDMA_CMDQ_ERR; +} + +int hmm_disable_roce_mpt(struct hmm_comp_priv *comp_priv, struct rdma_mpt *mpt, u16 channel) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct rdma_mpt_hw2sw_inbuf *mpt_hw2sw_inbuf = NULL; + + cqm_cmd_inbuf = cqm_cmd_alloc(comp_priv->hwdev); + if (cqm_cmd_inbuf == NULL) { + pr_err("%s: RoCE alloc cmd_buf failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + cqm_cmd_inbuf->size = (u16)sizeof(struct rdma_mpt_hw2sw_inbuf); + assemble_mpt_hw2sw(&mpt_hw2sw_inbuf, comp_priv, cqm_cmd_inbuf); + mpt_hw2sw_inbuf->com.index = cpu_to_be32(mpt->mpt_index); + mpt_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = (u16)cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + ret = cqm_send_cmd_box(comp_priv->hwdev, HINIC3_MOD_ROCE, RDMA_ROCE_CMD_HW2SW_MPT, + cqm_cmd_inbuf, NULL, NULL, RDMA_CMD_TIME_OUT_A, channel); + if (ret != 0) { + if (hinic3_get_heartbeat_status(comp_priv->hwdev) != PCIE_LINK_DOWN) { + pr_err("%s: Send cmd rdma_roce_cmd_hw2sw_mpt failed, ret(%d)\n", + __func__, ret); + cqm_cmd_free(comp_priv->hwdev, cqm_cmd_inbuf); + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + return -RDMA_CMDQ_TIMEOUT; + + return -RDMA_CMDQ_ERR; + } + pr_err("%s: Card not present, return ok\n", __func__); + cqm_cmd_free(comp_priv->hwdev, cqm_cmd_inbuf); + return 0; + } + cqm_cmd_free(comp_priv->hwdev, cqm_cmd_inbuf); + + ret = hmm_mpt_read_back_test(comp_priv, mpt); + return ret; +} + + +int hmm_modify_roce_mpt(void *hwdev, u32 mpt_index, u32 new_key, u64 length, u64 iova, u16 channel) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf; + struct rdma_mpt_modify_inbuf *mpt_modify_inbuf = NULL; + int ret; + + cqm_cmd_inbuf = cqm_cmd_alloc(hwdev); + if (cqm_cmd_inbuf == NULL) { + pr_err("%s: RoCE alloc cmd_buf failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + cqm_cmd_inbuf->size = (u16)sizeof(struct rdma_mpt_modify_inbuf); + mpt_modify_inbuf = (struct rdma_mpt_modify_inbuf *)cqm_cmd_inbuf->buf; + memset(mpt_modify_inbuf, 0, sizeof(*mpt_modify_inbuf)); + + mpt_modify_inbuf->com.dw0.bs.cmd_bitmask = (u16)cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + mpt_modify_inbuf->com.index = cpu_to_be32(mpt_index); + mpt_modify_inbuf->new_key = cpu_to_be32(new_key); + mpt_modify_inbuf->length = cpu_to_be64(length); + mpt_modify_inbuf->iova = cpu_to_be64(iova); + + ret = cqm_send_cmd_box(hwdev, HINIC3_MOD_ROCE, RDMA_ROCE_CMD_MODIFY_MPT, + cqm_cmd_inbuf, NULL, NULL, RDMA_CMD_TIME_OUT_A, channel); + if (ret != 0) { + if (hinic3_get_heartbeat_status(hwdev) != PCIE_LINK_DOWN) { + pr_err("%s: Send cmd rdma_roce_cmd_modify_mpt failed, ret(%d)\n", + __func__, ret); + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + return -RDMA_CMDQ_TIMEOUT; + + return -RDMA_CMDQ_ERR; + } + pr_err("%s: Card not present, return ok\n", __func__); + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + return 0; + } + + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h new file mode 100644 index 000000000..a509d701f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_COMP_H +#define HMM_COMP_H + +#ifndef ROCE_SERVICE +#include "ossl_knl.h" +#endif +#include <linux/delay.h> +#include <linux/types.h> +#include "roce_hmm_context.h" +#include "hinic3_crm.h" +#include "hinic3_cqm.h" +#include "hinic3_rdma.h" +#include "hmm_buddy.h" +#include "hmm_em.h" + +/* x86 */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#define PCIE_LINK_DOWN 0xFFFFFFFF + +#define RDMA_MPT_STATUS_INVALID 0xf +#define RDMA_MPT_STATUS_FREE 0x3 +#define RDMA_MPT_STATUS_VALID 0x1 +#define RDMA_MPT_STATUS_MEM_INIT 0xa + +#define RDMA_MS_DELAY 5 +#define RDMA_MAX_RETRY 200 + +#define RDMA_MPT_DMA_ATTR_IDX 0 +#define RDMA_MPT_SO_RO 0x1 + +#define RDMA_PA_SIZE ((u32)sizeof(dma_addr_t)) + +#define PAGE_SIZE_4k 4096 /* page size is 4K */ +#define PAGE_SHIFT_4K 12 /* page size is 1 left shift 12 */ + +#define PAGE_SIZE_64k (64 * 4096) /* page size is 64K */ +#define PAGE_SHIFT_64K 16 /* page size is 1 left shift 16 */ + +#define PAGE_SIZE_2M (2 * 1024 * 1024) /* page size is 2M */ +#define PAGE_SHIFT_2M 21 /* page size is 1 left shift 21 */ + +#define RDMA_MTT_PA_VALID 0x1 + +#define HMM_MTT_NUM_PER_CACHELINE 32 /* 256B Cache line has 32 records */ + +#define BLOCK_SIZE_DEVIDE_SECTOR 8 /* Chip logic: 8 */ + + +#define MPT_GPA_SIG_LEN 3 + +#define RDMA_CMDQ_ERR 1 +#define RDMA_CMDQ_TIMEOUT 2 + +#ifndef VERBS_CMD_TYPE_MR_BITMASK +#define VERBS_CMD_TYPE_MR_BITMASK (1u << 12) /* verbs_mr_bitmask */ +#endif + +enum { + RDMA_MTT_PAGE_SIZE_4K = 0, + RDMA_MTT_PAGE_SIZE_64K = 1, + RDMA_MTT_PAGE_SIZE_2M = 2 +}; + +#ifdef __EMU_X86__ +enum { + RDMA_CMD_TIME_OUT_A = 30000000, + RDMA_CMD_TIME_OUT_B = 40000000, + RDMA_CMD_TIME_OUT_C = 50000000 +}; +#else +enum { + RDMA_CMD_TIME_OUT_A = 30000, + RDMA_CMD_TIME_OUT_B = 40000, + RDMA_CMD_TIME_OUT_C = 50000 +}; +#endif + +enum rdma_mr_mw { + RDMA_MPT_MW = 0, + RDMA_MPT_MR = 1 +}; + +enum rdma_roce_cmd { + RDMA_ROCE_CMD_SW2HW_MPT = 0x70, + RDMA_ROCE_CMD_HW2SW_MPT = 0x71, + RDMA_ROCE_CMD_MODIFY_MPT = 0x72, + RDMA_ROCE_CMD_QUERY_MPT = 0x73, + RDMA_ROCE_CMD_FLUSH_TPT = 0x74, + RDMA_ROCE_CMD_SYNC_TPT = 0x75 +}; + + +enum mtt_layer { + RDMA_MTT_NO_LAYER = -1, /* dma mr needs no mtt */ + RDMA_MTT_ZERO_LAYER = 0, /* 1 page mr has 0 level mtt */ + RDMA_MTT_ONE_LAYER = 1, + RDMA_MTT_TWO_LAYER = 2, + RDMA_MTT_THREE_LAYER = 3 +}; + +struct rdma_verbs_cmd_com { + union { + __be32 value; + + struct { + __be32 version : 8; + __be32 rsvd : 8; + __be32 cmd_bitmask : 16; + } bs; + } dw0; + + __be32 index; +}; + +struct hmm_service_cap { + struct dev_rdma_svc_cap dev_rdma_cap; + /* + * 1. the number of MTT PA must be integer power of 2 + * 2. represented by logarithm. Each MTT table can + * contain 1, 2, 4, 8, and 16 PA) + */ + u8 log_mtt; + /* todo: need to check whether related to max_mtt_seg */ + /* + * Number of MTT table (4M), + * is actually MTT seg number + */ + u32 num_mtts; + /* todo: max value needs to be confirmed */ + /* MTT table number of Each MTT seg(3) */ + u32 log_mtt_seg; + u32 mtt_entry_sz; /* MTT table size 8B, including 1 PA(64bits) */ + u32 mpt_entry_sz; /* MPT table size (64B) */ + + u32 dmtt_cl_start; + u32 dmtt_cl_end; + u32 dmtt_cl_sz; + u32 mtt_page_size; /* 4K, 8K, 16K, 32K */ + u32 mtt_page_shift; /* 12, 13, 14, 15 */ +}; + + +struct hmm_comp_priv { + struct rdma_comp_resource rdma_comp_res; /* gid & guid */ + struct hmm_buddy mtt_buddy; + struct hmm_em_table mtt_em_table; + void *hwdev; + struct pci_dev *pdev; + struct rdma_mr rsvd_lkey; + struct rdma_mr fixed_mr; + u32 mtt_page_size; /* 4K, 8K, 16K, 32K */ + u32 mtt_page_shift; /* 12, 13, 14, 15 */ + + struct hmm_service_cap rdma_cap; +}; + +struct rdma_mpt_entry { + struct roce_mpt_context roce_mpt_ctx; +}; + +struct rdma_mpt_sw2hw_inbuf { + struct rdma_verbs_cmd_com com; + struct rdma_mpt_entry mpt_entry; +}; + +struct rdma_mpt_hw2sw_inbuf { + struct rdma_verbs_cmd_com com; + + __be32 dmtt_flags; + __be32 dmtt_num; + __be32 dmtt_cache_line_start; + __be32 dmtt_cache_line_end; + __be32 dmtt_cache_line_size; +}; + +struct rdma_mpt_modify_inbuf { + struct rdma_verbs_cmd_com com; + __be32 new_key; + __be64 length; + __be64 iova; +}; + + +/* for llt to set stub */ +int hmm_disable_roce_mpt(struct hmm_comp_priv *comp_priv, struct rdma_mpt *mpt, u16 channel); + +int hmm_modify_roce_mpt(void *hwdev, u32 mpt_index, + u32 new_key, u64 length, u64 iova, u16 channel); + +struct hmm_comp_priv *get_hmm_comp_priv(void *hwdev, u32 service_type); + +int hmm_rdma_mpt_alloc(void *hwdev, struct rdma_mpt *mpt, u32 service_type); + +int hmm_enable_roce_mpt(void *hwdev, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u16 channel); + +#ifdef RDMA_SIGN_MTT_EN +u64 hmm_gen_mtt_sign(u64 mtt_base_gpa, enum mtt_data_type_e type); + +#define RDMA_CMTT_SIGN_MASK 0x7ff +#define RDMA_CMTT_SIGN_SHIFT0 3 +#define RDMA_CMTT_SIGN_SHIFT1 14 +#define RDMA_CMTT_SIGN_SHIFT2 25 + +#define RDMA_DMTT_SIGN_MASK 0x3ff + +#define RDMA_DMTT_ADD_SHIFT0 7 +#define RDMA_DMTT_SIGN_SHIFT0 3 +#define RDMA_DMTT_SIGN_SHIFT1 6 +#define RDMA_DMTT_SIGN_SHIFT2 16 +#define RDMA_DMTT_SIGN_SHIFT3 26 + +#endif + +#endif // HMM_COMP_H diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c new file mode 100644 index 000000000..7e913f221 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_init.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include <linux/module.h> +#include <linux/netdevice.h> +#include "hinic3_hw.h" +#include "hinic3_hw_cfg.h" +#include "hinic3_crm.h" + +#include "hmm_comp.h" +#include "hinic3_hmm.h" + +#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 扩展表为12K */ + +u32 g_mtt_page_size; +module_param(g_mtt_page_size, uint, 0444); +MODULE_PARM_DESC(g_mtt_page_size, "0:4K,1:64K,2:2M,default:4K"); + +static int hmm_init_table(void *hwdev, struct hmm_comp_priv *comp_priv, u32 srv_type) +{ + int ret; + + ret = hmm_init_mtt_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize mtt's table failed, ret(%d)\n", __func__, ret); + kfree(comp_priv); + return ret; + } + + ret = hinic3_register_service_adapter((void *)hwdev, (void *)comp_priv, + (enum hinic3_service_type)srv_type); + if (ret != 0) { + pr_err("%s: put hmm_comp_res failed, ret(%d)\n", __func__, ret); + goto err_init; + } + pr_info("%s: Hmm init resource successful\n", __func__); + return 0; + +err_init: + hmm_cleanup_mtt_table(comp_priv); + kfree(comp_priv); + return ret; +} + +void hmm_cleanup_resource(void *hwdev, u32 service_type) +{ + struct rdma_service_cap rdma_cap; + struct hmm_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + if (!hinic3_support_rdma(hwdev, &rdma_cap)) { + pr_err("%s: Not support rdma service\n", __func__); + return; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + hmm_cleanup_mtt_table(comp_priv); + + kfree(comp_priv); + + hinic3_unregister_service_adapter((void *)hwdev, (enum hinic3_service_type)service_type); + + pr_info("%s: Rdma cleanup resource successful", __func__); +} + +int hmm_init_resource(void *hwdev, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct rdma_service_cap rdma_cap; + int ret; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + if (!hinic3_support_rdma(hwdev, &rdma_cap)) { + pr_info("%s: Don't support hmm dev\n", __func__); + return 0; + } + comp_priv = kzalloc(sizeof(struct hmm_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) + return -ENOMEM; + comp_priv->hwdev = hwdev; + comp_priv->pdev = (struct pci_dev *)((struct hinic3_hwdev *)hwdev)->pcidev_hdl; + comp_priv->rdma_cap.log_mtt = rdma_cap.log_mtt; + comp_priv->rdma_cap.log_mtt_seg = rdma_cap.log_mtt_seg; + comp_priv->rdma_cap.mtt_entry_sz = rdma_cap.mtt_entry_sz; + comp_priv->rdma_cap.mpt_entry_sz = rdma_cap.mpt_entry_sz; + comp_priv->rdma_cap.num_mtts = rdma_cap.num_mtts; + + comp_priv->rdma_cap.dmtt_cl_start = rdma_cap.dev_rdma_cap.roce_own_cap.dmtt_cl_start; + comp_priv->rdma_cap.dmtt_cl_end = rdma_cap.dev_rdma_cap.roce_own_cap.dmtt_cl_end; + comp_priv->rdma_cap.dmtt_cl_sz = rdma_cap.dev_rdma_cap.roce_own_cap.dmtt_cl_sz; + + switch (g_mtt_page_size) { + case RDMA_MTT_PAGE_SIZE_4K: + comp_priv->mtt_page_size = PAGE_SIZE_4k; /* page size is 4K */ + comp_priv->mtt_page_shift = PAGE_SHIFT_4K; /* page size is 1 left shift 12 */ + break; + case RDMA_MTT_PAGE_SIZE_64K: + comp_priv->mtt_page_size = PAGE_SIZE_64k; /* page size is 64K */ + comp_priv->mtt_page_shift = PAGE_SHIFT_64K; /* page size is 1 left shift 16 */ + break; + case RDMA_MTT_PAGE_SIZE_2M: + comp_priv->mtt_page_size = PAGE_SIZE_2M; /* page size is 2M */ + comp_priv->mtt_page_shift = PAGE_SHIFT_2M; /* page size is 1 left shift 21 */ + break; + default: + comp_priv->mtt_page_size = PAGE_SIZE_4k; /* page size is 4K */ + comp_priv->mtt_page_shift = PAGE_SHIFT_4K; /* page size is 1 left shift 12 */ + break; + } + ret = hmm_init_table(hwdev, comp_priv, service_type); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c new file mode 100644 index 000000000..906ca476e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mtt.c @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "hinic3_rdma.h" +#include "hmm_comp.h" +#include "hmm_mr.h" + +static int hmm_set_mtt_layer(const struct hmm_comp_priv *comp_priv, + struct rdma_mtt *mtt, u32 npages) +{ + u32 one_layer_flag = 0; + u64 two_layer_flag = 0; + u64 three_layer_flag = 0; + + one_layer_flag = comp_priv->mtt_page_size / RDMA_PA_SIZE; + two_layer_flag = ((u64)one_layer_flag) * ((u64)one_layer_flag); + three_layer_flag = (u64)one_layer_flag * two_layer_flag; + + if (npages <= 1) { + mtt->mtt_layers = RDMA_MTT_ZERO_LAYER; + return 0; + } else if (npages <= one_layer_flag) { + mtt->mtt_layers = RDMA_MTT_ONE_LAYER; + } else if (npages <= two_layer_flag) { + mtt->mtt_layers = RDMA_MTT_TWO_LAYER; + } else if ((u64)npages <= three_layer_flag) { + mtt->mtt_layers = RDMA_MTT_THREE_LAYER; + } else { + pr_err("%s: Npages(0x%x) over range, ret(%d)\n", __func__, npages, -EINVAL); + return -EINVAL; + } + + return 0; +} + +#ifdef RDMA_SIGN_MTT_EN + +u16 hmm_gen_cmtt_sign(u64 mtt_base_gpa) +{ + u16 sign0 = (mtt_base_gpa >> RDMA_CMTT_SIGN_SHIFT0) & RDMA_CMTT_SIGN_MASK; + u16 sign1 = (mtt_base_gpa >> RDMA_CMTT_SIGN_SHIFT1) & RDMA_CMTT_SIGN_MASK; + u16 sign2 = (mtt_base_gpa >> RDMA_CMTT_SIGN_SHIFT2) & RDMA_CMTT_SIGN_MASK; + u16 cmtt_sign = ~(sign0 ^ sign1 ^ sign2); + + cmtt_sign &= RDMA_CMTT_SIGN_MASK; + return cmtt_sign; +} + +u16 hmm_gen_dmtt_sign(u64 mtt_base_gpa) +{ + u16 sign0 = ((u16)(mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT0) << + RDMA_DMTT_ADD_SHIFT0) & RDMA_DMTT_SIGN_MASK; + u16 sign1 = (mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT1) & RDMA_DMTT_SIGN_MASK; + u16 sign2 = (mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT2) & RDMA_DMTT_SIGN_MASK; + u16 sign3 = (mtt_base_gpa >> RDMA_DMTT_SIGN_SHIFT3) & RDMA_DMTT_SIGN_MASK; + u16 dmtt_sign = ~(sign0 ^ sign1 ^ sign2 ^ sign3); + + dmtt_sign &= RDMA_DMTT_SIGN_MASK; + return dmtt_sign; +} + + +u64 hmm_gen_mtt_sign(u64 mtt_base_gpa, enum mtt_data_type_e type) +{ + if (type == MTT_CMTT_TYPE) + return hmm_gen_cmtt_sign(mtt_base_gpa); + + return (u64)hmm_gen_dmtt_sign(mtt_base_gpa) << 1; +} + +#endif + +static int hmm_find_mtt_page_list(struct hmm_comp_priv *comp_priv, struct rdma_mtt_seg *mtt_seg, + u32 npages, u64 *page_list) +{ + void *vaddr = NULL; + u32 i = 0; + u32 mtt_index = 0; + u32 mtts_per_page = 0; + + mtts_per_page = comp_priv->mtt_page_size / RDMA_PA_SIZE; + if ((mtt_seg->offset % mtts_per_page) != 0) { + pr_err("%s: First mtt isn't in the head of page, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + mtt_index = mtt_seg->offset; + for (i = 0; i < npages; i++) { + vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, mtt_index, &page_list[i]); + if (vaddr == NULL) { + pr_err("%s: Can't find va and pa of mtt entry, ret(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + mtt_index += comp_priv->mtt_page_size / RDMA_PA_SIZE; + } + + return 0; +} + +static int hmm_write_mtt_chunk(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, + u32 mtt_level_index, u32 start_index, u32 npages, const u64 *page_list) +{ + u32 i = 0; + u16 sign_val = 0; + __be64 *mtts = NULL; + + mtts = (__be64 *)hmm_em_table_find(&comp_priv->mtt_em_table, + mtt->mtt_seg[mtt_level_index]->offset + start_index, NULL); + if (mtts == NULL) { + pr_err("%s: Can't find va and pa of mtt entry, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } +#ifdef RDMA_SIGN_MTT_EN + sign_val = hmm_gen_mtt_sign(mtt->mtt_paddr, mtt->mtt_type); +#endif + for (i = 0; i < npages; i++) + mtts[i] = cpu_to_be64(page_list[i] | RDMA_MTT_PA_VALID | (sign_val << 1)); + + return 0; +} + +static int hmm_write_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, + u32 mtt_level_index, u32 start_index, u32 npages, u64 *page_list) +{ + int ret = 0; + u32 chunk = 0; + u32 mtts_per_page = 0; + u32 max_mtts_first_page = 0; + u32 tmp_npages = npages; + u32 tmp_start_index = start_index; + u64 *tmp_page_list = page_list; + + /* calculate how may mtts fit in the first page */ + mtts_per_page = comp_priv->mtt_page_size / RDMA_PA_SIZE; + max_mtts_first_page = mtts_per_page - ((mtt->mtt_seg[mtt_level_index]->offset + + tmp_start_index) % mtts_per_page); + + chunk = (tmp_npages < max_mtts_first_page) ? tmp_npages : max_mtts_first_page; + + while ((int)tmp_npages > 0) { + ret = hmm_write_mtt_chunk(comp_priv, mtt, mtt_level_index, + tmp_start_index, chunk, tmp_page_list); + if (ret != 0) { + pr_err("%s: Write mtt chunk failed, ret(%d)\n", __func__, ret); + return ret; + } + + tmp_npages -= chunk; + tmp_start_index += chunk; + tmp_page_list += chunk; + + chunk = (tmp_npages < mtts_per_page) ? tmp_npages : mtts_per_page; + } + + return 0; +} + +static int hmm_alloc_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt_seg *mtt_seg) +{ + int ret = 0; + u32 seg_offset = 0; + u32 seg_order = 0; + u32 log_mtts_per_seg = 0; + + log_mtts_per_seg = comp_priv->rdma_cap.log_mtt_seg; + + seg_order = (mtt_seg->order > log_mtts_per_seg) ? (mtt_seg->order - log_mtts_per_seg) : 0; + mtt_seg->order = seg_order + log_mtts_per_seg; + + seg_offset = hmm_buddy_alloc(&comp_priv->mtt_buddy, seg_order); + if (seg_offset == HMM_INVALID_INDEX) { + pr_err("%s: Alloc mtt index failed\n", __func__); + return -ENOMEM; + } + + mtt_seg->offset = seg_offset << log_mtts_per_seg; + + ret = hmm_em_table_get_range(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_seg->offset, + mtt_seg->offset + (u32)(1U << mtt_seg->order) - 1); + if (ret != 0) { + pr_err("%s: Alloc mtt entry failed, ret(%d)\n", __func__, ret); + goto err_get_entry; + } + + mtt_seg->vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, mtt_seg->offset, + &mtt_seg->paddr); + if (mtt_seg->vaddr == NULL) { + pr_err("%s: Can't find start address of mtt_seg\n", __func__); + goto err_find_entry; + } + + return 0; + +err_find_entry: + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_seg->offset, + mtt_seg->offset + (u32)(1U << mtt_seg->order) - 1); + +err_get_entry: + hmm_buddy_free(&comp_priv->mtt_buddy, seg_offset, seg_order); + + return -ENOMEM; +} + +static void hmm_free_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt_seg *mtt_seg) +{ + u32 seg_offset = 0; + u32 seg_order = 0; + int log_mtts_per_seg = 0; + + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_seg->offset, + mtt_seg->offset + (1U << mtt_seg->order) - 1); + + log_mtts_per_seg = (int)comp_priv->rdma_cap.log_mtt_seg; + seg_order = mtt_seg->order - (u32)log_mtts_per_seg; + seg_offset = mtt_seg->offset >> (unsigned int)log_mtts_per_seg; + + hmm_buddy_free(&comp_priv->mtt_buddy, seg_offset, seg_order); +} + +static int hmm_init_mtt_seg(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, u32 npages) +{ + u32 i; + int ret; + + if ((comp_priv == NULL) || (mtt == NULL)) { + pr_err("%s: Comp_priv or mtt is null\n", __func__); + return -EINVAL; + } + + if (npages >= comp_priv->rdma_cap.num_mtts) { + pr_err("%s: Npages(0x%x) over range, ret(%d)\n", __func__, npages, -EINVAL); + return -EINVAL; + } + + ret = hmm_set_mtt_layer(comp_priv, mtt, npages); + if (ret != 0) + return ret; + + mtt->mtt_seg = kcalloc(mtt->mtt_layers, sizeof(struct rdma_mtt_seg *), GFP_KERNEL); + if (mtt->mtt_seg == NULL) + return -ENOMEM; + + for (i = 0; i < mtt->mtt_layers; i++) { + mtt->mtt_seg[i] = kzalloc(sizeof(struct rdma_mtt_seg), GFP_KERNEL); + if (mtt->mtt_seg[i] == NULL) + goto err_out; + } + + return 0; + +err_out: + for (i = 0; i < mtt->mtt_layers; i++) { + kfree(mtt->mtt_seg[i]); + mtt->mtt_seg[i] = NULL; + } + + kfree(mtt->mtt_seg); + mtt->mtt_seg = NULL; + + return -ENOMEM; +} + +static int hmm_rdma_mtt_alloc_prepare(void *hwdev, u32 npages, struct rdma_mtt *mtt, + struct hmm_comp_priv **comp_priv, u32 service_type) +{ + int ret = 0; + + if ((hwdev == NULL) || (mtt == NULL)) { + pr_err("%s: Hwdev or mtt is null\n", __func__); + return -EINVAL; + } + + *comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (*comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + ret = hmm_init_mtt_seg(*comp_priv, mtt, npages); + if (ret != 0) { + pr_err("%s: Initialize mtt_seg failed, ret(%d)\n", __func__, ret); + return ret; + } + + return ret; +} + +static int hmm_enable_mtt_related(struct hmm_comp_priv *comp_priv, struct rdma_mtt *mtt, + u32 low_layer_index) +{ + u64 *page_list = NULL; + struct rdma_mtt_seg *low_mtt_seg = NULL; + u32 npages = 0; + int ret = 0; + + low_mtt_seg = mtt->mtt_seg[low_layer_index]; + npages = (u32)((1UL << low_mtt_seg->order) / (comp_priv->mtt_page_size / RDMA_PA_SIZE)); + page_list = kzalloc(npages * RDMA_PA_SIZE, GFP_KERNEL); + if (page_list == NULL) + return -ENOMEM; + + ret = hmm_find_mtt_page_list(comp_priv, low_mtt_seg, npages, page_list); + if (ret != 0) { + pr_err("%s: Can't find page_list of mtt_seg, ret(%d)\n", __func__, ret); + goto out; + } + + ret = hmm_write_mtt_seg(comp_priv, mtt, low_layer_index + 1, 0, npages, page_list); + if (ret != 0) { + pr_err("%s: Write mtt_seg failed, ret(%d)\n", __func__, ret); + goto out; + } + +out: + kfree(page_list); + + return ret; +} + +static void hmm_cleanup_mtt_seg(struct rdma_mtt *mtt) +{ + u32 i = 0; + + for (i = 0; i < mtt->mtt_layers; i++) { + kfree(mtt->mtt_seg[i]); + mtt->mtt_seg[i] = NULL; + } + + kfree(mtt->mtt_seg); + mtt->mtt_seg = NULL; +} + +int hmm_rdma_mtt_alloc(void *hwdev, u32 npages, u32 page_shift, struct rdma_mtt *mtt, + u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + int ret = 0; + u32 i = 0; + u32 cur_layer = 0; + u32 order = 0; + u32 tmp_npages = npages; + + ret = hmm_rdma_mtt_alloc_prepare(hwdev, npages, mtt, &comp_priv, service_type); + if (ret != 0) + return ret; + + for (cur_layer = 1; cur_layer <= mtt->mtt_layers; cur_layer++) { + tmp_npages = (tmp_npages < HMM_MTT_NUM_PER_CACHELINE) ? + HMM_MTT_NUM_PER_CACHELINE : tmp_npages; + for (i = 1; i < tmp_npages; i <<= 1) + order++; + + mtt->mtt_seg[cur_layer - 1]->order = order; + ret = hmm_alloc_mtt_seg(comp_priv, mtt->mtt_seg[cur_layer - 1]); + if (ret != 0) { + pr_err("%s: Alloc mtt_seg failed, npages(%d), ret(%d)\n", + __func__, tmp_npages, ret); + goto err_out; + } + + tmp_npages = (u32)(1U << mtt->mtt_seg[cur_layer - 1]->order) / + (comp_priv->mtt_page_size / RDMA_PA_SIZE); + order = 0; + } + if (mtt->mtt_layers > 0) { + mtt->mtt_vaddr = (__be64 *)mtt->mtt_seg[mtt->mtt_layers - 1]->vaddr; + mtt->mtt_paddr = mtt->mtt_seg[mtt->mtt_layers - 1]->paddr; + } + for (i = 1; i < mtt->mtt_layers; i++) { + ret = hmm_enable_mtt_related(comp_priv, mtt, i - 1); + if (ret != 0) { + pr_err("%s: Cant't get multi mtt_seg related, i(%d), ret(%d)\n", + __func__, i, ret); + goto err_out; + } + } + mtt->buf_page_shift = page_shift; + mtt->mtt_page_shift = comp_priv->mtt_page_shift; + return 0; +err_out: + for (i = cur_layer - 1; i > 0; i--) + hmm_free_mtt_seg(comp_priv, mtt->mtt_seg[i - 1]); + + hmm_cleanup_mtt_seg(mtt); + return -ENOMEM; +} + +void hmm_rdma_mtt_free(void *hwdev, struct rdma_mtt *mtt, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + u32 i = 0; + + if ((hwdev == NULL) || (mtt == NULL)) { + pr_err("%s: Hwdev or mtt is null\n", __func__); + return; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + if (mtt->mtt_layers == 0) + return; + + for (i = 0; i < mtt->mtt_layers; i++) + hmm_free_mtt_seg(comp_priv, mtt->mtt_seg[i]); + + hmm_cleanup_mtt_seg(mtt); +} + +int hmm_rdma_write_mtt(void *hwdev, struct rdma_mtt *mtt, u32 start_index, u32 npages, + u64 *page_list, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mtt == NULL)) { + pr_err("%s: Hwdev or mtt is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (mtt->mtt_layers == 0) { + mtt->mtt_paddr = page_list[0]; + return 0; + } + + ret = hmm_write_mtt_seg(comp_priv, mtt, 0, start_index, npages, page_list); + if (ret != 0) { + pr_err("%s: Write mtt seg failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +int hmm_init_mtt_table(struct hmm_comp_priv *comp_priv) +{ + int ret = 0; + u32 i = 0; + u32 max_order = 0; + u32 mtt_num = 0; + u32 mtt_size = 0; + u32 log_mtts_per_seg = 0; + + mtt_num = comp_priv->rdma_cap.num_mtts; + log_mtts_per_seg = comp_priv->rdma_cap.log_mtt_seg; + mtt_size = comp_priv->rdma_cap.mtt_entry_sz; + + for (i = 1; i < mtt_num; i <<= 1) + max_order++; + + max_order = (max_order > log_mtts_per_seg) ? (max_order - log_mtts_per_seg) : 0; + + ret = hmm_buddy_init(&comp_priv->mtt_buddy, max_order); + if (ret != 0) { + pr_err("%s: Initialize mtt's buddy failed, ret(%d)\n", __func__, ret); + return ret; + } + + ret = hmm_em_init_table(comp_priv->pdev, &comp_priv->mtt_em_table, mtt_size, mtt_num, 0, + (int)(comp_priv->mtt_page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + pr_err("%s: Initialize mtt's em_table failed, ret(%d)\n", __func__, ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->mtt_buddy); + + return ret; +} + +void hmm_cleanup_mtt_table(struct hmm_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->pdev, &comp_priv->mtt_em_table); + + hmm_buddy_cleanup(&comp_priv->mtt_buddy); +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c new file mode 100644 index 000000000..fce8f5c01 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_mw_mr.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "hmm_comp.h" + +static void hmm_roce_mpt_to_big_endian(struct roce_mpt_context *mpt_ctx) +{ + mpt_ctx->dw0.value = cpu_to_be32(mpt_ctx->dw0.value); + mpt_ctx->dw1.value = cpu_to_be32(mpt_ctx->dw1.value); + mpt_ctx->dw2.value = cpu_to_be32(mpt_ctx->dw2.value); + mpt_ctx->dw3.value = cpu_to_be32(mpt_ctx->dw3.value); + mpt_ctx->iova = cpu_to_be64(mpt_ctx->iova); + mpt_ctx->length = cpu_to_be64(mpt_ctx->length); + mpt_ctx->mtt_base_addr = cpu_to_be64(mpt_ctx->mtt_base_addr); + mpt_ctx->mtt_sz = cpu_to_be32(mpt_ctx->mtt_sz); +} + +static void hmm_set_roce_mr_access(struct roce_mpt_context *mpt_ctx, const struct rdma_mr *mr) +{ + mpt_ctx->dw0.bs.access_lr = 1; /* Local access enabled by default */ + + if ((RDMA_IB_ACCESS_LOCAL_WRITE & mr->access) != 0) + mpt_ctx->dw0.bs.access_lw = 1; + + if ((RDMA_IB_ACCESS_REMOTE_READ & mr->access) != 0) + mpt_ctx->dw0.bs.access_rr = 1; + + if ((RDMA_IB_ACCESS_REMOTE_WRITE & mr->access) != 0) + mpt_ctx->dw0.bs.access_rw = 1; + + if ((RDMA_IB_ACCESS_REMOTE_ATOMIC & mr->access) != 0) + mpt_ctx->dw0.bs.access_ra = 1; + + if ((RDMA_IB_ACCESS_MW_BIND & mr->access) != 0) + mpt_ctx->dw0.bs.access_bind = 1; +} + +static void hmm_set_mptc_type_above_phy_mr(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + switch (mr->mr_type) { + case RDMA_PHYS_MR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_RSVD_LKEY: + mpt_ctx->dw0.bs.rkey = 1; + mpt_ctx->dw0.bs.bpd = 0; + mpt_ctx->dw0.bs.invalid_en = 0; + mpt_ctx->dw0.bs.remote_invalid_en = 0; + mpt_ctx->dw0.bs.pa = 1; + mpt_ctx->mtt_base_addr = 0; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_SIG_MR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + break; + + case RDMA_INDIRECT_MR: + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + break; + default: + pr_err("%s: RoCE unsupport mr type(%d)\n", __func__, mr->mr_type); + break; + } +} + +static void hmm_set_mptc_type_below_phy_mr(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + switch (mr->mr_type) { + case RDMA_DMA_MR: + mpt_ctx->dw0.bs.pa = 1; + mpt_ctx->mtt_base_addr = 0; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_USER_MR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + + case RDMA_FRMR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw0.bs.fast_reg_en = 1; + mpt_ctx->dw0.bs.remote_access_en = 1; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + mpt_ctx->mtt_sz = (mr->mtt.mtt_layers > 0) ? 1U << + mr->mtt.mtt_seg[mr->mtt.mtt_layers - 1]->order : 0; + break; + + case RDMA_FMR: + mpt_ctx->mtt_base_addr = mr->mtt.mtt_paddr; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + break; + default: + pr_err("%s: RoCE unsupport mr type(%d)\n", __func__, mr->mr_type); + break; + } +} + +static void hmm_set_mptc_according_to_mr_type(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + if (mr->mr_type < RDMA_PHYS_MR) + hmm_set_mptc_type_below_phy_mr(mpt_ctx, mr); + else + hmm_set_mptc_type_above_phy_mr(mpt_ctx, mr); +} + +static void hmm_set_roce_mr_cmd_buf(struct roce_mpt_context *mpt_ctx, struct rdma_mr *mr) +{ + hmm_set_roce_mr_access(mpt_ctx, mr); + + mpt_ctx->dw0.bs.invalid_en = 1; + mpt_ctx->dw0.bs.remote_invalid_en = 1; + mpt_ctx->dw0.bs.r_w = RDMA_MPT_MR; + mpt_ctx->dw0.bs.bpd = 1; + mpt_ctx->dw2.bs.pdn = mr->pdn & 0x3ffff; + + if (mr->mr_type != RDMA_INDIRECT_MR) { + mpt_ctx->dw0.bs.mtt_page_size = + (mr->mtt.mtt_page_shift > PAGE_SHIFT_4K) ? + (mr->mtt.mtt_page_shift - PAGE_SHIFT_4K) : 0; + mpt_ctx->dw0.bs.mtt_layer_num = mr->mtt.mtt_layers; + mpt_ctx->dw0.bs.buf_page_size = + (mr->mtt.buf_page_shift > PAGE_SHIFT_4K) ? + (mr->mtt.buf_page_shift - PAGE_SHIFT_4K) : 0; + mpt_ctx->dw1.bs.dma_attr_idx = RDMA_MPT_DMA_ATTR_IDX; + mpt_ctx->dw1.bs.so_ro = 0; + mpt_ctx->dw2.bs.block_size = (mr->block_size / BLOCK_SIZE_DEVIDE_SECTOR) & 0x3f; + if (mr->block_size > 0) + mpt_ctx->dw3.bs.page_mode = 1; + + mpt_ctx->iova = mr->iova; + mpt_ctx->length = mr->size; + mpt_ctx->dw3.bs.fbo = 0; + if ((mr->access & RDMA_IB_ACCESS_ZERO_BASED) != 0) { + mpt_ctx->dw0.bs.zbva = 1; + mpt_ctx->dw3.bs.fbo = mr->iova & PAGE_MASK; + mpt_ctx->iova = 0; + } + } else { + mpt_ctx->dw2.bs.indirect_mr = 1; + } + + mpt_ctx->dw3.bs.mkey = mr->key & 0xFF; + + hmm_set_mptc_according_to_mr_type(mpt_ctx, mr); + hmm_roce_mpt_to_big_endian(mpt_ctx); +} + +int hmm_rdma_disable_mr_mpt(void *hwdev, struct rdma_mr *mr, u32 service_type, u16 channel) +{ + struct hmm_comp_priv *comp_priv = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mr == NULL)) { + pr_err("%s: Hwdev or mr is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (mr->enabled == RDMA_MPT_EN_HW) { + ret = hmm_disable_roce_mpt(comp_priv, &mr->mpt, channel); + if (ret != 0) { + pr_err("%s: Disable mr's mpt failed, ret(%d)\n", __func__, ret); + return ret; + } + + mr->enabled = RDMA_MPT_EN_SW; + } + return 0; +} + +int hmm_rdma_enable_mr_mpt(void *hwdev, struct rdma_mr *mr, u16 channel) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct rdma_mpt_entry *mpt_entry = NULL; + struct rdma_mpt_sw2hw_inbuf *mpt_sw2hw_inbuf = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mr == NULL)) { + pr_err("%s: Hwdev or mr is null\n", __func__); + return -EINVAL; + } + + cqm_cmd_inbuf = cqm_cmd_alloc(hwdev); + if (cqm_cmd_inbuf == NULL) { + pr_err("%s: Alloc cmd_buf failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + cqm_cmd_inbuf->size = (u16)sizeof(struct rdma_mpt_sw2hw_inbuf); + mpt_sw2hw_inbuf = (struct rdma_mpt_sw2hw_inbuf *)cqm_cmd_inbuf->buf; + memset(mpt_sw2hw_inbuf, 0, sizeof(*mpt_sw2hw_inbuf)); + mpt_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = (u16)cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); + mpt_sw2hw_inbuf->com.index = cpu_to_be32(mr->mpt.mpt_index); + mpt_entry = &mpt_sw2hw_inbuf->mpt_entry; + + hmm_set_roce_mr_cmd_buf(&mpt_entry->roce_mpt_ctx, mr); + ret = hmm_enable_roce_mpt(hwdev, cqm_cmd_inbuf, channel); + if (ret != 0) { + pr_err("%s: Enable mr's mpt failed, ret(%d)\n", __func__, ret); + goto out; + } + mr->enabled = RDMA_MPT_EN_HW; +out: + cqm_cmd_free(hwdev, cqm_cmd_inbuf); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c new file mode 100644 index 000000000..84f4c37d9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_comp_res.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "hmm_comp.h" + +int hmm_rdma_mpt_alloc(void *hwdev, struct rdma_mpt *mpt, u32 service_type) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct rdma_mpt_entry *mpt_entry = NULL; + u32 mpt_entry_size = 0; + + if ((hwdev == NULL) || (mpt == NULL)) { + pr_err("%s: Hwdev or mpt is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(hwdev, service_type); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + mpt_entry_size = comp_priv->rdma_cap.mpt_entry_sz; + + mpt->mpt_object = + (void *)cqm_object_qpc_mpt_create(hwdev, service_type, CQM_OBJECT_MPT, + mpt_entry_size, mpt, CQM_INDEX_INVALID, false); + if (mpt->mpt_object == NULL) { + pr_err("%s: Alloc mpt_object failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + mpt->mpt_index = ((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->xid; + mpt->vaddr = (void *)((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->vaddr; + if (!cqm_need_secure_mem(hwdev)) { + memset(mpt->vaddr, 0, sizeof(struct rdma_mpt_entry)); + + mpt_entry = (struct rdma_mpt_entry *)mpt->vaddr; + mpt_entry->roce_mpt_ctx.dw2.bs.status = RDMA_MPT_STATUS_MEM_INIT; + mpt_entry->roce_mpt_ctx.dw2.value = cpu_to_be32(mpt_entry->roce_mpt_ctx.dw2.value); + } + return 0; +} + + +void hmm_rdma_mpt_free(void *hwdev, struct rdma_mpt *mpt) +{ + if ((hwdev == NULL) || (mpt == NULL)) { + pr_err("%s: Hwdev or mpt is null\n", __func__); + return; + } + hiudk_cqm_object_delete(hwdev, &((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->object); + mpt->vaddr = NULL; + mpt->mpt_object = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c new file mode 100644 index 000000000..6830725c4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include <linux/mm.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> + +#include "hmm_em.h" + +static void hmm_em_chunk_free(struct pci_dev *pdev, struct hmm_em_chunk *em_chunk) +{ + struct hmm_em_buf *cur_buf = NULL; + struct hmm_em_buf *next_buf = NULL; + + cur_buf = &em_chunk->em_buf_list; + next_buf = cur_buf->next_buf; + + while (next_buf != NULL) { + cur_buf = next_buf; + next_buf = cur_buf->next_buf; + + if ((cur_buf->buf != NULL) && (cur_buf->length > 0)) { + memset(cur_buf->buf, 0, cur_buf->length); + dma_free_coherent(&pdev->dev, (unsigned long)cur_buf->length, + cur_buf->buf, cur_buf->dma_addr); + cur_buf->buf = NULL; + cur_buf->length = 0; + } + + kfree(cur_buf); + cur_buf = NULL; + } + + kfree(em_chunk); +} + +static int hmm_em_chunk_alloc_npages(struct pci_dev *pdev, struct hmm_em_chunk *em_chunk, + int min_order) +{ + int cur_order = 0; + int npages = 0; + unsigned int chunk_size = HMM_EM_CHUNK_SIZE; + struct hmm_em_buf *cur_buf = &em_chunk->em_buf_list; + struct hmm_em_buf *next_buf = NULL; + + cur_buf->next_buf = NULL; + cur_buf->length = 0; + cur_order = get_order(chunk_size); //lint !e834 !e587 + npages = (int)(1U << (unsigned int)cur_order); + while (npages > 0) { + if (next_buf == NULL) { + next_buf = kzalloc(sizeof(struct hmm_em_buf), GFP_KERNEL); + if (next_buf == NULL) + return (-ENOMEM); + + next_buf->length = 0; + next_buf->next_buf = NULL; + } + cur_buf->next_buf = next_buf; + + next_buf->buf = dma_alloc_coherent(&pdev->dev, (size_t)HMM_EM_PAGE_SIZE << + (unsigned int)cur_order, &next_buf->dma_addr, GFP_KERNEL); + if (next_buf->buf == NULL) { + cur_order--; + if (cur_order < min_order) { + dev_err(&pdev->dev, + "[HMM] %s:em_chunk alloc dma buf failed, err(%d)\n", + __func__, -ENOMEM); + return (-ENOMEM); + } + dev_err(&pdev->dev, + "[HMM, WARN] %s: em_chunk alloc %d dma failed, alloc small mem\n", + __func__, cur_order); + continue; + } + + next_buf->length = (u32)HMM_EM_PAGE_SIZE << (unsigned int)cur_order; + em_chunk->buf_num++; + npages -= (int)(1U << (unsigned int)cur_order); + + cur_buf = next_buf; + next_buf = NULL; + } + + return 0; +} +static struct hmm_em_chunk *hmm_em_chunk_alloc(struct pci_dev *pdev, int min_order) +{ + struct hmm_em_chunk *em_chunk = NULL; + int ret; + + em_chunk = kzalloc(sizeof(struct hmm_em_chunk), GFP_KERNEL); + if (em_chunk == NULL) + return (struct hmm_em_chunk *)ERR_PTR((long)-ENOMEM); + + em_chunk->buf_num = 0; + ret = hmm_em_chunk_alloc_npages(pdev, em_chunk, min_order); + if (ret != 0) { + hmm_em_chunk_free(pdev, em_chunk); + return (struct hmm_em_chunk *)ERR_PTR((long)ret); + } + + em_chunk->refcount = 0; + + return em_chunk; +} + +static void hmm_em_table_put(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 obj) +{ + u32 i = 0; + + if (obj >= em_table->obj_num) { + dev_err(&pdev->dev, "[HMM] %s: Obj over range, obj(0x%x), max(0x%x)\n", + __func__, obj, em_table->obj_num - 1); + return; + } + + i = obj / (HMM_EM_CHUNK_SIZE / em_table->obj_size); + + mutex_lock(&em_table->mutex); + + if ((em_table->em_chunk[i] == NULL) || (IS_ERR(em_table->em_chunk[i]))) { + dev_err(&pdev->dev, "[HMM] %s: Em_table->em_chunk[%d] not alloced, obj(0x%x)\n", + __func__, i, obj); + mutex_unlock(&em_table->mutex); + return; + } + + if (em_table->em_chunk[i]->refcount == 1) { + em_table->em_chunk[i]->refcount = 0; + hmm_em_chunk_free(pdev, em_table->em_chunk[i]); + em_table->em_chunk[i] = NULL; + } else { + --em_table->em_chunk[i]->refcount; + } + + mutex_unlock(&em_table->mutex); +} + +static int hmm_em_table_get(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 obj) +{ + int ret = 0; + u32 i; + + if (obj >= em_table->obj_num) { + dev_err(&pdev->dev, "[HMM] %s: Obj over range, obj(0x%x), max(0x%x)\n", + __func__, obj, em_table->obj_num - 1); + return -EINVAL; + } + + i = obj / (HMM_EM_CHUNK_SIZE / em_table->obj_size); + + mutex_lock(&em_table->mutex); + + if (em_table->em_chunk[i]) { + ++em_table->em_chunk[i]->refcount; + goto out; + } + + em_table->em_chunk[i] = hmm_em_chunk_alloc(pdev, em_table->min_order); + if (IS_ERR(em_table->em_chunk[i])) { + ret = (int)PTR_ERR(em_table->em_chunk[i]); + dev_err(&pdev->dev, "[HMM] %s: Alloc em_chunk failed, ret(%d)\n", + __func__, ret); + goto out; + } + + ++em_table->em_chunk[i]->refcount; + +out: + mutex_unlock(&em_table->mutex); + + return ret; +} + +void *hmm_em_table_find(struct hmm_em_table *em_table, u32 obj, dma_addr_t *dma_handle) +{ + void *vaddr = NULL; + struct hmm_em_chunk *em_chunk = NULL; + struct hmm_em_buf *cur_buf = NULL; + struct hmm_em_buf *next_buf = NULL; + u64 table_offset; + u32 offset; + + if (em_table == NULL) { + pr_err("%s: Em_table is null, err(%d)\n", __func__, -EINVAL); + return NULL; + } + + if (obj >= em_table->obj_num) { + pr_err("%s: Obj over range, obj(0x%x), max(0x%x)\n", + __func__, obj, em_table->obj_num - 1); + return NULL; + } + + mutex_lock(&em_table->mutex); + + table_offset = (u64)obj * em_table->obj_size; + em_chunk = em_table->em_chunk[table_offset / HMM_EM_CHUNK_SIZE]; + offset = table_offset % HMM_EM_CHUNK_SIZE; + + if (em_chunk == NULL) { + pr_err("%s: Em_chunk has not been alloced, err(%d)\n", __func__, -EINVAL); + goto err_out; + } + + cur_buf = &em_chunk->em_buf_list; + next_buf = cur_buf->next_buf; + + while (next_buf != NULL) { + cur_buf = next_buf; + if (offset < cur_buf->length) { + if (dma_handle) + *dma_handle = cur_buf->dma_addr + offset; + + vaddr = (void *)((char *)(cur_buf->buf) + offset); + mutex_unlock(&em_table->mutex); + return vaddr; + } + + offset -= cur_buf->length; + next_buf = cur_buf->next_buf; + } + +err_out: + mutex_unlock(&em_table->mutex); + + return NULL; +} + +void hmm_em_table_put_range(struct pci_dev *pdev, struct hmm_em_table *em_table, + u32 start, u32 end) +{ + int i = 0; + int inc = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null, err(%d)\n", + __func__, -EINVAL); + return; + } + + inc = (int)(HMM_EM_CHUNK_SIZE / em_table->obj_size); + for (i = (int)start; i <= (int)end; i += inc) + hmm_em_table_put(pdev, em_table, (u32)i); +} + +int hmm_em_table_get_range(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 start, u32 end) +{ + int ret = 0; + int i = 0; + int inc = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null, err(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + inc = (int)(HMM_EM_CHUNK_SIZE / em_table->obj_size); + + for (i = (int)start; i <= (int)end; i += inc) { + ret = hmm_em_table_get(pdev, em_table, (u32)i); + if (ret != 0) { + dev_err(&pdev->dev, + "[HMM] %s: Get entry failed, start(%d), end(%d), i(%d), ret(%d)\n", + __func__, start, end, i, ret); + goto err_out; + } + } + + return 0; + +err_out: + while (i > (int)start) { + i -= inc; + hmm_em_table_put(pdev, em_table, (u32)i); + } + + return ret; +} + +int hmm_em_init_table(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 obj_size, + u32 nobj, u32 reserved_bot, int min_order) +{ + u32 obj_per_chunk = 0; + u32 chunk_num = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null\n", __func__); + return -EINVAL; + } + + if (nobj == 0) { + dev_err(&pdev->dev, "[HMM] %s: Nobj is invalid\n", __func__); + return -EINVAL; + } + + /*lint -e587 */ + if (nobj != HMM_EM_ROUNDUP_POW_OF_TWO(nobj)) { + dev_err(&pdev->dev, "[HMM] %s: Obj isn't pow of two, nobj(0x%x)\n", + __func__, nobj); + return -EINVAL; + } + + if (obj_size != HMM_EM_ROUNDUP_POW_OF_TWO(obj_size)) { + dev_err(&pdev->dev, "[HMM] %s: Obj_size isn't pow of two, obj_size(0x%x)\n", + __func__, obj_size); + return -EINVAL; + } + /*lint +e587 */ + + obj_per_chunk = HMM_EM_CHUNK_SIZE / obj_size; + chunk_num = (nobj + obj_per_chunk - 1) / obj_per_chunk; + + em_table->em_chunk = kcalloc((size_t)chunk_num, + sizeof(struct hmm_em_chunk *), GFP_KERNEL); + if (em_table->em_chunk == NULL) + return -ENOMEM; + + em_table->chunk_num = chunk_num; + em_table->obj_num = nobj; + em_table->obj_size = obj_size; + em_table->min_order = min_order; + + mutex_init(&em_table->mutex); + + return 0; +} + +void hmm_em_cleanup_table(struct pci_dev *pdev, struct hmm_em_table *em_table) +{ + u32 i = 0; + + if ((pdev == NULL) || (em_table == NULL)) { + dev_err(&pdev->dev, "[HMM] %s: Pdev or em_table is null\n", __func__); + return; + } + + for (i = 0; i < em_table->chunk_num; i++) { + if (em_table->em_chunk[i]) { + hmm_em_chunk_free(pdev, em_table->em_chunk[i]); + em_table->em_chunk[i] = NULL; + } + } + + kfree(em_table->em_chunk); + em_table->em_chunk = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h new file mode 100644 index 000000000..328a28fef --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_em.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_EM_H +#define HMM_EM_H + +#include <linux/pci.h> +#include <linux/mutex.h> +#include "hmm_umem.h" + +#define HMM_EM_CHUNK_SIZE (1 << 21) +#define HMM_EM_PAGE_SIZE PAGE_SIZE // (4096UL) + +#define HMM_EM_ROUNDUP_POW_OF_TWO roundup_pow_of_two + +struct hmm_em_buf { + u32 length; + void *buf; + dma_addr_t dma_addr; + struct hmm_em_buf *next_buf; +}; + +struct hmm_em_chunk { + u32 buf_num; + u32 refcount; + struct hmm_em_buf em_buf_list; +}; + +struct hmm_em_table { + u32 chunk_num; + u32 obj_num; + u32 obj_size; + int min_order; + struct mutex mutex; + struct hmm_em_chunk **em_chunk; +}; + + +void *hmm_em_table_find(struct hmm_em_table *em_table, u32 obj, dma_addr_t *dma_handle); +void hmm_em_table_put_range(struct pci_dev *pdev, struct hmm_em_table *em_table, + u32 start, u32 end); +int hmm_em_table_get_range(struct pci_dev *pdev, struct hmm_em_table *em_table, u32 start, u32 end); +int hmm_em_init_table(struct pci_dev *pdev, struct hmm_em_table *em_table, + u32 obj_size, u32 nobj, u32 reserved_bot, int min_order); +void hmm_em_cleanup_table(struct pci_dev *pdev, struct hmm_em_table *em_table); + +#endif diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c new file mode 100644 index 000000000..3dd48a511 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.c @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/vmalloc.h> +#include <linux/semaphore.h> + +#include "hinic3_crm.h" +#include "hmm_umem.h" +#include "hmm_comp.h" +#include "hinic3_hmm.h" +#include "hmm_mr.h" + +/* + **************************************************************************** + Prototype : get_key_from_index + Description : mr key的计算算法，通过index移位计算得到 + Input : u32 mpt_index + Output : None +**************************************************************************** +*/ +static u32 get_key_from_index(u32 mpt_index) +{ + return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) | (mpt_index << MR_KEY_LEFT_SHIFT_OFS); +} + +/* + **************************************************************************** + Prototype : hmm_alloc_tpt + Description : alloc mpt and mtt + Input : struct hinic3_hwdev *hwdev + struct rdma_mr *mr + u32 npages + u32 page_shift + Output : None +**************************************************************************** +*/ +static int hmm_alloc_tpt(struct hinic3_hwdev *hwdev, struct rdma_mr *mr, + u32 npages, u32 page_shift, u32 service_type) +{ + int ret; + + ret = hmm_rdma_mpt_alloc(hwdev, &mr->mpt, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mpt, ret(%d), func_id(%d)\n", + __func__, __LINE__, ret, hinic3_global_func_id(hwdev)); + return ret; + } + mr->enabled = RDMA_MPT_EN_SW; + + /* npages = 0 or 1, means not need mtt */ + ret = hmm_rdma_mtt_alloc(hwdev, npages, page_shift, &mr->mtt, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mtt, ret(%d), func_id(%d)\n", + __func__, __LINE__, ret, hinic3_global_func_id(hwdev)); + goto err_alloc_mtt; + } + return 0; + +err_alloc_mtt: + hmm_rdma_mpt_free(hwdev, &mr->mpt); + mr->enabled = HMM_MPT_DISABLED; + + return ret; +} + +/* + **************************************************************************** + Prototype : hmm_free_tpt + Description : free mpt and mtt + Input : struct hinic3_hwdev *hwdev + struct rdma_mr *mr +**************************************************************************** +*/ +void hmm_free_tpt(void *hwdev, struct rdma_mr *mr, u32 service_type) +{ + hmm_rdma_mtt_free(hwdev, &mr->mtt, service_type); + hmm_rdma_mpt_free(hwdev, &mr->mpt); + mr->enabled = HMM_MPT_DISABLED; +} + +/* + **************************************************************************** + Prototype : hmm_set_rdma_mr + Description : set the member of rdma_mr + Input : struct rdma_mr *mr + enum rdma_mr_type mr_type + u32 pdn + u64 iova + u64 size + u32 access + Output : None +**************************************************************************** +*/ +static void hmm_set_rdma_mr(struct rdma_mr *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access) +{ + mr->iova = iova; + mr->size = size; + mr->pdn = pdn; + mr->access = access; + mr->key = get_key_from_index(mr->mpt.mpt_index); /* 由mpt index转换为key */ + mr->mr_type = mr_type; +} + +/* + **************************************************************************** + Prototype : hmm_alloc_mr + Description : register DMA_MR + Input : struct hinic3_hwdev *hwdev + enum rdma_mr_type mr_type + u32 max_num_sg + u32 service_type + Output : None +**************************************************************************** +*/ +struct hmm_mr *hmm_alloc_mr(struct hinic3_hwdev *hwdev, u32 pdn, enum rdma_mr_type mr_type, + u32 max_num_sg, u32 service_type, u16 channel) +{ + u32 access_flag; + int ret = 0; + struct hmm_mr *mr = NULL; + + if (hwdev == NULL) { + ret = -EINVAL; + pr_err("[HMM, ERR] %s(%d): dev is null\n", __func__, __LINE__); + goto err_out; + } +#ifndef PANGEA_V6 + if (mr_type != RDMA_DMA_MR && mr_type != RDMA_INDIRECT_MR) { +#else + if (mr_type != RDMA_DMA_MR) { +#endif + ret = -EINVAL; + pr_err("[HMM, ERR] %s(%d): mr_type is invalid\n", __func__, __LINE__); + goto err_out; + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + goto err_out; + } + + ret = hmm_alloc_tpt(hwdev->dev_hdl, &mr->rdmamr, 0, 0, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mpt and mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + access_flag = (RDMA_IB_ACCESS_REMOTE_READ | RDMA_IB_ACCESS_REMOTE_WRITE | + RDMA_IB_ACCESS_LOCAL_WRITE | RDMA_IB_ACCESS_REMOTE_ATOMIC); + + hmm_set_rdma_mr(&mr->rdmamr, mr_type, pdn, 0ULL, ROCE_DMA_MR_SIZE, access_flag); + + ret = hmm_rdma_enable_mr_mpt(hwdev->dev_hdl, &(mr->rdmamr), channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to enable mpt of DMA mr, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_enable_mpt; + } + + return mr; + +err_enable_mpt: + hmm_free_tpt(hwdev->dev_hdl, &mr->rdmamr, service_type); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct hmm_mr *)ERR_PTR((long)ret); +} + +static int hmm_umem_write_mtt_check(const void *hwdev, const struct rdma_mtt *mtt, + const struct hmm_umem *umem) +{ + if ((hwdev == NULL) || (mtt == NULL) || (umem == NULL)) { + pr_err("[HMM, ERR] %s(%d): hwdev or mtt or umem is null\n", __func__, __LINE__); + return -EINVAL; + } + return 0; +} + +static int hmm_umem_write_mtt_update(struct hinic3_hwdev *hwdev, struct rdma_mtt *mtt, + struct hmm_umem *umem, u64 *page_list, u32 service_type) +{ + int ret = 0; + int i = 0; + u32 j = 0; + u32 pages_in_chunk = 0; /* umem_chunk中单个内存块的页个数 */ + u32 npages = 0; /* 已经记录的页个数 */ + u32 start_index = 0; /* 要写入mtt的页 */ + struct scatterlist *sg = NULL; + u64 page_size = 0; + + page_size = BIT((unsigned int)umem->page_shift); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + /* cal page num in truck */ + pages_in_chunk = sg_dma_len(sg) >> mtt->buf_page_shift; + for (j = 0; j < pages_in_chunk; ++j) { + page_list[npages] = sg_dma_address(sg) + (page_size * j); + npages++; + + /* one page can hold (PAGE_SIZE / sizeof(u64)) addrs */ + if (npages == (PAGE_SIZE / sizeof(u64))) { + ret = hmm_rdma_write_mtt(hwdev, mtt, start_index, + npages, page_list, service_type); + start_index += npages; + npages = 0; + } + if ((npages == (PAGE_SIZE / sizeof(u64))) && (ret != 0)) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to write mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto out; + } + } + } + + if (npages != 0) { + ret = hmm_rdma_write_mtt(hwdev, mtt, start_index, npages, page_list, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, "[HMM, ERR] %s(%d): Failed to write mtt, ret(%d), start_index(%d), func_id(%d)\n", + __func__, __LINE__, ret, start_index, hinic3_global_func_id(hwdev)); + goto out; + } + } + +out: + kfree(page_list); + + return ret; +} + +/* + **************************************************************************** + Prototype : hmm_umem_write_mtt + Description : write mtt for umem(get from memory alloced by user) + Input : struct hinic3_hwdev *hwdev + struct rdma_mtt *mtt + struct hmm_umem *umem + Output : None +**************************************************************************** +*/ +int hmm_umem_write_mtt(struct hinic3_hwdev *hwdev, struct rdma_mtt *mtt, + struct hmm_umem *umem, u32 service_type) +{ + int ret; + u64 *page_list = NULL; /* 要写入mtt的page_list */ + + ret = hmm_umem_write_mtt_check(hwdev, mtt, umem); + if (ret != 0) + return ret; + + page_list = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) + return -ENOMEM; + ret = hmm_umem_write_mtt_update(hwdev, mtt, umem, page_list, service_type); + return ret; +} + +int hmm_reg_user_mr_update(struct hinic3_hwdev *hwdev, struct hmm_mr *mr, u32 pdn, + u64 length, u64 virt_addr, int access, u32 service_type, u16 channel) +{ + int ret = 0; + u32 npages = 0; + u32 page_shift = 0; + + if (hwdev == NULL) { + pr_err("[HMM, ERR] %s(%d): hwdev is null\n", __func__, __LINE__); + return 0; + } + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + npages = (u32)hmm_umem_page_count(mr->umem); + page_shift = (u32)(mr->umem->page_shift); + ret = hmm_alloc_tpt(hwdev, &mr->rdmamr, npages, page_shift, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to alloc mpt and mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + hmm_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length, (u32)access); + + ret = hmm_umem_write_mtt(hwdev, &mr->rdmamr.mtt, mr->umem, service_type); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to write mtt, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + ret = hmm_rdma_enable_mr_mpt(hwdev, &mr->rdmamr, channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to enable mpt of user mr, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_free_tpt(hwdev, &mr->rdmamr, service_type); + +err_alloc_tpt: + return ret; +} + +int hmm_dereg_mr_update(struct hinic3_hwdev *hwdev, struct rdma_mr *mr, + u32 service_type, u16 channel) +{ + int ret = 0; + + ret = hmm_rdma_disable_mr_mpt(hwdev, mr, service_type, channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to disable mpt of mr, ret(%d)\n", + __func__, __LINE__, ret); + return ret; + } + + hmm_free_tpt(hwdev, mr, service_type); + return ret; +} + +#ifndef ROCE_SERVICE +/* + **************************************************************************** + Prototype : hmm_reg_user_mr + Description : register MR for user + Input : struct hinic3_hwdev *hwdev + u64 start + u64 length + u64 virt_addr + int hmm_access + Output : None +**************************************************************************** +*/ +struct hmm_mr *hmm_reg_user_mr(struct hinic3_hwdev *hwdev, u64 start, u32 pd, + u64 length, u64 virt_addr, int hmm_access, u32 service_type, u16 channel) +{ + int ret = 0; + struct hmm_mr *mr = NULL; + + if (hwdev == NULL) { + pr_err("[HMM, ERR] %s(%d): hwdev is null\n", __func__, __LINE__); + goto err_out; + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + goto err_out; + } + + mr->hwdev = hwdev; + mr->rdmamr.iova = virt_addr; + mr->umem = hmm_umem_get(hwdev->dev_hdl, start, (size_t)length, hmm_access, 0); + if (IS_ERR(mr->umem)) { + ret = (int)PTR_ERR(mr->umem); + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to get ib umem, func_id(%d)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_empty; + } + + rcu_read_lock(); + mr->umem->context->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); + rcu_read_unlock(); + ret = hmm_reg_user_mr_update(hwdev, mr, pd, length, virt_addr, + hmm_access, service_type, channel); + if (ret != 0) + goto err_get_umem; + + return mr; + +err_get_umem: + hmm_umem_release(mr->umem); +err_empty: + kfree(mr); + +err_out: + return (struct hmm_mr *)ERR_PTR((long)ret); +} + +/* + **************************************************************************** + Prototype : hmm_dereg_mr + Description : dereg DMA_MR, user_MR or FRMR + Input : struct hmm_mr *mr + Output : None + +**************************************************************************** +*/ +int hmm_dereg_mr(struct hmm_mr *mr, u32 service_type, u16 channel) +{ + int ret = 0; + struct hinic3_hwdev *hwdev = NULL; + + if (mr == NULL) { + pr_err("[HMM, ERR] %s(%d): Ibmr is null\n", __func__, __LINE__); + return -EINVAL; + } + + hwdev = (struct hinic3_hwdev *)mr->hwdev; + ret = hmm_dereg_mr_update(hwdev, &(mr->rdmamr), service_type, channel); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[HMM, ERR] %s(%d): Failed to de-reg mr update, ret(%d), func_id(%d)\n", + __func__, __LINE__, ret, hinic3_global_func_id(hwdev)); + return ret; + } + + if (mr->umem) + hmm_umem_release(mr->umem); + + kfree(mr); + return ret; +} +#endif + diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h new file mode 100644 index 000000000..90a2b5e95 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_mr.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_MR_H +#define HMM_MR_H + +#include "hmm_umem.h" +#include "hmm_comp.h" + + +#define ROCE_DMA_MR_SIZE (~0ULL) +#define ROCE_FRMR_MAX_PAGES 512 +#define MR_KEY_RIGHT_SHIFT_OFS 24 +#define MR_KEY_LEFT_SHIFT_OFS 8 + +struct hmm_mr { + struct hmm_umem *umem; + struct rdma_mr rdmamr; + void *hwdev; +}; + +int hmm_rdma_enable_mr_mpt(void *hwdev, struct rdma_mr *mr, u16 channel); + +int hmm_rdma_disable_mr_mpt(void *hwdev, struct rdma_mr *mr, u32 service_type, u16 channel); + +void hmm_rdma_mpt_free(void *hwdev, struct rdma_mpt *mpt); + +int hmm_init_resource(void *hwdev, u32 service_type); + +void hmm_cleanup_resource(void *hwdev, u32 service_type); + +#endif // HMM_MR_H_ diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c new file mode 100644 index 000000000..c3a296568 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/mm.h> +#include <linux/dma-mapping.h> +#include <linux/signal.h> +#include <linux/sched/mm.h> +#include <linux/sched/signal.h> +#include <linux/hugetlb.h> +#include <linux/slab.h> +#include <linux/version.h> +#include "hinic3_rdma.h" +#include "hmm_umem.h" + +#ifndef ROCE_SERVICE +static void hmm_umemsg_release(struct device *device, struct hmm_umem *hmm_umem, int dirty) +{ + struct scatterlist *sg = NULL; + struct page *page = NULL; + int i; + + if (hmm_umem->nmap > 0) + dma_unmap_sg(device, hmm_umem->sg_head.sgl, hmm_umem->npages, DMA_BIDIRECTIONAL); + + for_each_sg(hmm_umem->sg_head.sgl, sg, hmm_umem->npages, i) { + page = sg_page(sg); + if (!PageDirty(page) && hmm_umem->writable && dirty) + set_page_dirty_lock(page); + put_page(page); + } + + sg_free_table(&hmm_umem->sg_head); +} + +/** + * hmm_umem_get - Pin and DMA map userspace memory. + * + * If access flags indicate ODP memory, avoid pinning. Instead, stores + * the mm for future page fault handling in conjunction with MMU notifiers. + * + * @context: userspace context to pin memory for + * @addr: userspace virtual address to start at + * @size: length of region to pin + * @access: RDMA_IB_ACCESS_xxx flags for memory being pinned + * @dmasync: flush in-flight DMA when the memory region is written + */ +struct hmm_umem *hmm_umem_get(struct device *device, unsigned long addr, + size_t size, int access, int dmasync) +{ + int ret; + int i; + struct hmm_umem *hmem = NULL; + struct page **page_list = NULL; + struct vm_area_struct **vma_list = NULL; + unsigned long locked_pages; + unsigned long lock_limit; + unsigned long current_base; + unsigned long npages; + +#ifdef HAVE_STRUCT_DMA_ATTRS + DEFINE_DMA_ATTRS(dma_attrs); +#else + unsigned long dma_attrs = 0; +#endif + struct scatterlist *sg = NULL, *sg_list_start = NULL; + int need_release = 0; +#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS + unsigned int gup_flags = FOLL_WRITE; +#endif + + /* + * If the combination of the addr and size requested for this memory + * region causes an integer overflow, return error. + */ + if (((addr + size) < addr) || PAGE_ALIGN(addr + size) < (addr + size)) + return ERR_PTR(-EINVAL); + + if (can_do_mlock() == 0) + return ERR_PTR(-EPERM); + + hmem = kzalloc(sizeof(*hmem), GFP_KERNEL); + if (hmem == NULL) + return ERR_PTR(-ENOMEM); + + hmem->context = kzalloc(sizeof(*(hmem->context)), GFP_KERNEL); + if ((hmem->context) == NULL) { + kfree(hmem); + return ERR_PTR(-ENOMEM); + } + hmem->context->device = device; + hmem->length = size; + hmem->address = addr; + hmem->page_shift = PAGE_SHIFT; + /* + * We ask for writable memory if any of the following + * access flags are set. "Local write" and "remote write" + * obviously require write access. "Remote atomic" can do + * things like fetch and add, which will modify memory, and + * "MW bind" can change permissions by binding a window. + */ + hmem->writable = !!(access & (RDMA_IB_ACCESS_LOCAL_WRITE | RDMA_IB_ACCESS_REMOTE_WRITE | + RDMA_IB_ACCESS_REMOTE_ATOMIC | RDMA_IB_ACCESS_MW_BIND)); + + if ((access & RDMA_IB_ACCESS_ON_DEMAND) != 0) { + kfree(hmem->context); + kfree(hmem); + dev_err(device, "[HMM, ERR] %s(%d): don't support odp\n", __func__, __LINE__); + return ERR_PTR(-ENOMEM); + } + hmem->odp_data = NULL; + + /* We assume the memory is from hugetlb until proved otherwise */ + hmem->hugetlb = 1; + page_list = (struct page **)__get_free_page(GFP_KERNEL); + if (page_list == NULL) { + kfree(hmem->context); + kfree(hmem); + return ERR_PTR(-ENOMEM); + } + + /* + * if we can't alloc the vma_list, it's not so bad; + * just assume the memory is not hugetlb memory + */ + vma_list = (struct vm_area_struct **)__get_free_page(GFP_KERNEL); + if (vma_list == NULL) + hmem->hugetlb = 0; + + npages = hmm_umem_num_pages(hmem); + mmap_write_lock(current->mm); + locked_pages = npages + atomic64_read(¤t->mm->pinned_vm); + lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if ((locked_pages > lock_limit) && !capable(CAP_IPC_LOCK)) { + ret = -ENOMEM; + goto out; + } + current_base = addr & PAGE_MASK; + if (npages == 0 || npages > UINT_MAX) { + ret = -EINVAL; + goto out; + } + ret = sg_alloc_table(&hmem->sg_head, (unsigned int)npages, GFP_KERNEL); + if (ret != 0) + goto out; + +#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS + if (hmem->writable == 0) + gup_flags |= FOLL_FORCE; +#endif + + need_release = 1; + sg_list_start = hmem->sg_head.sgl; + + while (npages != 0) { +#ifdef HAVE_GET_USER_PAGES_8_PARAMS + ret = get_user_pages(current, current->mm, current_base, + min_t(unsigned long, npages, PAGE_SIZE / sizeof(struct page *)), + 1, !hmem->writable, page_list, vma_list); +#else +#ifdef HAVE_GET_USER_PAGES_LONGTERM + ret = get_user_pages(current_base, +#else + ret = get_user_pages(current_base, +#endif + min_t(unsigned long, npages, PAGE_SIZE / sizeof(struct page *)), +#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS + gup_flags, page_list, vma_list); +#else + 1, !hmem->writable, page_list, vma_list); +#endif +#endif + + if (ret < 0) + goto out; + + hmem->npages += ret; + current_base += ret * PAGE_SIZE; + npages = (unsigned long)(npages - ret); + + for_each_sg(sg_list_start, sg, ret, i) { + if (vma_list != NULL && !is_vm_hugetlb_page(vma_list[i])) + hmem->hugetlb = 0; + + sg_set_page(sg, page_list[i], PAGE_SIZE, 0); + } + + /* preparing for next loop */ + sg_list_start = sg; + } + + hmem->nmap = dma_map_sg_attrs(device, hmem->sg_head.sgl, hmem->npages, DMA_BIDIRECTIONAL, +#ifdef HAVE_STRUCT_DMA_ATTRS + &dma_attrs); +#else + dma_attrs); +#endif + if (hmem->nmap <= 0) { + ret = -ENOMEM; + goto out; + } + ret = 0; + +out: + if (ret < 0) { + if (need_release != 0) + hmm_umemsg_release(device, hmem, 0); + + kfree(hmem->context); + kfree(hmem); + } else { + atomic64_set(¤t->mm->pinned_vm, locked_pages); + } + + mmap_write_unlock(current->mm); + if (vma_list != NULL) + free_page((unsigned long)(uintptr_t)vma_list); + + free_page((unsigned long)(uintptr_t)page_list); + return (ret < 0) ? ERR_PTR(ret) : hmem; +} + +/** + * hmm_umem_release - release memory pinned with ib_umem_get + * @hmem: umem struct to release + */ +void hmm_umem_release(struct hmm_umem *hmem) +{ + struct ib_ucontext *context = hmem->context; + struct mm_struct *mm = NULL; + struct task_struct *task = NULL; + unsigned long diff; + + if (hmem->odp_data) { + pr_err("[HMM, ERR] %s(%d): Don't support odp\n", __func__, __LINE__); + return; + } + + hmm_umemsg_release(context->device, hmem, 1); + task = get_pid_task(context->tgid, PIDTYPE_PID); + if (task == NULL) + goto out; + + mm = get_task_mm(task); + put_task_struct(task); + if (mm == NULL) + goto out; + + diff = hmm_umem_num_pages(hmem); + mmap_write_lock(mm); + atomic64_sub(diff, &mm->pinned_vm); + mmap_write_unlock(mm); + mmput(mm); +out: + kfree(context); + kfree(hmem); +} +#endif + +u32 hmm_umem_page_count(struct hmm_umem *hmem) +{ + u32 i; + u32 n; + struct scatterlist *sg = NULL; + + if (hmem->odp_data) + return (u32)(hmm_umem_num_pages(hmem)); + + n = 0; + for_each_sg(hmem->sg_head.sgl, sg, hmem->nmap, i) n += sg_dma_len(sg) >> + ((u32)hmem->page_shift); + + return n; +} diff --git a/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h new file mode 100644 index 000000000..61cc59f6e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/host/hmm/hmm_umem.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HMM_UMEM_H +#define HMM_UMEM_H + +#include <linux/list.h> +#include <linux/scatterlist.h> +#include <linux/workqueue.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/mmu_notifier.h> +#include <linux/kernel.h> + +#ifdef ROCE_SERVICE +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> +#endif + + +#ifndef ROCE_SERVICE + +enum rdma_remove_reason { + /* Userspace requested uobject deletion. Call could fail */ + RDMA_REMOVE_DESTROY, + /* Context deletion. This call should delete the actual object itself */ + RDMA_REMOVE_CLOSE, + /* Driver is being hot-unplugged. This call should delete the actual object itself */ + RDMA_REMOVE_DRIVER_REMOVE, + /* Context is being cleaned-up, but commit was just completed */ + RDMA_REMOVE_DURING_CLEANUP, +}; + +struct ib_uverbs_file; +struct ib_rdmacg_object {}; +struct rb_root_cached_struct { + struct rb_node *rb_root; + struct rb_node *rb_leftmost; +}; + +struct ib_ucontext { + struct device *device; + struct ib_uverbs_file *ufile; + int closing; + + /* locking the uobjects_list */ + struct mutex uobjects_lock; + struct list_head uobjects; + /* protects cleanup process from other actions */ + struct rw_semaphore cleanup_rwsem; + enum rdma_remove_reason cleanup_reason; + + struct pid *tgid; + struct rb_root_cached_struct umem_tree; + /* + * Protects .umem_rbroot and tree, as well as odp_mrs_count and + * mmu notifiers registration. + */ + struct rw_semaphore umem_rwsem; + void (*invalidate_range)(void *umem, unsigned long start, unsigned long end); + + struct mmu_notifier mn; + atomic_t notifier_count; + /* A list of umems that don't have private mmu notifier counters yet. */ + struct list_head no_private_counters; + int odp_mrs_count; + + struct ib_rdmacg_object cg_obj; +}; + +struct ib_umem_odp; + +struct hmm_umem *hmm_umem_get(struct device *device, unsigned long addr, + size_t size, int access, int dmasync); + +void hmm_umem_release(struct hmm_umem *hmem); + +#endif + +struct hmm_umem { + struct ib_ucontext *context; + size_t length; + unsigned long address; + int page_shift; + int writable; + int hugetlb; + struct work_struct work; + struct mm_struct *mm; + unsigned long diff; + struct ib_umem_odp *odp_data; + struct sg_table sg_head; + int nmap; + int npages; +}; + + +/* Returns the offset of the umem start relative to the first page. */ +static inline int hmm_umem_offset(const struct hmm_umem *umem) +{ + return umem->address & ~PAGE_MASK; +} + +/* Returns the first page of an ODP umem. */ +static inline unsigned long hmm_umem_start(struct hmm_umem *umem) +{ + return umem->address - hmm_umem_offset(umem); +} + +/* Returns the address of the page after the last one of an ODP umem. */ +static inline unsigned long hmm_umem_end(const struct hmm_umem *umem) +{ + return ALIGN(umem->address + umem->length, BIT((unsigned int)umem->page_shift)); +} + +static inline size_t hmm_umem_num_pages(struct hmm_umem *umem) +{ + return (size_t)(((unsigned long)(hmm_umem_end(umem) - + hmm_umem_start(umem))) >> (unsigned long)umem->page_shift); +} + +u32 hmm_umem_page_count(struct hmm_umem *hmem); + + +#endif /* HMM_UMEM_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h b/drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h new file mode 100644 index 000000000..009b9be04 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/hinic3_hmm.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HINIC_HMM_H__ +#define HINIC_HMM_H__ + +/* has no mpt entry */ +#define HMM_MPT_EN_SW 1 /* has mpt, state INVALID */ +#define HMM_MPT_EN_HW 2 /* has mpt, state FREE or VALID */ +#define HMM_MPT_DISABLED 0 /* has no mpt entry */ +#define HMM_MPT_FIX_BUG_LKEY 0 + +#include "hinic3_cqm.h" +#include "hinic3_hwdev.h" +#include "hmm_comp.h" +#include "hmm_mr.h" + +/* + **************************************************************************** + Prototype : hmm_reg_user_mr_update + Description : MR注册生成和更新MPT和MTT表 + Input : struct hinic3_hwdev *hwdev + hmm_mr *mr MR结构，包含已 + 经完成用户态内存的物理地址获取umem + u32 pdn PD号，如果不支持的pd的特性直接填0. + u64 length 需要注册的用户态地址长度 + u64 virt_addr 需要注册的IOV虚拟地址首地址 + int hmm_access填入enum rdma_ib_access的值 + u32 service_type enum hinic3_service_type的值 + Output : None +**************************************************************************** +*/ +int hmm_reg_user_mr_update(struct hinic3_hwdev *hwdev, struct hmm_mr *mr, u32 pdn, u64 length, + u64 virt_addr, int access, u32 service_type, u16 channel); + + +/* + **************************************************************************** + Prototype : hmm_reg_user_mr_update + Description : MR去注册删除MPT和MTT表 + Input : struct hinic3_hwdev *hwdev + rdma_mr *mr MR结构 + u32 service_type enum hinic3_service_type的值 + Output : None +**************************************************************************** +*/ +int hmm_dereg_mr_update(struct hinic3_hwdev *hwdev, struct rdma_mr *mr, + u32 service_type, u16 channel); + +#ifndef ROCE_SERVICE +/* + **************************************************************************** + Prototype : hmm_reg_user_mr + Description : register MR for user + Input : struct hinic3_hwdev *hwdev + u32 pdn PD�� + u64 start ע��memory����ʼ��ַ + u64 length ע���ڴ�ij��� + u64 virt_addr io�������ַ + int hmm_access ����enum rdma_ib_access��ֵ + u32 service_type enum hinic3_service_type��ֵ + Output : None +**************************************************************************** +*/ +struct hmm_mr *hmm_reg_user_mr(struct hinic3_hwdev *hwdev, u64 start, u32 pdn, u64 length, + u64 virt_addr, int hmm_access, u32 service_type, u16 channel); + +/* + **************************************************************************** + Prototype : hmm_dereg_mr + Description : dereg DMA_MR, user_MR or FRMR + Input : struct hmm_mr *mr + : u32 service_type enum hinic3_service_type的值 + Output : None + +**************************************************************************** +*/ +int hmm_dereg_mr(struct hmm_mr *mr, u32 service_type, u16 channel); +#endif + +int hmm_rdma_write_mtt(void *hwdev, struct rdma_mtt *mtt, u32 start_index, u32 npages, + u64 *page_list, u32 service_type); + +int hmm_rdma_mtt_alloc(void *hwdev, u32 npages, u32 page_shift, + struct rdma_mtt *mtt, u32 service_type); + +void hmm_rdma_mtt_free(void *hwdev, struct rdma_mtt *mtt, u32 service_type); + +int hmm_init_mtt_table(struct hmm_comp_priv *comp_priv); + +void hmm_cleanup_mtt_table(struct hmm_comp_priv *comp_priv); + +#endif /* HINIC_RDMA_H__ */ diff --git a/drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h b/drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h new file mode 100644 index 000000000..1de8ba98f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/hinic3_rdma.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef HINIC_RDMA_H__ +#define HINIC_RDMA_H__ + +#define RDMA_ROCE_ENABLE 1 +#define RDMA_IWARP_ENABLE 1 +#define RDMA_ROCE_DISABLE 0 +#define RDMA_IWARP_DISABLE 0 + +#define RDMA_MPT_DISABLED 0 /* has no mpt entry */ +#define RDMA_MPT_EN_SW 1 /* has mpt, state INVALID */ +#define RDMA_MPT_EN_HW 2 /* has mpt, state FREE or VALID */ + +#define RDMA_MPT_FIX_BUG_LKEY 0 + +struct mutex; +struct tag_cqm_qpc_mpt; +struct tag_cqm_object; +struct net_device; +struct rdma_gid_entry; + +#include "hinic3_cqm.h" + +enum mtt_check_type_e { + MTT_CHECK_TYPE_0 = 0, + MTT_CHECK_TYPE_1 +}; + +enum mtt_data_type_e { + MTT_DMTT_TYPE = 0, + MTT_CMTT_TYPE +}; + +enum rdma_ib_access { + RDMA_IB_ACCESS_LOCAL_WRITE = 1, + RDMA_IB_ACCESS_REMOTE_WRITE = (1 << 1), + RDMA_IB_ACCESS_REMOTE_READ = (1 << 2), + RDMA_IB_ACCESS_REMOTE_ATOMIC = (1 << 3), + RDMA_IB_ACCESS_MW_BIND = (1 << 4), + RDMA_IB_ACCESS_ZERO_BASED = (1 << 5), + RDMA_IB_ACCESS_ON_DEMAND = (1 << 6), +}; + +struct rdma_gid_entry { + union { + u8 raw[16]; + struct { + __be64 subnet_prefix; + __be64 interface_id; + } global; + }; + union { + struct { + u32 rsvd : 7; + u32 is_vroce : 1; + u32 cvlan : 12; /* 内层vlan customer vlan */ + u32 svlan : 12; /* 外层vlan */ + } bs; + u32 value; + } dw4; + + union { + u32 hdr_len_value; + }; + + union { + struct { + /* 0:没有vlan； 1：一层vlan； 2: 2层vlan； 3：stag */ + u16 tag : 2; + u16 tunnel : 1; // rsvd for ppe, don't use. 'tunnel' + u16 gid_type : 2; + u16 ppe_rsvd1 : 1; + u16 outer_tag : 2; // rsvd for ppe, don't use. 'outer_tag' + u16 ppe_rsvd3 : 1; // rsvd for ppe, don't use. 'stag' + u16 gid_update : 1; + u16 rsvd : 6; + } bs; + u16 value; + } dw6_h; + + u8 smac[6]; +}; + +struct rdma_comp_resource { + struct mutex mutex; /* gid_entry使用的互斥量 */ + __be64 node_guid; /* 与ibdev中的node_guid一致 */ + struct rdma_gid_entry **gid_table; /* gid_entry在rdma组件初始化时分配内存 */ +}; + +struct rdma_mpt { + u32 mpt_index; /* 封装cqm提供的mpt_index */ + void *vaddr; /* 封装cqm提供的mpt_entry的虚拟地址 */ + void *mpt_object; /* 封装的cqm提供的指针 */ +}; + + +struct rdma_mtt_seg { + u32 offset; /* 分配连续索引的首个索引 */ + u32 order; /* mtt索引个数为1<<order，每个索引对应一个mtt entry */ + void *vaddr; /* mtt_seg第一个MTT的起始虚拟地址 */ + dma_addr_t paddr; /* mtt_seg第一个MTT的起始物理地址 */ +}; + +struct rdma_mtt { + /* mtt的级数,该值为0时表示不使用mtt做地址转换 */ + u32 mtt_layers; + u32 mtt_page_shift; /* MTT的页大小 */ + u32 buf_page_shift; /* buffer页大小 */ + dma_addr_t mtt_paddr; /* 写入context中的物理地址 */ + __be64 *mtt_vaddr; /* 写入context中的虚拟地址 */ + struct rdma_mtt_seg **mtt_seg; /* 指向多级mtt */ + enum mtt_data_type_e mtt_type; +}; + +enum rdma_mr_type { + RDMA_DMA_MR = 0, + RDMA_USER_MR = 1, + RDMA_FRMR = 2, + RDMA_FMR = 3, + RDMA_PHYS_MR = 4, + RDMA_RSVD_LKEY = 5, + RDMA_SIG_MR = 6, + RDMA_INDIRECT_MR = 8, + RDMA_ODP_IMPLICIT_MR = 9, + RDMA_ODP_EXPLICIT_MR = 10, +}; + +struct rdma_mr { + struct rdma_mpt mpt; + struct rdma_mtt mtt; + u64 iova; /* mr指向内存的起始地址(虚拟地址,ZBVA时为0) */ + u64 size; /* mr指向内存的大小 */ + u32 key; /* mr对应的key */ + u32 pdn; /* mr绑定的pdn */ + u32 access; /* mr的访问权限 */ + int enabled; /* mr的状态,DISABLE、EN_SW、EN_HW */ + int mr_type; /* mr类型 */ + u32 block_size; +}; + +enum rdma_mw_type { + RDMA_MW_TYPE_1 = 1, + RDMA_MW_TYPE_2 = 2 +}; + +struct rdma_mw { + struct rdma_mpt mpt; + u32 key; /* mw对应的key */ + u32 pdn; /* mw绑定的pdn */ + enum rdma_mw_type type; /* mw的类型,type1,type2 */ + int enabled; /* mw的状态 */ +}; + +struct rdma_fmr { + struct rdma_mr mr; + u32 max_pages; /* fmr的最大映射页个数 */ + u32 max_maps; /* fmr的最大映射次数 */ + u32 maps; /* fmr的当前映射次数 */ + u32 page_shift; /* fmr指定的页偏移 */ +}; + +struct rdma_rdmarc { + u32 offset; /* 分配连续索引的首个索引 */ + u32 order; /* 分配的rdmarc的order,代表了个数 */ + u32 ext_order; /* 包含rc表和扩展表的个数 */ + dma_addr_t dma_addr; + void *vaddr; +}; + +int roce3_rdma_pd_alloc(void *hwdev, u32 *pdn); + +void roce3_rdma_pd_free(void *hwdev, u32 pdn); + +int roce3_rdma_enable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type); + +int roce3_rdma_disable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type); + +int roce3_rdma_map_phys_fmr(void *hwdev, struct rdma_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 service_type); + +int roce3_rdma_unmap_fmr(void *hwdev, struct rdma_fmr *fmr, u32 service_type); + +int roce3_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc); + +void roce3_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc); + +int roce3_rdma_update_gid_mac(void *hwdev, u32 port, struct rdma_gid_entry *gid_entry); +int roce3_rdma_update_gid(void *hwdev, u32 port, u32 update_index, + struct rdma_gid_entry *gid_entry); +int roce3_rdma_reset_gid_table(void *hwdev, u32 port); + +int roce3_rdma_get_gid(void *hwdev, u32 port, u32 gid_index, struct rdma_gid_entry *gid); + +/* 该接口在pf初始化时调用 */ +int roce3_rdma_init_resource(void *hwdev); + +/* 该接口在pf卸载时调用 */ +void roce3_rdma_cleanup_resource(void *hwdev); + +#endif /* HINIC_RDMA_H__ */ diff --git a/drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h b/drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h new file mode 100644 index 000000000..dbee090f3 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/nic/nic_mpu_cmd.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_MPU_CMD_H +#define NIC_MPU_CMD_H + +/** Commands between NIC to MPU + */ +enum hinic3_nic_cmd { + HINIC3_NIC_CMD_VF_REGISTER = 0, /**< Only for PFD and VFD @see > hinic3_cmd_register_vf */ + + /** FUNC CFG */ + HINIC3_NIC_CMD_SET_FUNC_TBL = 5, /**< Set function table @see > hinic3_cmd_set_func_tbl */ + HINIC3_NIC_CMD_SET_VPORT_ENABLE, /**< Enable a vport @see > hinic3_vport_state */ + HINIC3_NIC_CMD_SET_RX_MODE, /**< Set nic rx mode. @see > hinic3_rx_mode_config */ + HINIC3_NIC_CMD_SQ_CI_ATTR_SET, /**< Set SQ CI attr @see > hinic3_cmd_cons_idx_attr */ + /**< Get vport stat @see > hinic3_port_stats_info, < hinic3_cmd_vport_stats */ + HINIC3_NIC_CMD_GET_VPORT_STAT, + /**< Clean vport stat @see > hinic3_cmd_clear_vport_stats */ + HINIC3_NIC_CMD_CLEAN_VPORT_STAT, + /**< Clean queue pair resource @see > hinic3_cmd_clear_qp_resource */ + HINIC3_NIC_CMD_CLEAR_QP_RESOURCE, + HINIC3_NIC_CMD_CFG_FLEX_QUEUE, /**< Set flex queue @see > hinic3_cmd_cfg_qps */ + /** LRO CFG */ + HINIC3_NIC_CMD_CFG_RX_LRO, /**< Set rx LRO @see > hinic3_cmd_lro_config */ + HINIC3_NIC_CMD_CFG_LRO_TIMER, /**< Set LRO timer @see > hinic3_cmd_lro_timer */ + /**< negotiate features @see > hinic3_cmd_feature_nego */ + HINIC3_NIC_CMD_FEATURE_NEGO, + /**< Configure local LRO state @see > hinic3_cmd_local_lro_state */ + HINIC3_NIC_CMD_CFG_LOCAL_LRO_STATE, + + /**< Cache out queue pair resource @see > hinic3_cmd_cache_out_qp_resource */ + HINIC3_NIC_CMD_CACHE_OUT_QP_RES, + + /** MAC & VLAN CFG */ + HINIC3_NIC_CMD_GET_MAC = 20, /**< Get mac address @see > hinic3_port_mac_set */ + HINIC3_NIC_CMD_SET_MAC, /**< Set mac address @see > hinic3_port_mac_set */ + HINIC3_NIC_CMD_DEL_MAC, /**< Delete mac address @see > hinic3_port_mac_set */ + /**< Update mac address @see > hinic3_port_mac_update */ + HINIC3_NIC_CMD_UPDATE_MAC, + /**< Get all default mac address @see > cmd_mac_info_get_s */ + HINIC3_NIC_CMD_GET_ALL_DEFAULT_MAC, + + /**< Configure function vlan @see > hinic3_cmd_vlan_config */ + HINIC3_NIC_CMD_CFG_FUNC_VLAN, + /**< Enable vlan filter @see > hinic3_cmd_set_vlan_filter */ + HINIC3_NIC_CMD_SET_VLAN_FILTER_EN, + /**< Set rx vlan offload @see > hinic3_cmd_vlan_offload */ + HINIC3_NIC_CMD_SET_RX_VLAN_OFFLOAD, + HINIC3_NIC_CMD_SMAC_CHECK_STATE, + + /** SR-IOV */ + /**< Configure vf vlan @see > hinic3_cmd_vf_vlan_config */ + HINIC3_NIC_CMD_CFG_VF_VLAN = 40, + /**< Set snoopchk state @see > hinic3_cmd_spoofchk_set */ + HINIC3_NIC_CMD_SET_SPOOPCHK_STATE, + /* RATE LIMIT */ + /**< Set rate limit @see > HINIC3_NIC_CMD_SET_MAX_MIN_RATE */ + HINIC3_NIC_CMD_SET_MAX_MIN_RATE, + + /** RSS CFG */ + HINIC3_NIC_CMD_RSS_CFG = 60, /**< Set rss config @see > hinic3_cmd_rss_config */ + HINIC3_NIC_CMD_RSS_TEMP_MGR, /**< TODO: delete after implement nego cmd */ + HINIC3_NIC_CMD_GET_RSS_CTX_TBL, /**< TODO: delete: move to ucode cmd */ + /**< Set rss hash key @see > hinic3_cmd_rss_hash_key */ + HINIC3_NIC_CMD_CFG_RSS_HASH_KEY, + /**< Set rss hash engine type @see > hinic3_cmd_rss_engine_type */ + HINIC3_NIC_CMD_CFG_RSS_HASH_ENGINE, + /**< Set rss context table info @see > hinic3_rss_context_table */ + HINIC3_NIC_CMD_SET_RSS_CTX_TBL_INTO_FUNC, + /** IP checksum error packets, enable rss quadruple hash */ + /**< Set rss config @see > hinic3_ipcs_err_rss_enable_operation_s */ + HINIC3_NIC_CMD_IPCS_ERR_RSS_ENABLE_OP = 66, + + /** PPA/FDIR */ + HINIC3_NIC_CMD_ADD_TC_FLOW = 80, /**< Add tc flow @see > nic_cmd_fdir_add_rule */ + HINIC3_NIC_CMD_DEL_TC_FLOW, /**< Delete tc flow @see > nic_cmd_fdir_del_rules */ + HINIC3_NIC_CMD_GET_TC_FLOW, /**< Get tc flow @see > nic_cmd_fdir_get_rule */ + HINIC3_NIC_CMD_FLUSH_TCAM, /**< Flush TCAM @see > nic_cmd_flush_tcam_rules */ + /**< Configure TCAM block @see > nic_cmd_ctrl_tcam_block_out */ + HINIC3_NIC_CMD_CFG_TCAM_BLOCK, + /**< Enable TCAM @see > nic_cmd_set_tcam_enable */ + HINIC3_NIC_CMD_ENABLE_TCAM, + /**< Get TCAM block @see > nic_cmd_dfx_fdir_tcam_block_table */ + HINIC3_NIC_CMD_GET_TCAM_BLOCK, + /**< Configure PPA table id @see > hinic3_ppa_cfg_table_id_cmd */ + HINIC3_NIC_CMD_CFG_PPA_TABLE_ID, + /**< Set PPA enable @see > hinic3_ppa_cfg_ppa_en_cmd */ + HINIC3_NIC_CMD_SET_PPA_EN = 88, + /**< Configure PPA mode @see > hinic3_ppa_cfg_mode_cmd */ + HINIC3_NIC_CMD_CFG_PPA_MODE, + /**< Configure PPA flush @see > hinic3_ppa_cfg_flush_cmd */ + HINIC3_NIC_CMD_CFG_PPA_FLUSH, + /**< Set FDIR status @see > hinic3_set_fdir_ethertype_rule */ + HINIC3_NIC_CMD_SET_FDIR_STATUS, + /**< Get PPA counter @see > hinic3_ppa_fdir_query_cmd */ + HINIC3_NIC_CMD_GET_PPA_COUNTER, + /**< Set flow bifur status @see > cmd_flow_bifur_func_handle */ + HINIC3_NIC_CMD_SET_FUNC_FLOW_BIFUR_ENABLE, + /**< Set flow bifur bond @see > cmd_flow_bifur_bond_handle */ + HINIC3_NIC_CMD_SET_BOND_MASK, + /**< Get func tcam table @see > get_fdir_func_tcam_table */ + HINIC3_NIC_CMD_GET_BLOCK_TC_FLOWS, + /**< Get flow bifur bond @see > cmd_flow_bifur_bond_handle */ + HINIC3_NIC_CMD_GET_BOND_MASK, + + /** PORT CFG */ + HINIC3_NIC_CMD_SET_PORT_ENABLE = 100, /**< set port enable @see > hinic3_port_state */ + HINIC3_NIC_CMD_CFG_PAUSE_INFO, /**< Configure pause info @see > hinic3_cmd_pause_config */ + + HINIC3_NIC_CMD_SET_PORT_CAR, /**< Set port Car @see > hinic3_cmd_set_port_car */ + HINIC3_NIC_CMD_SET_ER_DROP_PKT, /**< Unused */ + + HINIC3_NIC_CMD_VF_COS, /**< Get vf CoS @see > hinic3_cmd_vf_dcb_state */ + HINIC3_NIC_CMD_SETUP_COS_MAPPING, /**< Unused */ + HINIC3_NIC_CMD_SET_ETS, /**< Unused */ + HINIC3_NIC_CMD_SET_PFC, /**< Unused */ + HINIC3_NIC_CMD_QOS_ETS, /**< Set QoS ETS @see > hinic3_cmd_ets_cfg */ + HINIC3_NIC_CMD_QOS_PFC, /**< Set QoS PFC @see > hinic3_cmd_set_pfc */ + HINIC3_NIC_CMD_QOS_DCB_STATE, /**< Get QoS DCB state @see > hinic3_cmd_set_dcb_state */ + HINIC3_NIC_CMD_QOS_PORT_CFG, /**< Get QoS port cfg @see > hinic3_cmd_qos_port_cfg */ + HINIC3_NIC_CMD_QOS_MAP_CFG, /**< Get QoS map cfg @see > hinic3_cmd_qos_map_cfg */ + HINIC3_NIC_CMD_FORCE_PKT_DROP, /**< Force pkt drop @see > hinic3_force_pkt_drop */ + /**< Configure nic tx promisc skip @see > hinic3_tx_promisc_cfg */ + HINIC3_NIC_CMD_CFG_TX_PROMISC_SKIP = 114, + /**< Set flow bifur port switch @see > cmd_flow_bifur_port_handle */ + HINIC3_NIC_CMD_SET_PORT_FLOW_BIFUR_ENABLE = 117, + /**< Set tx pause exc notice @see > nic_cmd_tx_pause_notice */ + HINIC3_NIC_CMD_TX_PAUSE_EXCP_NOTICE = 118, + /**< Inquirt pause cfg @see > nic_cmd_pause_inquiry_cfg_s */ + HINIC3_NIC_CMD_INQUIRT_PAUSE_CFG = 119, + + /** MISC */ + HINIC3_NIC_CMD_BIOS_CFG = 120, /**< Set QoS ETS @see > nic_cmd_bios_cfg */ + HINIC3_NIC_CMD_SET_FIRMWARE_CUSTOM_PACKETS_MSG, /**< Set QoS ETS @see > fault_msg_st */ + + /** BOND */ + /**< Create bond device @see > hinic3_cmd_create_bond */ + HINIC3_NIC_CMD_BOND_DEV_CREATE = 134, + HINIC3_NIC_CMD_BOND_DEV_DELETE, /**< Delete bond device @see > hinic3_cmd_delete_bond */ + /**<Open/close bond dev @see > hinic3_cmd_open_close_bond */ + HINIC3_NIC_CMD_BOND_DEV_OPEN_CLOSE, + HINIC3_NIC_CMD_BOND_INFO_GET, /**< Set QoS ETS @see > hinic3_bond_status_info */ + /**< Get bond active info @see > hinic3_bond_active_report_info */ + HINIC3_NIC_CMD_BOND_ACTIVE_INFO_GET, + /**< Bond active notice report @see > nic_cmd_bond_active_report_info */ + HINIC3_NIC_CMD_BOND_ACTIVE_NOTICE, + + /** DFX */ + HINIC3_NIC_CMD_GET_SM_TABLE = 140, /**< Get sm table @see > nic_cmd_dfx_sm_table */ + /**< Set RD line table @see > nic_mpu_lt_opera, < nic_mpu_lt_opera */ + HINIC3_NIC_CMD_RD_LINE_TBL, + + HINIC3_NIC_CMD_SET_UCAPTURE_OPT = 160, /**< TODO: move to roce */ + HINIC3_NIC_CMD_SET_VHD_CFG, /**< Set VHD configuration @see > hinic3_set_vhd_mode */ + + /** TODO: move to HILINK */ + /**< Get port stat @see > hinic3_port_stats_info, < hinic3_port_stats */ + HINIC3_NIC_CMD_GET_PORT_STAT = 200, + HINIC3_NIC_CMD_CLEAN_PORT_STAT, /**< Unused */ + HINIC3_NIC_CMD_CFG_LOOPBACK_MODE, /**< Unused */ + HINIC3_NIC_CMD_GET_SFP_QSFP_INFO, /**< Unused */ + HINIC3_NIC_CMD_SET_SFP_STATUS, /**< Unused */ + HINIC3_NIC_CMD_GET_LIGHT_MODULE_ABS, /**< Unused */ + HINIC3_NIC_CMD_GET_LINK_INFO, /**< Unused */ + HINIC3_NIC_CMD_CFG_AN_TYPE, /**< Unused */ + HINIC3_NIC_CMD_GET_PORT_INFO, /**< Get port info @see > hinic3_cmd_port_info */ + HINIC3_NIC_CMD_SET_LINK_SETTINGS, /**< Unused */ + HINIC3_NIC_CMD_ACTIVATE_BIOS_LINK_CFG, /**< Unused */ + HINIC3_NIC_CMD_RESTORE_LINK_CFG, /**< Unused */ + HINIC3_NIC_CMD_SET_LINK_FOLLOW, /**< Unused */ + HINIC3_NIC_CMD_GET_LINK_STATE, /**< Unused */ + HINIC3_NIC_CMD_LINK_STATUS_REPORT, /**< Unused */ + HINIC3_NIC_CMD_CABLE_PLUG_EVENT, /**< Unused */ + HINIC3_NIC_CMD_LINK_ERR_EVENT, /**< Unused */ + HINIC3_NIC_CMD_SET_LED_STATUS, /**< Unused */ + + HINIC3_NIC_CMD_MAX = 256, +}; + +#endif /* NIC_MPU_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h new file mode 100644 index 000000000..65b243a0e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_NPU_CMD_H +#define NIC_NPU_CMD_H + +/* NIC CMDQ MODE */ +enum hinic3_ucode_cmd { + /**< Modify queue context. @see > hinic3_sq_ctxt_block */ + HINIC3_UCODE_CMD_MODIFY_QUEUE_CTX = 0, + /**< Clean queue context. @see > hinic3_clean_queue_ctxt */ + HINIC3_UCODE_CMD_CLEAN_QUEUE_CONTEXT, + HINIC3_UCODE_CMD_ARM_SQ, /**< Unused */ + HINIC3_UCODE_CMD_ARM_RQ, /**< Unused */ + /**< Set RSS indir table. @see > nic_rss_indirect_tbl */ + HINIC3_UCODE_CMD_SET_RSS_INDIR_TABLE, + /**< Set RSS indir table. @see > nic_rss_context_tbl */ + HINIC3_UCODE_CMD_SET_RSS_CONTEXT_TABLE, + HINIC3_UCODE_CMD_GET_RSS_INDIR_TABLE, + HINIC3_UCODE_CMD_GET_RSS_CONTEXT_TABLE, /**< Unused */ + HINIC3_UCODE_CMD_SET_IQ_ENABLE, /**< Unused */ + HINIC3_UCODE_CMD_SET_RQ_FLUSH = 10, /**< Set RQ flush. @see > hinic3_cmd_set_rq_flush */ + HINIC3_UCODE_CMD_MODIFY_VLAN_CTX, /**< Get rxq info. @see > nic_vlan_ctx */ + HINIC3_UCODE_CMD_PPA_HASH_TABLE, + /**< Get rxq info. @see > hinic3_rxq_hw, < rxq_check_info */ + HINIC3_UCODE_CMD_RXQ_INFO_GET = 13, +}; + +#endif /* NIC_NPU_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h new file mode 100644 index 000000000..f128ba80d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/nic/nic_npu_cmd_defs.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef NIC_NPU_CMD_DEFS_H +#define NIC_NPU_CMD_DEFS_H + +#include "typedef.h" +#include "nic_cfg_comm.h" + +/* + * NIC Command Queue Header + * queues context follow this header should be consecutive + */ +struct nic_cmdq_header { + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + /* 0x0:SQ, 0x1:RQ */ + u16 queue_type; + /* queue number in buffer follow this header */ + u16 queue_num; +#else + u16 queue_num; + u16 queue_type; +#endif + } cmdq_ctx_dw0; + + u32 ctx_dw0; + }; + +#if (BYTE_ORDER == BIG_ENDIAN) + u16 rsvd; + u16 start_qid; +#else + u16 start_qid; + u16 rsvd; +#endif +}; + +struct nic_cmdq_context_modify_s { + struct nic_cmdq_header hdr; + u8 data[2016]; +}; + +struct nic_cmdq_clean_q_space { + /* + * queue_type = 0, TSO + * queue_type = 1, LRO + */ + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 queue_type; + u16 queue_num; +#else + u16 queue_num; + u16 queue_type; +#endif + } cmdq_space_dw0; + + u32 space_dw0; + }; + +#if (BYTE_ORDER == BIG_ENDIAN) + u16 rsvd; + u16 start_qid; +#else + u16 start_qid; + u16 rsvd; +#endif + + u32 rsvd1; +}; + +struct nic_cmdq_flush_rq_task { + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 q_id; + u16 glb_rq_id; +#else + u16 glb_rq_id; + u16 q_id; +#endif + } bs; + + u32 value; + } dw0; +}; + +/* arm sq/rq */ +union nic_cmdq_arm { + struct cmdq_arm_dw0_s { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 qpn; + u16 pi; +#else + u16 pi; + u16 qpn; +#endif + } dw0; + + u32 arm_dw0; +}; + + +/* rss */ +struct nic_rss_indirect_tbl { + u32 rsvd[4]; // Make sure that 16B beyond entry[] + u16 entry[NIC_RSS_INDIR_SIZE]; +}; + +struct nic_rss_glb_qid_indirect_tbl { + u32 group_index; + u32 offset; + u32 size; + u32 rsvd; /* Make sure that 16B beyond entry[] */ + u16 entry[NIC_RSS_INDIR_SIZE]; +}; + +struct nic_rss_context_tbl { + u32 rsvd[4]; + u32 ctx; +}; + +struct nic_vlan_ctx { + u32 func_id; + u32 qid; /* if qid = 0xFFFF, config current function all queue */ + u32 vlan_id; + u32 vlan_mode; + u32 vlan_sel; +}; + +#endif /* NIC_NPU_CMD_DEFS_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/node_id.h b/drivers/infiniband/hw/hiroce3/include/node_id.h new file mode 100644 index 000000000..84d276873 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/node_id.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ +#ifndef NODE_ID_H +#define NODE_ID_H + +#ifdef __cplusplus +#if __cplusplus +extern "C"{ +#endif +#endif /* __cplusplus */ + +/** RING NODE ID */ +enum { + NODE_ID_CPI = 0, + NODE_ID_MQM = 1, + NODE_ID_QUF = 2, + NODE_ID_Reserved0 = 3, + NODE_ID_SMF0 = 4, + NODE_ID_TILE_F0 = 5, + NODE_ID_TILE_F1 = 6, + NODE_ID_SMF1 = 7, + NODE_ID_DP_NETWORK = 8, + NODE_ID_CPB = 9, + NODE_ID_QUL = 10, + NODE_ID_TS = 11, + NODE_ID_TILE_L1 = 12, + NODE_ID_SML1 = 13, + NODE_ID_SML0 = 14, + NODE_ID_TILE_L0 = 15, + NODE_ID_SMF2 = 16, + NODE_ID_TILE_F2 = 17, + NODE_ID_TILE_F3 = 18, + NODE_ID_SMF3 = 19, + NODE_ID_TILE_L3 = 20, + NODE_ID_SML3 = 21, + NODE_ID_SML2 = 22, + NODE_ID_TILE_L2 = 23, + NODE_ID_CRYPTO = 24, + NODE_ID_LCAM = 25, + NODE_ID_MPU = 26, + NODE_ID_DP_HOST = 27, + NODE_ID_UP_HOST = 31 /* Used for API chain function in the CPI */ +}; + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ + +#endif /* NODE_ID_H */ + diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h new file mode 100644 index 000000000..d68648540 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_context_format.h @@ -0,0 +1,5181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_CONTEXT_FORMAT_H +#define RDMA_CONTEXT_FORMAT_H + +#include "roce_hmm_context.h" +#include "rdma_ext_ctx_format.h" + + +/* Align each field with 4bytes. */ +#pragma pack(4) + +/* **************** Macro Definition ****************** */ +#define ROCE_BASE_GID_IDX 1 +#define RDMA_SGID_TBL_KEY_VFID_OFFSET (4) +#define RDMA_SGID_TBL_KEY_SGIDIDX_OFFSET (0) +#define ROCE_SPU_HOST_ID (4) +#define ROCE_SPU_OQID_HOST_ID (5) +#define ROCE_SDI_SHOST_HOST_ID 0 +#define ROCE_SDI_MHOST_HOST_ID 1 +#define QU_OQID_RW_CHANNEL 1 +#define ROCE_GET_HOST_OQID(host_id, xid) \ + ((((128U * (host_id) + ((xid) & 0x7f)) << 2) & 0x7ff) | \ + (((host_id) & 0x7) << 11U) | (((xid) & 0x1) + 1)) +#define ROCE_GET_SPU_HOST_OQID(host_id, fid, xid) \ + (((((fid) & 0x1ff) << 2) & 0x7FF) | (((host_id) & 0x7) << 11U) | (QU_OQID_RW_CHANNEL)) + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#define ETH_ADDR_LEN (6) + +/* **************** Data Structure Definition ****************** */ +/* * QPC Format start */ + +/* Send Queue Context 64B */ +struct chip_seg_sqc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_pd : 18; + u32 sq_inline_en : 1; + u32 sq_pi_on_chip : 1; + u32 sq_wqebb_size : 3; + u32 sq_page_size : 4; + u32 sq_size : 5; +#else + /* + * Send WQ size is (2^sq_size)*WQEBBs; the + * maximum SQ size is 16K WQEs, so this field + * doesn't exceed 14. They have same configuration + * value: SQC.sq_size and SQAC.sq_size. + * Configured by Driver + */ + u32 sq_size : 5; + + /* + * Page size of SQ and RQ, equals to (2^sq_page_size)*4KB. + * The following three Page size have same configuration + * value: SQC.sq_page_size, + * SQAC.sq_page_size and RQC.rq_page_size. Configured by Driver + */ + u32 sq_page_size : 4; + /* + * Send WQE Basic Block (WQEBB) size in + * bytes is 16*(2^sq_weqbb_size). + * for the SQ, this field should be fixed to 2. + * They have same configuration value: SQC.sq_wqebb_size + * and SQAC.sq_wqebb_size. + * Configured by Driver + */ + u32 sq_wqebb_size : 3; + /* + * If set, the counter (PI index) of Send Queue is + * stored in the chip, the counter is + * absolute value. Configured by Driver + */ + u32 sq_pi_on_chip : 1; + /* + * Indicates if the SQ supports inline data operation. + * Configured by Driver + */ + u32 sq_inline_en : 1; + /* + * Protection Domain. + * The following five Protection Domains have + * same configuration value: QCC.rc_pd, + * QC.sq_pd, SQAC.sq_pd, RQC.rq_pd and RRWC.rrw_pd. + * Configured by Driver + */ + u32 sq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_so_ro : 2; + u32 sq_dma_attr_idx : 6; + u32 sq_wqecnt_rctl : 1; + u32 sq_prewqe_mode : 1; + u32 sq_pi_vld : 1; + u32 sq_wqecnt_rctl_en : 1; + u32 sq_wqecnt_lth : 4; + u32 rsvd : 16; +#else + u32 rsvd : 16; + /* + * WQE Counter Low Threshold, equals to (2^sq_wqecnt_lth). + * Configured by Driver + */ + u32 sq_wqecnt_lth : 4; + /* + * If this field is equal to zero, the SQ + * PI updating should be not controlled by + * the "sq_wqecnt_rctl" field; else the SQ + * RI updating can be performed only if + * the "sq_wqecnt_rctl" is equal to one. + * Configured by Driver + */ + u32 sq_wqecnt_rctl_en : 1; + /* + * If set, the SQ should use the Index Mechanism, + * else the SQ uses the Owner Bit + * Mechanism. Configured by Driver + */ + u32 sq_pi_vld : 1; + /* + * If set,the engine just performs prefetch WQE + * processing after completing the current WQE in + * the SQ_DMA_GEN_API flow. + */ + u32 sq_prewqe_mode : 1; + /* + * The hardware clear it to zero when performing a SQ + * PI updating, and driver set it + * to one to indicate the hardware can performing SQ PI updating. + * N/A + */ + u32 sq_wqecnt_rctl : 1; + /* + * bit[05:00] It specifies the outbound PCIe TLP + * header attribute of the DMA + * operation. This filed is only valid when processing + * SQ's WQEs. The following two + * "dma_attr_idx" have same configuration value: + * SQC.sq_dma_attr_idx and + * SQAC.sq_dma_attr_idx. Configured by Driver + */ + u32 sq_dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + * The following two "so_ro" have same configuration value: + * SQC.sq_so_ro and + * SQAC.sq_so_ro. Configured by Driver + */ + u32 sq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_state : 4; + u32 rsvd2 : 4; + u32 sq_rkey_en : 1; + u32 sq_wqe_check_en : 1; + u32 rsvd1 : 6; + u32 sq_pi : 16; +#else + /* The Producer Index (PI) of Send Queue (Step: WQEBB) 0x0 */ + u32 sq_pi : 16; + u32 rsvd1 : 6; + u32 sq_wqe_check_en : 1; + u32 sq_rkey_en : 1; + u32 rsvd2 : 4; + /* + * Send Queue State.0x0:ok;0x1:error; + * 0xf:hardware has no access right.Other:reserved. + * 0x0 + */ + u32 sq_state : 4; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_ci : 16; + u32 rsvd : 3; + u32 dsgl_en : 1; /* Whether to work in dual sgl mode */ + u32 sq_load_pi : 1; + u32 sq_load_page_gpa : 1; + u32 sq_wqe_curt_page_vld : 1; + u32 sq_wqe_next_page_vld : 1; + u32 sq_wqe_curt_page_gpa_h : 8; +#else + /* + * bit[63:56] Indicates the GPA of current SQ + * buffer's page pointed by + * "sq_ci". 0x0 + */ + u32 sq_wqe_curt_page_gpa_h : 8; + /* Indicates if the "sq_wqe_next_page_gpa" field is valid. */ + u32 sq_wqe_next_page_vld : 1; + /* + * Indicates if the "sq_wqe_curt_page_gpa" field is valid. + * 1: it is valid;0: it is invalid. + * 0x0 + */ + u32 sq_wqe_curt_page_vld : 1; + /* + * Indicates the thread is performing a + * prefetch for GPA of WQE page. 0x0 + */ + u32 sq_load_page_gpa : 1; + /* Indicates the thread is performing a PI prefetch processing */ + u32 sq_load_pi : 1; + u32 dsgl_en : 1; + u32 rsvd : 3; + /* bit[19:00] The CI index of Send Queue (Step: WQEBB) 0x0 */ + u32 sq_ci : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 sq_wqe_curt_page_gpa_m; /* bit[55:24] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_curt_page_gpa_l : 12; + u32 sq_wqe_next_page_gpa_h : 20; +#else + u32 sq_wqe_next_page_gpa_h : 20; /* bit[63:44] */ + u32 sq_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + /* + * bit[43:12] Indicates the page GPA of next + * SQ buffer's page pointed by "sq_ci". 0x0 + */ + u32 sq_wqe_next_page_gpa_l; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_prefetch_ci : 16; + u32 rsvd : 4; + u32 sq_signature : 3; + u32 sq_wqe_prefetch_curtmax : 3; + u32 sq_wqe_prefetch_maxnum : 3; + u32 sq_wqe_prefetch_minnum : 3; +#else + u32 sq_wqe_prefetch_minnum : 3; + u32 sq_wqe_prefetch_maxnum : 3; + u32 sq_wqe_prefetch_curtmax : 3; + u32 sq_signature : 3; + u32 rsvd : 4; + /* + * bit[15:00] The prefetch CI index of + * Send Queue (Step: WQEBB).0x0 + */ + u32 sq_wqe_prefetch_ci : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_curt_sge_vld : 1; + u32 rsvd : 3; + u32 sq_curt_sge_lkey : 28; +#else + /* + * bit[27:00] Indicates the L_Key of current SGE. + * Using sq_curt_sge_lkey[27:8] to + * access the corresponding MPT. 0x0 + */ + u32 sq_curt_sge_lkey : 28; + u32 rsvd : 3; + /* + * Indicates current SGE information is valid. + * The remaining sq_curt_sge_* field are + * only valid when the sq_curt_sge_vld is + * asserted.1: SGE is valid;,0: SGE is + * invalid. 0x0 + */ + u32 sq_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 sq_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 sq_curt_sge_va_hi; + u32 sq_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + /* bit[31:00] Indicates the remaining memory space of current SGE. 0x0 */ + u32 sq_curt_sge_remain_len; + + /* DW12 */ + /* + * bit[63:32] Indicates the GPA of current data buffer page pointed by + * "sq_curt_sge_va".0x0 + */ + u32 sq_curt_sge_dbuff_gpa_h; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_curt_sge_dbuff_gpa_l : 20; + u32 sq_curt_sge_dbuff_gpa_vld : 1; + u32 sq_curt_sge_dsgl : 1; + u32 sq_curt_sge_used : 1; + u32 sq_curt_sge_last : 1; + u32 rsvd : 8; +#else + u32 rsvd : 8; + /* + * Indicates the current SGE is the + * last SGE for the current WQE. 0x0 + */ + u32 sq_curt_sge_last : 1; + u32 sq_curt_sge_used : 1; + u32 sq_curt_sge_dsgl : 1; + /* + * Indicates if the "sq_curt_sge_dbuff_gpa" + * field is valid. 1: it is + * valid;0:it is invalid. + */ + u32 sq_curt_sge_dbuff_gpa_vld : 1; + /* + * bit[31:12] Indicates the GPA of current + * data buffer page pointed by + * "sq_curt_sge_va".0x0 + */ + u32 sq_curt_sge_dbuff_gpa_l : 20; +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_mtt_prefetch_cline_num : 5; + u32 sq_mtt_prefetch_cline_ptr : 6; + u32 sq_wqe_prefetch_finish : 1; + u32 rsvd : 4; + u32 sq_mtt_prefetch_wqe_ci : 16; +#else + /* bit[15:0] Indicates the WQE index of prefetch MTTs operation */ + u32 sq_mtt_prefetch_wqe_ci : 16; + u32 rsvd : 4; + u32 sq_wqe_prefetch_finish : 1; + u32 sq_mtt_prefetch_cline_ptr : 6; + u32 sq_mtt_prefetch_cline_num : 5; +#endif + } bs; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_cache_thd_sel : 2; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sq_wqe_prefetch_mode : 1; + u32 sq_prefetch_one_wqe : 1; + u32 sq_curt_sge_idx : 6; + u32 rsvd : 8; + u32 sq_mtt_prefetch_sge_idx : 6; + u32 sq_load_wqe : 1; + u32 sq_load_mtt : 1; + u32 sq_load_dbuff : 1; + u32 sq_prefetch_thread_num : 2; +#else + u32 sq_prefetch_thread_num : 2; + u32 sq_load_dbuff : 1; + u32 sq_load_mtt : 1; + u32 sq_load_wqe : 1; + u32 sq_mtt_prefetch_sge_idx : 6; + u32 rsvd : 8; + /* The SGE index of current accessed WQE. 0x0 */ + u32 sq_curt_sge_idx : 6; + u32 sq_prefetch_one_wqe : 1; + u32 sq_wqe_prefetch_mode : 1; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sq_wqe_cache_thd_sel : 2; +#endif + } bs; + u32 value; + } dw15; +}; + +/* Send Queue ACK Context 64B */ +struct chip_seg_sqac { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_pd : 18; + u32 rsvd : 1; + u32 vbs_sqa_pi_on_chip : 1; + u32 sq_wqebb_size : 3; + u32 sq_page_size : 4; + u32 sq_size : 5; +#else + /* + * Send WQ size is (2^sq_size)*WQEBBs; the maximum + * SQ size is 16K WQEs, so this field + * doesn't exceed 14. They have same configuration + * value: SQC.sq_size and SQAC.sq_size. + * Configured by Driver + */ + u32 sq_size : 5; + + /* + * Page size of SQ and RQ, equals to (2^sq_page_size)*4KB. + * The following three Page size have same configuration + * value: SQC.sq_page_size, + * SQAC.sq_page_size and RQC.rq_page_size. Configured by Driver + */ + u32 sq_page_size : 4; + /* Send WQE Basic Block (WQEBB) size in bytes is 16*(2^sq_weqbb_size). + * for the SQ, this field should be fixed to 2. + * They have same configuration value: SQC. + * sq_wqebb_size and SQAC.sq_wqebb_size. + * Configured by Driver + */ + u32 sq_wqebb_size : 3; + u32 vbs_sqa_pi_on_chip : 1; + u32 rsvd : 1; + /* + * Protection Domain. + * The following five Protection Domains have same + * configuration value: QCC.rc_pd, + * SQC.sq_pd, SQAC.sq_pd, RQC.rq_pd and RRWC.rrw_pd. + * Configured by Driver + */ + u32 sq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_so_ro : 2; + u32 sq_dma_attr_idx : 6; + u32 rsvd2 : 1; + u32 sqa_prewqe_mode : 1; + u32 vbs_sqa_pi_vld : 1; + u32 rsvd1 : 1; + u32 sqa_cqn : 20; +#else + /* bit[19:00] SQ's Completion Queue Number. */ + u32 sqa_cqn : 20; + u32 rsvd1 : 1; + u32 vbs_sqa_pi_vld : 1; + u32 sqa_prewqe_mode : 1; + u32 rsvd2 : 1; + /* + * bit[05:00] It specifies the outbound PCIe + * TLP header attribute of the DMA + * operation. This filed is only valid when + * processing SQ's WQEs. The following two + * "dma_attr_idx"have same configuration value: + * SQC. sq_dma_attr_idx and + * SQAC.sq_dma_attr_idx. Configured by Driver + */ + u32 sq_dma_attr_idx : 6; + u32 sq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_state : 4; + u32 rsvd2 : 4; + u32 sq_rkey_en : 1; + u32 sq_wqe_check_en : 1; + u32 rsvd1 : 6; + u32 sqa_wqe_index : 16; +#else + u32 sqa_wqe_index : 16; /* bit[15:00] */ + u32 rsvd1 : 6; + u32 sq_wqe_check_en : 1; + u32 sq_rkey_en : 1; + u32 rsvd2 : 4; + u32 sqa_state : 4; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_ci : 16; + u32 rsvd1 : 3; + u32 dsgl_en : 1; + u32 rsvd : 1; + u32 sqa_load_page_gpa : 1; + u32 sqa_wqe_curt_page_vld : 1; + u32 sqa_wqe_next_page_vld : 1; + u32 sqa_wqe_curt_page_gpa_h : 8; +#else + u32 sqa_wqe_curt_page_gpa_h : 8; /* bit[63:56] */ + u32 sqa_wqe_next_page_vld : 1; + u32 sqa_wqe_curt_page_vld : 1; + u32 sqa_load_page_gpa : 1; + u32 rsvd : 1; + u32 dsgl_en : 1; + u32 rsvd1 : 3; + u32 sqa_ci : 16; /* bit[15:00] */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 sqa_wqe_curt_page_gpa_m; /* bit[55:24] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_wqe_curt_page_gpa_l : 12; + u32 sqa_wqe_next_page_gpa_h : 20; +#else + u32 sqa_wqe_next_page_gpa_h : 20; /* bit[63:44] */ + u32 sqa_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + u32 sqa_wqe_next_page_gpa_l; /* bit[43:12] */ + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_wqe_prefetch_ci : 16; + u32 rsvd2 : 4; + u32 sqa_signature : 3; + u32 rsvd1 : 3; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 sqa_wqe_prefetch_minnum : 3; +#else + u32 sqa_wqe_prefetch_minnum : 3; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 rsvd1 : 3; + u32 sqa_signature : 3; + u32 rsvd2 : 4; + u32 sqa_wqe_prefetch_ci : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_curt_sge_vld : 1; + u32 rsvd : 3; + u32 sqa_curt_sge_lkey : 28; +#else + u32 sqa_curt_sge_lkey : 28; /* bit[27:00] */ + u32 rsvd : 3; + u32 sqa_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 sqa_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 sqa_curt_sge_va_hi; + u32 sqa_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + u32 sqa_curt_sge_remain_len; /* bit[31:00] */ + + /* DW12 */ + u32 sqa_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_curt_sge_dbuff_gpa_l : 20; + u32 sqa_curt_sge_dbuff_gpa_vld : 1; + u32 sqa_curt_sge_dsgl : 1; + u32 sqa_curt_sge_used : 1; + u32 sqa_curt_sge_last : 1; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 sqa_curt_sge_last : 1; + u32 sqa_curt_sge_used : 1; + u32 sqa_curt_sge_dsgl : 1; + u32 sqa_curt_sge_dbuff_gpa_vld : 1; + u32 sqa_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_mtt_prefetch_cline_num : 5; + u32 sqa_mtt_prefetch_cline_ptr : 6; + u32 sqa_mtt_prefetch_finish : 1; + u32 rsvd : 4; + u32 sqa_mtt_prefetch_wqe_ci : 16; +#else + u32 sqa_mtt_prefetch_wqe_ci : 16; + u32 rsvd : 4; + u32 sqa_mtt_prefetch_finish : 1; + u32 sqa_mtt_prefetch_cline_ptr : 6; + u32 sqa_mtt_prefetch_cline_num : 5; +#endif + } bs; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqa_wqe_cache_thd_sel : 2; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 rsvd2 : 2; + u32 sqa_curt_sge_idx : 6; + u32 rsvd1 : 8; + u32 sqa_mtt_prefetch_sge_idx : 6; + u32 sqa_load_wqe : 1; + u32 sqa_load_mtt : 1; + u32 sqa_load_dbuff : 1; + u32 sqa_prefetch_thread_num : 2; +#else + u32 sqa_prefetch_thread_num : 2; + u32 sqa_load_dbuff : 1; + u32 sqa_load_mtt : 1; + u32 sqa_load_wqe : 1; + u32 sqa_mtt_prefetch_sge_idx : 6; + u32 rsvd1 : 8; + u32 sqa_curt_sge_idx : 6; + u32 rsvd2 : 2; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 sqa_wqe_cache_thd_sel : 2; +#endif + } bs; + u32 value; + } dw15; +}; + +/* Receive Queue Context 64B */ +struct chip_seg_rqc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_pd : 18; + u32 rq_inline_en : 1; + u32 rq_pi_on_chip : 1; + u32 rq_wqebb_size : 3; + u32 rq_page_size : 4; + u32 rq_size : 5; +#else + /* + * Receive Queue size, equals to (2^rq_size)*WQEBB. + * The maximum RQ size is 16K WQEs, + * so this field doesn't exceed 14. + * Configured by Driver + */ + u32 rq_size : 5; + /* + * Page size of SQ and RQ, equals to (2^rq_page_size)*4KB. + * Configured by Driver + */ + u32 rq_page_size : 4; + /* + * Receive WQE Basic Block (WQEBB) size + * in bytes is (2^rq_wqebb_size)*16B. + * The minimum size is 16B and the values 4, 5, 6, 7 are reserved. + * Configured by Driver + */ + u32 rq_wqebb_size : 3; + /* + * If set, the counter (PI index) of Receive + * Queue is stored in the chip, the counter + * is absolute value. Configured by Driver + */ + u32 rq_pi_on_chip : 1; + /* + * If set, in-line scatter is enabled for this RQ. + * Configured by Driver + */ + u32 rq_inline_en : 1; + /* + * bit[17:00] Protection Domain.The following five + * Protection Domains have same configuration + * value: QCC.rc_pd, SQC.sq_pd, SQAC.sq_pd, + * RQC.rq_pd and RRWC.rrw_pd. Configured by + * Driver + */ + u32 rq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_so_ro : 2; + u32 rq_dma_attr_idx : 6; + u32 rq_rkey_en : 1; + u32 rq_signature : 3; + u32 rq_cqn : 20; +#else + /* + * bit[19:00] Receive Queue completions are + * to be reported to this CQ number. + * Configured by Driver + */ + u32 rq_cqn : 20; + u32 rq_signature : 3; + u32 rq_rkey_en : 1; + /* + * bit[05:00] It specifies the outbound PCIe + * TLP header attribute of the DMA + * operation. This filed is only valid when + * processing RQ's WQEs. Configured by + * Driver + */ + u32 rq_dma_attr_idx : 6; + /* It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing RQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + * Configured by Driver + */ + u32 rq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_state : 4; + u32 rsvd2 : 13; + u32 rq_wqe_cache_thd_sel : 2; + u32 rq_inlin_len : 4; + u32 rq_wqecnt_rctl : 1; + u32 rsvd1 : 2; + u32 rq_wqecnt_rctl_en : 1; + u32 rq_wqecnt_lth : 4; + u32 rq_srq_en : 1; +#else + /* + * If set, indicates it should use SRQ as the receive queue. + * Configured by Driver + */ + u32 rq_srq_en : 1; + /* + * WQE Counter Low Threshold, equals to (2^rq_wqecnt_lth). + * Configured by Driver + */ + u32 rq_wqecnt_lth : 4; + /* + * If this field is equal to zero, the RQ PI updating + * should be not controlled by + * the "rq_wqecnt_rctl" field; else the RQ RI updating + * can be performed only if + * the "rq_wqecnt_rctl" is equal to one. Configured by Driver + */ + u32 rq_wqecnt_rctl_en : 1; + u32 rsvd1 : 2; + /* + * The hardware clear it to zero when performing + * a RQ PI updating, and driver set it + * to one to indicate the hardware can performing + * RQ PI updating. N/A + */ + u32 rq_wqecnt_rctl : 1; + /* + * Inline Data Length, equals to (2^rq_inline_len) bytes. + * Configured by Driver + */ + u32 rq_inlin_len : 4; + u32 rq_wqe_cache_thd_sel : 2; + u32 rsvd2 : 13; + /* + * Receive Queue State.0xf: ok;0x1: error;0x0: + * hardware has no access right.Other: + * reserved. 0x0 + */ + u32 rq_state : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_state : 4; + u32 rsvd : 2; + u32 container_en : 1; + u32 rsvd0 : 1; + u32 rq_srqn : 18; + u32 rsvd1 : 5; + u32 rq_srq_en : 1; +#else + u32 rq_srq_en : 1; + u32 rsvd1 : 5; + u32 rq_srqn : 18; + u32 rsvd0 : 1; + u32 container_en : 1; + u32 rsvd : 2; + u32 rq_state : 4; +#endif + } srq; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_state : 4; + u32 container_size : 2; + u32 container_en : 1; + u32 rsvd : 6; + u32 rq_srqn : 18; + u32 rq_srq_en : 1; +#else + u32 rq_srq_en : 1; + u32 rq_srqn : 18; + u32 rsvd : 6; + u32 container_en : 1; + u32 container_size : 2; + u32 rq_state : 4; +#endif + } srq_c; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_ci : 16; + u32 rsvd : 4; + u32 rq_load_pi : 1; + u32 rq_load_page_gpa : 1; + u32 rq_wqe_curt_page_vld : 1; + u32 rq_wqe_next_page_vld : 1; + u32 rq_wqe_curt_page_gpa_h : 8; +#else + /* + * bit[63:56] Indicates the GPA of current + * RQ buffer's page pointed by + * "rq_ci". 0x0 + */ + u32 rq_wqe_curt_page_gpa_h : 8; + /* + * Indicates if the "rq_wqe_curt_page_gpa" field is valid. + * 1: it is valid;0: it is invalid. + * 0x0 + */ + u32 rq_wqe_next_page_vld : 1; + /* + * Indicates if the "rq_wqe_curt_page_gpa" field is valid. + * 1: it is valid;0: it is invalid. + * 0x0 + */ + u32 rq_wqe_curt_page_vld : 1; + /* + * Indicates the thread is performing a + * prefetch for GPA of WQE page.0x0 + */ + u32 rq_load_page_gpa : 1; + u32 rq_load_pi : 1; + u32 rsvd : 4; + /* bit[15:00] The CI index of Receive Queue (Step: WQEBB) 0x0 */ + u32 rq_ci : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 rq_wqe_curt_page_gpa_m; /* bit[55:24] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ + u32 rq_wqe_next_page_gpa_h : 20; /* bit[63:44] */ +#else + u32 rq_wqe_next_page_gpa_h : 20; /* bit[63:44] */ + u32 rq_wqe_curt_page_gpa_l : 12; /* bit[23:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + u32 rq_wqe_next_page_gpa_l; /* bit[43:12] */ + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_pi : 16; + u32 rq_wqe_prefetch_ci : 16; +#else + /* + * bit[15:00] The prefetch CI index of + * Receive Queue (Step: WQEBB).0x0 + */ + u32 rq_wqe_prefetch_ci : 16; + /* + * bit[15:00] The Producer Index (PI) of + * Receive Queue (Step: WQEBB).0x0 + */ + u32 rq_pi : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_curt_sge_vld : 1; + u32 rsvd : 3; + u32 rq_curt_sge_lkey : 28; +#else + /* + * bit[27:00] Indicates the L_Key of current SGE. + * Using rq_curt_sge_lkey[27:8] to + * access the corresponding MPT.0x0 + */ + u32 rq_curt_sge_lkey : 28; + u32 rsvd : 3; + /* + * Indicates current SGE information is valid. The remaining + * rq_curt_sge_* field are + * only valid when the rq_curt_sge_vld is asserted. + * 1: SGE is valid;0: SGE is + * invalid.0x0 + */ + u32 rq_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 rq_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 rq_curt_sge_va_hi; + u32 rq_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + /* bit[31:00] Indicates the remaining memory space of current SGE.0x0 */ + u32 rq_curt_sge_remain_len; + + /* DW12 */ + /* + * bit[63:32] Indicates the GPA of current data buffer page pointed by + * "rq_curt_sge_va".0x0 + */ + u32 rq_curt_sge_dbuff_gpa_h; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_curt_sge_dbuff_gpa_l : 20; /* bit[51:32] */ + /* + * Indicates if the "rq_curt_sge_dbuff_gpa" field is valid.1: it is + * valid;0: it is invalid.0x0 + */ + u32 rq_curt_sge_dbuff_gpa_vld : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rq_curt_sge_used : 1; + /* Indicates the current SGE is the last SGE for the current WQE.0x0 */ + u32 rq_curt_sge_last : 1; + /* + * Indicates the current Receive WQE's status. + * 0: the Receive WQE don't be + * accessed; 1: the Receive WQE has been accessed.0x0 + */ + u32 rq_curt_wqe_status : 1; + u32 rsvd : 3; + u32 rq_mtt_prefetch_sge_idx : 4; +#else + u32 rq_mtt_prefetch_sge_idx : 4; + u32 rsvd : 3; + u32 rq_curt_wqe_status : 1; + u32 rq_curt_sge_last : 1; + u32 rq_curt_sge_used : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rq_curt_sge_dbuff_gpa_vld : 1; + u32 rq_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_curt_sge_remain_len : 17; + u32 rq_curt_sge_meta_lkey : 3; + u32 rsvd1 : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rq_curt_sge_used : 1; + u32 rq_curt_sge_last : 1; + + u32 rq_curt_meta_used : 1; + u32 rq_curt_meta_last : 1; + u32 rq_curt_meta_vld : 1; + u32 rq_prefetch_thread_num : 2; + u32 rq_curt_meta_lkey : 3; +#else + + u32 rq_curt_meta_lkey : 3; + u32 rq_prefetch_thread_num : 2; + u32 rq_curt_meta_vld : 1; + u32 rq_curt_meta_last : 1; + u32 rq_curt_meta_used : 1; + + u32 rq_curt_sge_last : 1; + u32 rq_curt_sge_used : 1; + u32 rq_curt_sge_dsgl : 1; + u32 rsvd1 : 1; + u32 rq_curt_sge_meta_lkey : 3; + u32 rq_curt_sge_remain_len : 17; +#endif + } bs_dsgl; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_prefetch_thread_num : 2; + u32 rq_mtt_prefetch_cline_num : 4; + u32 rq_mtt_prefetch_cline_ptr : 6; + u32 rq_mtt_prefetch_finish : 1; + u32 rsvd : 3; + u32 rq_mtt_prefetch_wqe_ci : 16; +#else + u32 rq_mtt_prefetch_wqe_ci : 16; + u32 rsvd : 3; + u32 rq_mtt_prefetch_finish : 1; + u32 rq_mtt_prefetch_cline_ptr : 6; + u32 rq_mtt_prefetch_cline_num : 4; + u32 rq_prefetch_thread_num : 2; +#endif + } bs; + struct { + } bc_c; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_wqe_prefetch_maxnum : 3; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rq_curt_sge_idx : 4; + u32 rsvd : 11; + u32 rq_load_next_mtt : 1; + u32 rq_load_wqe : 1; + u32 rq_load_curt_mtt : 1; + u32 rq_load_dbuff : 1; +#else + u32 rq_load_dbuff : 1; + u32 rq_load_curt_mtt : 1; + u32 rq_load_wqe : 1; + u32 rq_load_next_mtt : 1; + u32 rsvd : 11; + /* The SGE index of current accessed WQE.0x0 */ + u32 rq_curt_sge_idx : 4; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_wqe_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw15; +}; + +/* SRQ info in QPC */ +struct chip_seg_srqc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_pd : 18; + u32 rsvd : 2; + u32 srq_wqebb_size : 3; + u32 srq_page_size : 4; + u32 srq_size : 5; +#else + u32 srq_size : 5; + u32 srq_page_size : 4; + /* + * Shared Receive WQE Basic Block (WQEBB) size in bytes is + * (2^rq_srq_wqebb_size)*16B. The minimum size is 32B and + * the values 0, 4, 5, 6, 7 are reserved. + * This field is updated by SRQC.srq_wqebb_size. + * 0x0 + */ + u32 srq_wqebb_size : 3; + u32 rsvd : 2; + /* + * bit[17:00]Protection Domain. If the QP is a + * XRC transport service type, + * this filed should be updated by the "srq_pd" + * in the SRQ context + * pointed by XRCSRQ; else this field is updated + * by the "srq_pd" in the + * SRQ context associated with the QP. + * The following five Protection Domains have + * same configuration value: + * QCC.rc_pd, SQC.sq_pd, SQAC.sq_pd, RQC.rq_pd and RRWC.rrw_pd. + * 0x0 + */ + u32 srq_pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_so_ro : 2; + u32 srq_dma_attr_idx : 6; + u32 srq_rkey_en : 1; + u32 srq_signature : 3; + u32 srq_cqn : 20; +#else + /* + * bit[19:00] Receive Queue completions are + * to be reported to this CQ number. + * If the QP is a XRC transport service type, + * this filed should be updated + * by the "srq_xrc_cqn" in the SRQ context + * pointed by XRCSRQ; else this + * field is configured by the driver. + * Configured by Driver + */ + u32 srq_cqn : 20; + u32 srq_signature : 3; + u32 srq_rkey_en : 1; + u32 srq_dma_attr_idx : 6; /* bit[05:00] */ + /* + * It specifies the ATTR[1:0] bits in the + * outbound PCIe TLP headers of the + * DMA operation. This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + * This field is updated by SRQC.srq_so_ro. + * 0x0 + */ + u32 srq_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_state : 4; + u32 rsvd2 : 4; + u32 srqn : 18; + u32 rsvd1 : 5; + u32 srq_en : 1; +#else + /* + * If set, indicates it should use SRQ as the receive queue. + * Configured by Driver + */ + u32 srq_en : 1; + u32 rsvd1 : 5; + /* + * SRQ number. If the QP is a XRC transport service type, this + * filed should be updated by the XRCSRQ; else this field is + * configured by the driver. Configured by Driver + */ + u32 srqn : 18; + u32 rsvd2 : 4; + /* + * Receive Queue State. + * 0x0: hardware has no access right;0x1: error;0xf: ok.Other: reserved. + * 0x0 + */ + u32 srq_state : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 container_sz : 2; + u32 container_en : 1; + u32 ep : 3; + u32 cos : 3; + u32 rq_srqn : 18; + u32 rq_srq_en : 1; +#else + u32 rq_srq_en : 1; + u32 rq_srqn : 18; + u32 cos : 3; + u32 ep : 3; + u32 container_en : 1; + u32 container_sz : 2; + u32 state : 4; +#endif + } bs_c; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 srq_xrcd : 16; +#else + u32 srq_xrcd : 16; + u32 rsvd : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 rsvd_dw4; + + /* DW5 */ + u32 srq_curt_wqe_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_curt_wqe_gpa_l : 24; /* bit[15:00] */ + u32 rsvd : 8; +#else + u32 rsvd : 8; /* bit[15:00] */ + u32 srq_curt_wqe_gpa_l : 24; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_gpa : 20; + u32 gpa_vld : 1; /* bit[15:00] */ + u32 wqe_gpa_flag : 1; + u32 warth_flag : 1; + u32 last_op_wqe : 1; + u32 link_wqe : 1; + u32 rqe_cnt_th : 4; + u32 rsvd : 3; +#else + u32 rsvd : 3; + u32 rqe_cnt_th : 4; + u32 link_wqe : 1; + u32 last_op_wqe : 1; + u32 warth_flag : 1; + u32 wqe_gpa_flag : 1; + u32 gpa_vld : 1; + u32 wqe_gpa : 20; +#endif + } bs_c; + u32 value; + } dw6; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pre_ci : 16; /* bit[15:00] */ + u32 pi : 16; +#else + u32 pi : 16; /* bit[15:00] */ + u32 pre_ci : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_curt_sge_vld : 1; + u32 srq_curt_sge_idx : 3; + u32 srq_curt_sge_lkey : 28; /* bit[27:00] */ +#else + u32 srq_curt_sge_lkey : 28; /* bit[27:00] */ + u32 srq_curt_sge_idx : 3; + u32 srq_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw8; + + /* DW9~10 */ + union { + u64 srq_curt_sge_va; /* lo:bit[31:00],hi:bit[63:32] */ + struct { + u32 srq_curt_sge_va_hi; + u32 srq_curt_sge_va_lo; + } dw9; + }; + + /* DW11 */ + u32 srq_curt_sge_remain_len; /* bit[31:00] */ + + /* DW12 */ + u32 srq_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_curt_sge_dbuff_gpa_l : 20; + u32 srq_curt_sge_dbuff_gpa_vld : 1; + u32 srq_curt_sge_dsgl : 1; + u32 srq_curt_sge_used : 1; + u32 srq_curt_sge_last : 1; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 srq_curt_sge_last : 1; + u32 srq_curt_sge_used : 1; + u32 srq_curt_sge_dsgl : 1; + u32 srq_curt_sge_dbuff_gpa_vld : 1; + u32 srq_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_prefetch_thread_num : 2; + u32 srq_mtt_prefetch_cline_num : 4; + u32 srq_mtt_prefetch_cline_ptr : 6; + u32 srq_mtt_prefetch_finish : 1; + u32 srq_mtt_prefetch_sge_idx : 3; + u32 rsvd : 16; +#else + u32 rsvd : 16; + u32 srq_mtt_prefetch_sge_idx : 3; + u32 srq_mtt_prefetch_finish : 1; + u32 srq_mtt_prefetch_cline_ptr : 6; + u32 srq_mtt_prefetch_cline_num : 4; + u32 srq_prefetch_thread_num : 2; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 prefetch_thread_num : 2; + u32 mtt_page_size : 4; + u32 rsvd1 : 10; + u32 xrcd : 16; +#else + u32 xrcd : 16; + u32 rsvd1 : 10; + u32 mtt_page_size : 4; + u32 prefetch_thread_num : 2; +#endif + } bs_c; + + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd2 : 11; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 rsvd1 : 17; + u32 srq_load_mtt : 1; + u32 srq_load_dbuff : 1; +#else + u32 srq_load_dbuff : 1; + u32 srq_load_mtt : 1; + u32 rsvd1 : 17; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 rsvd2 : 11; +#endif + } bs; + + u32 value; + } dw15; +}; + +/* Queue Pair Common Context Format */ +struct chip_seg_qpcc { + /* DW0~1 */ + union { + /* hi[63:32],lo[31:03],sq_rq_gpa_sign[02:00] */ + u64 sq_rq_l0mtt_gpa; + struct { + u32 sq_rq_l0mtt_gpa_hi; + u32 sq_rq_l0mtt_gpa_lo; + } dw0; + }; + + /* DW2~3 */ + union { + /* hi[63:32],lo[31:02],sq_rq_at_hop_num[01:00] */ + u64 sq_rq_pi_record_gpa_at_hop_num; + struct { + u32 sq_rq_pi_record_gpa_hi; + u32 sq_rq_pi_record_gpa_lo_at_hop_num; /* at_hop_num: bit[01:00] */ + } dw2; + }; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_base_ci : 5; + u32 rsvd : 2; + u32 rc_max_size : 3; + u32 sq_rq_mtt_page_size : 4; + u32 qp_pd : 18; +#else + u32 qp_pd : 18; + u32 sq_rq_mtt_page_size : 4; + u32 rc_max_size : 3; + u32 rsvd : 2; + u32 rq_base_ci : 5; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_entry_size : 2; + u32 rc_size : 4; + u32 qp_rkey_en : 1; + u32 qp_rwe : 1; + u32 qp_rae : 1; + u32 qp_rre : 1; + u32 rsvd : 1; + u32 qp_signature : 5; + u32 qp_xrcd : 16; +#else + u32 qp_xrcd : 16; + u32 qp_signature : 5; + u32 rsvd : 1; + u32 qp_rre : 1; + u32 qp_rae : 1; + u32 qp_rwe : 1; + u32 qp_rkey_en : 1; + /* + * RDMARC table size, equals to (2^rc_size)*Entry Size. + * 0x0: the depth of table is equal to 1; + * 0x1: the depth of table is equal to 2; + * 0x2: the depth of table is equal to 4; + * ... + * 0x7: the depth of table is equal to 128; + * Others: reserved. + * Configured by Driver + */ + u32 rc_size : 4; + /* + * Entry size of RDMARC table in bytes is (2^rc_entry_size)*16B. + * The minimum size is 32B and the maximum size is 64B; + * so the values 0 and 3 are + * reserved. Configured by Driver + */ + u32 rc_entry_size : 2; +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_entry_prefetch_maxnum : 3; + u32 rsvd : 5; + u32 rc_page_gpa_h : 24; +#else + /* + * bit[63:40] Indicates the start GPA of RDMARC table. + * The driver needs to allocate + * continuous physical address for the RDMARC table. + * Configured by Driver + */ + u32 rc_page_gpa_h : 24; + u32 rsvd : 5; + /* + * Maximum number of prefetch Entries for RDMARC table. + * 000: prefetch number + * equals to zero; Others: prefetch number equals to + * (2^(rc_entry_prefetch_maxnum-1)). Configured by Driver + */ + u32 rc_entry_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + u32 rc_page_gpa_l; /* bit[39:8] */ +}; + +/* RDMARC Context Format */ +struct chip_seg_rcc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_curt_sge_vld : 1; + u32 rsvd : 3; + u32 rc_curt_sge_rkey : 28; +#else + /* + * bit[27:00] Indicates the R_Key of current SGE. + * Using rc_curt_sge_rkey[27:8] to + * access the corresponding MPT.0x0 + */ + u32 rc_curt_sge_rkey : 28; + u32 rsvd : 3; + /* + * Indicates current SGE information is valid. + * The remaining rc_curt_sge_* field are + * only valid when the rc_curt_sge_vld is asserted. + * 1: SGE is valid;0: SGE is + * invalid. + */ + u32 rc_curt_sge_vld : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1~2 */ + union { + u64 rc_curt_sge_va; + struct { + u32 rc_curt_sge_va_hi; + u32 rc_curt_sge_va_lo; + } dw1; + }; + + /* DW3 */ + /* bit[31:00]Indicates the remaining memory space of current SGE.0x0 */ + u32 rc_curt_sge_remain_len; + + /* DW4 */ + u32 rc_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ + u32 rc_curt_sge_dbuff_gpa_vld : 1; + u32 rc_curt_sge_dsgl : 1; + u32 rc_curt_sge_used : 1; + u32 dsgl_en : 1; + u32 rc_pi : 8; +#else + /* + * bit[07:00] The Producer Index (PI) of + * RDMARC table (Step: Entry Size).0x0 + */ + u32 rc_pi : 8; + u32 dsgl_en : 1; + u32 rc_curt_sge_used : 1; + u32 rc_curt_sge_dsgl : 1; + /* + * Indicates if the "rc_curt_sge_dbuff_gpa" field is valid.1: it is + * valid;0: it is invalid.0x0 + */ + u32 rc_curt_sge_dbuff_gpa_vld : 1; + u32 rc_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_pd : 18; + u32 rc_signature : 3; + u32 rsvd : 3; + u32 rc_ci : 8; +#else + /* + * bit[07:00] The Consumer Index (CI) of + * RDMARC table (Step: Entry Size). + */ + u32 rc_ci : 8; + u32 rsvd : 3; + u32 rc_signature : 3; + /* + * bit[17:00] Protection Domain. If the QP is a XRC transport + * service type, this filed should + * be updated by the "srq_pd" in the SRQ context pointed by + * XRCSRQ; else this field should + * be equal to QCC.rc_pd.0x0 + */ + u32 rc_pd : 18; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_state : 4; + u32 rc_rae : 1; + u32 rc_rre : 1; + u32 rsvd2 : 1; + u32 rc_mtt_prefetch_maxlen : 2; + u32 rc_prefetch_thread_num : 2; + u32 rc_load_entry : 1; + u32 rc_load_mtt : 1; + u32 rc_load_dbuff : 1; + u32 rc_mtt_prefetch_finish : 1; + u32 rc_mtt_prefetch_cline_num : 4; + u32 rsvd1 : 5; + u32 rc_prefetch_ci : 8; +#else + u32 rc_prefetch_ci : 8; + u32 rsvd1 : 5; + u32 rc_mtt_prefetch_cline_num : 4; + u32 rc_mtt_prefetch_finish : 1; + u32 rc_load_dbuff : 1; + u32 rc_load_mtt : 1; + u32 rc_load_entry : 1; + u32 rc_prefetch_thread_num : 2; + u32 rc_mtt_prefetch_maxlen : 2; + u32 rsvd2 : 1; + u32 rc_rre : 1; + u32 rc_rae : 1; + u32 rc_state : 4; +#endif + } bs; + u32 value; + } dw7; +}; + +/* RQ RDMA Write Context */ +struct chip_seg_rrwc { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_curt_sge_vld : 1; + u32 rsvd : 3; + u32 rrw_curt_sge_rkey : 28; +#else + u32 rrw_curt_sge_rkey : 28; /* bit[27:00] */ + u32 rsvd : 3; /* bit[30:28] */ + u32 rrw_curt_sge_vld : 1; /* bit[31] */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1~2 */ + union { + u64 rrw_curt_sge_va; /* hi:bit[63:32],lo:bit[31:00] */ + struct { + u32 rrw_curt_sge_va_hi; + u32 rrw_curt_sge_va_lo; + } dw1; + }; + + /* DW3 */ + u32 rrw_curt_sge_remain_len; /* bit[31:00] */ + + /* DW4 */ + u32 rrw_curt_sge_dbuff_gpa_h; /* bit[63:32] */ + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ + u32 rrw_curt_sge_dbuff_gpa_vld : 1; + u32 rrw_curt_sge_dsgl : 1; + u32 rrw_curt_sge_used : 1; + u32 rsvd : 9; +#else + u32 rsvd : 9; + u32 rrw_curt_sge_used : 1; + u32 rrw_curt_sge_dsgl : 1; + u32 rrw_curt_sge_dbuff_gpa_vld : 1; + u32 rrw_curt_sge_dbuff_gpa_l : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_pd : 18; + u32 rrw_signature : 3; + u32 rsvd : 11; +#else + u32 rsvd : 11; + u32 rrw_signature : 3; + u32 rrw_pd : 18; + +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rrw_state : 4; + u32 rrw_rwe : 1; + u32 rsvd3 : 2; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 rrw_prefetch_thread_num : 2; + u32 rsvd2 : 1; + u32 rrw_load_mtt : 1; + u32 rrw_load_dbuff : 1; + u32 rrw_mtt_prefetch_finish : 1; + u32 rrw_mtt_prefetch_cline_num : 4; + u32 rsvd1 : 13; +#else + u32 rsvd1 : 13; + u32 rrw_mtt_prefetch_cline_num : 4; + u32 rrw_mtt_prefetch_finish : 1; + u32 rrw_load_dbuff : 1; + u32 rrw_load_mtt : 1; + u32 rsvd2 : 1; + u32 rrw_prefetch_thread_num : 2; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 rsvd3 : 2; + u32 rrw_rwe : 1; + u32 rrw_state : 4; +#endif + } bs; + u32 value; + } dw7; +}; + +/* + ***************************************************************************** + Data Structure: SQPC + Description: +***************************************************************************** +*/ +struct chip_seg_sqpc { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_signature : 5; + u32 rsvd0 : 14; + u32 sqp_ci_on_chip : 1; + u32 sqp_wqe_size : 3; + u32 sqp_page_size : 4; + u32 sqp_size : 5; +#else + u32 sqp_size : 5; + u32 sqp_page_size : 4; + u32 sqp_wqe_size : 3; + u32 sqp_ci_on_chip : 1; + u32 rsvd0 : 14; + u32 sqp_signature : 5; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_so_ro : 2; + u32 sqp_dma_attr_idx : 6; + u32 rsvd0 : 8; + u32 sqp_ci : 16; +#else + u32 sqp_ci : 16; + u32 rsvd0 : 8; + u32 sqp_dma_attr_idx : 6; + u32 sqp_so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_state : 4; + u32 rsvd0 : 12; + u32 sqp_pi : 16; +#else + u32 sqp_pi : 16; + u32 rsvd0 : 12; + u32 sqp_state : 4; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_wqecnt_lth : 4; + u32 sqp_wqecnt_rctl_en : 1; + u32 rsvd0 : 15; + u32 sqp_load_ci : 1; + u32 sqp_load_page_gpa : 1; + u32 sqp_wqe_curt_page_vld : 1; + u32 sqp_wqe_next_page_vld : 1; + u32 sqp_wqe_curt_page_gpa_h : 8; +#else + u32 sqp_wqe_curt_page_gpa_h : 8; + u32 sqp_wqe_next_page_vld : 1; + u32 sqp_wqe_curt_page_vld : 1; + u32 sqp_load_page_gpa : 1; + u32 sqp_load_ci : 1; + u32 rsvd0 : 15; + u32 sqp_wqecnt_rctl_en : 1; + u32 sqp_wqecnt_lth : 4; +#endif + } bs; + u32 value; + } dw3; + + u32 sqp_wqe_curt_page_gpa_m; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_wqe_curt_page_gpa_l : 12; + u32 sqp_wqe_next_page_gpa_h : 20; +#else + u32 sqp_wqe_next_page_gpa_h : 20; + u32 sqp_wqe_curt_page_gpa_l : 12; +#endif + } bs; + u32 value; + } dw5; + + u32 sqp_wqe_next_page_gpa_l; + + u32 rsvd_dw7; + + u32 rsvd_dw8; + + u32 rsvd_dw9; + + u32 rsvd_dw10; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 16; + u32 sqp_prefetch_thread_num : 2; + u32 rsvd1 : 10; + u32 sqp_mtt_page_size : 4; +#else + u32 sqp_mtt_page_size : 4; + u32 rsvd1 : 10; + u32 sqp_prefetch_thread_num : 2; + u32 rsvd0 : 16; +#endif + } bs; + u32 value; + } dw11; + + u32 sqp_l0mtt_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_l0mtt_gpa_l : 29; + u32 sqp_gpa_sign : 3; +#else + u32 sqp_gpa_sign : 3; + u32 sqp_l0mtt_gpa_l : 29; +#endif + } bs; + u32 value; + } dw13; + + u32 sqp_pi_record_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sqp_pi_record_gpa_l : 30; + u32 sqp_at_hop_num : 2; +#else + u32 sqp_at_hop_num : 2; + u32 sqp_pi_record_gpa_l : 30; +#endif + } bs; + u32 value; + } dw15; +}; + +/* Queue Pair Context */ +struct qpc_chip_seg { + /* RQ RDMA Write Context (32B) */ + struct chip_seg_rrwc rrwc; + /* RDMARDC Context (32B) */ + struct chip_seg_rcc rcc; + + /* Send Queue Context (64B) */ + struct chip_seg_sqc sqc; + /* Send Queue ACK Context (64B) */ + struct chip_seg_sqac sqac; + union { + /* Receive Queue Context (64B) */ + struct chip_seg_rqc rqc; + /* RQC Shared Receive Queue Mode(64B) */ + struct chip_seg_srqc srqc; + }; + + /* Queue Pair Common Context (32B) */ + struct chip_seg_qpcc qpcc; +}; + +/* Timer Section(32B) */ +struct qpc_timer_seg { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd3 : 1; + /* + * Indicate how many timers this timer section + * supports:0:rsvd,X:support up to X timers + */ + u32 tss_timer_num : 3; + /* + * Lock flag indicates whether this timer + * section is locked by one API or not + * 1: locked 0: unlock + */ + u32 tss_lock_flag : 1; + /* + * Modify flag indicates whether this timer section + * has been modified by other API + * without lock or not. 1: has been modified,0: has not been modified + */ + u32 tss_modify_flag : 1; + u32 rsvd2 : 1; + /* + * close flag indicates this connection needs to close + * and this XID context will be + * de-allocated by control plane; 0: not close,1: + * close,this timer section will be + * picked off from timer wheel and issue a special + * time-out API to tell ucode this + * context resource can be de-allocated. + */ + u32 tss_close_flag : 1; + u32 rsvd1 : 3; + u32 rsvd0 : 2; + /* + * Timer link valid, + * 1: this timer section is hung in timing wheel + * 0: this timer section is not hung in any timing wheel + */ + u32 tlwi_lv : 1; + /* indicate which timer is hung in timing wheel */ + u32 tlwi_timer_id : 3; + /* indicate which timing wheel this timer section is hung in */ + u32 tlwi_wheel_id : 3; + /* indicate which fire spoke this timer section is hung in */ + u32 tlwi_link_spoke : 12; +#else + u32 tlwi_link_spoke : 12; + u32 tlwi_wheel_id : 3; + u32 tlwi_timer_id : 3; + u32 tlwi_lv : 1; + u32 rsvd0 : 2; + u32 rsvd1 : 3; + + u32 tss_cl : 1; + u32 rsvd2 : 1; + u32 tss_modify_flag : 1; + u32 tss_lock_flag : 1; + u32 tss_timer_num : 3; + u32 rsvd3 : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + /* + * Previous link pointer, data structure is referred + * to timer pointer data structure + */ + u32 pre_link_ptr0; + + /* DW2 */ + u32 pre_link_ptr1; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pre_link_ptr : 8; + u32 next_link_ptr : 24; +#else + u32 next_link_ptr : 24; + u32 pre_link_ptr : 8; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + /* + * Next link pointer, data structure is referred + * to timer pointer data structure + */ + u32 next_link_ptr1; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 next_link_ptr : 16; + u32 rsvd : 16; +#else + u32 rsvd : 16; + u32 next_link_ptr : 16; +#endif + } bs; + u32 value; + } dw5; + + u32 ts_rsvd[2]; /* reserve 2DW */ +}; + +union ucode_rq_last { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_last_optype : 1; + u32 rq_last_opcode : 1; + u32 rsvd : 1; + u32 atomic_aloc_fail : 1; + u32 comm_est : 1; + u32 rnr_state : 1; + u32 err_state : 1; + u32 rq_ready_n : 1; + u32 last_msn : 24; +#else + u32 last_msn : 24; + u32 rq_ready_n : 1; + u32 err_state : 1; + u32 rnr_state : 1; + u32 comm_est : 1; + u32 atomic_aloc_fail : 1; + u32 rsvd : 1; + u32 rq_last_opcode : 1; + u32 rq_last_optype : 1; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_last_op : 2; + u32 rsv_com : 30; +#else + u32 rsv_com : 30; + u32 rq_last_op : 2; +#endif + } bs1; + + u32 value; +}; + +/* * sq ack ctx */ +struct ucode_sq_ack_ctx { + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 time_interval : 6; + u32 timestamp : 26; +#else + u32 timestamp : 26; + u32 time_interval : 6; // timer update interval +#endif + } bs; + u32 value; + } dw12; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 last_md_nrcv : 1; + u32 nof_sw : 1; + u32 rx_port : 3; + u32 credit_nlimit : 1; + u32 rsvd : 2; + u32 lsn : 24; +#else + u32 lsn : 24; + u32 rsvd : 2; + u32 credit_nlimit : 1; + u32 rx_port : 3; + u32 nof_sw : 1; + u32 last_md_nrcv : 1; // last metadata flag in dif +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 timer_type : 1; + u32 timer_en : 1; + u32 rnr_retry_cnt : 3; + u32 retry_cnt : 3; + u32 sq_rcv_psn : 24; +#else + u32 sq_rcv_psn : 24; + u32 retry_cnt : 3; + u32 rnr_retry_cnt : 3; + u32 timer_en : 1; + u32 timer_type : 1; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsv_com : 2; + u32 retry_reset : 6; + u32 rsv_com1 : 24; +#else + u32 rsv_com1 : 24; + u32 retry_reset : 6; + u32 rsv_com : 2; +#endif + } bs1; + u32 value; + } dw14; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 next_rdat : 1; + u32 sq_db : 1; + u32 rnr_state : 1; + u32 retry_state : 1; + u32 rsvd : 2; + u32 err_state : 1; + u32 sqa_ready_n : 1; + u32 sq_rcv_msn : 24; +#else + u32 sq_rcv_msn : 24; + u32 sqa_ready_n : 1; + u32 err_state : 1; + u32 rsvd : 2; + u32 retry_state : 1; + u32 rnr_state : 1; + u32 sq_db : 1; + u32 next_rdat : 1; +#endif + } bs; + u32 value; + } dw15; + + /* DW16 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sra_ack_cnt : 8; + u32 wqe_first_psn : 24; +#else + u32 wqe_first_psn : 24; + u32 sra_ack_cnt : 8; +#endif + } bs; + u32 value; + } dw16; + + /* DW17 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lcn : 8; + u32 max_fwd_psn : 24; +#else + u32 max_fwd_psn : 24; + u32 lcn : 8; +#endif + } bs; + u32 value; + } dw18; + + /* DW18 */ + u32 sq_rcv_len; + + /* DW19 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 rsvd : 18; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe2host_ci : 8; + u32 last_nak_psn : 24; +#else + u32 last_nak_psn : 24; + u32 wqe2host_ci : 8; +#endif + } nofaa; + u32 value; + } dw17; +}; + +struct ucode_sq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sra_cnt : 8; + u32 next_send_psn : 24; +#else + /* + * Initial value of the driver in BTH Packet + * Sequence Number(initial)RTR2RTS mode + */ + u32 next_send_psn : 24; + u32 sra_cnt : 8; /* Initiator depth count value */ +#endif + } bs; + u32 value; + } dw8; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lcn : 8; + u32 ssn : 24; +#else + u32 ssn : 24; + u32 lcn : 8; +#endif + } bs; + u32 value; + } dw9; + + /* DW2 */ + u32 send_left_len; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 17; + u32 last_md_nst : 1; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 last_md_nst : 1; + u32 rsvd : 17; +#endif + } dif; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 10; + u32 dci_state : 2; + u32 drctn : 20; +#else + u32 drctn : 20; + u32 dci_state : 2; + u32 rsvd : 10; +#endif + } drc; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sticky : 1; + u32 work_queue : 2; + u32 wqe2host_pi : 8; + u32 rsvd : 21; +#else + u32 rsvd : 21; + u32 wqe2host_pi : 8; + u32 work_queue : 2; + u32 sticky : 1; +#endif + } nofaa; + u8 vbs_sq[4]; + u32 value; + } dw11; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sync : 1; + u32 credit_ignore : 1; + u32 err_state : 1; + u32 sq_ready_n : 1; + u32 sq_flush_wait : 1; + u32 port_err : 1; + u32 ud_last_time : 26; +#else + u32 ud_last_time : 26; + u32 port_err : 1; + u32 sq_flush_wait : 1; + u32 sq_ready_n : 1; + u32 err_state : 1; + u32 credit_ignore : 1; + u32 sync : 1; +#endif + } bs; + u32 value; + } dw12; + + struct ucode_sq_ack_ctx ack_ctx; +}; + +/* * rq ack ctx */ +struct ucode_rq_ack_ctx { + /* DW24 */ + u32 rq_rsp_left; + + /* DW25 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rra_ack_cnt : 8; + u32 next_ack_psn : 24; +#else + u32 next_ack_psn : 24; + u32 rra_ack_cnt : 8; +#endif + } bs; + u32 value; + } dw25; + + /* DW26 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rqa_last_optype : 1; + u32 port_err : 1; + u32 nxt_rdat : 1; + u32 repeat_flag : 1; + u32 nak_rsp : 1; + u32 err_state : 1; + u32 rqa_ready_n : 1; + u32 remote_db : 1; + u32 last_ackpsn_low : 8; + u32 atomic_aloc_fail : 1; + u32 rq_flush_wait : 1; + u32 last_wqe_reached : 1; + u32 rq_recv_psn_cnt : 5; + u32 syndrome : 8; +#else + u32 syndrome : 8; + u32 rq_recv_psn_cnt : 5; + u32 last_wqe_reached : 1; + u32 rq_flush_wait : 1; + u32 atomic_aloc_fail : 1; + u32 last_ackpsn_low : 8; + u32 remote_db : 1; + u32 rqa_ready_n : 1; + u32 err_state : 1; + u32 nak_rsp : 1; + u32 repeat_flag : 1; + u32 nxt_rdat : 1; + u32 port_err : 1; + u32 rqa_last_optype : 1; +#endif + } bs; + + u32 value; + } dw26; + + /* DW27 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 1; + u32 credit : 7; + u32 msn : 24; +#else + u32 msn : 24; + u32 credit : 7; + u32 rsvd : 1; +#endif + } bs; + u32 value; + } dw27; + + /* DW28 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 32; +#else + u32 rsvd : 32; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 3; + u32 back_len : 13; + u32 last_md_nst : 1; + u32 rsvd1 : 1; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 rsvd1 : 1; + u32 last_md_nst : 1; + u32 back_len : 13; + u32 rsvd0 : 3; +#endif + } dif; + u32 value; + } dw28; +}; + +struct ucode_rq_ctx { + /* DW20 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rra_cnt : 8; + u32 next_rcv_psn : 24; +#else + u32 next_rcv_psn : 24; + u32 rra_cnt : 8; +#endif + } bs; + u32 value; + } dw20; + + /* DW21 */ + union { + u32 rq_rcv_len; + } dw21; + + /* DW22 */ + union ucode_rq_last dw22; + + /* DW23 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 last_md_nrcv : 1; + u32 rsvd : 7; + u32 mpt_index : 24; +#else + u32 mpt_index : 24; + u32 rsvd : 7; + u32 last_md_nrcv : 1; // last metadata flag in dif +#endif + } bs; + u32 value; + } dw23; + /* DW24 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 task_tag : 14; +#else + u32 task_tag : 14; + u32 rsvd : 18; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ctrl_rcv : 2; + u32 repeat : 1; + u32 rsvd : 29; +#else + u32 rsvd : 29; + u32 repeat : 1; + u32 ctrl_rcv : 2; +#endif + } nofaa; + u32 value; + } dw24; + /* DW24~27 */ + struct ucode_rq_ack_ctx ack_ctx; +}; + +struct ucode_dcqcn_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_flag : 1; + u32 f_cnt : 3; + u32 cnp_receive : 1; + u32 ai_cnt : 3; + u32 reinq_times : 12; + u32 rsvd : 2; + u32 alpha : 10; +#else + u32 alpha : 10; + u32 rsvd : 2; + u32 reinq_times : 12; + u32 ai_cnt : 3; + u32 cnp_receive : 1; + u32 f_cnt : 3; + u32 cnp_flag : 1; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 token_period : 4; + u32 rate_period : 4; + u32 cur_rate : 24; +#else + u32 cur_rate : 24; + u32 rate_period : 4; + u32 token_period : 4; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 target_rate : 24; +#else + u32 target_rate : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 token : 24; /* valid token */ +#else + u32 token : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw3; + + u32 rcnp_timestamp; /* last receive cnp */ + u32 alpha_timestamp; /* last update alpha */ + u32 rate_timestamp; /* last update rate */ +}; + +struct ucode_common_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsp_ready_n : 1; + u32 req_ready_n : 1; + u32 rq_flush : 1; + u32 sq_flush : 1; + u32 rsvd : 3; + u32 ccf_appid : 3; + u32 port : 2; + /* for user plugin extension */ + u32 ulp_type : 4; + u32 fake : 1; + /* Cq overflow Indicates whether WQ CATAS is generated. */ + u32 cq_ovl_flag : 1; + u32 sqd_event : 1; + u32 vf_id : 13; +#else + u32 vf_id : 13; + u32 sqd_event : 1; + /* Cq overflow Indicates whether WQ CATAS is generated. */ + u32 cq_ovl_flag : 1; + u32 fake : 1; + u32 ulp_type : 4; + u32 port : 2; + u32 ccf_appid : 3; + u32 rsvd : 3; + u32 sq_flush : 1; + u32 rq_flush : 1; + u32 req_ready_n : 1; + u32 rsp_ready_n : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 unit_time : 8; + u32 rsvd : 11; + u32 dst_vfid : 13; +#else + u32 dst_vfid : 13; + u32 rsvd : 11; + u32 unit_time : 8; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 pd : 18; +#else + u32 pd : 18; + u32 rsvd : 14; +#endif + } ud; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 drcin : 20; +#else + u32 drcin : 20; + u32 rsvd : 12; +#endif + } drct; + u32 drci_drc_key_h; + u32 value; + } dw1; + + /* DW2 */ + union { + u32 qkey; + u32 drci_drc_key_l; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_tx_filter_ts : 12; + u32 rq_cqn : 20; +#else + u32 rq_cqn : 20; + u32 cnp_tx_filter_ts : 12; +#endif + } bs; + u32 value; + } dw3; +}; + +struct drv_ucode_all_info { + /* DW0~DW4 */ + struct ucode_common_ctx common; + + /* DW5~DW18 */ + struct ucode_sq_ctx sq_ctx; + + /* DW19~DW27 */ + struct ucode_rq_ctx rq_ctx; +}; + +struct drv_path_info { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 bond_tx_hash_value : 16; + u32 dmac_h16 : 16; +#else + u32 dmac_h16 : 16; + u32 bond_tx_hash_value : 16; +#endif + } bs; + u32 value; + } dw0; + + u32 dmac_l32; + + /* DW2~5 */ + u8 dgid[16]; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ext_mtu : 1; + u32 ext_md : 1; + u32 rcq_lb1 : 1; + u32 scq_lb1 : 1; + u32 tclass : 8; + u32 flow_label : 20; +#else + u32 flow_label : 20; /* GRH flow lable */ + u32 tclass : 8; + u32 scq_lb1 : 1; + u32 rcq_lb1 : 1; + u32 ext_md : 1; + u32 ext_mtu : 1; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sl : 3; + u32 loop : 1; + u32 udp_src_port : 8; + u32 rsvd : 4; + u32 base_sgid_n : 1; + u32 sgid_index : 7; + u32 hoplmt : 8; +#else + u32 hoplmt : 8; + u32 sgid_index : 7; + u32 base_sgid_n : 1; + u32 rsvd : 4; + u32 udp_src_port : 8; + u32 loop : 1; + u32 sl : 3; +#endif + } bs; + u32 value; + } dw7; +}; + +/* Driver use segment ,16B */ +struct qpc_drv_seg { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 service_type : 3; + u32 fre : 1; + u32 state : 4; + u32 dest_qp : 24; +#else + /* + * Destination QP number, which is extended to 24 + * bits in consideration of interconnection + * with commercial devices. + */ + u32 dest_qp : 24; + /* + * QP state 0000:RST 0001:INIT 0010:RTR 0011:RTS 0100:SQEr + * 0101:SQD(Drained) 0110:ERR + * 0111:Draining This field is not modified for the + * *1XXX:Reserved driver verbs and is + * modified by the microcode based on the command type. + */ + u32 state : 4; + /* Indicates whether the local FRPMR is enabled. */ + u32 fre : 1; + /* + * Transmission Type + * 000:RC + * 001:UC + * 010:RD + * 011 UD + * 101:XRC + * Other:Reserved + */ + u32 service_type : 3; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_flag : 1; + u32 dif_en : 1; + u32 datagram : 1; + u32 local_trans : 1; + u32 srq_en : 1; + u32 drc : 1; + u32 vroce_en : 1; + u32 rsvd : 1; + u32 xrc_vld : 1; + u32 ep : 4; + u32 db_cos : 3; + u32 host_oqid : 16; +#else + u32 host_oqid : 16; + u32 db_cos : 3; + u32 ep : 4; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 vroce_en : 1; + u32 drc : 1; + u32 srq_en : 1; + u32 local_trans : 1; + u32 datagram : 1; + u32 dif_en : 1; + u32 rc_flag : 1; /* Reliable service : */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ack_to : 5; + /* + * NAK code of RNR. This parameter is mandatory when INIT2RNR and + * RTR2RTS\SQE2RTS\SQD2SQD\SQD2RTS is optional. + */ + u32 min_rnr_nak : 5; + /* + * The maximum number of RNR retransmissions is 7. + * The value 7 indicates that the + * maximum number of retransmissions is 7, + * and the value 0 indicates that the + * retransmission is not performed. + */ + u32 rnr_retry_limit : 3; + /* + * Number of ACK retransmissions. The value 7 + * indicates unlimited times, and the + * value 0 indicates no retransmission. + */ + u32 to_retry_limit : 3; + u32 ack_timer_step : 6; + u32 rsvd : 2; + u32 base_mtu_n : 1; + u32 mtu_code : 4; + u32 pmtu : 3; +#else + u32 pmtu : 3; + u32 mtu_code : 4; + u32 base_mtu_n : 1; + u32 rsvd : 2; + u32 ack_timer_step : 6; + u32 to_retry_limit : 3; + u32 rnr_retry_limit : 3; + u32 min_rnr_nak : 5; + u32 ack_to : 5; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 com_rsvd0 : 10; + u32 retry_reset : 6; + u32 com_rsvd : 16; +#else + u32 com_rsvd : 16; + u32 retry_reset : 6; + u32 com_rsvd0 : 10; +#endif + } bs1; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* The maximum value of initiator depth is 128. */ + u32 sra_max : 3; + /* The maximum value of responser resource is 128. */ + u32 rra_max : 3; + /* + * Indicates whether the RQ disables the credit. + * The SRQ is disabled by default and + * must be set to 1. + */ + u32 invalid_credit : 1; + u32 ackto_overstep : 1; + u32 rsvd : 2; + u32 srq_container : 1; + u32 dsgl : 1; + u32 local_qp : 20; /* Local QP number */ +#else + u32 local_qp : 20; + u32 dsgl : 1; + u32 srq_container : 1; + u32 rsvd : 2; + u32 ackto_overstep : 1; + u32 invalid_credit : 1; + u32 rra_max : 3; + u32 sra_max : 3; +#endif + } bs; + u32 value; + } dw3; +}; + +/* QPC Struct */ +struct qpc_sw_seg { + /* driver seg, DW0 ~ DW3 */ + struct qpc_drv_seg drv_seg; + + /* path seg, DW4 ~ DW11 */ + struct drv_path_info path_seg; + + /* 112B(DW12~DW39) */ + struct drv_ucode_all_info ucode_seg; + + /* 32B(DW40~DW47) */ + struct ucode_ext_ctx ext_seg; +}; + +/* QPC Struct */ +struct roce_qp_context { + struct qpc_chip_seg chip_seg; + struct qpc_timer_seg timer_seg; + struct qpc_sw_seg sw_seg; +}; + +struct roce_aa_qp_context { + struct qpc_chip_seg chip_seg; + struct qpc_timer_seg timer_seg; + struct qpc_sw_seg sw_seg; + struct ucode_nofaa_ctx nofaa; +}; + +/* *QPC Format end */ + +/* * SRQC Format start */ +struct roce_srq_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pdn : 18; /* Protection Domain. */ + u32 lth_pre_en : 1; + /* + * If set, the producer counter of Shared Receive Queue + * is stored in the chip, the counter is absolute value. + */ + u32 pcnt_on_chip : 1; + /* + * Shared Receive WQE Basic Block (WQEBB) size + * in bytes is (2^rq_wqebb_size)*16B. + * The minimum size is 32B and the + * values 0, 4, 5, 6, 7 are reserved + */ + u32 wqebb_size : 3; + /* Page size of SRQ, equals to (2^srq_page_size)*4KB */ + u32 page_size : 4; + /* + * Shared Receive Queue size, equals to (2^srq_size)*WQEBB, + * the maximum SRQ size is 16K WQEs, so this field doesn't exceed 14. + */ + u32 size : 5; +#else + /* + * Shared Receive Queue size, equals to (2^srq_size)*WQEBB, + * the maximum SRQ size is 16K WQEs, so this field doesn't exceed 14. + */ + u32 size : 5; + /* Page size of SRQ, equals to (2^srq_page_size)*4KB */ + u32 page_size : 4; + /* + * Shared Receive WQE Basic Block (WQEBB) size + * in bytes is (2^rq_wqebb_size)*16B. + * The minimum size is 32B and the + * values 0, 4, 5, 6, 7 are reserved + */ + u32 wqebb_size : 3; + /* + * If set, the producer counter of Shared Receive Queue + * is stored in the chip, the counter is absolute value. + */ + u32 pcnt_on_chip : 1; + u32 lth_pre_en : 1; + u32 pdn : 18; /* Protection Domain. */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * This filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + u32 rkey_en : 1; + u32 srq_wqe_check_en : 1; + u32 rsvd : 2; + /* + * Completion Queue to report XRC messages + * directed to this SRQ (XRC only) + */ + u32 xrc_cqn : 20; +#else + /* + * Completion Queue to report XRC messages + * directed to this SRQ (XRC only) + */ + u32 xrc_cqn : 20; + u32 rsvd : 2; + u32 srq_wqe_check_en : 1; + u32 rkey_en : 1; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * This filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * his filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + u32 rkey_en : 1; + u32 xrc_cqn2 : 7; + u32 ccnt : 16; +#else + u32 ccnt : 16; + u32 xrc_cqn2 : 7; + u32 rkey_en : 1; + /* + * It specifies the outbound PCIe TLP header + * attribute of the DMA operation. + * This filed is only valid when processing SRQ's WQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing SRQ's WQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; +#endif + } bs_c; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; + u32 rsvd2 : 2; + u32 container_en : 1; + u32 ep : 3; + u32 cos : 3; + u32 rsvd1 : 1; + u32 srqn : 18; /* SRQ number. */ +#else + u32 srqn : 18; /* SRQ number. */ + u32 rsvd1 : 1; + u32 cos : 3; + u32 ep : 3; + u32 container_en : 1; + u32 rsvd2 : 2; + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; + /* + * 0:15 normal + 1 link wqe; 1:7 normal + 1 linkwqe + * 2:3normal + 1 linkwqe 3:1 + * normal + 1 linkwqe + */ + u32 container_size : 2; + u32 container_en : 1; + u32 xrc_cqn1 : 3; + u32 warn_th : 4; /* warn thresthod */ + u32 srqn : 18; /* SRQ number. */ +#else + u32 srqn : 18; /* SRQ number. */ + u32 warn_th : 4; + u32 xrc_cqn1 : 3; + u32 container_en : 1; + /* + * 0:15 normal + 1 link wqe; 1:7 normal + 1 linkwqe + * 2:3normal + 1 linkwqe 3:1 + * normal + 1 linkwqe + */ + u32 container_size : 2; + /* + * SRQ status. + * 0x0: hardware has no access right; + * 0x1: error; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; +#endif + } bs_c; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Limit Threshold. If the "srq_lth" is not zero, and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is de-queued from the SRQ, + * then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 lth : 16; + /* + * The Scalable RC Domain. The received message + * can be served only if the XRC + * Domain of the transport QP of received message + * matches this field. + */ + u32 xrcd : 16; +#else + /* + * The Scalable RC Domain. The received message + * can be served only if the XRC + * Domain of the transport QP of received message + * matches this field. + */ + u32 xrcd : 16; + /* + * Limit Threshold. If the "srq_lth" is not + * zero, and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is + * de-queued from the SRQ, then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 lth : 16; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* Limit Threshold. If the "srq_lth" is not zero, + * and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is de-queued from the + * SRQ, then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 head_index : 16; + /* + * The Scalable RC Domain. The received + * message can be served only if the XRC + * Domain of the transport QP of received + * message matches this field. + */ + u32 pcnt : 16; +#else + /* + * The Scalable RC Domain. The received message + * can be served only if the XRC + * Domain of the transport QP of + * received message matches this field. + */ + u32 pcnt : 16; + /* + * Limit Threshold. If the "srq_lth" is + * not zero, and the "srq_wqe_cnt" + * drops below the "srq_lth" when a WQE is + * de-queued from the SRQ, then an SRQ + * limit event is fired and the "srq_lth" is set to zero. + */ + u32 head_index : 16; +#endif + } bs_c; + + u32 value; + } dw3; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * A 16-bit counter incremented for + * each WQE posted to the SRQ. + */ + u32 pcnt : 16; + u32 ccnt : 16; /* WQE count on the SRQ. */ +#else + u32 ccnt : 16; /* WQE count on the SRQ. */ + /* + * A 16-bit counter incremented for + * each WQE posted to the SRQ. + */ + u32 pcnt : 16; +#endif + } bs; + + u32 head_gpa; + + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_prefetch_ccnt : 16; + /* + * The current WQE index; uses this field to + * get the corresponding WQE from SRQ. + */ + u32 next_wqe_idx : 16; +#else + /* + * The current WQE index; uses this field + * to get the corresponding WQE from SRQ. + */ + u32 next_wqe_idx : 16; + u32 wqe_prefetch_ccnt : 16; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 head_gpa : 20; + u32 rsvd : 1; + u32 pcnt_on_chip : 1; + u32 xrc_cqn0 : 10; +#else + u32 xrc_cqn0 : 10; + u32 pcnt_on_chip : 1; + u32 rsvd : 1; + u32 head_gpa : 20; +#endif + } bs_c; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 head_gpa : 24; + u32 rsvd : 1; + u32 pcnt_on_chip : 1; + /* + * The current WQE index; uses this field to + * get the corresponding WQE from SRQ. + */ + u32 resd : 6; +#else + u32 resd : 6; + u32 pcnt_on_chip : 1; + /* + * The current WQE index; uses this field to + * get the corresponding WQE from SRQ. + */ + u32 rsvd : 1; + u32 head_gpa : 24; +#endif + } bs_osd; + + u32 value; + } dw5; + + /* DW6 */ + union { + u32 next_wqe_gpa; /* bit[63:32] */ + u32 pcnt_record_gpa_h; /* bit[63:32] */ + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 next_wqe_gpa : 20; + u32 next_wqe_gpa_vld : 1; + u32 load_wqe_gpa : 1; + u32 load_pcnt : 1; + u32 rsvd : 4; + u32 wqecnt_lth : 4; + u32 wqecnt_ctrl_en : 1; +#else + u32 wqecnt_ctrl_en : 1; + u32 wqecnt_lth : 4; + u32 rsvd : 4; + u32 load_pcnt : 1; + u32 load_wqe_gpa : 1; + u32 next_wqe_gpa_vld : 1; + u32 next_wqe_gpa : 20; + +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 pcnt_record_gpa_l : 30; + u32 rsvd : 2; +#else + u32 rsvd : 30; + u32 pcnt_record_gpa_l : 2; +#endif + } bs_c; + u32 value; + } dw7; + + /* DW8~9 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the SRQ's buffer directly. + * low 3bits srq_gpa_sign + */ + u64 l0mtt_gpa; + struct { + u32 l0mtt_gpa_hi; + u32 l0mtt_gpa_lo; + } dw8; + }; + + /* DW10~11 */ + union { + /* + * bit[63:2]:The GPA of stored producer counters of Shared Receive Queue. + * bit[1:0]: Address translation hop numbers. + * 0x0: the "srq_l0mtt_gpa" points to the buffer of SRQ directly. + * 0x1: it need to perform one hop address translation to get the + * buffer's address of SRQ; 0x2: there is two hop address translation to get + * the buffer's address of SRQ; 0x3: reserved. + */ + u64 pcnt_record_gpa_at_hop_num; + struct { + u32 pcnt_record_gpa_hi; + /* bit[1:0]: Address translation hop numbers. */ + u32 pcnt_record_gpa_lo_at_hop_num; + } dw10; + + struct { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 xrcd : 16; +#else + u32 xrcd : 16; + u32 rsvd : 16; +#endif + } bs; + + u32 value; + } dw10; + + u32 dw11; + } dw10_container; + }; + + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The hardware clear it to zero when performing + * a SRQ PCnt updating, and driver + * set it to one to indicate the hardware can performing SRQ PCnt updating. + */ + u32 wqecnt_rctl : 1; + u32 rsvd3 : 7; + /* + * Maximum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^srq_wqe_prefetch_maxnum). + */ + u32 wqe_prefetch_max_num : 3; + /* + * Minimum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^(srq_wqe_prefetch_minnum-1)). + */ + u32 wqe_prefetch_min_num : 3; + /* + * Maximum length of prefetch MTTs for SRQ. + * 000: prefetch length equals to zero; + * Others: prefetch length equals to + * (2^(srq_mtt_prefetch_maxlen-1)*1KB). + */ + u32 wqe_cache_thd_sel : 2; + u32 mtt_prefetch_maxlen : 2; + u32 rsvd2 : 5; + /* Page size of MTT for SRQ, equals to (2^srq_mtt_page_size)*4KB. */ + u32 mtt_page_size : 4; + u32 rsvd1 : 4; + u32 load_wqe : 1; +#else + u32 load_wqe : 1; + u32 rsvd1 : 4; + /* Page size of MTT for SRQ, equals to (2^srq_mtt_page_size)*4KB. */ + u32 mtt_page_size : 4; + u32 rsvd2 : 5; + u32 mtt_prefetch_maxlen : 2; + u32 wqe_cache_thd_sel : 2; + /* + * Minimum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^(srq_wqe_prefetch_minnum-1)). + */ + u32 wqe_prefetch_min_num : 3; + /* + * Maximum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^srq_wqe_prefetch_maxnum). + */ + u32 wqe_prefetch_max_num : 3; + u32 rsvd3 : 7; + /* + * The hardware clear it to zero when performing + * a SRQ PCnt updating, and driver + * set it to one to indicate the hardware can performing SRQ PCnt updating. + */ + u32 wqecnt_rctl : 1; +#endif + } bs; + u32 value; + } dw12; + + /* DW13 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 wqe_prefetch_idx : 16; /* The WQE index of prefetch MTTs operation. */ +#else + u32 wqe_prefetch_idx : 16; /* The WQE index of prefetch MTTs operation. */ + u32 rsvd : 16; +#endif + } bs; + u32 value; + } dw13; + + /* DW14 */ + u32 srq_prefetch_wqe_gpa_hi; /* bit[63:32] */ + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mtt_prefetch_wqe_gpa_lo : 20; + u32 srq_prefetch_wqe_gpa_vld : 1; + u32 rsvd : 9; + u32 srq_prefetch_thread_num : 2; +#else + u32 srq_prefetch_thread_num : 2; + u32 rsvd : 9; + u32 srq_prefetch_wqe_gpa_vld : 1; + u32 mtt_prefetch_wqe_gpa_lo : 20; /* bit[31:12] */ +#endif + } bs; + u32 value; + } dw15; +}; + +/* *SRQC Format end */ + +struct roce_ssgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma sequence preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to the mw. */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 rsvd : 1; + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 rsvd : 1; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa) */ + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mtt_base_addr_lo; /* Lower 32 bits of mtt base address (pa) */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + u32 task_id; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 9; + u32 ext_mtu : 1; + u32 tp2_fst_sct : 1; + u32 smd_size : 1; /* metadata size[0]:0 is 512, 1'b1 is 4096 */ + u32 smd_type : 4; /* smd_type[3:2]:reserved. */ + u32 task_id : 16; /* This parameter is valid when SIG MR. */ +#else + u32 task_id : 16; /* This parameter is valid when SIG MR. */ + /* + * This is sector metadata type: 2b'0x is 8B; + * 2b'10 is [8B+56B];2b'11 is [56B+8B] + */ + u32 smd_type : 4; + u32 smd_size : 1; /* metadata size[0]:0 is 512, 1'b1 is 4096 */ + u32 tp2_fst_sct : 1; + u32 ext_mtu : 1; + u32 rsvd : 9; +#endif + } dw10; + }; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_cmp_app_tag_msk : 16; + u32 m_cmp_app_tag_msk : 16; +#else + u32 m_cmp_app_tag_msk : 16; + u32 w_cmp_app_tag_msk : 16; +#endif + } bs; + u32 value; + } dw11; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_app_tag : 16; + u32 m_app_tag : 16; +#else + u32 m_app_tag : 16; + u32 w_app_tag : 16; +#endif + } bs; + u32 value; + } dw12; + + u32 m_ref_tag; + u32 w_ref_tag; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_dif_en : 1; /* Dif enable in the wire domain */ + u32 rx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 rx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 rx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 w_app_esc : 1; + u32 w_ref_esc : 1; + + u32 m_dif_en : 1; /* mem */ + u32 tx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 tx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 tx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 m_app_esc : 1; + u32 m_ref_esc : 1; + + u32 grd_verify : 1; /* Indicates whether to verify the guard. */ + u32 grd_ctrl : 2; /* guard tag ctrl */ + /* The host memory is CRC(0) or IP CheckSum(1). */ + u32 m_bg_type : 1; + /* Indicates whether the line is CRC(0) or IP CheckSum(1). */ + u32 w_bg_type : 1; + u32 grd_agm_ini_ctl : 3; /* GRD_AGM_INI_CTRL */ + + u32 ref_tag_verify : 1; /* Indicates whether to verify the ref_tag. */ + u32 ref_ctrl : 2; /* ref tag ctrl */ + u32 m_ref_tag_mode : 1; /* Obtained from mem.dif.ref_remap */ + u32 w_ref_tag_mode : 1; /* Obtained from wire.dif.ref_remap */ + u32 app_tag_verify : 1; /* Indicates whether to verify the app_tag. */ + u32 app_ctrl : 2; /* app tag ctrl */ +#else + u32 app_ctrl : 2; /* app tag ctrl */ + u32 app_tag_verify : 1; /* Indicates whether to verify the app_tag. */ + u32 w_ref_tag_mode : 1; /* Obtained from wire.dif.ref_remap */ + u32 m_ref_tag_mode : 1; /* Obtained from mem.dif.ref_remap */ + u32 ref_ctrl : 2; /* ref tag ctrl */ + u32 ref_tag_verify : 1; /* Indicates whether to verify the ref_tag. */ + + u32 grd_agm_ini_ctl : 3; /* GRD_AGM_INI_CTRL */ + u32 w_bg_type : 1; /* Is CRC(0) or IP CheckSum(1) on the line? */ + /* The host memory is CRC(0) or IP CheckSum(1). */ + u32 m_bg_type : 1; + u32 grd_ctrl : 2; /* guard tag ctrl */ + u32 grd_verify : 1; /* Indicates whether to verify the guard. */ + + u32 m_ref_esc : 1; + u32 m_app_esc : 1; + u32 tx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 tx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 tx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 m_dif_en : 1; /* mem */ + + u32 w_ref_esc : 1; + u32 w_app_esc : 1; + u32 rx_dif_en : 1; /* IO_1ST_SCT 1 */ + u32 rx_m_dif_type : 2; /* type 0 ~ 3,def:1 */ + u32 rx_w_dif_type : 2; /* type 0 ~ 3,def:1 */ + /* Indicates whether dif is enabled in the wire domain. */ + u32 w_dif_en : 1; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u8 w_dif_ctl; + u8 m_dif_ctl; + u8 m_grd_ctl; + u8 m_ref_ctl; +#else + u8 m_ref_ctl; + u8 m_grd_ctl; + u8 m_dif_ctl; + u8 w_dif_ctl; +#endif + } ctrl; + u32 value; + } dw15; +}; + +struct roce_dsgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 signature : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + u32 access_bind : 1; /* Whether mr can be bound to mw */ +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 signature : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma order-preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to mw */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 indirect_mr : 1; + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 sectfbo : 22; +#else + u32 sectfbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 data_mr_key : 28; +#else + u32 data_mr_key : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw4; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 mtt_offset : 28; +#else + u32 mtt_offset : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw5; + + /* DW6~DW9 */ + u32 rsvd[4]; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + u32 task_id; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 smd_type : 4; /* smd_type[3:2]:reserved. */ + u32 task_id : 16; /* This parameter is valid when SIG MR. */ +#else + + u32 task_id : 16; /* This parameter is valid when SIG MR. */ + /* + * This is sector metadata type: 2b'0x is 8B; + * 2b'10 is [8B+56B];2b'11 is [56B+8B] + */ + u32 smd_type : 4; + u32 rsvd : 12; + +#endif + } dw10; + }; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_cmp_app_tag_msk : 16; + u32 m_cmp_app_tag_msk : 16; +#else + u32 m_cmp_app_tag_msk : 16; + u32 w_cmp_app_tag_msk : 16; +#endif + } bs; + u32 value; + } dw11; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_app_tag : 16; + u32 m_app_tag : 16; +#else + /* + * TX: Verify ref tag domain + * RX: Insert&replace ref tag domain + */ + u32 m_app_tag : 16; + + /* + * TX: Insert&replace ref tag domain + * RX: Verify ref tag domain + */ + u32 w_app_tag : 16; +#endif + } bs; + u32 value; + } dw12; + + /* + * TX: Verify ref tag domain + * RX: Insert&replace ref tag domain + */ + u32 m_ref_tag; + + /* + * TX: Insert&replace ref tag domain + * RX: Verify ref tag domain + */ + u32 w_ref_tag; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 w_dif_en : 1; // Wire domain dif enable + u32 rx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 rx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 rx_dif_en : 1; // IO_1ST_SCT 1 + u32 w_app_esc : 1; + u32 w_ref_esc : 1; + + u32 m_dif_en : 1; // mem + u32 tx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 tx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 tx_dif_en : 1; // IO_1ST_SCT 1 + u32 m_app_esc : 1; + u32 m_ref_esc : 1; + + u32 grd_verify : 1; // Indicates whether to verify the guard. + u32 grd_ctrl : 2; // guard tag ctrl; ucode + u32 m_bg_type : 1; // The host memory is CRC(0) or IP CheckSum(1). + // Indicates whether the line is CRC(0) or IP CheckSum(1). + u32 w_bg_type : 1; + u32 grd_agm_ini_ctl : 3; // GRD_AGM_INI_CTRL + + u32 ref_tag_verify : 1; // Indicates whether to verify the ref_tag. + u32 ref_ctrl : 2; // ref tag ctrl; ucode + u32 m_ref_tag_mode : 1; // Obtained from mem.dif.ref_remap + u32 w_ref_tag_mode : 1; // Obtained from wire.dif.ref_remap + u32 app_tag_verify : 1; // Indicates whether to verify the app_tag. + u32 app_ctrl : 2; // app tag ctrl; ucode +#else + /* REF APP CTRL segment */ + u32 app_ctrl : 2; // app tag ctrl + u32 app_tag_verify : 1; // Indicates whether to verify the app_tag. + u32 w_ref_tag_mode : 1; // Obtained from wire.dif.ref_remap + u32 m_ref_tag_mode : 1; // Obtained from mem.dif.ref_remap + u32 ref_ctrl : 2; // ref tag ctrl + u32 ref_tag_verify : 1; // Indicates whether to verify the ref_tag. + + /* GRD CTRL segment */ + u32 grd_agm_ini_ctl : 3; // GRD_AGM_INI_CTRL + u32 w_bg_type : 1; // Is CRC(0) or IP CheckSum(1) on the line? + u32 m_bg_type : 1; // The host memory is CRC(0) or IP CheckSum(1). + u32 grd_ctrl : 2; // guard tag ctrl + u32 grd_verify : 1; // Indicates whether to verify the guard. + + u32 m_ref_esc : 1; + u32 m_app_esc : 1; + u32 tx_dif_en : 1; // IO_1ST_SCT 1 + u32 tx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 tx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 m_dif_en : 1; // mem + + u32 w_ref_esc : 1; + u32 w_app_esc : 1; + u32 rx_dif_en : 1; // IO_1ST_SCT 1 + u32 rx_m_dif_type : 2; // type 0 ~ 3,def:1 + u32 rx_w_dif_type : 2; // type 0 ~ 3,def:1 + u32 w_dif_en : 1; // Dif enable in the wire domain +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u8 w_dif_ctl; + u8 m_dif_ctl; + u8 m_grd_ctl; + u8 m_ref_ctl; +#else + u8 m_ref_ctl; + u8 m_grd_ctl; + u8 m_dif_ctl; + u8 w_dif_ctl; + +#endif + } ctrl; + u32 value; + } dw15; +}; + +struct iw_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 rsvd1 : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + u32 access_bind : 1; /* Can mr be bound to mw? */ +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 rsvd1 : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* + * The value 1 indicates that the remote + * Atomic permission is supported. + */ + u32 access_ra : 1; + /* 1: The remote write permission is supported. */ + u32 access_rw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* + * The value 1 indicates that the local + * write permission is supported. + */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma order-preserving flag */ + u32 dma_attr_idx : 6; /* Indicates the dma attribute index. */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 rsvd : 3; + u32 qpn : 20; /* Qp bound to mw */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 rsvd : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 rsvd : 4; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 rsvd : 4; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* If set, indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa) */ + + struct { + u32 mtt_base_addr_hi; /* Upper 32 bits of the base address (pa) of mtt */ + u32 mtt_base_addr_lo; /* Lower 32 bits of mtt base address (pa) */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + + /* DW11 */ + u32 mtt_sz; /* This parameter is valid when FRMR. */ + + /* DW12 ~ DW13 */ + u32 rsvd[2]; + + /* DW14 */ + u32 mw_base_addr_hi; + u32 mw_base_addr_lo; +}; + +/* * MPT Format end */ + +/* * CQC Format start */ +struct roce_cq_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 signature : 5; + u32 last_solicited_pi : 9; + u32 ceqn_low : 5; + u32 ci_on_chip : 1; + u32 cqe_size : 3; + u32 page_size : 4; + u32 size : 5; +#else + /* + * Completion Queue size, equals to (2^cq_size)*CQE. + * The maximum CQ size is 2^23 CQEs. + */ + u32 size : 5; + u32 page_size : 4; /* Page size of CQ, equals to (2^cq_page_size)*4KB. */ + /* + * Completion Queue Entry (CQE) size in bytes is (2^cq_cqe_size)*16B. + * The minimum size is 32B and the values 0, 3, 4, 5, 6, 7 are reserved. + */ + u32 cqe_size : 3; + /* + * If set, the CI of Complete Queue is stored in the chip, + * the counter is absolute value. + */ + u32 ci_on_chip : 1; + /* + * Completion Event Queue, this CQ reports + * Completion Events to this CQE (cq_ceqn[04:00]) + */ + u32 ceqn_low : 5; + u32 last_solicited_pi : 9; /* bit[23:15] */ + u32 signature : 5; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; + u32 dma_attr_idx : 6; + u32 ci : 24; +#else + /* The CI index of Receive Queue (Step: CQE). */ + u32 ci : 24; + /* + * It specifies the outbound PCIe TLP header attribute + * of the DMA operation. + * This filed is only valid when processing CQ's CQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound + * PCIe TLP headers of the DMA operation. + * This field is only valid when processing CQ's CQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 rsvd : 1; + u32 ceqn_high : 3; + u32 pi : 24; +#else + /* The PI index of Completion Queue (Step: CQE) */ + u32 pi : 24; + /* + * Completion Event Queue, this CQ reports + * Completion Events to this CQE (cq_ceqn[07:05]) + */ + u32 ceqn_high : 3; + u32 rsvd : 1; + /* + * CQ status. + * 0x0: hardware has no access right; + * 0x1: CQ is error; + * 0x2: CQ is overflow; + * 0xf: ok. + * Other: reserved. + */ + u32 state : 4; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cqecnt_lth : 4; + u32 cqecnt_rctl_en : 1; + u32 last_solicited_pi : 15; + u32 load_ci : 1; + u32 load_page_gpa : 1; + u32 cqe_cur_pg_vld : 1; + u32 cqe_next_pg_vld : 1; + u32 cqe_cur_pg_gpa_h : 8; +#else + u32 cqe_cur_pg_gpa_h : 8; /* curt_pg_gpa[63:56]bit */ + /* + * Indicates if the "cq_cqe_next_page_gpa" field is valid. + * 1: it is valid; + * 0: it is invalid. + */ + u32 cqe_next_pg_vld : 1; + /* + * Indicates if the "cq_cqe_curt_page_gpa" field is valid. + * 1: it is valid; + * 0: it is invalid. + */ + u32 cqe_cur_pg_vld : 1; + /* Indicates the thread is performing a prefetch for GPA of CQE page. */ + u32 load_page_gpa : 1; + u32 load_ci : 1; + u32 last_solicited_pi : 15; /* bit[14:00] */ + u32 cqecnt_rctl_en : 1; + u32 cqecnt_lth : 4; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + /* + * Indicates the page GPA of current CQ buffer's page pointed by "cq_pi". + * curt_pg_gpa[55:24]bit + */ + u32 cqe_curt_pg_gpa_m; + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cqe_curt_pg_gpa_l : 12; /* curt_pg_gpa[23:12]bit */ + u32 cqe_next_pg_gpa_h : 20; /* next_pg_gpa[63:44]bit */ +#else + u32 cqe_next_pg_gpa_h : 20; /* next_pg_gpa[63:44]bit */ + u32 cqe_curt_pg_gpa_l : 12; /* curt_pg_gpa[23:12]bit */ +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + /* + * Indicates the page GPA of next CQ buffer's page pointed by "cq_pi". + * next_pg_gpa[43:12]bit + */ + u32 cqe_next_pg_gpa_l; + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Completion Event Moderation counters. + * 0x0: interrupt moderation disabled. + */ + u32 max_cnt : 16; + /* + * Completion Event Moderation timer in microseconds. + * 0x0: interrupt moderation disabled. + */ + u32 timeout : 16; +#else + /* + * Completion Event Moderation timer in microseconds. + * 0x0: interrupt moderation disabled. + */ + u32 timeout : 16; + /* + * Completion Event Moderation counters. + * 0x0: interrupt moderation disabled. + */ + u32 max_cnt : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cmd_sn : 2; + u32 arm_state : 2; + u32 rsvd : 12; + u32 cqe_cnt : 16; +#else + u32 cqe_cnt : 16; + u32 rsvd : 12; + /* + * The CQ ARM State machine. + * 0x0: idle state; + * 0x1: solicited state; + * 0x2: next state; + * Others: reserved. + */ + u32 arm_state : 2; + u32 cmd_sn : 2; /* The last received command sequence number. */ +#endif + } bs; + u32 value; + } dw8; + + /* DW9 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 timer_mode : 1; + u32 arm_timer_en : 1; + u32 solicited_cqe_en : 1; + u32 rsvd : 3; + u32 cnt_clear_en : 1; + u32 cnt_adjust_en : 1; + u32 last_notified_pi : 24; /* The last notified PI of CQ. */ +#else + u32 last_notified_pi : 24; /* The last notified PI of CQ. bit[23:00] */ + u32 cnt_adjust_en : 1; + u32 cnt_clear_en : 1; + u32 rsvd : 3; + u32 solicited_cqe_en : 1; + u32 arm_timer_en : 1; + u32 timer_mode : 1; +#endif + } bs; + u32 value; + } dw9; + + /* DW10 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 idle_max_count : 16; +#else + u32 idle_max_count : 16; + u32 rsvd : 16; +#endif + } bs; + u32 value; + } dw10; + + /* DW11 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cqecnt_rctl : 1; + u32 rsvd3 : 15; + u32 prefetch_thread_num : 2; + u32 rsvd2 : 6; + u32 ceqe_en : 1; + u32 arm_ceqe_en : 1; + u32 rsvd1 : 2; + u32 mtt_page_size : 4; +#else + u32 mtt_page_size : 4; + u32 rsvd1 : 2; + u32 arm_ceqe_en : 1; + u32 ceqe_en : 1; + u32 rsvd2 : 6; + u32 prefetch_thread_num : 2; + u32 rsvd3 : 15; + u32 cqecnt_rctl : 1; +#endif + } bs; + u32 value; + } dw11; + + /* DW12~13 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the CQ's buffer directly. + * low 3bits(cq_gpa_sign) + */ + u64 cqc_l0mtt_gpa; + struct { + u32 cqc_l0mtt_gpa_hi; + u32 cqc_l0mtt_gpa_lo; + } dw12; + }; + + /* DW14 */ + union { + /* + * The GPA of stored CI of Complete Queue. + * Address translation hop numbers. + * 0x0: the "cq_l0mtt_gpa" points to the buffer of CQ directly. + * 0x1: it need to perform one hop address translation to get the buffer's + * address of CQ; 0x2: there is two hop address translation to get the buffer's + * address of CQ; 0x3: reserved. + */ + u64 ci_record_gpa_at_hop_num; + struct { + u32 ci_record_gpa_hi; + /* bit[1:0] Address translation hop numbers */ + u32 ci_record_gpa_lo_at_hop_num; + } dw14; + }; + + /* timer seg info */ + struct qpc_timer_seg timer_seg; + + u32 rsvd[8]; /* The 32B is reserved, and the entire CQC is aligned with the 128B. */ +}; +/* * CQC Format end */ + +/* * GID Format start */ +struct roce_gid_context { + /* DW0~3 */ + u32 gid[4]; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 svlan : 12; /* outer vlan */ + u32 cvlan : 12; /* inner vlan */ + u32 is_vroce : 1; + u32 rsvd : 7; +#else + u32 rsvd : 7; + u32 is_vroce : 1; + u32 ppe_rsvd0 : 12; + u32 svlan : 12; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 svlan : 12; /* outer vlan */ + u32 cvlan : 12; /* inner vlan */ + u32 rsvd : 4; + u32 ip_ctrl_bmp : 4; +#else + u32 ip_ctrl_bmp : 4; + u32 rsvd : 4; + u32 ppe_rsvd0 : 12; + u32 svlan : 12; +#endif + } nofaa_master_gid; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 svlan : 12; /* outer vlan */ + u32 ppe_rsvd0 : 12; /* inner vlan */ + u32 rsvd : 1; + u32 master_gid_index : 7; +#else + u32 master_gid_index : 7; + u32 rsvd : 1; + u32 ppe_rsvd0 : 12; + u32 svlan : 12; +#endif + } nofaa_slave_gid; + + u32 value; + } dw4; + + /* DW5 */ + union { + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 add_value : 6; + u32 gid_update : 1; + u32 ppe_rsvd3 : 1; + u32 outer_tag : 2; + u32 ppe_rsvd1 : 1; + u32 gid_type : 2; + u32 tunnel : 1; + u32 tag : 2; + u32 smac_hi16 : 16; +#else + u32 smac_hi16 : 16; + u32 tag : 2; + u32 tunnel : 1; // tunnel + u32 gid_type : 2; + u32 ppe_rsvd1 : 1; // outer_ip_type + u32 outer_tag : 2; // outer_tag + u32 ppe_rsvd3 : 1; // stag + u32 gid_update : 1; + u32 add_value : 6; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd_com : 8; + u32 outer_value : 3; + u32 misc_idx : 5; + u32 smac_hi16 : 16; +#else + u32 smac_hi16 : 16; + u32 misc_idx : 5; + u32 outer_value : 3; + u32 rsvd_com : 8; +#endif + } bs1; + u32 value; + } dw6; + + u32 smac_lo32; +}; +/* * GID Format end */ + +/* * sqrq db Format start */ +struct roce_sqrq_db { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 _wqe_counter : 16; +#else + u32 _wqe_counter : 16; + u32 rsvd : 16; +#endif +}; +/* * sqrq db Format end */ + +/* * srq db Format start */ +struct roce_srq_db { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 _wqe_counter : 16; +#else + u32 _wqe_counter : 16; + u32 rsvd : 16; +#endif +}; +/* * srq db Format end */ + +/* * VTEP Format start */ +struct roce_vtep_context { + u32 dmac_h32; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 dmac_l16 : 16; + u32 smac_h16 : 16; +#else + u32 smac_h16 : 16; + u32 dmac_l16 : 16; +#endif + } bs; + u32 value; + } dw1; + + u32 smac_l32; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 portid : 8; + u32 vni : 24; +#else + u32 vni : 24; + u32 portid : 8; +#endif + } bs; + u32 value; + } dw3; + + u32 sip[4]; // dw4 ~ 7 + u32 dip[4]; // dw8 ~ 11 + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 src_port : 16; + u32 dst_port : 16; +#else + u32 dst_port : 16; + u32 src_port : 16; +#endif + } bs; + u32 value; + } dw12; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 base_id : 8; /* vf cfg id */ + u32 cvlan : 12; + u32 svlan : 12; +#else + u32 svlan : 12; + u32 cvlan : 12; + u32 base_id : 8; +#endif + } bs; + u32 value; + } dw13; + + u32 racl_id; + + /* DW15 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 5; + u32 vm_dp_capture_en : 1; + u32 vm_dp_chk_en : 1; + u32 vm_dp_chk_invld : 1; + u32 dscp_rc : 8; + u32 dscp_ud : 8; + u32 dscp_xrc : 8; +#else + u32 dscp_xrc : 8; + u32 dscp_ud : 8; + u32 dscp_rc : 8; + u32 vm_dp_chk_invld : 1; + u32 vm_dp_chk_en : 1; + u32 vm_dp_capture_en : 1; + u32 rsvd : 5; +#endif + } bs; + u32 value; + } dw15; +}; +/* * vtep Format end */ + +union tag_roce_qu_db_dw4 { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; + u32 udf_3 : 12; + u32 rsvd0 : 8; + u32 pi : 8; +#else + u32 pi : 8; + u32 rsvd0 : 8; + u32 udf_3 : 12; + u32 sub_type : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; + u32 sgid_index : 7; + u32 mtu_shift : 3; /* 256B:0;512B:1;1024B:2;2048B:3;4096B:4 */ + u32 rsvd : 1; + u32 xrc_vld : 1; + u32 rsvd0 : 8; + u32 pi : 8; +#else + u32 pi : 8; + u32 rsvd0 : 8; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 mtu_shift : 3; + u32 sgid_index : 7; + u32 sub_type : 4; +#endif + } bs1; + u32 value; +}; + +struct tag_roce_cq_mtt_info { + /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + u32 mtt_flags; + /* + * Number of cmtt, which needs to be assigned by the + * driver when the is kicked out by queue. + */ + u32 mtt_num; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_start; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_end; + u32 mtt_cache_line_size; /* 0:256B,1:512B */ +}; + +struct tag_timer_section { + union { + u32 value; + struct { + u32 timer_num : 4; // last immeth+cmdlen + u32 rsvd : 9; + u32 lv : 1; + u32 link_timer_id : 3; + u32 link_wheel_id : 3; + u32 link_spoke : 12; + } bs; + } dw0; + + union { + u32 value; + struct { + u32 v : 1; // last immeth+cmdlen + u32 service_type : 5; + u32 rsvd : 9; + u32 instance_id : 5; + u32 vf : 12; + } bs; + } dw1; + union { + u32 value; + struct { + u32 xid : 20; + u32 cid : 12; + } bs; + } dw2; + union { + u32 value; + struct { + u32 cid : 8; + u32 v : 1; // last immeth+cmdlen + u32 service_type : 5; + u32 rsvd : 9; + u32 instance_id : 5; + u32 vf : 4; + } bs; + } dw3; + union { + u32 value; + struct { + u32 vf : 8; + u32 xid : 20; + u32 cid : 4; + } bs; + } dw4; + + u16 cid; + u16 te0; + u16 te1; + u16 te2; + u16 te3; + u16 te4; +}; + +struct tag_spoke_info { + union { + u32 value; + struct { + u32 v : 1; + u32 rsvd : 19; + u32 vf : 12; + } bs; + } dw0; + + union { + u32 value; + struct { + u32 xid : 20; + u32 cid : 12; + } bs; + } dw1; + union { + u32 value; + struct { + u32 cid : 8; + u32 v : 1; // last immeth+cmdlen + u32 service_type : 5; + u32 rsvd : 9; + u32 instance_id : 5; + u32 vf : 4; + } bs; + } dw2; + union { + u32 value; + struct { + u32 vf : 8; + u32 xid : 20; + u32 cid : 4; + } bs; + } dw3; + + u16 cid; + u16 rsvd; + u32 total_cnt; + u32 scan_cnt; + u32 rsvd1; +}; + +struct racl_value_rsp { + u32 vni; + u32 vtep_ip; + u16 vf_id; + u8 mode; + u8 gw_mac[ETH_ADDR_LEN]; + u8 vm_mac[ETH_ADDR_LEN]; + u8 rsvd; +}; + +struct racl_value { + u32 vip; + u32 vni; + u32 vtep_ip; + + union { + struct { + u32 gw_mac_h : 16; + u32 gw_mac_m : 16; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 gw_mac_l : 16; + u32 vf_id : 8; + u32 rsv0 : 7; + u32 mode : 1; + } bs; + u32 value; + } dw4; +}; +#pragma pack(0) + +#endif // RDMA_CONTEXT_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h new file mode 100644 index 000000000..7d0252849 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/rdma_ext_ctx_format.h @@ -0,0 +1,379 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_EXT_CTX_FORMAT_H +#define RDMA_EXT_CTX_FORMAT_H + +#include "roce_compile_macro.h" + +#include "roce_ccf_format.h" + +#ifdef ROCE_VBS_EN +#include "roce_vbs_format.h" +#endif + +#include "roce_dif_format.h" +/* Align each field with 4bytes. */ +#pragma pack(4) + + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +/***********************************/ +struct ucode_dcc_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 back_token : 9; + u32 token : 13; + u32 alpha : 10; +#else + u32 alpha : 10; + u32 token : 13; + u32 back_token : 9; +#endif + } dcqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 less_mtu_cnt : 2; + u32 alpha : 2; + u32 beta : 2; + u32 set_flag : 1; + u32 rsvd : 1; + u32 sw_wnd_fraction : 12; + u32 sw_wnd : 12; +#else + u32 sw_wnd : 12; + u32 sw_wnd_fraction : 12; + u32 rsvd : 1; + u32 set_flag : 1; + u32 beta : 2; + u32 alpha : 2; + u32 less_mtu_cnt : 2; +#endif + } ldcpw; + + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 cnp_receive : 1; + u32 cur_rate : 15; + u32 target_rate : 15; +#else + u32 target_rate : 15; + u32 cur_rate : 15; + u32 cnp_receive : 1; + u32 ccf_en : 1; +#endif + } dcqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 dip_valid : 1; + u32 dip_index : 16; + u32 rsvd : 14; +#else + u32 rsvd : 14; + u32 dip_index : 16; + u32 dip_valid : 1; + u32 ccf_en : 1; +#endif + } ipqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 mtu_thr_cnt : 2; + u32 rsvd : 3; + u32 sw_wnd_just_halve : 1; + u32 rr_wnd_just_halve : 1; + u32 rr_wnd_fraction : 12; + u32 rr_wnd : 12; +#else + u32 rr_wnd : 12; + u32 rr_wnd_fraction : 12; + u32 rr_wnd_just_halve : 1; + u32 sw_wnd_just_halve : 1; + u32 rsvd : 3; + u32 mtu_thr_cnt : 2; + u32 ccf_en : 1; +#endif + } ldcpw; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_en : 1; + u32 rsvd : 31; +#else + u32 rsvd : 31; + u32 ccf_en : 1; +#endif + } cc_common; + + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_timestamp : 13; + u32 alpha_timestamp : 13; + u32 cal_token_l : 6; +#else + u32 cal_token_l : 6; + u32 alpha_timestamp : 13; + u32 rate_timestamp : 13; +#endif + } dcqcn; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sw_left_ce_num : 12; + u32 last_ack_event_timestamp : 20; +#else + u32 last_ack_event_timestamp : 20; + u32 sw_left_ce_num : 12; +#endif + } ldcpw; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rcnp_timestamp : 13; + u32 token_timestamp : 13; + u32 f_cnt : 4; + u32 cal_token_h : 2; +#else + u32 cal_token_h : 2; + u32 f_cnt : 4; + u32 token_timestamp : 13; + u32 rcnp_timestamp : 13; +#endif + } dcqcn; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wnd_ctrl_rtt : 16; + u32 rr_left_ce_num : 12; + u32 gamma : 2; + u32 eta : 2; +#else + u32 eta : 2; + u32 gamma : 2; + u32 rr_left_ce_num : 12; + u32 wnd_ctrl_rtt : 16; +#endif + } ldcpw; + + u32 value; + } dw3; +}; + +/*************************************/ +union ucode_ext_sq_ctx { + struct ucode_ccf_sq_ctx ccf_sq; + + u32 rsvd; +}; + +union ucode_ext_sqa_ctx { + struct ucode_ccf_sqa_ctx ccf_sqa; + + u32 rsvd[2]; +}; + +union ucode_ext_rq_ctx { + struct ucode_ccf_rq_ctx ccf_rq; + u8 vbs_rq[4]; + u32 rsvd; +}; + +union ucode_ext_rqa_ctx { + struct ucode_ccf_rqa_ctx ccf_rqa; + + u32 rsvd; +}; + +struct ucode_ext_ctx { + /* DW0 */ + union ucode_ext_sq_ctx sq; + + /* DW1~2 */ + union ucode_ext_sqa_ctx sqa; + + /* DW3 */ + union ucode_ext_rq_ctx rq; + + /* DW4 */ + union ucode_ext_rqa_ctx rqa; + + /* DW5~8 */ + struct ucode_dcc_ctx dcc; +}; + +struct tag_nofaa_io_qpn_entry { + u32 rsvd : 8; + u32 io_qpn_2 : 12; + u32 io_qpn_1 : 12; +}; + +struct ucode_nofaa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 admin_qp : 1; + u32 rts_bmp : 4; + u32 nof_bmp : 4; + u32 disconn : 1; + u32 rsvd : 6; + u32 cm_bmp : 16; +#else + u32 cm_bmp : 16; + u32 rsvd : 6; + u32 disconn : 1; + u32 nof_bmp : 4; + u32 rts_bmp : 4; + u32 admin_qp : 1; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 nof_vport : 2; + u32 gid_index_3 : 6; + u32 rsvd2 : 2; + u32 gid_index_2 : 6; + u32 rsvd1 : 2; + u32 gid_index_1 : 6; + u32 rsvd0 : 2; + u32 gid_index_0 : 6; +#else + u32 gid_index_0 : 6; + u32 rsvd0 : 2; + u32 gid_index_1 : 6; + u32 rsvd1 : 2; + u32 gid_index_2 : 6; + u32 rsvd2 : 2; + u32 gid_index_3 : 6; + u32 nof_vport : 2; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_2 : 12; + u32 io_qpn_1 : 12; +#else + u32 io_qpn_1 : 12; + u32 io_qpn_2 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 admin_qpn : 20; + u32 rsvd : 4; + u32 qid : 8; +#else + u32 qid : 8; + u32 rsvd : 4; + u32 admin_qpn : 20; +#endif + } io_q; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_4 : 12; + u32 io_qpn_3 : 12; +#else + u32 io_qpn_3 : 12; + u32 io_qpn_4 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { + u32 rsvd; + } io_q; + u32 value; + } dw3; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_6 : 12; + u32 io_qpn_5 : 12; +#else + u32 io_qpn_5 : 12; + u32 io_qpn_6 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd; +#else + u32 rsvd; +#endif + } io_q; + u32 value; + } dw4; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 io_qpn_8 : 12; + u32 io_qpn_7 : 12; +#else + u32 io_qpn_7 : 12; + u32 io_qpn_8 : 12; + u32 rsvd : 8; +#endif + } admin_q; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd; +#else + u32 rsvd; +#endif + } io_q; + u32 value; + } dw5; + + u32 rsvd[2]; +}; + +#pragma pack(0) + +#endif // RDMA_EXT_CTX_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h new file mode 100644 index 000000000..cc8c6f55e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ccf_format.h @@ -0,0 +1,722 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CCF_FORMAT_H +#define ROCE_CCF_FORMAT_H + +/* Align each field with 4bytes. */ +#pragma pack(4) + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +enum ROCE3_CCF_PARAM_ALGO_ID { + ROCE_CCF_ALGO_ID, + ROCE_DCQCN_ALGO_ID, + ROCE_HC3_2_0_ALGO_ID, + ROCE_LDCP_ALGO_ID, + ROCE_MIBO_ALGO_ID, + ROCE_BW_CTRL_ALGO_ID, + ROCE_CCF_PARAM_MAX_ID = 7 +}; +#define ROCE_IPQCN_ALGO_ID ROCE_DCQCN_ALGO_ID +#define ROCE_CCF_PARAM_INDEX_GET(vf_id, algo_id) (((vf_id) << 3) | (algo_id)) + +enum ROCE_CC_ALGO_E { + ROCE_CC_DISABLE = 0, + ROCE_CC_DCQCN_ALGO, + ROCE_CC_LDCP_ALGO, + ROCE_CC_IPQCN_ALGO, + ROCE_CC_MIBO_ALGO +}; + +#define CTX_TBL_WR_KEY_OFFSET 16 +#define CTX_TBL_CPY_BYTE_TYPE 48 + +/* *************************** EVENT *************************** */ +enum dcc_event { + DCC_EVENT_NEW_FLOW = 0, + DCC_EVENT_DATA_TX = 1, + DCC_EVENT_RETRANSMIT, + DCC_EVENT_NACK, + DCC_EVENT_RTT_RESP, + DCC_EVENT_CNP_TX = 5, + DCC_EVENT_CNP_RX, + DCC_EVENT_ACK = 7, + DCC_EVENT_REPEAT_READ, + DCC_EVENT_NORMAL_READ, + DCC_EVENT_RSVD4 = 10, + DCC_EVENT_RSVD5, + DCC_EVENT_TIMER, + DCC_EVENT_RX, + DCC_EVENT_ACK_TX, + DCC_EVENT_ACK_DATA_TX = 15, + DCC_EVENT_ACK_DATA_RX, + DCC_EVENT_LESS_MTU_TX, + DCC_EVENT_LESS_MTU_RSPTX, + DCC_EVENT_UPDATE_TOKEN, + DCC_EVENT_VERBS_INIT = 20, + DCC_EVENT_VERBS_DEINIT, + DCC_EVENT_RSVD1, + DCC_EVENT_RSVD2, + DCC_EVENT_RSVD3, + + DCC_MATCH_RX_CTL = 25, + DCC_MOD_TX_SW, + DCC_MOD_TX_READ, + DCC_MOD_ACK_TX_ACK, + DCC_MOD_ACK_TX_RR, + DCC_CTR_AVA_WND = 30, + DCC_PKT_ACK_EXT, + DCC_PKT_DATA_EXT, + DCC_PKT_ACK_DATA_EXT, + DCC_PKT_ACK_EXT_RX, + DCC_PKT_DATA_EXT_RX = 35, + DCC_PKT_ACK_DATA_EXT_RX +}; + +/* *************************** QPC *************************** */ +struct ucode_ccf_sq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 retrans_read_req : 1; + u32 retrans_mark : 1; + u32 sw_send_sn : 16; + u32 sw_wnd_timer_stat : 2; +#else + u32 sw_wnd_timer_stat : 2; + u32 sw_send_sn : 16; + u32 retrans_mark : 1; + u32 retrans_read_req : 1; + u32 rsvd : 12; +#endif + } ldcpw; + + u32 value; + } dw0; +}; + +struct ucode_ccf_sqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 6; + u32 sw_zero_avail_wnd : 1; + u32 rr_zero_avail_wnd : 1; + u32 change2rqa_stg_cnt : 8; + u32 rr_rcv_sn : 16; +#else + u32 rr_rcv_sn : 16; + u32 change2rqa_stg_cnt : 8; + u32 rr_zero_avail_wnd : 1; + u32 sw_zero_avail_wnd : 1; + u32 rsvd : 6; +#endif + } ldcpw; + + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sw_ack_sn : 16; + u32 rr_ack_sn : 16; +#else + u32 rr_ack_sn : 16; + u32 sw_ack_sn : 16; +#endif + } ldcpw; + + u32 value; + } dw1; +}; + +struct ucode_ccf_rq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sw_rcv_sn : 16; + u32 rcv_timestamp : 16; +#else + u32 rcv_timestamp : 16; + u32 sw_rcv_sn : 16; +#endif + } ldcpw; + + u32 value; + } dw0; +}; + +struct ucode_ccf_rqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rr_send_sn : 16; + u32 rr_send_ce_sn : 11; + u32 rr_rsp_ack_cnt : 1; + u32 rcv_read_rsp : 1; + u32 ext_hdr_flip : 1; + u32 rr_wnd_timer_stat : 2; +#else + u32 rr_wnd_timer_stat : 2; + u32 ext_hdr_flip : 1; + u32 rcv_read_rsp : 1; + u32 rr_rsp_ack_cnt : 1; + u32 rr_send_ce_sn : 11; + u32 rr_send_sn : 16; +#endif + } ldcpw; + + u32 value; + } dw0; +}; + +/* *************************** EXT TABLE *************************** */ +/* ccf common param tbl */ +struct roce_ccf_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 np_enable : 8; + u32 rp_enable : 8; + u32 ecn_ver : 4; + u32 cnp_prio_enable : 1; + u32 ip_enable : 8; + u32 port_mode : 1; + u32 rsvd : 2; +#else + u32 rsvd : 2; + u32 port_mode : 1; + u32 ip_enable : 8; + u32 cnp_prio_enable : 1; + u32 ecn_ver : 4; + u32 rp_enable : 8; + u32 np_enable : 8; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_cos : 3; + u32 cnp_prio : 3; + u32 ccf_appid : 8; + u32 rsvd : 18; +#else + u32 rsvd : 18; + u32 ccf_appid : 8; + u32 cnp_prio : 3; + u32 cnp_cos : 3; +#endif + } bs; + u32 value; + } dw1; + + u32 rsvd[2]; +}; + +struct roce_dcqcn_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 flow_min_rate : 6; + u32 token_period : 8; +#else + u32 token_period : 8; + u32 flow_min_rate : 6; + u32 rsvd : 18; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 10; + u32 rsvd : 3; + u32 cnp_cnt_threshold : 4; + u32 alpha_dec_period : 10; + u32 alpha_threshold : 5; +#else + u32 alpha_threshold : 5; + u32 alpha_dec_period : 10; + u32 cnp_cnt_threshold : 4; + u32 rsvd : 3; + u32 rate_inc_period : 10; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 16; + u32 alpha_dec_period : 16; +#else + u32 alpha_dec_period : 16; + u32 rate_inc_period : 16; +#endif + } bs1; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_ai : 8; + u32 rate_inc_hai : 8; + u32 rate_dec_period : 8; + u32 min_cnp_period : 8; +#else + u32 min_cnp_period : 8; + u32 rate_dec_period : 8; + u32 rate_inc_hai : 8; + u32 rate_inc_ai : 8; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 factor_gita : 4; + u32 rt_clamp : 1; + u32 rsvd : 1; + u32 initial_alpha : 10; + u32 rate_first_set : 16; +#else + u32 rate_first_set : 16; + u32 initial_alpha : 10; + u32 rsvd : 1; + u32 rt_clamp : 1; + u32 factor_gita : 4; +#endif + } bs; + u32 value; + } dw3; +}; + +struct roce_ipqcn_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 18; + u32 flow_min_rate : 6; + u32 token_period : 8; +#else + u32 token_period : 8; + u32 flow_min_rate : 6; + u32 rsvd : 18; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 10; + u32 rsvd : 3; + u32 cnp_cnt_threshold : 4; + u32 alpha_dec_period : 10; + u32 alpha_threshold : 5; +#else + u32 alpha_threshold : 5; + u32 alpha_dec_period : 10; + u32 cnp_cnt_threshold : 4; + u32 rsvd : 3; + u32 rate_inc_period : 10; +#endif + } bs; + + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_period : 16; + u32 alpha_dec_period : 16; +#else + u32 alpha_dec_period : 16; + u32 rate_inc_period : 16; +#endif + } bs1; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rate_inc_ai : 8; + u32 rate_inc_hai : 8; + u32 rate_dec_period : 8; + u32 min_cnp_period : 8; +#else + u32 min_cnp_period : 8; + u32 rate_dec_period : 8; + u32 rate_inc_hai : 8; + u32 rate_inc_ai : 8; +#endif + } bs; + u32 value; + } dw2; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 factor_gita : 4; + u32 rt_clamp : 1; + u32 rsvd : 1; + u32 initial_alpha : 10; + u32 rate_first_set : 16; +#else + u32 rate_first_set : 16; + u32 initial_alpha : 10; + u32 rsvd : 1; + u32 rt_clamp : 1; + u32 factor_gita : 4; +#endif + } bs; + u32 value; + } dw3; +}; + +struct roce_ldcp_param { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha : 2; + u32 beta : 2; + u32 gamma : 2; + u32 eta : 2; + u32 set_flag : 1; + u32 rsvd : 23; +#else + u32 rsvd : 23; + u32 set_flag : 1; + u32 eta : 2; + u32 gamma : 2; + u32 beta : 2; + u32 alpha : 2; +#endif + } bs; + u32 value; + } dw0; + + u32 rsvd1[3]; +}; + +struct ucode_dcc_ip_table_rate_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha : 14; + u32 ip_rate : 18; /* cur_rate, unit:1M */ +#else + u32 ip_rate : 18; /* cur_rate, unit:1M */ + u32 alpha : 14; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cnp_flag : 1; + u32 rsvd : 6; + u32 cnp_receive : 1; + u32 f_cnt : 3; + u32 ai_cnt : 3; + u32 ip_target_rate : 18; +#else + u32 ip_target_rate : 18; + u32 ai_cnt : 3; + u32 f_cnt : 3; + u32 cnp_receive : 1; + u32 rsvd : 6; + u32 cnp_flag : 1; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha_timestamp : 16; + u32 rate_timestamp : 16; /* last update rate */ +#else + u32 rate_timestamp : 16; /* last update rate */ + u32 alpha_timestamp : 16; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rcnp_timestamp : 32; +#else + u32 rcnp_timestamp : 32; +#endif + } bs; + u32 value; + } dw3; +}; + +struct ucode_dcc_bw_ctrl_ctx { + union { + struct { + u32 bw_ctrl_thd_def : 32; + } bs; + + u32 value; + } dw0; + + u32 rsvd1[3]; +}; + +/* *************************** RC EXT TABLE *************************** */ +/* dcc ext qpc table struct */ +struct ucode_ext_table_sq_ctx { + /* DW0 */ + union { + u32 ldcpw_rsvd; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_sqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 21; + u32 rr_rcv_ce_sn : 11; +#else + u32 rr_rcv_ce_sn : 11; + u32 rsvd : 21; +#endif + } ldcpw; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_rq_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 20; + u32 sw_rcv_ce_sn : 12; +#else + u32 sw_rcv_ce_sn : 12; + u32 rsvd : 20; +#endif + } ldcpw; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_rqa_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 20; + u32 sw_send_ce_sn : 12; +#else + u32 sw_send_ce_sn : 12; + u32 rsvd : 20; +#endif + } ldcpw; + + u32 value; + } dw0; + + u32 rsvd; +}; + +struct ucode_ext_table_dcc_ctx { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha : 12; + u32 cur_rate : 20; +#else + u32 cur_rate : 20; + u32 alpha : 12; +#endif + } hc3_1_0; + + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 target_rate : 20; + u32 ai_cnt : 3; + u32 f_cnt : 3; + u32 flow_min_rate : 6; +#else + u32 flow_min_rate : 6; + u32 f_cnt : 3; + u32 ai_cnt : 3; + u32 target_rate : 20; +#endif + } hc3_1_0; + + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 alpha_timestamp : 16; + u32 rate_timestamp : 16; +#else + u32 rate_timestamp : 16; + u32 alpha_timestamp : 16; +#endif + } hc3_1_0; + + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 cnp_flag : 1; + u32 cnp_receive : 1; + u32 rcnp_timestamp : 16; +#else + u32 rcnp_timestamp : 16; + u32 cnp_receive : 1; + u32 cnp_flag : 1; + u32 rsvd : 14; +#endif + } hc3_1_0; + + u32 value; + } dw3; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 token_timestamp : 16; +#else + u32 token_timestamp : 16; + u32 rsvd : 16; +#endif + } hc3_1_0; + + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 16; + u32 cnp_tx_filter_ts : 16; +#else + u32 cnp_tx_filter_ts : 16; + u32 rsvd : 16; +#endif + } hc3_1_0; + + u32 value; + } dw5; +}; + +struct smf_rdma_dcc_ip_table { + struct ucode_dcc_ip_table_rate_ctx algo_para; // dw0-dw3 +}; + +struct ucode_ext_table_qpc_ctx { + struct ucode_ext_table_sq_ctx sq; + struct ucode_ext_table_sqa_ctx sqa; + struct ucode_ext_table_rq_ctx rq; + struct ucode_ext_table_rqa_ctx rqa; +}; + + +struct tag_iqpcn_hash_entry { + u32 dw0; + union { + struct { + u32 rsvd_key[7]; + u8 dgid[16]; + u32 srctagl; + } bs; + u32 value[12]; + } key; + union { + struct { + u32 rsvd : 4; + u32 cnt : 28; + } bs; + u32 item; + } dw13; +}; + +struct tag_ipqcn_hash_value { + union { + struct { + u32 code : 2; + u32 subcode : 2; + u32 rsvd : 13; + u32 node_index : 15; + } bs; + u32 value; + } dw0; + union { + struct { + u32 w : 1; + u32 sm_id : 3; + u32 cnt : 28; + } bs; + u32 value; + } dw1; + u32 rsvd[2]; +}; + +#pragma pack(0) + +#endif // ROCE_CCF_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h new file mode 100644 index 000000000..54eb98cb6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_compile_macro.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_COMPILE_MACRO_H +#define ROCE_COMPILE_MACRO_H + +#include "node_id.h" + +#ifdef ROCE_LB_MODE1_EN +#define ROCE_LBF_MODE1 +#endif + +#ifdef ROCE_LB_MODE0_EN +#define ROCE_LBF_MODE0 +#endif + +#ifdef TBL_ID_ROCE_QDFX_CTR_SM_NODE +#define ROCE_SRQ_QDFX_EN +#endif + +/* ******************************************************** */ +enum ROCE_LBF_E { + ROCE_LBF_DIS = 0, + ROCE_LBF_TYPE1, + ROCE_LBF_TYPE2, +}; + +#ifdef ROCE_LB_MODE1_ECO_EN +#define ROCE_LB_XID_MASK 0x1 +#else +#define ROCE_LB_XID_MASK 0x3 +#endif + +#if defined(ROCE_LBF_MODE0) || defined(ROCE_LBF_MODE1) + +#ifdef ROCE_LB_MODE1_ECO_EN +#define NODE_SMF_0 NODE_ID_SMF0 +#define NODE_SMF_1 NODE_ID_SMF0 +#define NODE_SMF_2 NODE_ID_SMF2 +#define NODE_SMF_3 NODE_ID_SMF2 +#else +#define NODE_SMF_0 NODE_ID_SMF0 +#define NODE_SMF_1 NODE_ID_SMF1 +#define NODE_SMF_2 NODE_ID_SMF2 +#define NODE_SMF_3 NODE_ID_SMF3 +#endif /* ROCE_LB_MODE1_ECO_EN */ + +#define ROCE_LBF_MODE ROCE_LBF_TYPE1 +#else +#define NODE_SMF_0 NODE_ID_SMF0 +#define NODE_SMF_1 NODE_ID_SMF0 +#define NODE_SMF_2 NODE_ID_SMF0 +#define NODE_SMF_3 NODE_ID_SMF0 + +#define ROCE_LBF_MODE ROCE_LBF_DIS +#endif /* defined(ROCE_LBF_MODE0) || defined(ROCE_LBF_MODE1) */ + +#define ROCE_LBF_INLINE_SMFPG23(idx) ((0 == ((idx) & 0x2)) ? NODE_SMF_2 : NODE_SMF_3) +#define ROCE_LBF_INLINE_SMFPG01(idx) ((0 == ((idx) & 0x2)) ? NODE_SMF_0 : NODE_SMF_1) +#define ROCE_LBF_INLINE_SMFID(idx) ((0 == ((idx) & 0x1)) ? \ + ROCE_LBF_INLINE_SMFPG01(idx) : ROCE_LBF_INLINE_SMFPG23(idx)) +/* ******************************************************** */ + +#define ROCE_FUNC_CHECK(roce_en) ((roce_en) ^ 0x1) + +#endif /* ROCE_COMPILE_MACRO_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h new file mode 100644 index 000000000..7ac693aad --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ctx_api.h @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CTX_API_H +#define ROCE_CTX_API_H + +#include "roce_compile_macro.h" +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" +#include "rdma_context_format.h" + + +enum ROCE_TIMER_ID_E { + ROCE_TIMER_ID_ACK = 0, + ROCE_TIMER_ID_DCQCN_RCNP, + ROCE_TIMER_ID_WND_LESS_THAN_MTU, + ROCE_TIMER_ID_MAX = 4, + ROCE_TIMER_ID_QP_DEL = 7 +}; + +enum roce_qpc_ulp_type { + ROCE_ULP_STD = 0, + ROCE_ULP_PLOG, + ROCE_ULP_VBS, + ROCE_ULP_OSD, + ROCE_ULP_NOFAA +}; + +enum roce_req_rsp_side { + ROCE_REQ_SIDE = 0, + ROCE_RSP_SIDE +}; + +enum roce_rq_cqe_mask { + ROCE_CQEMASK_WRITE = 0, + ROCE_CQEMASK_SEND +}; + +enum ROCE_FLOWID_E { + ROCE_FLOWID_0 = 0, + ROCE_FLOWID_1, + ROCE_FLOWID_2 +}; + +enum ROCE_STGID_E { + ROCE_STGID_0 = 0, + ROCE_STGID_1, + ROCE_STGID_2, + ROCE_STGID_3, + ROCE_STGID_4, + ROCE_STGID_5, + ROCE_STGID_6, + ROCE_STGID_7 +}; + +enum ROCE_EVENT_STG_E { + ROCE_EVENT_STG_SQ = 0, + ROCE_EVENT_STG_SQA, + ROCE_EVENT_STG_RQ, + ROCE_EVENT_STG_RQA, + ROCE_EVENT_STG_SQA_2HOST, + ROCE_EVENT_STG_DCC, + ROCE_EVENT_STG_VERBS, + ROCE_EVENT_STG_NO +}; + +#define ROCE_SQ_STGID ROCE_STGID_0 +#define ROCE_RQ_STGID ROCE_STGID_1 +#define ROCE_VERBS_STGID ROCE_STGID_2 +#define ROCE_DCC_STGID ROCE_STGID_3 +#define ROCE_RQ_ACK_STGID ROCE_STGID_4 +#define ROCE_SQ_ACK_STGID ROCE_STGID_5 +#define ROCE_SQA_2HOST_STGID ROCE_STGID_6 +#define ROCE_REBIND_STGID QU_STG_ID_RSVD + +/* *********TABLE SIZE******* */ +#define WQEBB_SIZE_SHIFT 6 // 64b +#define ROCE_GET_WQBB_NUM_PERWQE(wqe_len_8b) (BYTE8_TO_BYTES(wqe_len_8b) >> WQEBB_SIZE_SHIFT) + +#define ROCE_GID_TABLE_SIZE (32) +#define ROCE_MPT_TABLE_SIZE (64) +#define ROCE_MTT_TABLE_SIZE (8) +#define ROCE_CQC_TABLE_SIZE (128) +#define ROCE_SRQC_TABLE_SIZE (64) +#define ROCE_RDMARC_TABLE_SIZE (32) +#define ROCE_QPC_TABLE_SIZE (512) +#define ROCE_WQE_MAX_SIZE (256) +#define ROCE_CM_MAD_MEG_LEN (64) + +#define ROCE_MTT_QUERY_SIZE (16) +#define ROCE_PCNT_QUERY_SIZE (16) + +#define ROCE_CQE_LEN (64) +#define ROCE_CQE_LEN_SHIFT (6) +#define ROCE_RESIZE_CQE_LEN (64) + +#define ROCE_QPC_RRWC_SEG_SIZE (32) +#define ROCE_QPC_RCC_SEG_SIZE (32) +#define ROCE_QPC_SQPC_SEG_SIZE (64) +#define ROCE_QPC_SQC_SEG_SIZE (64) +#define ROCE_QPC_SQAC_SEG_SIZE (64) +#define ROCE_QPC_RQC_SEG_SIZE (64) +#define ROCE_QPC_QPCC_SEG_SIZE (32) +#define ROCE_QPC_HW_SEG_SIZE (ROCE_QPC_RRWC_SEG_SIZE + ROCE_QPC_RCC_SEG_SIZE + \ + ROCE_QPC_SQC_SEG_SIZE + ROCE_QPC_SQAC_SEG_SIZE + ROCE_QPC_RQC_SEG_SIZE + \ + ROCE_QPC_QPCC_SEG_SIZE) +#define ROCE_QPC_TIMER_SEG_SIZE (32) +#define ROCE_QPC_SW_SEG_SIZE (192) +#define ROCE_QPC_SW_DCQCN_SIZE (16) +#define ROCE_QPC_SW_NOFAA_SIZE ROCE_QPC_QPCC_SEG_SIZE +#define ROCE_QPC_SW_SQC_SIZE (20) +#define ROCE_QPC_SW_SQAC_SIZE (36) +#define ROCE_QPC_SW_RQC_SIZE (20) +#define ROCE_QPC_SW_RQAC_SIZE (20) +#define ROCE_QPC_SW_STGID_SIZE (68) + +#define ROCE_QPC_EXT_SW_SEG_SIZE (32) + +#define ROCE_QPC_RRWC_DW_SIZE (8) +#define ROCE_QPC_RCC_DW_SIZE (8) +#define ROCE_QPC_SQPC_DW_SIZE (16) +#define ROCE_QPC_SQC_DW_SIZE (16) +#define ROCE_QPC_SQAC_DW_SIZE (16) +#define ROCE_QPC_RQC_DW_SIZE (16) +#define ROCE_QPC_QPCC_DW_SIZE (8) +#define ROCE_QPC_TIMER_DW_SIZE (8) +#define ROCE_QPC_SW_DW_SIZE (48) +#define ROCE_QPC_SW_DCQCN_DW_SIZE (8) +#define ROCE_QPC_SW_NOFAA_DW_SIZE ROCE_QPC_QPCC_DW_SIZE + +#define ROCE_QPC_SW_SEG_DWORD_LEN (ROCE_QPC_SW_SEG_SIZE >> 2) +#define ROCE_QPC_HW_SEG_DWORD_LEN (ROCE_QPC_HW_SEG_SIZE >> 2) +#define ROCE_QPC_SW_DCQCN_DWORD_LEN (ROCE_QPC_SW_DCQCN_SIZE >> 2) +#define ROCE_QPC_SW_NOFAA_DWORD_LEN (ROCE_QPC_SW_NOFAA_SIZE >> 2) +#define ROCE_QPC_SW_STGID_DWORD_LEN (ROCE_QPC_SW_STGID_SIZE >> 2) +#define ROCE_QPC_DWORD_LEN (ROCE_QPC_TABLE_SIZE >> 2) +#define ROCE_MPT_DWORD_LEN (ROCE_MPT_TABLE_SIZE >> 2) +#define ROCE_CQC_DWORD_LEN (ROCE_CQC_TABLE_SIZE >> 2) +#define ROCE_SRQC_DWORD_LEN (ROCE_SRQC_TABLE_SIZE >> 2) +#define ROCE_GID_DWORD_LEN (ROCE_GID_TABLE_SIZE >> 2) +#define ROCE_CM_MAD_DWORD_LEN (ROCE_CM_MAD_MEG_LEN >> 2) + +#define ROCE_RDMARC_MAX_DATA_DW_LEN (ROCE_RDMARC_TABLE_SIZE >> 2) + +#define ROCE_SQ_WQE_SEND_TASK_SEG_LEN (16) +#define ROCE_SQ_WQE_UD_SEND_TASK_SEG_LEN (64) +#define ROCE_SQ_WQE_RDMA_TASK_SEG_LEN (32) +#define ROCE_SQ_WQE_ATOMIC_TASK_SEG_LEN (32) +#define ROCE_SQ_WQE_MASK_ATOMIC_CMPSWP_TASK_SEG_LEN (48) +#define ROCE_SQ_WQE_SEND_WRITE_COM_TASK_SEG_LEN (16) + +/* ******************************************************** */ +enum ROCE_QP_CTX_OFF_E { + ROCE_CTX_RRWC_OFF = 0, + ROCE_CTX_RCC_OFF = 32, + ROCE_CTX_SQC_OFF = 64, + ROCE_CTX_SQAC_OFF = 128, + ROCE_CTX_RQC_OFF = 192, + ROCE_CTX_QPCC_OFF = 256, + ROCE_CTX_TIMER_OFF = 288, + ROCE_CTX_SW_OFF = 320, + ROCE_CTX_SW_SQC_OFF = 400, + ROCE_CTX_SW_SQAC_OFF = 416, + ROCE_CTX_SW_RQC_OFF = 448, + ROCE_CTX_SW_RQAC_OFF = 464 +}; +#define ROCE_CTX_SQPC_OFF ROCE_CTX_RRWC_OFF + +#define ROCE_MISC_RRWC_OFFSET_16B_ALIGN (ROCE_CTX_RRWC_OFF >> 4) +#define ROCE_MISC_RCC_OFFSET_16B_ALIGN (ROCE_CTX_RCC_OFF >> 4) +#define ROCE_MISC_SQC_OFFSET_16B_ALIGN (ROCE_CTX_SQC_OFF >> 4) +#define ROCE_MISC_SQAC_OFFSET_16B_ALIGN (ROCE_CTX_SQAC_OFF >> 4) +#define ROCE_MISC_RQC_OFFSET_16B_ALIGN (ROCE_CTX_RQC_OFF >> 4) +#define ROCE_MISC_QPCC_OFFSET_16B_ALIGN (ROCE_CTX_QPCC_OFF >> 4) +#define ROCE_MISC_TIMERC_OFFSET_16B_ALIGN (ROCE_CTX_TIMER_OFF >> 4) +#define ROCE_MISC_SW_OFFSET_16B_ALIGN (ROCE_CTX_SW_OFF >> 4) +#define ROCE_MISC_SW_OFFSET64_16B_ALIGN ((ROCE_CTX_SW_OFF + 64) >> 4) +#define ROCE_MISC_SW_OFFSET128_16B_ALIGN ((ROCE_CTX_SW_OFF + 128) >> 4) +#define ROCE_MISC_SQPC_OFFSET_16B_ALIGN (ROCE_CTX_SQPC_OFF >> 4) + +#ifdef NOF_ROCE_AA_MODE_EN +#define ROCE_WB_CTX_SIZE QU_WB_CTX_SIZE_224 +#else +#define ROCE_WB_CTX_SIZE QU_WB_CTX_SIZE_192 +#endif + +#define ROCE_RDMARC_ATOMIC_DATA_LEN (8) + +#define ROCE_RDMARC_ATOMIC_DATA_DW_LEN (16 >> 2) +#define ROCE_RDMARC_ATOMIC_STATUS_DW_LEN (4 >> 2) +#define ROCE_RDMARC_ATOMIC_DATA_DW_OFF (16 >> 2) +#define ROCE_RDMARC_ATOMIC_STATUS_DW_OFF (28 >> 2) +/* ************************************************* */ +#define ROCE_QPCC_RXE_DW_OFF (4) +#define ROCE_QPCC_RXE_DW_BITMASK (0X01C00000) +#define ROCE_RCC_RXE_DW_OFF (7) +#define ROCE_RRWC_RXE_DW_OFF (7) +#define ROCE_RCC_RAE_DW_BITMASK (~0X08000000) +#define ROCE_RCC_RRE_DW_BITMASK (~0X04000000) +#define ROCE_RRWC_RWE_DW_BITMASK (~0X08000000) + +#define ROCE_QP_QPCC_GPAH_DW_OFF (6) +#define ROCE_QP_QPCC_GPAH_DW_BITMASK (~0X00FFFFFF) +#define ROCE_QP_QPCC_GPAL_DW_BITMASK (~0XFFFFFFFF) + +#define ROCE_QP_RCC_RCPI_DW_OFF (5) +#define ROCE_QP_RCC_RCPI_DW_BITMASK (~0X000000FF) +#define ROCE_QP_RCC_RCCI_DW_BITMASK (~0X000000FF) + +#define ROCE_QP_COM_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_RCC_STATE_DW_OFF (7) +#define ROCE_QP_RCC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_SQC_STATE_DW_OFF (2) +#define ROCE_QP_SQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_SQAC_STATE_DW_OFF (2) +#define ROCE_QP_SQAC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_SQAC_WQEIDX_DW_OFF (2) +#define ROCE_QP_SQAC_WQEIDX_DW_BITMASK (~0X0000ffff) + +#define ROCE_QP_RQC_STATE_DW_OFF (2) +#define ROCE_QP_RQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_RRWC_STATE_DW_OFF (7) +#define ROCE_QP_RRWC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_QP_RQC_SGEVLD_DW_OFF (0) +#define ROCE_QP_RQC_SGEVLD_DW_BITMASK (~0X80000000) + +#define ROCE_MPT_STATE_DW_OFF (2) +#define ROCE_MPT_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_MPT_MWCNT_DW_OFF (10) +#define ROCE_MPT_MWCNT_DW_BITMASK (~0Xffffffff) + +#define ROCE_SRQC_STATE_DW_OFF (2) +#define ROCE_SRQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_CQC_STATE_DW_OFF (2) +#define ROCE_CQC_STATE_DW_BITMASK (~0Xf0000000) + +#define ROCE_CQC_TIMEOUT_DW7_OFF (7) +#define ROCE_CQC_TIMEOUT_DW7_BITMASK (~0Xffffffff) + +#define ROCE_CQC_TMR_TWID_DW10_OFF (10) +#define ROCE_CQC_TMR_TWID_DW_BITMASK (~0Xffff0000) + +#define ROCE_SRQC_ERR_STATE (0x10000000) +#define ROCE_QPC_RQ_COMM_EST_OFF (27) /* *keep the same with qpc rq DW22 */ + +#define ROCE_RRWC_SGE_VLD (0x80000000) + +#define ROCE_VLAN 1 + +#endif /* ROCE_CTX_API_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h new file mode 100644 index 000000000..87e4e303f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_dif_format.h @@ -0,0 +1,492 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DIF_FORMAT_H +#define ROCE_DIF_FORMAT_H + +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" +#include "roce_verbs_mr_attr.h" + +struct tag_roce_dif_cqe_s { + // roce_cqe_s roce_cqe; + u32 common[8]; /* dif multiplexes the first 32 Bits of standard cqe */ + u32 timestamp_h; + u32 timestamp_l; + u32 mr_lkey; + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd1 : 4; + u32 app_err : 1; + u32 ref_err : 1; + u32 grd_err : 1; + u32 rsvd2 : 17; + u32 rcv_pi_vb : 8; +#else + u32 rcv_pi_vb : 8; + u32 rsvd2 : 17; + u32 grd_err : 1; + u32 ref_err : 1; + u32 app_err : 1; + u32 rsvd1 : 4; +#endif + } bs; + u32 value; + } dw11; + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rcv_pi_h; + u32 rcv_pi_l; +#else + u32 rcv_pi_l; + u32 rcv_pi_h; +#endif + } bs; + u64 value; + } rcv_pi; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 vrf_agm_imm : 16; + u32 ri_agm_imm : 16; +#else + u32 ri_agm_imm : 16; + u32 vrf_agm_imm : 16; +#endif + } bs; + u32 value; + } dw14; + u32 rsvd3; +}; + +struct tag_roce_dif_com_s { + struct tag_roce_dif_user_data_s *dif_data; + + u32 sd_pdu_ofs; + u32 sd_pdu_len; + + u16 difx_cnt; + u16 mss; + u16 metadata_len; + u16 sector_len; +}; + +union tag_roce_dif_misc_rx_s { + struct { + u32 rsvd : 18; + u32 fst_bt_tp : 1; + u32 fst_sector : 1; + u32 fst_bt_ofs : 12; + } bs; + u32 value; +}; + +struct tag_roce_dif_wqe_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 imm_data; + + /* DW3 */ + u32 rsvd1; + + /* DW4~5 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw4; + }; + + /* DW6 */ + u32 rkey; + struct tag_roce_dif_user_data_s dif_data; + /* DW7 */ + // u32 sig_key; +}; + +struct tag_roce_dif_wqe_snd_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW1 */ + /* This parameter is valid for the immediate data operation or SEND invalidate. */ + u32 immdata_invkey; + + struct tag_roce_dif_user_data_s dif_data; +}; + + +/* REG SIG MR Local WQE */ +struct roce_wqe_reg_sig_mr_seg { + union roce3_wqe_tsk_com_seg common; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rre : 1; /* Remote read enable */ + u32 rwe : 1; /* Remote write enable */ + u32 lwe : 1; /* Local write enable */ + u32 sgl_mode : 1; /* 0: Single SGL, 1: Dual SGL */ + u32 sector_size : 1; /* 0 : 512B, 1: 4096B */ + u32 rsvd1 : 3; + /* Block size, which is the same as the MPT data definition. */ + u32 block_size : 6; + u32 rsvd2 : 18; +#else + u32 rsvd2 : 18; + /* Block size, which is the same as the MPT data definition. */ + u32 block_size : 6; + u32 rsvd1 : 3; + u32 sector_size : 1; /* 0 : 512B, 1: 4096B */ + u32 sgl_mode : 1; /* 0: Single SGL, 1: Dual SGL */ + u32 lwe : 1; /* Local write enable */ + u32 rwe : 1; /* Remote write enable */ + u32 rre : 1; /* Remote read enable */ +#endif + } bs; + u32 value; + } dw0; + + u32 sig_mr_mkey; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 10; + u32 mtt_offset : 22; +#else + u32 mtt_offset : 22; + u32 rsvd : 10; +#endif + } bs; + u32 value; + } dw2; + struct tag_roce_dif_user_data_s dif_data; + u32 data_mr_key; + u32 metadata_mr_key; + u32 rsvd1; +}; + + +struct roce_wqe_reg_mr_dsgl_seg { + struct roce_wqe_reg_sig_mr_seg reg_mr_seg; + struct roce3_sge_info data_sge; + struct roce3_sge_info prot_sge; +}; + +struct roce_dif_ssgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 rsvd1 : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 rsvd1 : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; /* Dma sequence preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to the mw. */ +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 indirect_mr : 1; + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; /* If set, indicates this MPT is double SGL type. */ + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa) */ + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mtt_base_addr_lo; /* Lower 32 bits of mtt base address (pa) */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + struct tag_roce_dif_user_data_s dif_data; +}; + +struct roce_dif_dsgl_mpt_context { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd6 : 11; + u32 signature : 4; + u32 rsvd5 : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + u32 rsvd4 : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 rsvd3 : 1; + u32 rkey : 1; + u32 rsvd2 : 1; + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + u32 rsvd0 : 1; + u32 rsvd1 : 1; + u32 access_bind : 1; /* Whether mr can be bound to mw */ +#else + u32 access_bind : 1; /* Whether the mr supports the binding of the mw */ + u32 rsvd0 : 1; + u32 rsvd1 : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 rsvd2 : 1; + u32 rkey : 1; + u32 rsvd3 : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + u32 rsvd4 : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + u32 rsvd5 : 1; + u32 signature : 4; + u32 rsvd6 : 11; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { + u32 rsvd; + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 status : 4; + u32 indirect_mr : 1; /* If set, indicates this MPT is double SGL type. */ + u32 rsvd : 27; +#else + u32 rsvd : 27; + u32 indirect_mr : 1; /* If set, indicates this MPT is double SGL type. */ + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; /* The index is not included. */ + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 rsvd : 23; +#else + u32 rsvd : 23; + u32 sw_dif_en : 1; /* If set, indicates dif mode. */ + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 data_mr_key : 28; +#else + u32 data_mr_key : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw4; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 metadata_mr_key : 28; +#else + u32 metadata_mr_key : 28; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw5; + + /* DW6~DW9 */ + u32 rsvd[4]; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + struct tag_roce_dif_user_data_s dif_data; +}; + +#endif diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h new file mode 100644 index 000000000..f0a84b8ae --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_err_type.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_ERR_TYPE_H +#define ROCE_ERR_TYPE_H + +#include "typedef.h" +#include "roce_pub_cmd.h" +#include "roce_api.h" + + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif /* __cplusplus */ + +/* **********************macro define************************ */ +enum ROCE_ERR_STATE_E { + ROCE_AEQE_NOERR_STATE = 0, + ROCE_AEQE_ERR_STATE, +}; + +enum ROCE_QP_ERR_STATE_E { + ROCE_QP_NOERR_STATE = 0, + ROCE_QP_ERR_STATE, +}; + +/* *********ROCE_AEQE_SUBTYPE_E******* */ +enum ROCE_AEQE_SUBTYPE_E { + ROCE_AEQE_SUBTYPE_PATH_MIGRATED = 0, /* * APM */ + ROCE_AEQE_SUBTYPE_COMM_ESTABLISHED, /* *RQ RTR RCV FIRST PKT */ + ROCE_AEQE_SUBTYPE_SQ_DRAINED, /* * sq empty and change to SQ DRANNING STATE */ + ROCE_AEQE_SUBTYPE_SRQ_LIMIT_REACHED, /* *SRQ LIMIT_REACHED from RXDMA api rsp */ + ROCE_AEQE_SUBTYPE_QP_LAST_WQE_REACHED, /* *srq en & LAST_WQE_REACHED */ + ROCE_AEQE_SUBTYPE_CQ_ERR, /* *api rsp cq err */ + ROCE_AEQE_SUBTYPE_QP_FATAL, /* *LOC_WORK_QUEUE_CATASTROPHIC_ERR */ + ROCE_AEQE_SUBTYPE_QP_REQ_ERR, /* *INVALID_REQ_LOC_WORK_QUEUE_ERR */ + ROCE_AEQE_SUBTYPE_QP_ACCESS_ERR, /* *LOC_ACCESS_VIOLATION_WORK_QUEUE_ERR */ + ROCE_AEQE_SUBTYPE_PATH_MIG_ERR, + ROCE_AEQE_SUBTYPE_GID_CHANGE_EVENT, + ROCE_AEQE_SUBTYPE_CQ_OVERRUN, + ROCE_AEQE_SUBTYPE_SRQ_CATASTROPHIC_ERR, + ROCE_AEQE_SUBTYPE_CFG_ERR, + ROCE_AEQE_SUBTYPE_CHIP_ERR, + ROCE_AEQE_SUBTYPE_DIF_ERR, + ROCE_AEQE_SUBTYPE_XRC_DOMAIN_VIOLATION_ERR, + ROCE_AEQE_SUBTYPE_INVALID_XRCETH_ERR, + ROCE_AEQE_SUBTYPE_UCODE_FATAL_ERR, + ROCE_AEQE_SUBTYPE_ODP_PAGE_FAULT, + ROCE_AEQE_SUBTYPE_RSVD, +}; + +/* *********ROCE SQ ERR type******* */ +enum ROCE_COMPLETION_EVENT_ERR_TYPE_E { + ROCE_COMPLTE_EVENT_NO_ERR = 0, + ROCE_COMPLTE_EVENT_LOC_LEN_ERR = 0x1, + ROCE_COMPLTE_EVENT_LOC_QP_OPERATION_ERR, + ROCE_COMPLTE_EVENT_LOC_PROTECTION_ERR = 0x4, + ROCE_COMPLTE_EVENT_LOC_MEM_MANAGEMENT_ERR = 0x4, + ROCE_COMPLTE_EVENT_WR_FLUSH_ERR, + ROCE_COMPLTE_EVENT_MW_BIND_ERR, + ROCE_COMPLTE_EVENT_BAD_RESP_ERR = 0x10, + ROCE_COMPLTE_EVENT_LOC_ACCESS_ERR, + ROCE_COMPLTE_EVENT_REM_INV_REQ_ERR, + ROCE_COMPLTE_EVENT_REM_ACCESS_ERR, + ROCE_COMPLTE_EVENT_REM_OPERATION_ERR, + ROCE_COMPLTE_EVENT_RETRY_CTR_EXCEED_ERR, + ROCE_COMPLTE_EVENT_RNR_RETRY_CTR_EXCEED_ERR, + ROCE_COMPLTE_EVENT_REM_ABORTED_ERR = 0x22, + ROCE_COMPLTE_EVENT_XRC_VIOLATION_ERR, +}; + +enum ROCE_AEQE_EVENT_ERR_TYPE_E { + ROCE_AEQE_QP_FATAL_SQ_FETCH_WQE_STATUS_ERR = 0, + ROCE_AEQE_QP_FATAL_SQ_DMA_GEN_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_DMA_GEN_MPT_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_DMA_GEN_CQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_COMPLETE_WQE_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SQ_COMPLETE_CQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_MPT_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_CQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RXDMA_SRQ_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RDMA_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RQ_RDMA_MPT_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_CQE_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_ARM_STATUS_ERR, + ROCE_AEQE_QP_FATAL_SRQ_ARM_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RDMARC_STATUS_ERR, + ROCE_AEQE_QP_FATAL_RDMARC_RSP_PSN_ERR = 0x10, + ROCE_AEQE_QP_FATAL_MINVLD_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_RESIZE_STATUS_ERR, + ROCE_AEQE_QP_FATAL_CQ_OVERFLOW, + ROCE_AEQE_CFG_TYPE_STATELESS_MAC = 0x20, + ROCE_AEQE_CFG_TYPE_STATELESS_FUNC, + ROCE_AEQE_CFG_TYPE_QPC_PREFETCH_RSP_ERR, + ROCE_AEQE_CFG_TYPE_QPC_STATE_ERR, + ROCE_AEQE_CFG_TYPE_QPC_STATE_NOT_RTS, + ROCE_AEQE_CHIP_TYPE_CPB_BUF_FULL, + ROCE_AEQE_CHIP_TYPE_IPSU_RSP_ERR, + ROCE_AEQE_CHIP_TYPE_QU_QUERY_RSP_ERR, + ROCE_AEQE_LOC_QP_REQ_ERR_TOO_MANY_READ_ATOMIC = 0x30, + ROCE_AEQE_LOC_QP_REQ_ERR_OPCODE_MISMATCH, + ROCE_AEQE_LOC_QP_REQ_ERR_LEN_ERR, + ROCE_AEQE_LOC_QP_OPERATION_ERR, + ROCE_AEQE_LOC_QP_NUM_RANGE_ERR, + ROCE_AEQE_MPT_NUM_RANGE_ERR, + ROCE_AEQE_MASTER_QP_MODIFY_ERR, +}; + +#define ROCE_AEQE_EVENT(module_type, subtype, type, err_code) \ + (((u32)(module_type) << 28) | ((subtype) << 20) | ((type) << 8) | (err_code)) + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ + + +#endif /* ROCE_ERR_TYPE_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h new file mode 100644 index 000000000..93cdad34e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_hmm_context.h @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_HMM_CONTEXT_H +#define ROCE_HMM_CONTEXT_H + +/* **************** Macro Definition ****************** */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +/* **************** Data Structure Definition ****************** */ +/* * MPT Format start */ +struct roce_mpt_context { + /* DW0 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 access_bind : 1; /* Whether mr can be bound to mw */ + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 signature : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#else + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 signature : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify the remote authority. */ + u32 remote_access_en : 1; + u32 access_bind : 1; /* Whether mr can be bound to mw */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#else + u32 so_ro : 2; /* Dma sequence preserving flag */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 ep : 3; + u32 qpn : 20; /* Qp bound to mw */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; + /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 status : 4; +#else + /* Mpt status. Valid values are VALID, FREE, and INVALID. */ + u32 status : 4; + u32 indirect_mr : 1; /* indirect mr flag */ + u32 cos : 3; + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 pdn : 18; /* Pd bound to mr or mw */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 fbo : 22; + u32 page_mode : 1; + u32 sgl_mode : 1; /* Set indicates this MPT is double SGL type. */ + u32 mkey : 8; /* The index is not included. */ +#else + u32 mkey : 8; /* The index is not included. */ + u32 sgl_mode : 1; /* Set indicates this MPT is double SGL type. */ + u32 page_mode : 1; + u32 fbo : 22; +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + /* Mtt base address (pa)hi:bit[63:32], lo:bit[31:03], gpa_sign[02:00] */ + u64 mtt_base_addr; + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mtt_base_addr_lo; /* Mtt base address (pa) lower 32 bits */ + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + + /* DW11 */ + u32 mtt_sz; /* This parameter is valid when FRMR. */ + + /* DW12~DW13 */ + u32 rsvd[2]; + + /* DW14~15 */ + union { + /* Mtt base address (pa)hi:bit[63:32], lo:bit[31:03], gpa_sign[02:00] */ + u64 mw_vaddr; + struct { + u32 mw_vaddr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mw_vaddr_lo; /* Mtt base address (pa) lower 32 bits */ + } dw14; + struct { + u32 mw_vaddr_hi; /* Mtt base address (pa) upper 32 bits */ + u32 mw_vaddr_lo; /* Mtt base address (pa) lower 32 bits */ + } dw15; + }; +}; + +#endif + diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h new file mode 100644 index 000000000..13bc6ef88 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_mpu_common.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MPU_COMMON_H +#define ROCE_MPU_COMMON_H + +#include "mpu_cmd_base_defs.h" + +enum { + /* FUNC CFG */ + ROCE_MPU_CMD_SET_FUNC_STATE = 0, + ROCE_MPU_CMD_SET_CPU_ENDIAN, + ROCE_MPU_CMD_GET_CFG_INFO, + ROCE_MPU_CMD_DEL_FUNC_RES, + ROCE_MPU_CMD_GET_FUNC_TABLE, + ROCE_MPU_CMD_ADD_MAC = 10, + ROCE_MPU_CMD_DEL_MAC, + ROCE_MPU_CMD_ADD_IPSU_MAC, + ROCE_MPU_CMD_DEL_IPSU_MAC, + ROCE_MPU_CMD_GET_MAC_VNI, + + /* BOND */ + ROCE_MPU_CMD_BOND_SET_STATE = 20, + ROCE_MPU_CMD_BOND_GET_ER_FWD_ID, + ROCE_MPU_CMD_BOND_SET_ER_FWD_ID, + ROCE_MPU_CMD_BOND_ER_FWD_ID_COMBINE, + ROCE_MPU_CMD_BOND_ER_FWD_ID_COMPACT, + ROCE_MPU_CMD_GET_GROUP_ID, + + /* CC */ + ROCE_MPU_CMD_CC_CFG_CCF_PARAM = 40, + ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM, + ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM, + ROCE_MPU_CMD_CC_CFG_LDCP_PARAM, + ROCE_MPU_CMD_CC_SET_BW_CTRL, + ROCE_MPU_CMD_CC_QUERY_BW_CTRL, + + /* DFX */ + ROCE_MPU_CMD_DFX_CACHE_OUT = 55, + ROCE_MPU_CMD_DFX_SET_CAP_CFG, + ROCE_MPU_CMD_DFX_GET_CAP_CFG, + ROCE_MPU_CMD_DFX_READ_CAP_CTR, + ROCE_MPU_CMD_DFX_CLEAR_CAP_CTR, + + /* ULP */ + ROCE_MPU_CMD_ULP_AA_SET_DD_CFG = 128, + ROCE_MPU_CMD_ULP_AA_CTRL_READY, + ROCE_MPU_CMD_ULP_AA_SWITCH_IO, + ROCE_MPU_CMD_ULP_AA_FAKE_DATA, + ROCE_MPU_CMD_ULP_AA_CLAER_ACT_CTRL_BMP, + + ROCE_MPU_CMD_MAX +}; + +#define MAX_ROCE_PHY_PORT 8 + +/* *********************** Func cmd between driver and mpu ************************** */ +struct roce_set_func_state_cmd { + struct comm_info_head head; + + u16 func_id; + u8 func_en; + u8 tag; +}; + +struct roce_get_func_table_cmd { + struct comm_info_head head; + + u16 func_id; + u8 rsvd[2]; +}; + +struct roce_get_func_table_rsp { + struct comm_info_head head; + u32 func_tbl_value; +}; + +struct roce_get_cfg_info_cmd { + struct comm_info_head head; + + u16 func_id; + u8 rsvd[2]; +}; + +struct roce_get_cfg_info_resp { + struct comm_info_head head; + + u8 scence_id; + u8 lb_en; + u8 lb_mode; + u8 container_mode; + + u8 fake_en; + u8 pf_start_bit; + u8 pf_end_bit; + u8 page_bit; + + u8 port_num; + u8 host_num; + u16 rsvd; +}; + +struct roce_set_cpu_endian_cmd { + struct comm_info_head head; + + u16 func_id; + u8 cpu_endian; + u8 rsvd; +}; + +struct roce_cfg_mac_cmd { + struct comm_info_head head; + + u16 func_id; + u16 rsvd; + u16 er_fwd_id; + u8 mac[6]; + u8 vni_en; + u32 vlan_id; +}; + +struct roce_cfg_ipsu_mac_cmd { + struct comm_info_head head; + + u16 func_id; + u8 er_id; + u8 vni_en; /* vni enable */ + u8 mac[6]; + u8 rsvd[2]; + u32 vlanid_vni; +}; + +struct vroce_mac_cfg_vni_info { + struct comm_info_head head; + u16 func_id; + u16 rsvd; + + u32 vlan_vni : 24; + u32 vni_en : 8; +}; + +/* ************************* Bond cmd between driver and mpu ************************* */ +struct roce_bond_cfg_state_cmd { + struct comm_info_head head; + + u16 func_id; + u8 bond_en; + u8 rsvd; +}; + +struct roce_bond_set_ipsu_mac_cmd { + struct comm_info_head head; + + u16 func_id; + u16 vlan_id; + u16 er_fwd_id; + u8 mac[6]; +}; + +struct roce_bond_cfg_er_fwd_id_cmd { + struct comm_info_head head; + + u16 func_id; + u16 er_fwd_id; + u32 bond_tbl_val; +}; + +struct roce_bond_combine_er_fwd_id_cmd { + struct comm_info_head head; + u16 func_id_src; + u16 func_id_dst; + u16 er_fwd_id; + u8 rsvd[2]; +}; + +struct roce_bond_compact_er_fwd_id_cmd { + struct comm_info_head head; + + u16 func_id; + u16 rsvd; + u32 value; /* dw14 value */ +}; + +/* ************************* CC cmd between driver and mpu ************************** */ +struct roce_cc_cfg_param_cmd { + struct comm_info_head head; + u16 func_id; + u16 rsvd; + u32 param[4]; +}; + +struct roce_cc_cfg_bw_ctrl_cmd { + struct comm_info_head head; + + u16 func_id; + u8 cmd; + u8 rsvd; + + u8 color_type; + u16 ptype; + u8 hw_wred_mode; + + u32 cir; + u32 pir; + u32 cbs; + u32 xbs; + u32 cnp; + u32 enable; +}; + +/* *************************** DFX cmd between driver and mpu ***************************** */ +struct roce_dfx_cache_out_cmd { + struct comm_info_head head; + + u16 func_idx; + u8 cache_index; + u8 rsvd; +}; + +struct roce_dfx_cfg_cap_param_cmd { + struct comm_info_head head; + u16 index; + u16 rsvd; + u32 param[4]; +}; + +struct roce_dfx_cap_ctr_cmd { + struct comm_info_head head; + u16 index; + u16 rsvd; + u32 value; +}; + +#endif /* ROCE_MPU_COMMON_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h new file mode 100644 index 000000000..dfcc6e7bc --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub.h @@ -0,0 +1,259 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_PUB_H +#define ROCE_PUB_H + +#include "rdma_context_format.h" + + +/* ********************** sync info ************************ */ +#define ROCE_SQC_RST_STATE (0x0) +#define ROCE_SQAC_RST_STATE (0x0) +#define ROCE_RQC_RST_STATE (0x0) +#define ROCE_RCC_RST_STATE (0x0) +#define ROCE_RRWC_RST_STATE (0x0) + +#define ROCE_QPC_DELED_STATE (0xCE) +#define ROCE_RC_KICKOFF_VALUE (0x5a5acece) +#define ROCE_CACHE_LINE_SIZE (0x100) + +#define ROCE_CQC_MID_STATE_VALUE (0xe) +#define ROCE_CQC_MID_STATE (0xe0000000) + +#define ROCE_CPU_ENDIAN_LITTLE 0 + +#define ROCE_MAX_ARM_SN_NUM 4 + +#define ROCE_MKEY_MPT_IDX_OFF (8) +#define ROCE_MKEY_GET_MPT_IDX(mkey) ((mkey) >> ROCE_MKEY_MPT_IDX_OFF) + +#define ROCE_PAGE_SIZE 4096 +#define ROCE_RC_ENTRY_NUM_PER_CACELINE 8 +#define ROCE_MTT_BUFLEN_TOTAL(mtt_num) ((mtt_num) << 3) + +#define ROCE_SQWQE_CI_MASK (0xffff) + +#define ROCE_GID_LEN 16 + +#define ROCE_LOCAL_DEFAULT_PMTU 5 +#define ROCE_LOCAL_DEFAULT_MTU_CODE 0xf + +enum { + ROCE_QP_ST_RC = 0x0, /* 000 */ + ROCE_QP_ST_UC = 0x1, /* 001 */ + ROCE_QP_ST_RD = 0x2, /* 010 */ + ROCE_QP_ST_UD = 0x3, /* 011 */ + ROCE_QP_ST_XRC = 0x6, /* 110 */ + ROCE_QP_ST_PRIV = 0x7 /* 111 */ +}; +/* ********************** Macro Definition ************************ */ + +enum ROCE_SERVICE_TYPE_E { + ROCE_RC_TYPE = 0, + ROCE_UC_TYPE, + ROCE_RD_TYPE, + ROCE_UD_TYPE, + ROCE_RSVD_TYPE, + ROCE_XRC_5_TYPE = 0x5, + ROCE_XRC_TYPE = 0x6, + ROCE_XRC_6_TYPE = 0x6 +}; + +enum ROCE_RC_FLAG_E { + ROCE_UC_UD = 0, + ROCE_RC_XRC +}; + +/* Description of parameters in the *********ROCE table ******* */ +/* the same with stp */ +enum ROCE_GID_TYPE_E { + ROCE_GID_TYPE_V2_IPV4 = 0, + ROCE_GID_TYPE_V2_IPV6 = 1, + ROCE_GID_TYPE_V1 = 2 +}; + +enum ROCE_SQWQE_MSGTYPE_E { + ROCE_TX_MSGTYPE_SEND = 0, + ROCE_TX_MSGTYPE_WRITE, + ROCE_TX_MSGTYPE_READ, + ROCE_TX_MSGTYPE_ATOMIC, + ROCE_TX_MSGTYPE_LOCAL, + ROCE_TX_MSGTYPE_RESIZECQ = 5, + ROCE_TX_MSGTYPE_RSVD6 = 6, + ROCE_TX_MSGTYPE_ERR = 7 +}; + +enum ROCE_SQWQE_OPTYPE_E { + ROCE_TX_SEND = 0, + ROCE_TX_SEND_INVALIDATE, + ROCE_TX_SEND_IMMEDIATE, + ROCE_TX_OPTYPE_RSVD3, + + ROCE_TX_WRITE = 4, + ROCE_TX_WRITE_IMMEDIATE, + ROCE_TX_OPTYPE_RSVD6, + ROCE_TX_OPTYPE_RSVD7, + + ROCE_TX_READ = 8, + ROCE_TX_OPTYPE_RSVD9, + ROCE_TX_OPTYPE_RSVD10, + ROCE_TX_EXT_ATOMIC_COMPARE_SWAP, + + ROCE_TX_ATOMIC_COMPARE_SWAP = 0xc, + ROCE_TX_ATOMIC_FETCH_ADD, + ROCE_TX_ATOMIC_MASKED_COMPARE_SWAP, + ROCE_TX_ATOMIC_MASKED_FETCH_ADD, + + ROCE_FAST_REG_PMR = 0x10, + ROCE_LOCAL_INVALIDATE, + ROCE_BIND_MW_TYPE1_TYPE2, + ROCE_REG_SIG_MR, + ROCE_LOCAL_EXT, + ROCE_TX_OPTYPE_RSVD15, + ROCE_RESIZE_TYPE = 0x16, + ROCE_TX_OPTYPE_RSVD17, + ROCE_TX_OPTYPE_RSVD18, + ROCE_TX_OPTYPE_RSVD19, + ROCE_TX_OPTYPE_RSVD1A, + ROCE_TX_OPTYPE_RSVD1B, + ROCE_TX_OPTYPE_RSVD1C, + ROCE_TX_OPTYPE_RSVD1D, + ROCE_ERR_TYPE = 0x1e, + ROCE_TX_OPTYPE_RSVD1F +}; + +enum ROCE_SQPC_STATE_E { + ROCE_SQPC_STATE_SW = 0, + ROCE_SQPC_STATE_ERR = 1, + ROCE_SQPC_STATE_HW = 0xF +}; + +enum ROCE_SRQC_STATE_E { + ROCE_SRQC_STATE_SW = 0, + ROCE_SRQC_STATE_ERR = 1, + ROCE_SRQC_STATE_HW = 0xF +}; + +enum ROCE_CQC_STATE_E { + ROCE_CQC_STATE_SW = 0, + ROCE_CQC_STATE_ERR = 1, + ROCE_CQC_STATE_OVERFLOW = 2, + ROCE_CQC_STATE_HW = 0xF +}; + +enum ROCE_MPT_STATE_E { + ROCE_MPT_STATE_VALID = 1, + ROCE_MPT_STATE_FREE = 3, + ROCE_MPT_STATE_ABORT = 4, + ROCE_MPT_STATE_INVALID = 0xF +}; + +enum ROCE_MPT_R_W_E { + ROCE_MPT_MW = 0, + ROCE_MPT_MR +}; + +enum ROCE_MPT_BQP_E { + ROCE_MPT_BQP_TYPE1 = 0, + ROCE_MPT_BQP_TYPE2B +}; + +enum ROCE_MPT_DIF_MODE_E { + ROCE_MPT_DIF_MODE_INS = 0, + ROCE_MPT_DIF_MODE_DEL, + ROCE_MPT_DIF_MODE_FWD +}; + +enum ROCE_MPT_SECTOR_SIZE_E { + ROCE_MPT_SECTOR_SIZE_512 = 0, + ROCE_MPT_SECTOR_SIZE_4KB +}; + +enum ROCE_MPT_METEDATA_SIZE_E { + ROCE_MPT_METEDATA_SIZE_8 = 0, + ROCE_MPT_METEDATA_SIZE_64 +}; + + +enum ROCE_MPT_SGL_MODE_E { + ROCE_MPT_SGL_MODE_SINGLE = 0, + ROCE_MPT_SGL_MODE_DOUBLE +}; + +enum ROCE_QPC_STATE_E { + ROCE_QPC_STATE_RST = 0, + ROCE_QPC_STATE_INIT, + ROCE_QPC_STATE_RTR, + ROCE_QPC_STATE_RTS, + ROCE_QPC_STATE_SQD, + ROCE_QPC_STATE_SQEr, + ROCE_QPC_STATE_ERR, + ROCE_QPC_STATE_DRAINING, + ROCE_QPC_STATE_RSV +}; + + +enum ROCE_MTT_LAYER_NUM_E { + ROCE_MTT_LAYER_NUM_0 = 0, + ROCE_MTT_LAYER_NUM_1, + ROCE_MTT_LAYER_NUM_2, + ROCE_MTT_LAYER_NUM_3, + ROCE_MTT_LAYER_NUM_RSVD +}; + +enum ROCE_CQE_TYPE_E { + /* *1-Send Completion; 0-Receive Completion */ + ROCE_RQ_CQE = 0, + ROCE_SQ_CQE +}; + +enum ROCE_CQE_SIZE_E { + ROCE_CQE_SIZE_0 = 0, /* * 4B */ + ROCE_CQE_SIZE_1 = 1, /* * 8B */ + ROCE_CQE_SIZE_2 = 2, /* * 16B */ + ROCE_CQE_SIZE_3 = 3 /* * 32B, RoCE fix 3 */ +}; + +enum ROCE_CQE_RQ_OPTYPE_E { + ROCE_CQE_RQ_OPTYPE_WRITE_IMMEDIATE = 0, + ROCE_CQE_RQ_OPTYPE_SEND, + ROCE_CQE_RQ_OPTYPE_SEND_IMMEDIATE, + ROCE_CQE_RQ_OPTYPE_SEND_INVALIDATE, + ROCE_CQE_RQ_OPTYPE_WRITE, + ROCE_CQE_RQ_OPTYPE_READ +}; + +enum ROCE_CQE_COMMON_OPTYPE_E { + ROCE_CQE_COMMON_OPTYPE_RESIZE = 0x16, + ROCE_CQE_COMMON_OPTYPE_ERR = 0x1e +}; + +enum ROCE_QPC_MTU_E { + ROCE_PMTU_256B = 1, + ROCE_PMTU_512B = 2, + ROCE_PMTU_1KB = 3, + ROCE_PMTU_2KB = 4, + ROCE_PMTU_4KB = 5 +}; + +enum { + ROCE_DB_MTU_256B_SHIFT = 0, + ROCE_DB_MTU_512B_SHIFT = 1, + ROCE_DB_MTU_1K_SHIFT = 2, + ROCE_DB_MTU_2K_SHIFT = 3, + ROCE_DB_MTU_4K_SHIFT = 4, +}; + +enum ROCE_FLOW_ID_E { + ROCE_STL_TIMER_FLOW = 0, + ROCE_PKT_FLOW = 1, + ROCE_DB_FLOW = 2, + ROCE_TIMER_FLOW = 3, + ROCE_ARM_FLOW = 4, + ROCE_CMDQ_FLOW = 5, + ROCE_NRET_FLOW = 6, + ROCE_STL2STF_FLOW = 7, +}; + +#endif /* ROCE_PUB_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h new file mode 100644 index 000000000..eefd53404 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_pub_cmd.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_PUB_CMD_H +#define ROCE_PUB_CMD_H + +#ifndef CQM_AEQ_BASE_T_ROCE +#define CQM_AEQ_BASE_T_ROCE 16 +#endif + +/* RoCE EVENT */ +enum { + ROCE_EVENT_TYPE_OFED_NO_DEF = (CQM_AEQ_BASE_T_ROCE + 0x00), + ROCE_EVENT_TYPE_PATH_MIG = (CQM_AEQ_BASE_T_ROCE + 0x01), /* RSVD */ + ROCE_EVENT_TYPE_COMM_EST = (CQM_AEQ_BASE_T_ROCE + 0x02), + ROCE_EVENT_TYPE_SQ_DRAINED = (CQM_AEQ_BASE_T_ROCE + 0x03), + ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE = (CQM_AEQ_BASE_T_ROCE + 0x04), + + /* 5 */ + ROCE_EVENT_TYPE_WQ_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x05), + ROCE_EVENT_TYPE_PATH_MIG_FAILED = (CQM_AEQ_BASE_T_ROCE + 0x06), + ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x07), + ROCE_EVENT_TYPE_WQ_ACCESS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x08), + ROCE_EVENT_TYPE_CQ_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x09), + + /* 10 */ + ROCE_EVENT_TYPE_SRQ_LIMIT = (CQM_AEQ_BASE_T_ROCE + 0x0a), + ROCE_EVENT_TYPE_SRQ_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x0b), + ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x0c), /* RSVD */ + ROCE_EVENT_TYPE_MR_SIG_ERR = (CQM_AEQ_BASE_T_ROCE + 0x0e), + + ROCE_EVENT_TYPE_ODP_PAGE_FAULT = (CQM_AEQ_BASE_T_ROCE + 0x0f), + /* ROCE NOF AA */ + ROCE_EVENT_TYPE_SWITCH_HOST = (CQM_AEQ_BASE_T_ROCE + 0x10), + ROCE_EVENT_TYPE_HOST_FAILOVER = (CQM_AEQ_BASE_T_ROCE + 0x11), + ROCE_EVENT_NOFAA_STUB = (CQM_AEQ_BASE_T_ROCE + 0x12), + + /* * RSVE */ + ROCE_EVENT_TYPE_EEC_CATAS_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x30), + ROCE_EVENT_TYPE_PORT_CHANGE = (CQM_AEQ_BASE_T_ROCE + 0x31), + ROCE_EVENT_TYPE_ECC_DETECT = (CQM_AEQ_BASE_T_ROCE + 0x32), + ROCE_EVENT_TYPE_VEP_UPDATE = (CQM_AEQ_BASE_T_ROCE + 0x33), + ROCE_EVENT_TYPE_FATAL_WARNING = (CQM_AEQ_BASE_T_ROCE + 0x34), + ROCE_EVENT_TYPE_FLR_EVENT = (CQM_AEQ_BASE_T_ROCE + 0x35), + ROCE_EVENT_TYPE_PORT_MNG_CHG_EVENT = (CQM_AEQ_BASE_T_ROCE + 0x36), + ROCE_EVENT_TYPE_XRC_DOMAIN_VIOLATION_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x37), + ROCE_EVENT_TYPE_INVALID_XRCETH_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x38), + ROCE_EVENT_TYPE_FLUSH_WQE_ERROR = (CQM_AEQ_BASE_T_ROCE + 0x39), + + ROCE_EVENT_TYPE_EQ_OVERFLOW = (CQM_AEQ_BASE_T_ROCE + 0x40), + ROCE_EVENT_TYPE_CMD = (CQM_AEQ_BASE_T_ROCE + 0x41), + ROCE_EVENT_TYPE_COMM_CHANNEL = (CQM_AEQ_BASE_T_ROCE + 0x42), + ROCE_EVENT_TYPE_OP_REQUIRED = (CQM_AEQ_BASE_T_ROCE + 0x43), + + ROCE_EVENT_TYPE_NONE = (CQM_AEQ_BASE_T_ROCE + 0x50) +}; + + +enum { + /* GID CMD */ + ROCE_CMD_UPDATE_GID = 0x60, + ROCE_CMD_QUERY_GID = 0x61, + ROCE_CMD_CLEAR_GID = 0x62, + + /* IP CMD */ + ROCE_CMD_ADD_IP_ENTRY = 0x68, + ROCE_CMD_DEL_IP_ENTRY = 0x69, + ROCE_CMD_LOOKUP_IP_ENTRY = 0x6a, + ROCE_CMD_UPDATE_IP_ENTRY = 0x6b, + + /* TPT CMD */ + ROCE_CMD_SW2HW_MPT = 0x70, + ROCE_CMD_HW2SW_MPT = 0x71, + ROCE_CMD_MODIFY_MPT = 0x72, + ROCE_CMD_QUERY_MPT = 0x73, + ROCE_CMD_FLUSH_TPT = 0x74, + ROCE_CMD_SYNC_TPT = 0x75, + + /* CQ CMD */ + ROCE_CMD_SW2HW_CQ = 0x80, + ROCE_CMD_RESIZE_CQ = 0x81, + ROCE_CMD_MODIFY_CQ = 0x82, + ROCE_CMD_HW2SW_CQ = 0x83, + ROCE_CMD_QUERY_CQ = 0x84, + + /* SRQ CMD */ + ROCE_CMD_SW2HW_SRQ = 0x90, + ROCE_CMD_ARM_SRQ = 0x91, + ROCE_CMD_HW2SW_SRQ = 0x92, + ROCE_CMD_QUERY_SRQ = 0x93, + + /* QP CMD */ + ROCE_CMD_QP_BASE = 0xa0, + ROCE_CMD_RST2INIT_QP = 0xa0, + ROCE_CMD_INIT2INIT_QP = 0xa1, + ROCE_CMD_INIT2RTR_QP = 0xa2, + ROCE_CMD_RTR2RTS_QP = 0xa3, + ROCE_CMD_RTS2RTS_QP = 0xa4, + ROCE_CMD_SQERR2RTS_QP = 0xa5, + ROCE_CMD_2ERR_QP = 0xa6, + ROCE_CMD_RTS2SQD_QP = 0xa7, + ROCE_CMD_SQD2SQD_QP = 0xa8, + ROCE_CMD_SQD2RTS_QP = 0xa9, + ROCE_CMD_2RST_QP = 0xaa, + ROCE_CMD_QUERY_QP = 0xab, + ROCE_CMD_MIRROR_QP = 0xac, /* DFX RoCE +/- packet mirroring */ + ROCE_CMD_MODIFY_HASH_VALUE_QP = 0xad, + ROCE_CMD_GET_QP_RX_PORT = 0xae, + ROCE_CMD_MODIFY_UDP_SRC_PORT_QP = 0xaf, + ROCE_CMD_GET_UDP_SRC_PORT_QP = 0xb0, + ROCE_CMD_GET_QP_FUNC_TABLE = 0xb1, + ROCE_CMD_QP_MAX = 0xb1, + /* Miscellaneous CMD */ + // ROCE_CMD_NOP = 0xb0, + + /* MTT commands */ + ROCE_CMD_QUERY_MTT = 0xc0, + + ROCE_CMD_MODIFY_CONTEXT = 0xd0, + ROCE_CMD_EN_QP_CAP_PKT = 0xd1, + ROCE_CMD_DIS_QP_CAP_PKT = 0xd2, + + ROCE_CMD_MISC_CACHE_INVLD = 0xe0, + + ROCE_CMD_MISC_CQ_CACHE_INVLD = 0xf0, + + ROCE_CMD_CREAT_SLAVE_QP = 0xf1, + ROCE_CMD_QUERY_MASTER_QP_BITMAP = 0xf2, + ROCE_CMD_GET_MASTER_QPN = 0xf3, + ROCE_CMD_SET_CONN_STAT = 0xf4, + ROCE_CMD_DISCONNECT_QP = 0xf5, + ROCE_CMD_SET_SHARD_CFG = 0xf6 +}; + +enum roce3_cmd_type { + COMMON_CMD_TX_INFO = 1, + COMMON_CMD_Q_NUM, + COMMON_CMD_TX_WQE_INFO, + COMMON_CMD_TX_MAPPING, + COMMON_CMD_RX_INFO, + COMMON_CMD_RX_WQE_INFO, + COMMON_CMD_RX_CQE_INFO, + COMMON_CMD_UPRINT_FUNC_EN, + COMMON_CMD_UPRINT_FUNC_RESET, + COMMON_CMD_UPRINT_SET_PATH, + COMMON_CMD_UPRINT_GET_STATISTICS, + COMMON_CMD_FUNC_TYPE, + COMMON_CMD_GET_FUNC_IDX, + COMMON_CMD_GET_INTER_NUM, + COMMON_CMD_CLOSE_TX_STREAM, + COMMON_CMD_GET_DRV_VERSION, + COMMON_CMD_CLEAR_FUNC_STASTIC, + COMMON_CMD_GET_HW_STATS, + COMMON_CMD_CLEAR_HW_STATS, + COMMON_CMD_GET_SELF_TEST_RES, + COMMON_CMD_GET_CHIP_FAULT_STATS, + COMMON_CMD_NIC_RSVD1, + COMMON_CMD_NIC_RSVD2, + COMMON_CMD_NIC_RSVD3, + COMMON_CMD_GET_CHIP_ID, + COMMON_CMD_GET_SINGLE_CARD_INFO, + COMMON_CMD_GET_FIRMWARE_ACTIVE_STATUS, + COMMON_CMD_ROCE_DFX_FUNC, + COMMON_CMD_GET_DEVICE_ID, + COMMON_CMD_GET_PF_DEV_INFO, + COMMON_CMD_CMD_FREE_MEM, + COMMON_CMD_GET_LOOPBACK_MODE = 32, + COMMON_CMD_SET_LOOPBACK_MODE, + COMMON_CMD_SET_LINK_MODE, + COMMON_CMD_SET_PF_BW_LIMIT, + COMMON_CMD_GET_PF_BW_LIMIT, + COMMON_CMD_ROCE_CMD, + COMMON_CMD_GET_POLL_WEIGHT, + COMMON_CMD_SET_POLL_WEIGHT, + COMMON_CMD_GET_HOMOLOGUE, + COMMON_CMD_SET_HOMOLOGUE, + COMMON_CMD_GET_SSET_COUNT, + COMMON_CMD_GET_SSET_ITEMS, + COMMON_CMD_IS_DRV_IN_VM, + COMMON_CMD_LRO_ADPT_MGMT, + COMMON_CMD_SET_INTER_COAL_PARAM, + COMMON_CMD_GET_INTER_COAL_PARAM, + COMMON_CMD_GET_CHIP_INFO, + COMMON_CMD_GET_NIC_STATS_LEN, + COMMON_CMD_GET_NIC_STATS_STRING, + COMMON_CMD_GET_NIC_STATS_INFO, + COMMON_CMD_GET_PF_ID, + COMMON_CMD_NIC_RSVD4, + COMMON_CMD_NIC_RSVD5, + COMMON_CMD_DCB_QOS_INFO, + COMMON_CMD_DCB_PFC_STATE, + COMMON_CMD_DCB_ETS_STATE, + COMMON_CMD_DCB_STATE, + COMMON_CMD_QOS_DEV, + COMMON_CMD_NIC_RSVD7, + COMMON_CMD_GET_ULD_DEV_NAME, + + COMMON_CMD_RSS_CFG = 0x40, + COMMON_CMD_RSS_INDIR, + COMMON_CMD_PORT_ID, + + COMMON_CMD_GET_FUNC_CAP = 0x50, + + COMMON_CMD_GET_WIN_STAT = 0x60, + COMMON_CMD_WIN_CSR_READ = 0x61, + COMMON_CMD_WIN_CSR_WRITE = 0x62, + COMMON_CMD_WIN_API_CMD_RD = 0x63, + + ROCE_CMD_GET_QPC_FROM_CACHE = 0x80, + ROCE_CMD_GET_QPC_FROM_HOST = 0x81, + ROCE_CMD_GET_CQC_FROM_CACHE = 0x82, + ROCE_CMD_GET_CQC_FROM_HOST = 0x83, + ROCE_CMD_GET_SRQC_FROM_CACHE = 0x84, + ROCE_CMD_GET_SRQC_FROM_HOST = 0x85, + ROCE_CMD_GET_MPT_FROM_CACHE = 0x86, + ROCE_CMD_GET_MPT_FROM_HOST = 0x87, + ROCE_CMD_GET_GID_FROM_CACHE = 0x88, + ROCE_CMD_GET_QPC_CQC_PI_CI = 0x89, + ROCE_CMD_GET_QP_COUNT = 0x8a, + ROCE_CMD_GET_DEV_ALGO = 0x8b, + + ROCE_CMD_START_CAP_PACKET = 0x90, + ROCE_CMD_STOP_CAP_PACKET = 0x91, + ROCE_CMD_QUERY_CAP_INFO = 0x92, + ROCE_CMD_ENABLE_QP_CAP_PACKET = 0x93, + ROCE_CMD_DISABLE_QP_CAP_PACKET = 0x94, + ROCE_CMD_QUERY_QP_CAP_INFO = 0x95, + + ROCE_CMD_ENABLE_BW_CTRL = 0xa0, + ROCE_CMD_DISABLE_BW_CTRL = 0xa1, + ROCE_CMD_CHANGE_BW_CTRL_PARAM = 0xa2, + ROCE_CMD_QUERY_BW_CTRL_PARAM = 0xa3, + + ROCE_CMD_TIMEOUT_ALARM = 0xb0, + COMMON_CMD_VM_COMPAT_TEST = 0xFF +}; + +enum roce3_cmd_ctx_type { + ROCE_CMD_CTX_MODIFY_QPC = 0X0, + ROCE_CMD_CTX_MODIFY_MPT = 0X1, + ROCE_CMD_CTX_MODIFY_CQC = 0X2, + ROCE_CMD_CTX_MODIFY_SRQC = 0X3, + ROCE_CMD_CTX_MODIFY_WQE_PI_CI = 0X4, + ROCE_CMD_CTX_FETCH_WQE = 0X5, + ROCE_CMD_CTX_DEL_TIMER = 0X6, + ROCE_CMD_CTX_LB_SQDB = 0X7 +}; + +enum roce3_modify_fetch_wqe_cmd_type { + ROCE_CMD_FCH_WQE_ACKCI = 1, + ROCE_CMD_FCH_WQE_UPDATE = 2 +}; + +enum roce3_gid_type { + ROCE_IPv4_ROCEv2_GID = 0, + ROCE_IPv6_ROCEv2_GID = 1, + ROCE_ROCEv1_GID = 2, + ROCE_INV_GID +}; + +#endif /* ROCE_PUB_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h new file mode 100644 index 000000000..bc1b9cb01 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_ulp.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_ULP_H +#define ROCE_ULP_H + +#include "pub_base_defs.h" + +/* *********ECN alpha SHIFT******* */ +#define ECN_ALPHA_MAGNIFICATION_SHIFT (21) + +/* ********************** NOFAA start ************************ */ +enum ROCE_NOFAA_QP_CTX_OFF_E { + ROCE_NOFAA_CTX_TIMER_OFF = 0, + ROCE_NOFAA_CTX_SW_OFF = 32, + ROCE_NOFAA_CTX_SW_SQC_OFF = 96, + ROCE_NOFAA_CTX_SW_SQAC_OFF = 116, + ROCE_NOFAA_CTX_SW_RQC_OFF = 148, + ROCE_NOFAA_CTX_SW_RQAC_OFF = 168 +}; + +#define ROCE_VERBS_NOFAA_TIMERC_OFFSET_16B_ALIGN (ROCE_NOFAA_CTX_TIMER_OFF >> 4) +#define ROCE_VERBS_NOFAA_SW_OFFSET_16B_ALIGN (ROCE_NOFAA_CTX_SW_OFF >> 4) + +/* API */ +#define ROCE_NOFAA_WQE2HOST_SM_NODE NODE_ID_SMF0 +#define ROCE_NOFAA_CMIDINFO_SM_MODE NODE_ID_SMF0 +#define SML_INST_MASTER_QPC 20 + +#define ROCE_BMP_LOAD_BYTE 64 +#define ROCE_BMP_ENTRY_BY_64B ((ROCE_QPC_TABLE_SIZE>>6) + 1) +#define ROCE_BMP_RSVD_BYTE 32 +#define ROCE_NOFAA_QPN_RSVD 0x2 +#define ROCE_REVERSE_PER_64B_SHIFT 4 + +#define ROCE_MQP_LT_RSVD 100 // (64 + 12kbit / 8) / 16 +#define ROCE_ACT_HOST_LT_RSVD 4 // (64) / 16 +#define ROCE_HASH_LT_RSVD 17920 // (0xA1000 - 0x5B000) / 16 +#define ROCE_SHARD_HASH_LT_RSVD 8960 // (0x8DC00 - 0x6AC00) / 16 +#define ROCE_HASH_LT_RSVD_MUL_SML (ROCE_HASH_LT_RSVD >> ROCE_NOFAA_PORT_NUM_SHIFT) +#define ROCE_LT_64B_IDX_SHIFT 2 +#define ROCE_LT_64B_ENTRY_NUM 4 + +#define ROCE_NOFAA_HASH_WORD_EN 0x400 +#define ROCE_NOFAA_HASH_API_KEY_LEN 4 +#define ROCE_NOFAA_BITHEAP_ID0 0 +#define ROCE_NOFAA_RSVD_BMPIDX_NUM 2 +#define ROCE_NOFAA_MASTER_BMPIDX_BASE 0x3002 +#define ROCE_NOFAA_CM_QPN_INDEX 0x3001 +#define ROCE_NOFAA_FLR_STUB_INDEX 0x2fff +#define ROCE_NOFAA_QPN2BMPIDX(xid) ((xid) + ROCE_NOFAA_MASTER_BMPIDX_BASE - ROCE_NOFAA_QPN_RSVD) +#define ROCE_NOFAA_BMPIDX2QPN(bmp_idx) \ + ((bmp_idx) + ROCE_NOFAA_QPN_RSVD - ROCE_NOFAA_MASTER_BMPIDX_BASE) +#define ROCE_NOFAA_GET_ACT_HOST_IDX(xid, host_id) (((xid) << 2) + (((host_id) & 0x3) ^ 0x3)) + +#define ROCE_NOFAA_GIDIDX_MASK 0x3f +#define ROCE_NOFAA_GIDIDX_RSVD_MASK 0xc0 + +/* cfg */ +#define NOFAA_SRC_TAG_L (global_share_space.global_config.dw6.bs.master_func) +#define ROCE_NOFAA_HOSTID_MASK (global_share_space.global_config.dw6.bs.host_mask) +#define ROCE_NOFAA_HOST_NUM (global_share_space.global_config.dw6.bs.host_num) +#define ROCE_NOFAA_PORT_NUM (global_share_space.global_config.dw6.bs.port_num) +#define ROCE_NOFAA_HOST_NUM_SHIFT (global_share_space.global_config.dw6.bs.host_shift) +#define ROCE_NOFAA_PORT_NUM_SHIFT (global_share_space.global_config.dw6.bs.port_shift) +#define ROCE_NOFAA_PORT_NUM_MASK (global_share_space.global_config.dw6.bs.port_mask) +#define ROCE_NOFAA_MAX_HOST_NUM 4 +#define ROCE_NOFAA_HASH_POOL_IO_NUM (ROCE_NOFAA_MAX_HASH_NUM >> ROCE_NOFAA_PORT_NUM_SHIFT) +#define ROCE_NOFAA_SRQN_BASE 2 +#define ROCE_NOFAA_SRQN_MASK 0x7 +#define ROCE_NOFAA_SRQN_NUM 8 + +#define ROCE_NOFAA_MAX_QPN_SHIFT 12 +#define ROCE_NOFAA_PORT_QP_NUM_SHIFT (ROCE_NOFAA_MAX_QPN_SHIFT - ROCE_NOFAA_PORT_NUM_SHIFT) +#define ROCE_NOFAA_PORT_QP_NUM (1u << ROCE_NOFAA_PORT_QP_NUM_SHIFT) +#define ROCE_NOFAA_PORT_QP_NUM_MASK (ROCE_NOFAA_PORT_QP_NUM - 1) +#define ROCE_AA_SWITCH_QP_NUM 4 +#define ROCE_AA_SWITCH_QPN (4095 + 1) +#define ROCE_AA_SWITCH_MAX_QPN (ROCE_AA_SWITCH_QPN + ROCE_AA_SWITCH_QP_NUM) +#define ROCE_QP0 0x0 +#define ROCE_CM_QPN 0x1 +#define ROCE_AA_QID_NUM 9 +#define ROCE_NVME_QID_GP_SHIFT 1 +#define ROCE_NVME_QID_GP_MASK 1 + +#define NEXT_IO_SPF_GET(start_spf, index) ((start_spf) + ((index) << 7)) +#define NEXT_SPF_GET(start_spf, index) ((start_spf) + ((index) << 6)) + +#define ROCE_NOFAA_VFID2HOSTIDX(vf_id) \ + (((vf_id) >> ROCE_NOFAA_PORT_NUM_SHIFT) & ROCE_NOFAA_HOSTID_MASK) +#define ROCE_NOFAA_VFID2PORTID(vf_id) ((vf_id) & ROCE_NOFAA_PORT_NUM_MASK) +#define ROCE_NOFAA_SLAVE_ALIVE_GET(active_bmp, slave_id) \ + ((active_bmp) & (1u << ((slave_id) & ROCE_NOFAA_HOSTID_MASK))) +#define ROCE_AA_SRCCHNL2HOST(src_chnl) (((src_chnl) >> 3) & 0x3) +#define ROCE_NOFAA_GET_GID_IDX(qpc, host_id) \ + (*((u8 *)&((qpc)->nofaa_ctx.dw1.value) + (host_id)) & 0x3f) +/* ROCE NOFAA(QPC512):CID[19:0] = {XID[12:4],XID[12:4],XID[3:2]} */ +#define roce_nofaa_calc_cid_by_xid(xid) ((((xid)&0x3ffe0) << 6) | (((xid)&0x7ff8) >> 3)) +#define ROCE_NOFAA_BITMAP_CLEAR(bmp, idx) ((bmp) &= ((1u << (idx)) ^ 0xffffffff)) +#define ROCE_NOFAA_BITMAP_SET(bmp, idx) ((bmp) |= (1u << (idx))) +#define ROCE_NOFAA_BITMAP_GET(bmp, idx) ((bmp) & (1u << (idx))) +#define ROCE_NOFAA_BITWISE_COMPARE(x, y) ((x) ^ (y)) +/* GET CMINFO TABLE */ +#define ROCE_GET_CMINFO_ADDR(sp_addr, index) \ + ((struct tag_roce_nof_cm_ctx *)(sp_addr) + \ + (((index) >> ROCE_LT_XID_LB_SHIFT) & ROCE_CMINFO_PER_16B_MASK)) + +#define ROCE_NOFAA_QUERY_BMP_SIZE ((ROCE_NOFAA_PORT_QP_NUM) >> 3) +#define ROCE_NOFAA_GET_PORT_QPNUM_BYTE(port_id) \ + ((u32)((port_id) << ROCE_NOFAA_PORT_QP_NUM_SHIFT) >> 3) + +/* switch */ +#define ROCE_SWITCH_COUNTER_BASE 0x4000 + +/* timer */ +#define ROCE_NOFAA_TIME_DLY_STEP 2047 +#define ROCEAA_TIMER_DLY_STEP(dly_step) (((dly_step) < ROCE_NOFAA_TIME_DLY_STEP) ? \ + (dly_step) : ROCE_NOFAA_TIME_DLY_STEP) + +/* data struct */ +struct tag_roce_nof_cm_ctx { + u32 local_comm_id; + u32 remote_comm_id; +}; + +struct tag_nofaa_shard_hash_key { + u32 xid : 16; + u32 lun_id : 16; +}; + +struct tag_nofaa_shard_hash_item { + struct { + u32 volume_id : 24; + u32 dd_id : 8; + } bs; + + u8 hash_low_cnt; + u8 hash_high_cnt; + u8 hash_low_offset; + u8 hash_high_offset; +}; + +struct tag_nofaa_shard_hash_entry { + struct tag_nofaa_shard_hash_key key; + struct tag_nofaa_shard_hash_item item; +}; + +struct tag_nofaa_shard_hash_value { + union { + struct { + u32 code : 2; + u32 rsvd0 : 30; + } bs; + u32 value; + } dw0; + u32 rsvd0; + struct tag_nofaa_shard_hash_item item; +}; + +struct tag_nofaa_shard_lt_value { + union { + struct { + u32 v : 1; + u32 s : 1; + u32 b : 1; + u32 rsvd0 : 5; + u32 hash_value : 16; + u32 rsvd1 : 8; + } bs; + u32 value; + } dw0; + struct tag_nofaa_shard_hash_key key; + struct tag_nofaa_shard_hash_item item; +}; + +/* ********************** NOFAA end ************************ */ + +#endif /* ROCE_ULP_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h new file mode 100644 index 000000000..f43accd09 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_vbs_format.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VBS_FORMAT_H +#define ROCE_VBS_FORMAT_H + +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" + +#ifdef ROCE_VBS_EN + +struct roce_wqe_vbs_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW2 */ + u32 dw2; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; +}; +#else + +union roce_wqe_vbs_tsk_misc_seg { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 request_id : 16; + u32 xts_vld : 1; + u32 rq_buf_shift : 4; + u32 sio_grp_num : 3; + u32 sio_grp4_len : 8; + + u32 sio_grp3_len : 8; + u32 sio_grp2_len : 8; + u32 sio_grp1_len : 8; + u32 sio_grp0_len : 8; +#else + u32 sio_grp0_len : 8; + u32 sio_grp1_len : 8; + u32 sio_grp2_len : 8; + u32 sio_grp3_len : 8; + + u32 sio_grp4_len : 8; + u32 sio_grp_num : 3; + u32 rq_buf_shift : 4; + u32 xts_vld : 1; + u32 request_id : 16; +#endif + } bs; + u64 value; +}; + +struct roce_wqe_vbs_aad_seg { + union { + struct { + u32 pdu_difx_en : 1; + u32 pad_len_vld : 1; + u32 pdu_pad_len : 2; + u32 smd_tp : 4; + u32 rsvd0 : 1; + u32 msg_hdr_vld : 1; + u32 msg_hdr_len : 2; + u32 rd_difx_rslt : 1; + u32 vbs_difx : 1; + u32 rsvd1 : 1; + u32 pdu_difx_cnt : 9; + u32 rsvd2 : 4; + u32 sio_num : 4; + } bs; + u32 value; + } dw0; + + union { + struct { + u32 fst_pdu : 1; + u32 fst_sct : 1; + u32 last_pdu : 1; + u32 rsp_indiv : 1; + u32 sct_sz : 1; + u32 dsgl : 1; + u32 rsp_vld : 1; + u32 rsp_len : 5; + u32 app_esc : 1; + u32 ref_esc : 1; + u32 sct_v_tp : 2; + u32 grd_v_en : 1; + u32 grd_rid : 2; + u32 grd_v_agm : 1; + u32 grd_ri_agm : 1; + u32 grd_agm_ini : 1; + u32 crc16_ini : 1; + u32 ipcs_ini : 1; + u32 ref_v_en : 1; + u32 ref_rid : 2; + u32 ref_v_inc : 1; + u32 ref_ri_inc : 1; + u32 app_v_en : 1; + u32 app_rid : 2; + } bs; + u32 value; + } dw1; + + union { + struct { + u32 task_tag : 16; + u32 rep_app_tag : 16; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 cmp_app_tag : 16; + u32 cmp_app_tag_msk : 16; + } bs; + u32 value; + } dw3; + + u32 cmp_ref_tag; + + u32 rep_ref_tag; + + union { + struct { + u32 pdu_sd_ofs : 13; + u32 rsvd0 : 1; + u32 pdu_sd_len : 18; + } bs; + u32 value; + } dw6; +}; + +struct roce_wqe_vbs_subinfo { + union { + struct { + u32 sio0_size : 4; + u32 sio1_size : 4; + u32 sio2_size : 4; + u32 sio3_size : 4; + u32 sio4_size : 4; + u32 sio5_size : 4; + u32 sio6_size : 4; + u32 sio7_size : 4; + } bs; + u32 value; + } dw0; + + union { + struct { + u32 sio0_crc2_ini : 16; + u32 sio1_crc2_ini : 16; + } bs; + u32 value; + } dw1; + + union { + struct { + u32 sio2_crc2_ini : 16; + u32 sio3_crc2_ini : 16; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 sio4_crc2_ini : 16; + u32 sio5_crc2_ini : 16; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 sio6_crc2_ini : 16; + u32 sio7_crc2_ini : 16; + } bs; + u32 value; + } dw4; +}; + +struct roce_wqe_vbs_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW2 */ + /* This parameter is valid for the immediate data operation or SEND invalidate. */ + u32 wqe_ext_len; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; + + /* DW4~5 */ + union roce_wqe_vbs_tsk_misc_seg misc; + + u32 rsvd[2]; + + struct roce_wqe_vbs_aad_seg vbs_aad; +}; + +#endif + +#endif diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h new file mode 100644 index 000000000..05b3363c0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr.h @@ -0,0 +1,413 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_ATTR_H +#define ROCE_VERBS_ATTR_H + +#include "roce_verbs_mr_attr.h" +#include "roce_verbs_gid_attr.h" +#include "roce_verbs_cq_attr.h" +#include "roce_verbs_srq_attr.h" +#include "roce_verbs_attr_qpc_chip.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#define ROCE_VERBS_SQ_WQEBB_SIZE (2) +#define ROCE_VERBS_SQ_PI_VLD (1) + +#pragma pack(4) +/* qpc_attr_com info ,12*4B */ +struct tag_roce_verbs_qpc_attr_com { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 service_type : 3; + u32 fre : 1; + u32 rwe : 1; + u32 rre : 1; + u32 rae : 1; + u32 rkey_en : 1; + u32 dest_qp : 24; +#else + /* + * Destination QP number, which is extended to 24 bits in + * consideration of interconnection with commercial devices. + */ + u32 dest_qp : 24; + u32 rkey_en : 1; + u32 rae : 1; + u32 rre : 1; + u32 rwe : 1; + /* Indicates whether the local FRPMR is enabled. */ + u32 fre : 1; + /* Transmission Type + * 000:RC + * 001:UC + * 010:RD + * 011 UD + * 101:XRC + * Other:Reserved + */ + u32 service_type : 3; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sra_max : 3; + u32 rra_max : 3; + u32 rnr_retry_limit : 3; + u32 to_retry_limit : 3; + u32 local_qp : 20; +#else + u32 local_qp : 20; /* Local QP number */ + /* + * Number of ACK retransmissions. The value 7 indicates unlimited + * times, and the value 0 indicates no retransmission. + */ + u32 to_retry_limit : 3; + /* + * The maximum number of RNR retransmissions is 7. The value 7 + * indicates that the maximum number of retransmissions is 7, and + * the value 0 indicates that the retransmission is not performed. + */ + u32 rnr_retry_limit : 3; + /* The maximum value of responser resource is 128. */ + u32 rra_max : 3; + /* The maximum value of initiator depth is 128. */ + u32 sra_max : 3; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ack_to : 5; + u32 min_rnr_nak : 5; + u32 cont_size : 2; + u32 cont_en : 1; + u32 srq_en : 1; + u32 xrc_srq_en : 1; + u32 vroce_en : 1; + u32 host_oqid : 16; +#else + u32 host_oqid : 16; + u32 vroce_en : 1; + u32 xrc_srq_en : 1; + u32 srq_en : 1; + u32 cont_en : 1; + u32 cont_size : 2; + /* + * NAK code of RNR. This parameter is mandatory when INIT2RNR and + * RTR2RTS\SQE2RTS\SQD2SQD\SQD2RTS is optional. + */ + u32 min_rnr_nak : 5; + u32 ack_to : 5; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 tss_timer_num : 3; + u32 xrc_vld : 1; + u32 srq_container : 1; + u32 invalid_credit : 1; + u32 ext_md : 1; + u32 ext_mtu : 1; + u32 dsgl_en : 1; + u32 dif_en : 1; + u32 pmtu : 3; + u32 base_mtu_n : 1; + u32 pd : 18; +#else + u32 pd : 18; + u32 base_mtu_n : 1; + u32 pmtu : 3; + u32 dif_en : 1; + u32 dsgl_en : 1; + u32 ext_mtu : 1; + u32 ext_md : 1; + u32 invalid_credit : 1; + u32 srq_container : 1; + u32 xrc_vld : 1; + u32 tss_timer_num : 3; +#endif + } bs; + + u32 value; + } dw3; + + /* DW4 */ + u32 q_key; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ulp_type : 8; + u32 oor_en : 1; + u32 capture_en : 1; + u32 rsvd : 1; + u32 acx_mark : 1; + u32 mtu_code : 4; + u32 port : 2; + u32 ep : 3; + u32 cos : 3; + u32 so_ro : 2; + u32 dma_attr_idx : 6; +#else + u32 dma_attr_idx : 6; + u32 so_ro : 2; + u32 cos : 3; + u32 ep : 3; + u32 port : 2; + u32 mtu_code : 4; + u32 acx_mark : 1; + u32 rsvd : 1; + u32 capture_en : 1; + u32 oor_en : 1; + u32 ulp_type : 8; +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sd_mpt_idx : 12; + u32 rsvd : 20; +#else + u32 rsvd : 20; + u32 sd_mpt_idx : 12; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 10; + u32 sq_cqn_lb : 1; + u32 rq_cqn_lb : 1; + u32 rq_cqn : 20; +#else + u32 rq_cqn : 20; + u32 rq_cqn_lb : 1; + u32 sq_cqn_lb : 1; + u32 rsvd : 10; +#endif + } bs; + u32 value; + } dw7; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 next_send_psn : 24; +#else + u32 next_send_psn : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw8; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 next_rcv_psn : 24; +#else + u32 next_rcv_psn : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw9; + + /* DW10 */ + u32 lsn; + + /* DW11 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 17; + u32 set_mpt_indx : 1; + u32 fake : 1; + u32 vf_id : 13; +#else + u32 vf_id : 13; + u32 fake : 1; + u32 set_mpt_indx : 1; + u32 rsvd : 17; +#endif + } bs; + u32 value; + } dw11; + + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ccf_app_id : 8; + u32 rsvd : 24; +#else + u32 rsvd : 24; + u32 ccf_app_id : 8; +#endif + } bs; + u32 value; + } dw12; +}; + +struct tag_roce_verbs_qpc_attr_path { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 bond_tx_hash_value : 16; + u32 dmac_h16 : 16; +#else + u32 dmac_h16 : 16; + u32 bond_tx_hash_value : 16; +#endif + } bs; + u32 value; + } dw0; + + u32 dmac_l32; + + /* DW2~5 */ + u8 dgid[16]; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 tclass : 8; + u32 flow_label : 20; +#else + u32 flow_label : 20; /* GRH flow lable */ + u32 tclass : 8; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sl : 3; + u32 loop : 1; + u32 udp_src_port : 8; + u32 rsvd : 4; + u32 base_sgid_n : 1; + u32 sgid_index : 7; + u32 hoplmt : 8; +#else + u32 hoplmt : 8; + u32 sgid_index : 7; + u32 base_sgid_n : 1; + u32 rsvd : 4; + u32 udp_src_port : 8; + u32 loop : 1; + u32 sl : 3; +#endif + } bs; + u32 value; + } dw7; +}; + +struct roce_verbs_qpc_attr_nof_aa { + u32 gid_index; + u32 qid; + u32 local_comm_id; + u32 remote_comm_id; +}; + +struct roce_verbs_qpc_attr_vbs { + u32 sqpc_ci_record_addr_h; + u32 sqpc_ci_record_addr_l; +}; + +struct roce_verbs_qpc_attr_ext { + struct roce_verbs_qpc_attr_nof_aa nof_aa_info; + struct roce_verbs_qpc_attr_vbs vbs_info; +}; + +/* QPC Struct */ +struct tag_roce_verbs_qp_attr { + /* com seg, DW0 ~ DW11 */ + struct tag_roce_verbs_qpc_attr_com com_info; + + /* path seg, DW0 ~ DW7 */ + struct tag_roce_verbs_qpc_attr_path path_info; + + /* chip seg, DW0 ~ DW19 */ + struct tag_roce_verbs_qpc_attr_chip chip_seg; + + /* ext seg */ + struct roce_verbs_qpc_attr_ext ext_seg; +}; + + +struct tag_roce_verbs_qp_hw2sw_info { + /* DW0~1 */ + u32 sq_buf_len; /* Buffer length of the SQ queue */ + u32 rq_buf_len; /* Buffer length of the RQ queue */ + + /* DW2~6 */ + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; + + /* DW7~8 */ + union { + u64 wb_gpa; /* Address written back by the ucode after processing */ + + struct { + u32 syn_gpa_hi32; /* Upper 32 bits of the start address of mr or mw */ + u32 syn_gpa_lo32; /* Lower 32 bits of the start address of mr or mw */ + } gpa_dw; + }; + union { + struct { + u32 rsvd : 16; + u32 host_oqid : 16; + } bs; + + u32 value; + } dw9; + struct tag_roce_verbs_wqe_cacheout_info wqe_cache; +}; + +struct tag_roce_verbs_modify_ctx_info { + u32 ctx_type; + u32 offset; + u32 value; + u32 mask; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h new file mode 100644 index 000000000..8eb2dcf6f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_attr_qpc_chip.h @@ -0,0 +1,360 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_ATTR_QPC_CHIP_H +#define ROCE_VERBS_ATTR_QPC_CHIP_H + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_qpc_attr_chip { + /* DW0~1 */ + union { + u64 sq_rq_l0mtt_gpa; /* hi[63:32],lo[31:03],sq_rq_gpa_sign[02:00] */ + struct { + u32 sq_rq_l0mtt_gpa_hi; + u32 sq_rq_l0mtt_gpa_lo; + } bs; + } dw0; + + /* DW2~3 */ + union { + /* hi[63:32],lo[31:02],sq_rq_at_hop_num[01:00] */ + u64 sq_rq_pi_record_gpa_at_hop_num; + struct { + u32 sq_rq_pi_record_gpa_hi; + u32 sq_rq_pi_record_gpa_lo_at_hop_num; /* at_hop_num: bit[01:00] */ + } bs; + } dw2; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 qp_page_size : 4; + u32 sq_rq_mtt_page_size : 4; + u32 rsvd1 : 3; + u32 qp_signature : 5; + u32 rsvd0 : 1; + u32 dsgl : 1; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 rc_size : 4; + u32 rc_max_size : 3; + u32 rq_base_ci : 5; +#else + u32 rq_base_ci : 5; + u32 rc_max_size : 3; + u32 rc_size : 4; + u32 rrw_mtt_prefetch_maxlen : 2; + u32 dsgl : 1; + u32 rsvd0 : 1; + u32 qp_signature : 5; + u32 rsvd1 : 3; + u32 sq_rq_mtt_page_size : 4; + u32 qp_page_size : 4; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_prefetch_maxnum : 3; + u32 sq_wqe_prefetch_minnum : 3; + u32 sq_wqe_cache_thd_sel : 2; + u32 sq_wqecnt_lth : 4; + u32 sq_wqecnt_rctl_en : 1; + u32 sq_wqecnt_rctl : 1; + u32 sq_prefetch_one_wqe : 1; + u32 sq_prewqe_mode : 1; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 sqa_wqe_prefetch_minnum : 3; + u32 sqa_wqe_cache_thd_sel : 2; + u32 sq_wqe_check_en : 1; + u32 sq_pi_on_chip : 1; + u32 sq_inline_en : 1; + u32 sq_size : 5; +#else + u32 sq_size : 5; + u32 sq_inline_en : 1; + u32 sq_pi_on_chip : 1; + u32 sq_wqe_check_en : 1; + u32 sqa_wqe_cache_thd_sel : 2; + u32 sqa_wqe_prefetch_minnum : 3; + u32 sqa_wqe_prefetch_maxnum : 3; + u32 sq_prewqe_mode : 1; + u32 sq_prefetch_one_wqe : 1; + u32 sq_wqecnt_rctl : 1; + u32 sq_wqecnt_rctl_en : 1; + u32 sq_wqecnt_lth : 4; + u32 sq_wqe_cache_thd_sel : 2; + u32 sq_wqe_prefetch_minnum : 3; + u32 sq_wqe_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw5; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sq_wqe_prefetch_mode : 1; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 srq_pd : 18; + u32 rsvd : 7; +#else + u32 rsvd : 7; + u32 srq_pd : 18; + u32 sqa_mtt_prefetch_maxlen : 3; + u32 sq_mtt_prefetch_maxlen : 3; + u32 sq_wqe_prefetch_mode : 1; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rq_wqe_prefetch_maxnum : 3; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_wqe_cache_thd_sel : 2; + u32 rq_wqecnt_lth : 4; + u32 rq_wqecnt_rctl_en : 1; + u32 rq_wqecnt_rctl : 1; + u32 srqn : 18; +#else + u32 srqn : 18; + u32 rq_wqecnt_rctl : 1; + u32 rq_wqecnt_rctl_en : 1; + u32 rq_wqecnt_lth : 4; + u32 rq_wqe_cache_thd_sel : 2; + u32 rq_wqe_prefetch_minnum : 3; + u32 rq_wqe_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 srq_wqe_rthd_sel : 2; + u32 srq_rqecnt_th : 4; + u32 rq_pi_on_chip : 1; + u32 rq_inline_en : 1; + u32 rq_wqebb_size : 3; + u32 rq_size : 5; + u32 xrcd : 16; +#else + u32 xrcd : 16; + u32 rq_size : 5; + u32 rq_wqebb_size : 3; + u32 rq_inline_en : 1; + u32 rq_pi_on_chip : 1; + u32 srq_rqecnt_th : 4; + u32 srq_wqe_rthd_sel : 2; +#endif + } bs; + u32 value; + } dw8; + + /* DW9 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 container_en : 1; + u32 container_sz : 2; + u32 srq_warth_flag : 1; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rsvd : 19; +#else + u32 rsvd : 19; + u32 rq_mtt_prefetch_maxlen1 : 2; + u32 rq_mtt_prefetch_maxlen0 : 2; + u32 rq_mtt_prefetch_maxwqe : 3; + u32 srq_mtt_prefetch_maxlen1 : 2; + u32 srq_warth_flag : 1; + u32 container_sz : 2; + u32 container_en : 1; +#endif + } bs; + u32 value; + } dw9; + + /* DW10 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rc_entry_prefetch_maxnum : 3; + u32 rc_mtt_prefetch_maxlen : 2; + u32 rsvd : 1; + u32 rc_entry_size : 2; + u32 rc_page_gpa_h : 24; +#else + /* + * bit[63:40] Indicates the start GPA of RDMARC table. + * The driver needs to allocate continuous physical address for + * the RDMARC table.Configured by Driver + */ + u32 rc_page_gpa_h : 24; + u32 rc_entry_size : 2; + u32 rsvd : 1; + u32 rc_mtt_prefetch_maxlen : 2; + /* + * Maximum number of prefetch Entries for RDMARC table.000: prefetch + * number equals to zero; Others: prefetch number equals to + * (2^(rc_entry_prefetch_maxnum-1)). Configured by Driver + */ + u32 rc_entry_prefetch_maxnum : 3; +#endif + } bs; + u32 value; + } dw10; + + /* DW11 */ + u32 rc_page_gpa_l; /* bit[39:8] */ + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 1; + u32 srq_pd : 18; + u32 srq_wqebb_size : 3; + u32 srq_page_size : 4; + u32 srq_size : 5; + u32 srq_rkey_en : 1; +#else + u32 srq_rkey_en : 1; + u32 srq_size : 5; + u32 srq_page_size : 4; + u32 srq_wqebb_size : 3; + u32 srq_pd : 18; + u32 rsvd : 1; +#endif + } bs; + u32 value; + } dw12; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 srq_cqn : 20; + u32 srq_state : 4; +#else + u32 srq_state : 4; + u32 srq_cqn : 20; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw13; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 15; + u32 qp_rkey_en : 1; + u32 srq_xrcd : 16; +#else + u32 srq_xrcd : 16; + u32 qp_rkey_en : 1; + u32 rsvd : 15; +#endif + } bs; + u32 value; + } dw14; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 4; + u32 rq_page_size : 4; + u32 rq_pd : 18; + u32 rq_rkey_en : 1; + u32 rq_size : 5; +#else + u32 rq_size : 5; + u32 rq_rkey_en : 1; + u32 rq_pd : 18; + u32 rq_page_size : 4; + u32 rsvd : 4; +#endif + } bs; + u32 value; + } dw15; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 6; + u32 sq_wqebb_size : 3; + u32 sq_pd : 18; + u32 sq_page_size : 4; + u32 sq_rkey_en : 1; +#else + u32 sq_rkey_en : 1; + u32 sq_page_size : 4; + u32 sq_pd : 18; + u32 sq_wqebb_size : 3; + u32 rsvd : 6; +#endif + } bs; + u32 value; + } dw16; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 12; + u32 sqa_cqn : 20; +#else + u32 sqa_cqn : 20; + u32 rsvd : 12; +#endif + } bs; + u32 value; + } dw17; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 ud_pd : 18; +#else + u32 ud_pd : 18; + u32 rsvd : 14; +#endif + } bs; + u32 value; + } dw18; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 qp_pd : 18; +#else + u32 qp_pd : 18; + u32 rsvd : 14; +#endif + } bs; + u32 value; + } dw19; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_ATTR_QPC_CHIP_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h new file mode 100644 index 000000000..422995c86 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cmd.h @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_CMD_H +#define ROCE_VERBS_CMD_H + +#include "rdma_context_format.h" +#include "roce_verbs_pub.h" + + +/* ************************************************* */ +struct tag_roce_verbs_cmd_com { + union { + u32 value; + + struct { + u32 version : 8; + u32 rsvd : 8; + u32 cmd_bitmask : 16; // CMD_TYPE_BITMASK_E + } bs; + } dw0; + + u32 index; // qpn/cqn/srqn/mpt_index/gid idx +}; + +struct tag_roce_cmd_gid { + struct tag_roce_verbs_cmd_com com; + + u32 port; + u32 rsvd; + struct roce_gid_context gid_entry; +}; + +struct tag_roce_clear_gid { + struct tag_roce_verbs_cmd_com com; + + u32 port; + u32 gid_num; +}; + +struct tag_roce_qurey_gid { + struct tag_roce_verbs_cmd_com com; + + u32 port; + u32 rsvd; +}; + +struct tag_roce_flush_mpt { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_flush_mpt { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_mpt_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_sw2hw_mpt { + struct tag_roce_verbs_cmd_com com; + /* When creating a MR/MW, you need to enter the content of the MPT Context. */ + struct roce_mpt_context mpt_entry; +}; + +struct tag_roce_cmd_modify_mpt { + struct tag_roce_verbs_cmd_com com; + + u32 new_key; + + /* DW2~3 */ + union { + u64 length; /* Length of mr or mw */ + + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw2; + }; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; +}; + +struct tag_roce_cmd_mpt_hw2sw { + struct tag_roce_verbs_cmd_com com; + + u32 dmtt_flags; + u32 dmtt_num; + u32 dmtt_cache_line_start; + u32 dmtt_cache_line_end; + u32 dmtt_cache_line_size; +}; + +struct tag_roce_cmd_query_mtt { + struct tag_roce_verbs_cmd_com com; + + u32 mtt_addr_start_hi32; + u32 mtt_addr_start_lo32; + u32 mtt_num; + u32 rsvd; +}; + +struct tag_roce_cmd_creat_cq { + struct tag_roce_verbs_cmd_com com; + struct roce_cq_context cqc; +}; + +struct tag_roce_cmd_resize_cq { + struct tag_roce_verbs_cmd_com com; + + u32 rsvd; + u32 page_size; /* Size of the resize buf page. */ + u32 log_cq_size; /* Cq depth after resize */ + u32 mtt_layer_num; /* Number of mtt levels after resize */ + /* DW4~5 */ + union { + u64 mtt_base_addr; /* Start address of mr or mw */ + u32 cqc_l0mtt_gpa[2]; + }; + u32 mtt_page_size; /* Size of the mtt page after resize. */ + struct tag_roce_cq_mtt_info mtt_info; +}; + +struct tag_roce_cmd_modify_cq { + struct tag_roce_verbs_cmd_com com; + u32 max_cnt; + u32 timeout; +}; + +struct tag_roce_cmd_cq_hw2sw { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_cq_cache_invalidate { + struct tag_roce_verbs_cmd_com com; + struct tag_roce_cq_mtt_info mtt_info; +}; + +struct tag_roce_cmd_roce_cq_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_creat_srq { + struct tag_roce_verbs_cmd_com com; + struct roce_srq_context srqc; +}; + +struct tag_roce_cmd_srq_arm { + struct tag_roce_verbs_cmd_com com; + union { + u32 limitwater; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lwm : 16; + u32 warth : 4; + u32 th_up_en : 1; + u32 cont_en : 1; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 cont_en : 1; + u32 th_up_en : 1; + u32 warth : 4; + u32 lwm : 16; +#endif + } bs; + }; +}; + +struct tag_roce_cmd_srq_hw2sw { + struct tag_roce_verbs_cmd_com com; + struct tag_roce_cq_mtt_info mtt_info; + u32 srq_buf_len; + u32 wqe_cache_line_start; + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; +}; + +struct tag_roce_cmd_srq_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_modify_qpc { + struct tag_roce_verbs_cmd_com com; + + u32 opt; + u32 rsvd[3]; + struct roce_qp_context qpc; +}; + +struct tag_roce_cmd_qp_modify2rst { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_qp_modify_rts2sqd { + struct tag_roce_verbs_cmd_com com; + + u32 sqd_event_en; + u32 rsvd; +}; + +struct tag_roce_cmd_qp_query { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_cmd_modify_ctx { + struct tag_roce_verbs_cmd_com com; + u32 ctx_type; + u32 offset; + u32 value; + u32 mask; +}; + +struct tag_roce_cmd_cap_pkt { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_modify_hash_value { + struct tag_roce_verbs_cmd_com com; + u32 hash_value; +}; + +struct tag_roce_modify_udp_src_port { + struct tag_roce_verbs_cmd_com com; + u32 udp_src_port; +}; + +struct roce_get_qp_udp_src_port { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_get_qp_rx_port { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_get_qp_func_table { + struct tag_roce_verbs_cmd_com com; +}; + +#endif /* ROCE_VERBS_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h new file mode 100644 index 000000000..19c275024 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_cq_attr.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_CQ_ATTR_H +#define ROCE_VERBS_CQ_ATTR_H + +#include "roce_verbs_mr_attr.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct roce_verbs_cq_attr { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 signature : 5; + u32 ci_on_chip : 1; + u32 cnt_clear_en : 1; + u32 cnt_adjust_en : 1; + u32 rsvd : 3; + u32 timer_mode : 1; + u32 arm_timer_en : 1; + u32 tss_timer_num : 3; + u32 mtt_page_size : 4; + u32 cqe_size : 3; + u32 page_size : 4; + u32 size : 5; +#else + /* + * Completion Queue size, equals to (2^cq_size)*CQE. + * The maximum CQ size is 2^23 CQEs. + */ + u32 size : 5; + u32 page_size : 4; /* Page size of CQ, equals to (2^cq_page_size)*4KB. */ + /* + * Completion Queue Entry (CQE) size in bytes is (2^cq_cqe_size)*16B. + * The minimum size is 32B and the values 0, 3, 4, 5, 6, 7 are reserved. + */ + u32 cqe_size : 3; + u32 mtt_page_size : 4; + u32 tss_timer_num : 3; + u32 arm_timer_en : 1; + u32 timer_mode : 1; + u32 rsvd : 3; + u32 cnt_adjust_en : 1; + u32 cnt_clear_en : 1; + /* + * If set, the CI of Complete Queue is stored in the chip, + * the counter is absolute value. + */ + u32 ci_on_chip : 1; + u32 signature : 5; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 rsvd : 14; + u32 ep : 3; + u32 cos : 3; + u32 so_ro : 2; + u32 dma_attr_idx : 6; +#else + /* + * It specifies the outbound PCIe TLP header attribute of the + * DMA operation.This filed is only valid when processing CQ's CQEs. + */ + u32 dma_attr_idx : 6; + /* + * It specifies the ATTR[1:0] bits in the outbound PCIe + * TLP headers of the DMA operation. + * This field is only valid when processing CQ's CQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + u32 cos : 3; + u32 ep : 3; + u32 rsvd : 14; + u32 state : 4; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ceqn : 8; + u32 rsvd : 1; + u32 arm_ceqe_en : 1; + u32 ceqe_en : 1; + u32 cqecnt_rctl_en : 1; + u32 cqecnt_lth : 4; + u32 idle_max_count : 16; +#else + u32 idle_max_count : 16; + u32 cqecnt_lth : 4; + u32 cqecnt_rctl_en : 1; + u32 ceqe_en : 1; + u32 arm_ceqe_en : 1; + u32 rsvd : 1; + u32 ceqn : 8; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 max_cnt : 16; + u32 timeout : 16; +#else + /* + * Completion Event Moderation timer in microseconds. + * 0x0: interrupt moderation disabled. + */ + u32 timeout : 16; + /* + * Completion Event Moderation counters. + * 0x0: interrupt moderation disabled. + */ + u32 max_cnt : 16; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 - DW5 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the CQ's buffer directly. + * low 3bits(cq_gpa_sign) + */ + u64 cqc_l0mtt_gpa; + struct { + u32 cqc_l0mtt_gpa_hi; + u32 cqc_l0mtt_gpa_lo; + } dw4_dw5; + }; + + /* DW6 - DW7 */ + union { + /* The GPA of stored CI of Complete Queue. + * Address translation hop numbers. + * 0x0: the 'cq_l0mtt_gpa' points to the buffer of CQ directly. + * 0x1: it need to perform one hop address translation to get the buffer's + * address of CQ; 0x2: there is two hop address translation to get the buffer's + * address of CQ; 0x3: reserved. + */ + u64 ci_record_gpa_at_hop_num; + struct { + u32 ci_record_gpa_hi; + /* bit[1:0] Address translation hop numbers */ + u32 ci_record_gpa_lo_at_hop_num; + } dw6_dw7; + }; +}; + +struct tag_roce_verbs_cq_resize_info { + /* DW0~3 */ + u32 mtt_page_size; /* Size of the mtt page after resize. */ + u32 page_size; /* Size of the resize buf page. */ + u32 log_cq_size; /* Cq depth after resize */ + u32 mtt_layer_num; /* Number of mtt levels after resize */ + + /* DW4~5 */ + union { + u64 mtt_base_addr; /* Start address of mr or mw */ + u32 cqc_l0mtt_gpa[2]; + }; + + /* DW6~10 */ + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; +}; + +struct tag_roce_verbs_modify_cq_info { + u32 max_cnt; + u32 timeout; +}; +#pragma pack() + +#endif /* ROCE_VERBS_CQ_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h new file mode 100644 index 000000000..471824a0f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ext_attr.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_EXT_ATTR_H +#define ROCE_VERBS_EXT_ATTR_H + +#include "roce_verbs_pub.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +enum VERBS_ATTR_EXT_TYPE_E { + VERBS_ATTR_EXT_TYPE_NONE = 0, + VERBS_ATTR_EXT_TYPE_SQP, + VERBS_ATTR_EXT_TYPE_RSVD = 0xff +}; + +#define ROCE_VERBS_SQP_WQE_SIZE (2) + +#pragma pack(4) + +struct tag_roce_verbs_sqp_attr { + /* DW0 */ + union tag_roce_verbs_seg_hdr seg_hdr; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 26; + u32 sqp_wqecnt_lth : 4; + u32 sqp_wqecnt_rctl_en : 1; + u32 sqp_ci_on_chip : 1; +#else + u32 sqp_ci_on_chip : 1; + u32 sqp_wqecnt_rctl_en : 1; + u32 sqp_wqecnt_lth : 4; + u32 rsvd0 : 26; +#endif + } bs; + u32 value; + } dw1; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_EXT_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h new file mode 100644 index 000000000..75b14cc25 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_format.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_FORMAT_H +#define ROCE_VERBS_FORMAT_H + +#include "roce_verbs_pub.h" +#include "roce_verbs_attr.h" + + +/* ********************************************************************************** */ +/* * verbs struct */ +struct tag_roce_uni_cmd_gid { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_gid_attr gid_attr; +}; + +struct tag_roce_uni_cmd_clear_gid { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_clear_gid_info gid_clear; +}; + +struct tag_roce_uni_cmd_qurey_gid { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_flush_mpt { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_mpt_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_sw2hw_mpt { + struct tag_roce_verbs_cmd_header com; + /* When creating a MR/MW, you need to enter the content of the MPT Context. */ + struct tag_roce_verbs_mr_attr mr_attr; +}; + +struct tag_roce_uni_cmd_modify_mpt { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_mr_sge mr_sge; +}; + +struct tag_roce_uni_cmd_mpt_hw2sw { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_mtt_cacheout_info dmtt_cache; +}; + +struct tag_roce_uni_cmd_query_mtt { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_query_mtt_info mtt_query; +}; + +struct tag_roce_uni_cmd_creat_cq { + struct tag_roce_verbs_cmd_header com; + struct roce_verbs_cq_attr cq_attr; +}; + +struct tag_roce_uni_cmd_resize_cq { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_cq_resize_info cq_resize; +}; + +struct tag_roce_uni_cmd_modify_cq { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_modify_cq_info cq_modify; +}; + +struct tag_roce_uni_cmd_cq_hw2sw { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; +}; + +struct tag_roce_uni_cmd_roce_cq_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_creat_srq { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_srq_attr srq_attr; +}; + +struct tag_roce_uni_cmd_srq_arm { + struct tag_roce_verbs_cmd_header com; + union tag_roce_verbs_arm_srq_info srq_arm; +}; + +struct tag_roce_uni_cmd_srq_hw2sw { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_srq_hw2sw_info srq_cache; +}; + +struct tag_roce_uni_cmd_srq_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_modify_qpc { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_qp_attr qp_attr; +}; + +struct tag_roce_uni_cmd_qp_modify2rst { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_qp_modify_rts2sqd { + struct tag_roce_verbs_cmd_header com; + u32 sqd_event_en; +}; + +struct tag_roce_uni_cmd_qp_query { + struct tag_roce_verbs_cmd_header com; +}; + +struct tag_roce_uni_cmd_qp_cache_invalid { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_qp_hw2sw_info qp_cache; +}; + +struct tag_roce_uni_cmd_modify_ctx { + struct tag_roce_verbs_cmd_header com; + struct tag_roce_verbs_modify_ctx_info ctx_modify; +}; + +struct tag_roce_uni_cmd_cap_pkt { + struct tag_roce_verbs_cmd_header com; +}; + + +#endif /* ROCE_VERBS_FORMAT_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h new file mode 100644 index 000000000..b4d58bd8d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_gid_attr.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_GID_ATTR_H +#define ROCE_VERBS_GID_ATTR_H + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_gid_ipv4_attr { + /* DW0 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 udpudp_len_gap : 8; + u32 rsvd : 8; + u32 bth_addr : 16; +#else + u32 bth_addr : 16; + u32 rsvd : 8; + u32 udpudp_len_gap : 8; +#endif + u32 value; + } dw0; + + /* DW1 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ip_addr : 16; + u32 udp_addr : 16; +#else + u32 udp_addr : 16; + u32 ip_addr : 16; +#endif + u32 value; + } dw1; +}; + +struct tag_roce_verbs_gid_attr { + /* DW0~3 */ + u32 gid[4]; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd0 : 12; + u32 cvlan : 12; + u32 rsvd : 8; +#else + u32 rsvd : 8; + u32 cvlan : 12; + u32 rsvd0 : 12; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ppersp_pad_len : 8; + u32 l2_len : 8; + u32 ipv4_hdr_len : 8; + u32 pkthdr_len : 8; +#else + u32 pkthdr_len : 8; + u32 ipv4_hdr_len : 8; + u32 l2_len : 8; + u32 ppersp_pad_len : 8; +#endif + u32 value; + } dw5; + + /* DW6 */ + union { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 7; + u32 o_ip_type : 1; + u32 o_tag : 2; + u32 gid_update : 1; + u32 rsvd0 : 1; + u32 gid_type : 2; + u32 tag : 2; + u32 smac_hi16 : 16; +#else + u32 smac_hi16 : 16; + u32 tag : 2; + /* 0:ROCE V1; 1:ROCE V2 IPV4; 2:ROCE V2 IPV6; other:rsvd */ + u32 gid_type : 2; + u32 rsvd0 : 1; + u32 gid_update : 1; + u32 o_tag : 2; + u32 o_ip_type : 1; + u32 rsvd : 7; +#endif + u32 value; + } dw6; + + u32 smac_lo32; + + struct tag_roce_verbs_gid_ipv4_attr ipv4; +}; + +struct tag_roce_verbs_clear_gid_info { + u32 gid_num; +}; +#pragma pack() + +#endif /* ROCE_VERBS_GID_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h new file mode 100644 index 000000000..d684dd778 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_mr_attr.h @@ -0,0 +1,330 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_MR_ATTR_H +#define ROCE_VERBS_MR_ATTR_H + + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_mr_sge { + u32 rsvd; + u32 new_key; + + /* DW2~3 */ + union { + u64 length; /* Length of mr or mw */ + + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw2; + }; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; +}; + +struct tag_roce_verbs_mtt_cacheout_info { + u32 mtt_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + /* + * Number of cmtt, which needs to be assigned by + * the driver when the is kicked out by queue. + */ + u32 mtt_num; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_start; + /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_end; + u32 mtt_cache_line_size; /* 0:256B,1:512B */ +}; + +struct tag_roce_verbs_wqe_cacheout_info { + u32 wqe_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + /* + * Number of wqe, which needs to be assigned by the + * driver when the is kicked out by queue. + */ + u32 wqe_num; + /* The driver needs to read the driver from the configuration file. */ + u32 wqe_cache_line_start; + /* The driver needs to read the driver from the configuration file. */ + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; /* 0:256B,1:512B */ +}; + + +struct tag_roce_verbs_query_mtt_info { + u32 mtt_addr_start_hi32; + u32 mtt_addr_start_lo32; + u32 mtt_num; + u32 rsvd; +}; + +struct tag_roce_dif_user_data_s { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 smd_tp : 2; + u32 app_esc : 1; + u32 ref_esc : 1; + u32 sct_v_tp : 2; + u32 sct_sz : 1; + u32 md_sz : 1; + u32 hdr_vld : 1; + u32 sec_num : 23; +#else + u32 sec_num : 23; + u32 hdr_vld : 1; // tx: 0->no nvme hdr, 1: hdr + u32 md_sz : 1; + u32 sct_sz : 1; + u32 sct_v_tp : 2; + u32 ref_esc : 1; + u32 app_esc : 1; + u32 smd_tp : 2; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rep_app_tag : 16; + u32 grd_v_en : 1; + u32 grd_rid : 2; + u32 grd_v_agm : 1; + u32 grd_ri_agm : 1; + u32 grd_agm_ini : 1; + u32 crc16_ini : 1; + u32 ipcs_ini : 1; + u32 ref_v_en : 1; + u32 ref_rid : 2; + u32 ref_v_inc : 1; + u32 ref_ri_inc : 1; + u32 app_v_en : 1; + u32 app_rid : 2; +#else + u32 app_rid : 2; + u32 app_v_en : 1; + u32 ref_ri_inc : 1; + u32 ref_v_inc : 1; + u32 ref_rid : 2; + u32 ref_v_en : 1; + u32 ipcs_ini : 1; + u32 crc16_ini : 1; + u32 grd_agm_ini : 1; + u32 grd_ri_agm : 1; + u32 grd_v_agm : 1; + u32 grd_rid : 2; + u32 grd_v_en : 1; + u32 rep_app_tag : 16; +#endif + } bs; + u32 value; + } dw1; + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cmp_app_tag : 16; + u32 cmp_app_tag_mask : 16; +#else + u32 cmp_app_tag_mask : 16; + u32 cmp_app_tag : 16; +#endif + } bs; + u32 value; + } dw2; + + u32 cmp_ref_tag; + u32 rep_ref_tag; +}; + +struct tag_roce_verbs_mr_attr { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 buf_page_size : 4; /* Page_size of the buffer */ + u32 mtt_layer_num : 3; /* Mtt level */ + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 rsvd2 : 4; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 bpd : 1; /* 1: Bound to pd */ + u32 bqp : 1; /* 1: Bound to qp */ + u32 dif_mode : 1; + u32 rkey : 1; + u32 pa : 1; /* Flag bit of DMA_MR */ + /* Mr or mw. The value 1 indicates MR, and the value 0 indicates MW. */ + u32 r_w : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; +#else + /* Whether the mr supports the binding of the mw */ + u32 access_bind : 1; + /* Indicates whether the FRMR can specify remote rights. */ + u32 remote_access_en : 1; + /* Indicates whether the FRMR operation is supported. */ + u32 fast_reg_en : 1; + /* Indicates whether to support the INVALID operation. */ + u32 invalid_en : 1; + /* Indicates whether to support the remote INVALID operation. */ + u32 remote_invalid_en : 1; + u32 r_w : 1; /* Mr or mw */ + u32 pa : 1; /* Flag bit of DMA_MR */ + u32 rkey : 1; + u32 dif_mode : 1; + u32 bqp : 1; /* 1: Bound to qp */ + u32 bpd : 1; /* 1: Bound to pd */ + /* The value 1 indicates that the remote Atomic permission is supported. */ + u32 access_ra : 1; + u32 access_rw : 1; /* 1: The remote write permission is supported. */ + /* 1: Indicates that the remote read permission is supported. */ + u32 access_rr : 1; + /* The value 1 indicates that the local write permission is supported. */ + u32 access_lw : 1; + /* 1: Indicates that the local read permission is supported. */ + u32 access_lr : 1; + /* The value 1 indicates that ZBVA is supported, that is, iova = 0. */ + u32 zbva : 1; + u32 rsvd2 : 4; + u32 mtt_page_size : 4; /* Page_size of mtt */ + u32 mtt_layer_num : 3; /* Number of mtt levels */ + u32 buf_page_size : 4; /* Page_size of the buffer */ +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 so_ro : 2; + u32 dma_attr_idx : 6; + u32 sector_size : 1; + u32 ep : 3; + u32 qpn : 20; +#else + u32 qpn : 20; /* Qp bound to mw */ + u32 ep : 3; + u32 sector_size : 1; /* 0:512B, 1:4KB */ + u32 dma_attr_idx : 6; /* Dma attribute index */ + u32 so_ro : 2; /* Dma order-preserving flag */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 status : 4; + u32 indirect_mr : 1; + u32 cos : 3; + u32 block_size : 6; + u32 pdn : 18; +#else + u32 pdn : 18; /* Pd bound to mr or mw */ + u32 block_size : 6; /* 2^(page_size+12) + 8*block_size */ + u32 cos : 3; + u32 indirect_mr : 1; + u32 status : 4; /* Mpt status. Valid values are VALID, FREE, and INVALID. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 mkey : 8; + u32 sw_dif_en : 1; + u32 page_mode : 1; + u32 fbo : 22; +#else + u32 fbo : 22; + u32 page_mode : 1; + u32 sw_dif_en : 1; + u32 mkey : 8; /* The index is not included. */ +#endif + } bs; + u32 value; + } dw3; + + /* DW4~5 */ + union { + u64 iova; /* Start address of mr or mw */ + struct { + u32 iova_hi; /* Upper 32 bits of the start address of mr or mw */ + u32 iova_lo; /* Lower 32 bits of the start address of mr or mw */ + } dw4; + }; + + /* DW6~7 */ + union { + u64 length; /* Length of mr or mw */ + struct { + u32 length_hi; /* Length of mr or mw */ + u32 length_lo; /* Length of mr or mw */ + } dw6; + }; + + /* DW8~9 */ + union { + u64 mtt_base_addr; /* Mtt base address (pa),low 3bits(gpa_sign) */ + struct { + u32 mtt_base_addr_hi; /* Mtt base address (pa) upper 32 bits */ + /* Lower 32 bits of mtt base address (pa),low 3bits(gpa_sign) */ + u32 mtt_base_addr_lo; + } dw8; + }; + + /* DW10 */ + union { + u32 mr_mkey; /* This parameter is valid for MW. */ + u32 mw_cnt; /* This parameter is valid when the MR is used. */ + }; + + /* DW11 */ + u32 mtt_sz; + + /* DW12~16 */ + struct tag_roce_dif_user_data_s dif_info; + + /* DW17 */ + u32 rsvd; +}; +#pragma pack() + +#endif /* ROCE_VERBS_MR_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h new file mode 100644 index 000000000..77bf31694 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_pub.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_PUB_H +#define ROCE_VERBS_PUB_H + + +enum CMD_TYPE_BITMASK_E { + CMD_TYPE_BITMASK_EXT = 0, + CMD_TYPE_BITMASK_RSVD1, + CMD_TYPE_BITMASK_RSVD0, + CMD_TYPE_BITMASK_PLUGIN_RET, + CMD_TYPE_BITMASK_VBS, + CMD_TYPE_BITMASK_DSW, + CMD_TYPE_BITMASK_NOFAA, + CMD_TYPE_BITMASK_PLOG, + CMD_TYPE_BITMASK_DBG = 8, + CMD_TYPE_BITMASK_VROCE, + CMD_TYPE_BITMASK_CCF, + CMD_TYPE_BITMASK_GID, + CMD_TYPE_BITMASK_MR, + CMD_TYPE_BITMASK_SRQ, + CMD_TYPE_BITMASK_CQ, + CMD_TYPE_BITMASK_QP = 15 +}; + +#define VERBS_CMD_TYPE_QP_BITMASK (1u << CMD_TYPE_BITMASK_QP) +#define VERBS_CMD_TYPE_CQ_BITMASK (1u << CMD_TYPE_BITMASK_CQ) +#define VERBS_CMD_TYPE_SRQ_BITMASK (1u << CMD_TYPE_BITMASK_SRQ) +#define VERBS_CMD_TYPE_MR_BITMASK (1u << CMD_TYPE_BITMASK_MR) +#define VERBS_CMD_TYPE_GID_BITMASK (1u << CMD_TYPE_BITMASK_GID) +#define VERBS_CMD_TYPE_CCF_BITMASK (1u << CMD_TYPE_BITMASK_CCF) +#define VERBS_CMD_TYPE_VROCE_BITMASK (1u << CMD_TYPE_BITMASK_VROCE) +#define VERBS_CMD_TYPE_DEBUG_BITMASK (1u << CMD_TYPE_BITMASK_DBG) +#define VERBS_CMD_TYPE_PLOG_BITMASK (1u << CMD_TYPE_BITMASK_PLOG) +#define VERBS_CMD_TYPE_NOFAA_BITMASK (1u << CMD_TYPE_BITMASK_NOFAA) +#define VERBS_CMD_TYPE_DSW_BITMASK (1u << CMD_TYPE_BITMASK_DSW) +#define VERBS_CMD_TYPE_VBS_BITMASK (1u << CMD_TYPE_BITMASK_VBS) +#define VERBS_CMD_TYPE_EXT_BITMASK (1u << CMD_TYPE_BITMASK_EXT) + +enum roce3_cmd_ret_status { + ROCE_CMD_RET_SUCCESS = 0x0, + + ROCE_CMD_RET_FLR_ERR = 0x1, + ROCE_CMD_RET_FUNC_INVLD, + ROCE_CMD_RET_CMDMISC_UNSUPPORT, + ROCE_CMD_RET_CMDTYPE_UNSUPPORT, + ROCE_CMD_RET_RET_ADDR_INVLD, + ROCE_CMD_RET_INDEX_INVLD, + ROCE_CMD_RET_LOCAL_UNSUPPORT, + ROCE_CMD_RET_QPC_RSP_ERR = 0x10, + ROCE_CMD_RET_QPC_STATE_UNEXPECT, + ROCE_CMD_RET_MODIFY_QP_OPT_ERR, + ROCE_CMD_RET_MODIFY_QP_EXTEND_OP_API_ERR, + ROCE_CMD_RET_SQD_QP_FETCH_WQE_ERR, + ROCE_CMD_RET_RDMARC_SYNC_EXTEND_OP_API_ERR, + ROCE_CMD_RET_MODIFY_QP_OQID_ERR, + ROCE_CMD_RET_CQC_STATE_UNEXPECT = 0x18, + ROCE_CMD_RET_CQC_MISC_API_ERR, + ROCE_CMD_RET_CQC_CREATE_CACHE_OUT_ERR, + ROCE_CMD_RET_CQ_RESIZE_API_ERR, + ROCE_CMD_RET_CQ_EXTEND_OP_API_ERR, + ROCE_CMD_RET_CQ_TIEMR_DEL_ERR, + ROCE_CMD_RET_CQ_TIEMR_REM_ERR, + ROCE_CMD_RET_SRQC_STATE_UNEXPECT = 0x20, + ROCE_CMD_RET_SRQ_ARM_SRQ_CONT_EN_ERR, + ROCE_CMD_RET_SRQ_ARM_SRQ_API_ERR, + ROCE_CMD_RET_SRQ_EXTEND_OP_API_ERR, + ROCE_CMD_RET_MR_STATE_ERR = 0x28, + ROCE_CMD_RET_MR_BIND_MW_API_ERR, + ROCE_CMD_RET_MR_EXTEND_OP_API_ERR, + ROCE_CMD_RET_MR_MISC_LOAD_API_ERR, + ROCE_CMD_RET_MR_MPTC_SYNC_ERR, + ROCE_CMD_RET_MR_MISC_STORE_API_ERR, + ROCE_CMD_RET_MR_CACHE_OUT_MPT_ERR, + ROCE_CMD_RET_MR_CACHE_OUT_MTT_ERR, + ROCE_CMD_RET_DBG_EXTEND_OP_API_ERR = 0x30, + ROCE_CMD_RET_DBG_WQE_UPDATE_PI_ERR, + ROCE_CMD_RET_QU_FLUSH_API_ERR, + ROCE_CMD_RET_CC_HASH_API_ERR, + ROCE_CMD_RET_CMD_OP_LB_ERR, + ROCE_CMD_RET_CMD_OP_SRQN_ERR, + ROCE_CMD_RET_DIF_TASKID_ALLOC_ERR, + ROCE_CMD_RET_QP_EXTEND_OP_ERR, + ROCE_CMD_RET_LOAD_ERR, + ROCE_CMD_RET_STORE_ERR, + ROCE_CMD_RET_LOAD_HOST_GPA_ERR, + ROCE_CMD_RET_CACHE_OUT_ERR, + ROCE_CMD_RET_CACHE_OUT_MTT_ERR, + ROCE_CMD_RET_BANK_GPA_FLUSH_ERR, + ROCE_CMD_RET_INVALID_PARAM_ERR, + ROCE_CMD_RET_RSVD_ERR = 0xff, +}; + +enum roce3_cmd_ret_extend_op_err { + ROCE_CMD_RET_EXTEND_OP_NONE = 0x0, + + ROCE_CMD_RET_EXTEND_OP_RCC_RRE = 0x1, + ROCE_CMD_RET_EXTEND_OP_RCC_RAE, + ROCE_CMD_RET_EXTEND_OP_RRWC_RWE, + ROCE_CMD_RET_EXTEND_OP_RRWC_STA, + ROCE_CMD_RET_EXTEND_OP_RCC_STA, + ROCE_CMD_RET_EXTEND_OP_SQC_STA, + ROCE_CMD_RET_EXTEND_OP_SQAC_STA, + ROCE_CMD_RET_EXTEND_OP_RQC_STA, + ROCE_CMD_RET_EXTEND_OP_CQC_TIMEOUT = 0x10, + ROCE_CMD_RET_EXTEND_OP_CQC_STA, + ROCE_CMD_RET_EXTEND_OP_SRQC_STA = 0x18, + ROCE_CMD_RET_EXTEND_OP_MPT_STA = 0x20, + ROCE_CMD_RET_EXTEND_OP_QPC_DBG_EDIT = 0x28, + ROCE_CMD_RET_EXTEND_OP_CQC_SRQC_DBG_EDIT +}; + +/* Mask description of *********ROCE verbs modify qp ******* */ +/* + ****************************************************************** +opt INIT2INIT INIT2RTR RTR2RTS RTS2RTS SQERR2RTS SQD2SQD SQD2RTS +QP_OPTPAR_ALT_ADDR_PATH 0 √ √ √ √ √ +QP_OPTPAR_RRE 1 √ √ √ √ √ √ √ +QP_OPTPAR_RAE 2 √ √ √ √ √ √ √ +QP_OPTPAR_RWE 3 √ √ √ √ √ √ √ +QP_OPTPAR_PKEY_INDEX 4 +QP_OPTPAR_Q_KEY 5 √ √ √ √ √ √ √ +QP_OPTPAR_RNR_TIMEOUT 6 √ √ √ √ √ +QP_OPTPAR_PRIMARY_ADDR_PATH 7 √ √ +QP_OPTPAR_SRA_MAX 8 √ √ +QP_OPTPAR_RRA_MAX 9 √ √ +QP_OPTPAR_PM_STATE 10 √ √ √ √ +QP_OPTPAR_RETRY_COUNT 11 √ √ +QP_OPTPAR_RNR_RETRY 12 √ √ +QP_OPTPAR_ACK_TIMEOUT 13 √ √ +QP_OPTPAR_SCHED_QUEUE 14 +QP_OPTPAR_COUNTER_INDEX 15 +******************************************************************** +*/ +enum QP_OPTPAR_E { + QP_OPTPAR_ALT_ADDR_PATH = 0, + QP_OPTPAR_RRE, + QP_OPTPAR_RAE, + QP_OPTPAR_RWE, + QP_OPTPAR_PKEY_INDEX, + QP_OPTPAR_Q_KEY, + QP_OPTPAR_RNR_TIMEOUT, + QP_OPTPAR_PRIMARY_ADDR_PATH, + QP_OPTPAR_SRA_MAX = 8, + QP_OPTPAR_RRA_MAX, + QP_OPTPAR_PM_STATE, + QP_OPTPAR_RETRY_COUNT, + QP_OPTPAR_RNR_RETRY, + QP_OPTPAR_ACK_TIMEOUT, + QP_OPTPAR_SCHED_QUEUE, + QP_OPTPAR_COUNTER_INDEX = 15 +}; + +#define ROCE_QP_ALT_ADDR_PATH_OPT (1u << QP_OPTPAR_ALT_ADDR_PATH) +#define ROCE_QP_RRE_OPT (1u << QP_OPTPAR_RRE) +#define ROCE_QP_RAE_OPT (1u << QP_OPTPAR_RAE) +#define ROCE_QP_RWE_OPT (1u << QP_OPTPAR_RWE) +#define ROCE_QP_PKEY_INDEX_OPT (1u << QP_OPTPAR_PKEY_INDEX) +#define ROCE_QP_Q_KEY_OPT (1u << QP_OPTPAR_Q_KEY) +#define ROCE_QP_RNR_TIMEOUT_OPT (1u << QP_OPTPAR_RNR_TIMEOUT) +#define ROCE_QP_PRIMARY_ADDR_PATH_OPT (1u << QP_OPTPAR_PRIMARY_ADDR_PATH) +#define ROCE_QP_SRA_MAX_OPT (1u << QP_OPTPAR_SRA_MAX) +#define ROCE_QP_RRA_MAX_OPT (1u << QP_OPTPAR_RRA_MAX) +#define ROCE_QP_PM_STATE_OPT (1u << QP_OPTPAR_PM_STATE) +#define ROCE_QP_RETRY_COUNT_OPT (1u << QP_OPTPAR_RETRY_COUNT) +#define ROCE_QP_RNR_RETRY_OPT (1u << QP_OPTPAR_RNR_RETRY) +#define ROCE_QP_ACK_TIMEOUT_OPT (1u << QP_OPTPAR_ACK_TIMEOUT) +#define ROCE_QP_SCHED_QUEUE_OPT (1u << QP_OPTPAR_SCHED_QUEUE) +#define ROCE_QP_COUNTER_INDEX_OPT (1u << QP_OPTPAR_COUNTER_INDEX) + +#define ROCE_MODIFY_QP_INIT2INIT_OPT (~0x402e) +#define ROCE_MODIFY_QP_INIT2RTR_OPT (~0x2ee) +#define ROCE_MODIFY_QP_RTR2RTS_OPT (~0x3d6e) +#define ROCE_MODIFY_QP_RTS2RTS_OPT (~0x46e) +#define ROCE_MODIFY_QP_SQERR2RTS_OPT (~0x2e) +#define ROCE_MODIFY_QP_SQD2SQD_OPT (~0x7fee) +#define ROCE_MODIFY_QP_SQD2RTS_OPT (~0x46e) +#define ROCE_MODIFY_QP_RTS2RTS_EXP_OPT (~0x10000) + +#define ROCE_MODIFY_QP_RTS2SQD_SQD_EVENT_OPT (0x80000000) +/* ************************************************* */ + +#pragma pack(4) +struct tag_roce_verbs_cmd_header { + union { + u32 value; + + struct { + u32 version : 8; + u32 rsvd : 8; + u32 cmd_bitmask : 16; // CMD_TYPE_BITMASK_E + } bs; + } dw0; + + u32 index; // qpn/cqn/srqn/mpt_index/gid idx + + u32 opt; // + + union { + u32 value; + + struct { + u32 cmd_type : 8; + u32 rsvd : 7; + u32 seg_ext : 1; + u32 cmd_len : 16; // verbs cmd total len(include cmd_com),unit:byte + } bs; + } dw3; +}; + +union tag_roce_verbs_seg_hdr { + u32 value; + + struct { + u32 seg_type : 16; + u32 rsvd : 7; + u32 last_seg : 1; + u32 seg_len : 8; // seg info len,unit:4B + } bs; +}; + +#pragma pack() + +#endif /* ROCE_VERBS_PUB_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h new file mode 100644 index 000000000..a0e6412f5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_srq_attr.h @@ -0,0 +1,264 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_SRQ_ATTR_H +#define ROCE_VERBS_SRQ_ATTR_H + +#include "roce_verbs_mr_attr.h" + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#pragma pack(4) +struct tag_roce_verbs_srq_cont_attr { + /* DW0 */ + u32 head_gpa_h; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 head_gpa_l : 20; + u32 rsvd : 11; + u32 head_gpa_vld : 1; +#else + u32 head_gpa_vld : 1; + u32 rsvd : 11; + u32 head_gpa_l : 20; +#endif + } bs; + u32 value; + } dw1; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cont_size : 2; + u32 rsvd : 10; + u32 warn_th : 4; + u32 head_idx : 16; +#else + u32 head_idx : 16; + u32 warn_th : 4; + u32 rsvd : 10; + u32 cont_size : 2; +#endif + } bs; + u32 value; + } dw2; +}; + +struct tag_roce_verbs_srq_attr { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 xrcd : 16; + u32 mtt_page_size : 4; + u32 wqebb_size : 3; + u32 page_size : 4; + u32 size : 5; +#else + /* + * Shared Receive Queue size, equals to (2^srq_size)*WQEBB, + * the maximum SRQ size is 16K WQEs, so this field doesn't exceed 14. + */ + u32 size : 5; + /* Page size of SRQ, equals to (2^srq_page_size)*4KB */ + u32 page_size : 4; + /* + * Shared Receive WQE Basic Block (WQEBB) size in bytes + * is (2^rq_wqebb_size)*16B. + * The minimum size is 32B and the values 0, 4, 5, 6, 7 are reserved + */ + u32 wqebb_size : 3; + /* Page size of MTT for SRQ, equals to (2^srq_mtt_page_size)*4KB. */ + u32 mtt_page_size : 4; + u32 xrcd : 16; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 state : 4; + u32 rsvd : 14; + u32 ep : 3; + u32 cos : 3; + u32 so_ro : 2; + u32 dma_attr_idx : 6; +#else + /* + * It specifies the outbound PCIe TLP header attribute of the + * DMA operation.This filed is only valid when processing CQ's CQEs. + */ + u32 dma_attr_idx : 6; + /* It specifies the ATTR[1:0] bits in the outbound PCIe TLP headers + * of the DMA operation.This field is only valid when processing + * CQ's CQEs. + * 2'b00: Strict Ordering; + * 2'b01: Relaxed Ordering; + * 2'b10: ID Based Ordering; + * 2'b11: Both Relaxed Ordering and ID Based Ordering. + */ + u32 so_ro : 2; + u32 cos : 3; + u32 ep : 3; + u32 rsvd : 14; + u32 state : 4; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_prefetch_max_num : 3; + u32 wqe_prefetch_min_num : 3; + u32 wqe_cache_thd_sel : 2; + u32 wqecnt_lth : 4; + u32 wqecnt_ctrl_en : 1; + u32 wqecnt_rctl : 1; + u32 mtt_prefetch_maxlen : 2; + u32 next_wqe_idx : 16; +#else + /* + * The current WQE index; uses this field to get + * the corresponding WQE from SRQ. + */ + u32 next_wqe_idx : 16; + u32 mtt_prefetch_maxlen : 2; + /* + * The hardware clear it to zero when performing a SRQ PCnt updating, + * and driver set it to one to indicate the hardware can performing + * SRQ PCnt updating. + */ + u32 wqecnt_rctl : 1; + u32 wqecnt_ctrl_en : 1; + u32 wqecnt_lth : 4; + /* + * Maximum length of prefetch MTTs for SRQ. + * 000: prefetch length equals to zero; + * Others: prefetch length equals to + * (2^(srq_mtt_prefetch_maxlen-1)*1KB). + */ + u32 wqe_cache_thd_sel : 2; + /* + * Minimum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^(srq_wqe_prefetch_minnum-1)). + */ + u32 wqe_prefetch_min_num : 3; + /* + * Maximum number of prefetch WQEBBs for SRQ. + * 000: prefetch number equals to zero; + * Others: prefetch number equals to (2^srq_wqe_prefetch_maxnum). + */ + u32 wqe_prefetch_max_num : 3; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 container : 1; + u32 pcnt_on_chip : 1; + u32 lth_pre_en : 1; + u32 rkey_en : 1; + u32 wqe_check_en : 1; + u32 lth_gap : 4; + u32 rsvd : 5; + u32 pd : 18; +#else + u32 pd : 18; + u32 rsvd : 5; + u32 lth_gap : 4; + u32 wqe_check_en : 1; + u32 rkey_en : 1; + u32 lth_pre_en : 1; + u32 pcnt_on_chip : 1; + u32 container : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 srqn; + + /* DW5 */ + u32 xrc_cqn; + + /* DW6~7 */ + union { + /* + * The GPA of Layer 0 MTT. It may point to the CQ's buffer directly. + * low 3bits(cq_gpa_sign) + */ + u64 l0mtt_gpa; + struct { + u32 l0mtt_gpa_hi; + u32 l0mtt_gpa_lo; + } dw6_dw7; + }; + + /* DW8~9 */ + union { + /* + * The GPA of stored CI of Complete Queue. + * Address translation hop numbers. + * 0x0: the 'cq_l0mtt_gpa' points to the buffer of CQ directly. + * 0x1: it need to perform one hop address translation to get the buffer's + * address of CQ; 0x2: there is two hop address translation to get the buffer's + * address of CQ; 0x3: reserved. + */ + u64 record_gpa_at_hop_num; + struct { + u32 record_gpa_hi; + u32 record_gpa_lo_at_hop_num; /* bit[1:0] Address translation hop numbers */ + } dw8_dw9; + }; + + struct tag_roce_verbs_srq_cont_attr cont; +}; + +struct tag_roce_verbs_srq_hw2sw_info { + /* DW0~3 */ + u32 srq_buf_len; + u32 wqe_cache_line_start; + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; + + /* DW4~8 */ + struct tag_roce_verbs_mtt_cacheout_info cmtt_cache; +}; + +union tag_roce_verbs_arm_srq_info { + u32 limitwater; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lwm : 16; + u32 warth : 4; + u32 th_up_en : 1; + u32 cont_en : 1; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 cont_en : 1; + u32 th_up_en : 1; + u32 warth : 4; + u32 lwm : 16; +#endif + } bs; +}; +#pragma pack() + +#endif /* ROCE_VERBS_SRQ_ATTR_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h new file mode 100644 index 000000000..fc49948a0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_verbs_ulp_format.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_VERBS_ULP_FORMAT_H +#define ROCE_VERBS_ULP_FORMAT_H +#include "roce_verbs_cmd.h" +#include "roce_ccf_format.h" + +/*******************AA*******************/ +struct tag_roce_aa_master_modify_qpc { + struct tag_roce_verbs_cmd_com com; + u32 opt; + u32 qid; + u32 local_comm_id; + u32 remote_comm_id; + + struct roce_qp_context qpc; +}; + +struct tag_roce_aa_slave_modify_qpc { + struct tag_roce_verbs_cmd_com com; + u32 gid_index; + u32 qid; + u32 local_comm_id; + u32 remote_comm_id; + + struct roce_qp_context qpc; +}; + +struct roce_aa_set_conn_stat_inbuf { + struct tag_roce_verbs_cmd_com com; + u32 qid; + u32 io_qpn[9]; +}; + +struct roce_aa_get_master_qpn_inbuf { + struct tag_roce_verbs_cmd_com com; +}; + +struct roce_aa_disconnect_inbuf { + struct tag_roce_verbs_cmd_com com; + u32 tid_h; + u32 tid_l; + u32 rsvd4; + u32 rsvd5; + u32 local_cmid; + u32 remote_cmid; + u32 remote_qpn; +}; + +struct roce_shard_map_entry { + u32 volume_id; + u32 hash_low_cnt : 8; + u32 dd_id : 8; + u32 lun_id : 16; + + u32 rsvd : 8; + u32 hash_high_offset : 8; + u32 hash_low_offset : 8; + u32 hash_high_cnt : 8; + u32 admin_qpn; +}; + +struct roce_aa_set_shard_cfg_inbuf { + struct tag_roce_verbs_cmd_com com; + u32 op; + u32 entry_num; + struct roce_shard_map_entry shard_info[64]; +}; + +struct roce_master_qp_modify2rst_inbuf { + struct tag_roce_verbs_cmd_com com; +}; + +struct tag_roce_aa_query_master_qp_bitmap { + struct tag_roce_verbs_cmd_com com; +}; + +/*******************VBS*******************/ +struct tag_roce_vbs_master_modify_qpc { + struct tag_roce_verbs_cmd_com com; + + u32 opt; + u32 sqpc_ci_record_addr_h; + u32 sqpc_ci_record_addr_l; + u32 rsvd; + + struct roce_qp_context qpc; +}; + +#endif /* ROCE_VERBS_ULP_FORMAT_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h new file mode 100644 index 000000000..7e964a8cd --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_wqe_format.h @@ -0,0 +1,930 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_WQE_FORMAT_H +#define ROCE_WQE_FORMAT_H + +/* **************************************************************** */ +struct roce3_wqe_ctrl_seg { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 owner : 1; + u32 ctrlsl : 2; + u32 csl : 2; + u32 difsl : 3; + u32 cr : 1; + u32 df : 1; + u32 va : 1; + u32 tsl : 5; + u32 cf : 1; + u32 wf : 1; + u32 rsvd : 4; + u32 drvsl : 2; + u32 bdsl : 8; +#else + /* + * Data segment length, in the unit of 1B. When inline is used, + * the length of inline is described. The total length of the + * data segment must be aligned to 8B. + */ + u32 bdsl : 8; + /* + * Indicates the length of the driver section. The value is + * counted by 8B, and the value of RoCE is 0. + */ + u32 drvsl : 2; + u32 rsvd : 4; + /* 0-Normal WQE, 1-link WQE */ + u32 wf : 1; + /* + * Complete segment format flag. The 0-complete segment directly + * contains the status information. 1: SGL + */ + u32 cf : 1; + /* + * Length of the task segment. The value ranges from 8 to 48. + * The value of RoCE ranges from 8 to 48. + */ + u32 tsl : 5; + /* + * SGE address format flag. The 0-SGE contains the physical + * address and length. The 1-SGE contains the virtual address, + * length, and key. The value of RoCE is 1. + */ + u32 va : 1; + /* + * Data segment format flag bit. 0-describes data in SGE format. + * 1: Data is described in inline format. + */ + u32 df : 1; + /* + * The CQE generates the request flag. The 0-does not + * generate the CQE. 1: Generate CQE + */ + u32 cr : 1; + /* + * DIF segment length. The value is counted by 8 bytes. + * The value of RoCE is 0. + */ + u32 difsl : 3; + /* + * Length of the completed segment. The value is counted by 8B. + * The value of RoCE is 0. + */ + u32 csl : 2; + /* + * Length of the control segment. The value is 8 bytes + * and the value of RoCE is 16. + */ + u32 ctrlsl : 2; + /* + * Owner bit. The value 1 indicates all hardware, and the value 0 + * indicates all software. The meaning of the queue owner bit is + * reversed every time the queue owner is traversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 cl : 4; + u32 signature : 8; + u32 rsvd : 4; + u32 mask_pi : 16; +#else + /* + * Pi is the value of the queue depth mask. + * It is valid when direct wqe. + */ + u32 mask_pi : 16; + u32 rsvd : 4; + u32 signature : 8; + u32 cl : 4; /* The length of CQE is generated in the task segment. */ +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 type : 2; + u32 rsvd1 : 3; + u32 cos : 3; + u32 cp_flag : 1; + u32 rsvd : 1; + u32 ctx_size : 2; + u32 qpn : 20; +#else + u32 qpn : 20; + u32 ctx_size : 2; /* RoCE QPC size, 512B */ + u32 rsvd : 1; + u32 cp_flag : 1; /* control plane flag */ + u32 cos : 3; /* Scheduling priority. The value source is SL. */ + u32 rsvd1 : 3; + u32 type : 2; /* Set RoCE SQ Doorbell to 2 and RoCE Arm CQ Doorbell to 3. */ +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; /* */ + u32 sgid_index : 7; /* gid index */ + /* 256B:0;512B:1;1024B:2;2048B:3;4096B:4 */ + u32 mtu_shift : 3; + u32 rsvd : 1; + /* 1:XRC service type */ + u32 xrc_vld : 1; + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 16; +#else + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 16; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 mtu_shift : 3; + u32 sgid_index : 7; + u32 sub_type : 4; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 queue_id : 4; + u32 rsvd : 12; + u32 pi : 16; +#else + u32 pi : 16; + u32 rsvd : 12; + u32 queue_id : 4; +#endif + } bs1; + u32 value; + } dw3; +}; + +union roce3_wqe_tsk_com_seg { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 se : 1; + u32 f : 1; + u32 c : 1; + u32 op_type : 5; + u32 so : 1; + u32 cs_len_mul : 3; + u32 dif_en : 1; + u32 ext_vld : 1; + u32 xrc_srqn : 18; +#else + /* The XRC is valid, and the remote SRQN is specified. */ + u32 xrc_srqn : 18; + u32 ext_vld : 1; + u32 dif_en : 1; + /* * atomic cmp swap N len:atomic datalen=8*(cs_len_mul + 1) */ + u32 cs_len_mul : 3; + /* + * Strong order-preserving flag, valid only for Local invalidate\Type + * 2 Bind MW and FRPMR (consider the implementation at the bottom layer) + */ + u32 so : 1; + /* + * Operation type of the SQ WQE. + * 8'h00-Send + * 8'h01-Send with Invalidate + * 8'h02-Send with Immediate Data + * 8'h03-rsvd + * 8'h04-RDMA Write + * 8'h05-RDMA Write with Immediate Data + * 8'h06-RDMA WRITE CMD64 + * 8'h07-rsvd + * 8'h08-RDMA READ + * 8'h09-ATOMIC WRITE + * 8'h0a-FLUSH + * 8'h0b-rsvd + * 8'h0c-Atomic compare & swap + * 8'h0d-Atomic Fetch & ADD + * 8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation) + * 8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation) + * 8'h10-Fast Register PMR + * 8'h11-Local Invalidate + * 8'h12-Bind Memory Window Type1/2 + * 8'h13-Local operation(extended for further local operation) + * other-Reserved + */ + u32 op_type : 5; + /* + * Indicates whether the SQ generates the CQE, + * which is required by the microcode. + */ + u32 c : 1; + /* Indicates whether the SQ requires order-preserving. */ + u32 f : 1; + /* Indicates whether the packet carries the SE flag. */ + u32 se : 1; +#endif + } bs; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 se : 1; + u32 f : 1; + u32 c : 1; + u32 op_type : 5; + u32 so : 1; + u32 rsvd : 2; + u32 vbs_vld : 1; + u32 dif_en : 1; + u32 ext_vld : 1; + u32 xrc_srqn : 18; +#else + u32 xrc_srqn : 18; + u32 ext_vld : 1; + u32 dif_en : 1; + u32 vbs_vld : 1; + u32 rsvd : 2; + u32 so : 1; + u32 op_type : 5; + u32 c : 1; + u32 f : 1; + u32 se : 1; +#endif + } bs1; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 14; + u32 sticky : 1; + u32 repeat : 1; + u32 cmdid : 16; +#else + u32 cmdid : 16; + u32 repeat : 1; + u32 sticky : 1; + u32 rsvd : 14; +#endif + } nofaa; + u32 value; +}; + +union roce3_wqe_tsk_misc_seg { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 ext_last : 1; + u32 nxt_ext_hdr : 7; + u32 cmd_len : 8; + u32 rsvd : 8; + u32 last_ext_len : 8; +#else + u32 last_ext_len : 8; + u32 rsvd : 8; + u32 cmd_len : 8; + u32 nxt_ext_hdr : 7; + u32 ext_last : 1; +#endif + } bs; + + u32 value; +}; + +struct roce3_wqe_tsk_rdma_sge_seg { + /* DW4~5 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw4; + }; + + /* DW6 */ + u32 rkey; + + /* DW7 */ + u32 sig_key; +}; + +/* * + * Struct name: sq_wqe_task_remote_common. + * @brief : SQ WQE common. + * Description: + */ +struct tag_sq_wqe_task_remote_common { + union roce3_wqe_tsk_com_seg common; + + u32 data_len; /* SQ WQE DMA LEN */ + + u32 immdata_invkey; /* SEND invalidate or immediate */ + + union roce3_wqe_tsk_misc_seg dw3; +}; + +/* Send WQE/Send with imme WQE/Send with invalid(inline or not inline) */ +struct roce3_wqe_send_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; /* Length of the data sent by the SQ WQE */ + + /* DW2 */ + /* + * This parameter is valid for the immediate + * data operation or SEND invalidate. + */ + u32 immdata_invkey; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; +}; + +/* RDMA Read WQE/RDMA Write WQE/RDMA Write with imme WQE(inline or non-inline) */ +struct roce3_wqe_rdma_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 imm_data; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; + + /* DW4~5 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw4; + }; + + /* DW6 */ + u32 rkey; + + /* DW7 */ + u32 sig_key; +}; + +/* RDMA Read WQE/RDMA Write WQE/RDMA Write with imme WQE(inline or non-inline) */ +struct roce3_wqe_rdma_ext_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 imm_data; + + /* DW3 */ + union roce3_wqe_tsk_misc_seg dw3; + + /* DW4~5 */ + struct roce3_wqe_tsk_rdma_sge_seg rdma; + + u32 ulp_cmd[24]; +}; + +/* Atomic WQE */ +struct roce3_wqe_atomic_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 key; + + /* DW2~3 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw2; + }; + + /* DW4~5 */ + union { + u64 swap_add_data; + struct { + u32 swap_add_data_h32; + u32 swap_add_data_l32; + } dw4; + }; + + /* DW6~7 */ + union { + u64 cmp_data; + struct { + u32 cmp_data_h32; + u32 cmp_data_l32; + } dw6; + }; +}; + +/* ext Atomic WQE */ +#define ROCE_WQE_ATOMIC_DATA_SIZE 32 +#define ROCE_WQE_ATOMIC_DATA_SIZE_2B_ALIGN (ROCE_WQE_ATOMIC_DATA_SIZE >> 1) +struct roce3_wqe_ext_atomic_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 key; + + /* DW2~3 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw2; + }; + + u32 atomic_data[ROCE_WQE_ATOMIC_DATA_SIZE]; +}; + +/* Mask Atomic WQE */ +struct roce3_wqe_mask_atomic_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 rkey; + + /* DW2~3 */ + union { + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } dw2; + }; + + /* DW4~5 */ + union { + u64 swap_add_data; + struct { + u32 swap_add_data_h32; + u32 swap_add_data_l32; + } dw4; + }; + + /* DW6~7 */ + union { + u64 cmp_data; + struct { + u32 cmp_data_h32; + u32 cmp_data_l32; + } dw6; + }; + + /* DW8~9 */ + union { + u64 swap_msk; + struct { + u32 swap_msk_h32; + u32 swap_msk_l32; + } dw8; + }; + + /* DW9~10 */ + union { + u64 cmp_msk; + struct { + u32 cmp_msk_h32; + u32 cmp_msk_l32; + } dw10; + }; +}; + +/* UD send WQE */ +struct roce3_wqe_ud_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW1 */ + u32 data_len; + + /* DW2 */ + u32 immdata_invkey; + + union roce3_wqe_tsk_misc_seg dw2; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 fl : 1; + /* DB cos or Path cos or MQM cos */ + u32 wqe_cos : 3; + u32 stat_rate : 4; + u32 rsvd : 6; + u32 pd : 18; +#else + u32 pd : 18; /* Used to verify the PD in the QPC. */ + u32 rsvd : 6; + /* Maximum static rate control 0: + * No limit on the static rate (100% port speed) + * 1-6: reserved + * 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s. + * 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. + */ + u32 stat_rate : 4; + /* DB cos or Path cos or MQM cos */ + u32 wqe_cos : 3; + u32 fl : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 tc : 8; + u32 rsvd : 4; + u32 port : 4; + u32 vlan_en : 1; + u32 sgid_idx : 7; + u32 hop_limit : 8; +#else + u32 hop_limit : 8; + u32 sgid_idx : 7; + u32 vlan_en : 1; + u32 port : 4; + u32 rsvd : 4; + u32 tc : 8; +#endif + } bs; + u32 value; + } dw4; + + /* DW5 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 2; + u32 smac_index : 10; + u32 flow_label : 20; +#else + u32 flow_label : 20; + u32 smac_index : 10; + u32 rsvd : 2; +#endif + } bs; + u32 value; + } dw5; + + /* DW6~9 */ + u8 dgid[16]; + + /* DW10 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + u32 dst_qp : 24; +#else + u32 dst_qp : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw10; + + /* DW11 */ + u32 qkey; + + /* DW12 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 vlan_pri : 3; /* send pkt pri */ + u32 cfi : 1; + u32 vlan_id : 12; + u32 dmac_h16 : 16; +#else + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 cfi : 1; + u32 vlan_pri : 3; /* send pkt pri */ +#endif + } bs; + u32 value; + } dw12; + + /* DW14 */ + u32 dmac_l32; + + /* DW15 */ + u32 rsvd; +}; + +struct roce3_wqe_data_seg { + /* DW0~1 */ + union { + u64 addr; + struct { + u32 addr_h32; + u32 addr_l32; + } bs; + }; + + /* DW2~3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 1; + u32 len : 31; +#else + u32 len : 31; + u32 rsvd : 1; +#endif + } bs; + u32 value; + } dw2; + + /* DW4 */ + u32 key; +}; + +/* Type1/2 MW Bind WQE */ +struct roce3_wqe_bind_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0~2 */ + u32 rsvd0[3]; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rae : 1; + u32 rwe : 1; + u32 rre : 1; + u32 type : 1; + u32 rsvd : 28; +#else + u32 rsvd : 28; + /* MW type: 0-Type1 Window 1-Type2B Window */ + u32 type : 1; + u32 rre : 1; /* Remote read enable */ + u32 rwe : 1; /* Remote write enable */ + /* Indicates whether remote Atomic is enabled. */ + u32 rae : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + /* + * The MW corresponds to the MPT key and is indexed to the MPT + * through the New_Rkey. For the type1 MW, This parameter is + * valid only when the value of New_Rkey is the same as the + * value of mem_key in the MPT. For type 2 MW, it is valid only + * when New_Rkey is equal to mem_key+1 in MPT. If this parameter + * is valid, the corresponding field in the MPT is replaced with + * the entire section. + */ + u32 new_rkey; + + /* DW5 */ + /* Indicates the mem_key of the MR bound to the MW. */ + u32 lkey; + + /* DW6 */ + u32 rsvd1; + + /* DW7~8 */ + union { + /* Indicates the start virtual IP address of the MW. */ + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } bs; + } dw7; + + /* DW9~10 */ + union { + /* Indicates the length of the data corresponding to the MW. */ + u64 len; + struct { + u32 len_h32; + u32 len_l32; + } bs; + } dw9; +}; + +/* Fast Register PMR WQE */ +struct roce3_wqe_frmr_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0~1 */ + u32 rsvd[2]; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 fbo : 22; + u32 rsvd : 10; +#else + u32 rsvd : 10; + /* This parameter is valid when ZERO_BASE is set to 1. */ + u32 fbo : 22; +#endif + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rae : 1; + u32 rwe : 1; + u32 rre : 1; + u32 lwe : 1; + u32 lre : 1; + u32 be : 1; + u32 zbva : 1; + u32 block : 1; + u32 rsvd : 2; + u32 page_size : 6; + u32 pa_num : 16; +#else + /* + * Number of registered PAs, which is calculated + * by the length/page_size. + */ + u32 pa_num : 16; + u32 page_size : 6; /* Memory Region page size */ + u32 rsvd : 2; + u32 block : 1; + u32 zbva : 1; /* ZERO_BASED Permission */ + /* Indicates whether the binding operation is enabled. */ + u32 be : 1; + u32 lre : 1; /* Local read enable */ + u32 lwe : 1; /* Local write enable */ + u32 rre : 1; /* Remote read enable */ + u32 rwe : 1; /* Remote write enable */ + /* Indicates whether remote Atomic is enabled. */ + u32 rae : 1; +#endif + } bs; + u32 value; + } dw3; + + /* DW4 */ + u32 m_key; /* Key of the Memory Region. */ + + /* DW5~6 */ + union { + /* + * Start virtual address of Memory Region. + * This parameter is valid only when ZBVA is not set to 1. + */ + u64 va; + struct { + u32 va_h32; + u32 va_l32; + } bs; + } dw5; + + /* DW7~8 */ + union { + u64 len; /* Length of Memory Region */ + struct { + u32 len_h32; + u32 len_l32; + } bs; + } dw7; + + /* DW9~10 */ + union { + /* Physical address for storing the cache of the PA table. */ + u64 pbl_addr; + struct { + u32 pbl_addr_h32; + u32 pbl_addr_l32; + } bs; + } dw9; +}; + +struct roce3_sge_info { + union { + u64 sge_vaddr; + struct { + u32 sge_vaddr_hi; + u32 sge_vaddr_lo; + } bs; + } dw0; + + u32 sge_length; + u32 sge_key; +}; + + +/* Local Invalidate WQE */ +struct roce3_wqe_local_inv_tsk_seg { + union roce3_wqe_tsk_com_seg common; + + /* DW0 */ + u32 rsvd; + + /* DW1 */ + u32 inv_key; /* Mem_Key for invalidate */ + + /* DW2 */ + u32 rsvd1; +}; + +/* * SRQ Data Format start */ +struct roce3_wqe_srq_data_seg { + /* DW0~1 */ + union { + u64 addr; + + struct { + u32 addr_h32; + u32 addr_l32; + } bs; + }; + + /* DW2 */ + union { + struct { + /* Reserved field */ + u32 rsv : 1; + /* Data length. The value can be [0 or 2G-1]. */ + u32 len : 31; + } bs; + u32 length; + } dw2; + + /* DW3 */ + union { + struct { + /* + * Last flag. The 0-also has the next SGE. + * 1: The current SGE is the last one. + */ + u32 last : 1; + /* + * Extended flag. The value 0-is normal and does not need to be extended. + * 1: Extended mode. The address of the current SGE points to SGL. + * The key and len are invalid. For RoCE, the value is fixed to 0. + */ + u32 ext : 1; + /* + * Local_key. The least significant eight bits are keys, + * and the most significant 22 bits are + * indexes. The most significant two bits are reserved and only + * 20 bits are used. + */ + u32 key : 30; + } bs; + u32 lkey; + } dw3; +}; +/* * SRQ Data Format end */ + +struct roce_osd_srq_link_wqe { + u32 cont_gpa_h; + + union { + struct { + u32 cont_gpa_l : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw1; + + u32 rsvd; + + union { + struct { + u32 rsvd : 1; + u32 link_flag : 1; + u32 rsvd1 : 30; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 cur_container_idx : 16; + u32 container_idx : 16; + } bs; + u32 value; + } dw4; +}; + + +#endif // ROCE_WQE_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h b/drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h new file mode 100644 index 000000000..4ad437ec2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/rdma/roce_xqe_format.h @@ -0,0 +1,722 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_XQE_FORMAT_H +#define ROCE_XQE_FORMAT_H + +/* * SQ DB Format start */ +struct roce_sq_db_seg { + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 type : 5; + u32 cos : 3; + u32 cp_flag : 1; + u32 rsvd : 1; + u32 ctx_size : 2; + u32 qpn : 20; +#else + u32 qpn : 20; + /* RoCE QPC size, 512B */ + u32 ctx_size : 2; + u32 rsvd : 1; + /* control plane flag */ + u32 cp_flag : 1; + /* Scheduling priority. The value source is SL. */ + u32 cos : 3; + /* Set RoCE SQ Doorbell to 2 and RoCE Arm CQ Doorbell to 3. */ + u32 type : 5; +#endif + } bs; + u32 value; + } dw0; + + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 sub_type : 4; + /* gid index */ + u32 sgid_index : 7; + /* 256B:0;512B:1;1024B:2;2048B:3;4096B:4 */ + u32 mtu_shift : 3; + u32 rsvd : 1; + /* 1:XRC service type */ + u32 xrc_vld : 1; + u32 resv : 8; + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 8; +#else + /* host sw write the sq produce index high 8bit to this section; */ + u32 pi : 8; + u32 resv : 8; + u32 xrc_vld : 1; + u32 rsvd : 1; + u32 mtu_shift : 3; + u32 sgid_index : 7; + u32 sub_type : 4; +#endif + } bs; + u32 value; + } dw1; +}; + +union roce_sq_db { + struct roce_sq_db_seg sq_db_seg; + union { + u64 sq_db_val; + struct { + u32 sq_db_val_h32; + u32 sq_db_val_l32; + } dw2; + }; +}; + +/* *SQ DB Format end */ + +/* * ARM CQ DB Format start */ +struct roce_db_cq_arm { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* dB type. The value of RoCE ARM CQ DB is 3. */ + u32 type : 5; + /* ARM CQ DB Not required */ + u32 cos : 3; + /* the control plane flag of a DB */ + u32 cp : 1; + u32 non_filter : 1; + /* Cq type, 0: RDMA/1:T/IFOE/2.3: Rsv */ + u32 cqc_type : 2; + u32 cqn : 20; +#else + u32 cqn : 20; + /* Cq type, 0: RDMA/1:T/IFOE/2.3: Rsv */ + u32 cqc_type : 2; + u32 non_filter : 1; + /* the control plane flag of a DB */ + u32 cp : 1; + /* ARM CQ DB Not required */ + u32 cos : 3; + /* dB type. The value of RoCE ARM CQ DB is 3. */ + u32 type : 5; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The ArmCQ DB carries the CmdSn, which is compared with the CmdSn in the + * chip. If the values are different, the is valid. If the values are the + * same, the needs to be checked. -- Each time a CEQE is generated, + * the CmdSn of the chip is updated to the CmdSn of the latest ArmCq DB. + */ + u32 cmd_sn : 2; + /* + * Run the Arm command. 0-non-Arm After receiving the next CQE with the SE, + * the 1-ARM SOLICITED generates the CEQE. The * 2-ARM NEXT generates the + * CEQE after receiving the next CQE. + */ + u32 cmd : 2; + u32 rsv1 : 4; + /* Consumer pointer */ + u32 ci : 24; +#else + /* Consumer pointer */ + u32 ci : 24; + u32 rsv1 : 4; + /* + * Run the Arm command. 0-non-Arm After receiving the next CQE with the SE, + * the 1-ARM SOLICITED generates the CEQE. The * 2-ARM NEXT generates the + * CEQE after receiving the next CQE. + */ + u32 cmd : 2; + /* + * The ArmCQ DB carries the CmdSn, which is compared with the CmdSn in the + * chip. If the values are different, the is valid. If the values are the + * same, the needs to be checked. -- Each time a CEQE is generated, + * the CmdSn of the chip is updated to the CmdSn of the latest ArmCq DB. + */ + u32 cmd_sn : 2; +#endif + } bs; + u32 value; + } dw1; +}; +/* * ARM CQ DB Format end */ + +/* * CQE Format start */ +struct roce_cqe { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is modified when the hardware writes the CQE. + * The meaning of each queue owner bit is reversed. + */ + u32 owner : 1; + /* For roce, this field is reserved. */ + u32 size : 2; + /* For roce, this field is reserved. */ + u32 dif_en : 1; + /* For roce, this field is reserved. */ + u32 wq_id : 4; + /* For roce, this field is reserved. */ + u32 error_code : 4; + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; +#else + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; + /* For roce, this field is reserved. */ + u32 error_code : 4; + /* For roce, this field is reserved. */ + u32 wq_id : 4; + /* For roce, this field is reserved. */ + u32 dif_en : 1; + /* For roce, this field is reserved. */ + u32 size : 2; + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is modified when the hardware writes the CQE. + * The meaning of each queue owner bit is reversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The same as what is for SQ WQE. For details, + * see the enumeration roce_cqe_opcode + */ + u32 op_type : 5; + /* Indicates SQ CQE or RQ CQE. 1-Send Completion; 0-Receive Completion */ + u32 s_r : 1; + /* Indicates whether RQ inline; SQ ignores this bit */ + u32 inline_r : 1; + u32 flush_op : 1; + /* + * Indicates whether the CQE is a fake one. + * When fake, optype & syndronme should be 0 + */ + u32 fake : 1; + u32 rsvd : 3; + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; +#else + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; + u32 rsvd : 3; + /* + * Indicates whether the CQE is a fake one. + * When fake, optype & syndronme should be 0 + */ + u32 fake : 1; + u32 flush_op : 1; + /* Indicates whether RQ inline; SQ ignores this bit */ + u32 inline_r : 1; + /* Indicates SQ CQE or RQ CQE. 1-Send Completion; 0-Receive Completion */ + u32 s_r : 1; + /* + * The same as what is for SQ WQE. For details, + * see the enumeration roce_cqe_opcode + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + /* + * Indicates the number of transmitted bytes. This field is valid + * for the RDMA read and receive + * operations. For the recv of RDMA write imm, the value is 0. + */ + u32 byte_cnt; + + /* DW3 */ + u32 imm_invalid_rkey; /* The receiving is complete and valid. */ + + /* DW4 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 vlan_id : 12; + u32 rsvd : 1; + u32 vlan_pri : 3; + u32 smac_h : 16; +#else + u32 smac_h : 16; + u32 vlan_pri : 3; + u32 rsvd : 1; + u32 vlan_id : 12; +#endif + } bs; /* for ud only */ + u32 value; + } dw4; + + /* DW5 */ + u32 smac_l; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The for ud only:UD receive end is valid. + * The 00-packet does not contain a VLAN ID. The + * 01-packet contains a VLAN ID. + */ + u32 vlan_pre : 2; + /* This field is valid only for UD. Force loopback */ + u32 fl : 1; + u32 stp : 2; + u32 rsvd : 3; + /* + * The XRC at the receive end is valid. When the XRCSRQ at + * the receive end is received, the ;UD at the remote end + * refers to the QPN at the remote end. + */ + u32 srqn_rqpn : 24; +#else + /* + * The XRC at the receive end is valid. When the XRCSRQ at + * the receive end is received, the ;UD at the remote end + * refers to the QPN at the remote end. + */ + u32 srqn_rqpn : 24; + u32 rsvd : 3; + u32 stp : 2; + /* This field is valid only for UD. Force loopback */ + u32 fl : 1; + /* + * The for ud only:UD receive end is valid. + * The 00-packet does not contain a VLAN ID. The + * 01-packet contains a VLAN ID. + */ + u32 vlan_pre : 2; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_cnt : 16; /* WQE index, valid SQ */ + u32 rsvd : 8; + /* + * 0 indicates that the operation is complete. This parameter + * is valid when Op_type is set to ROCE_OPCODE_ERR. + * For details about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; +#else + /* + * 0 indicates that the operation is complete. This parameter + * is valid when Op_type is set to ROCE_OPCODE_ERR. + * For details about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; + u32 rsvd : 8; + u32 wqe_cnt : 16; /* WQE index, valid SQ */ +#endif + } bs; + u32 value; + } dw7; + + u32 timestamp_h; + u32 timestamp_l; + u32 common_rsvd[6]; +}; + +struct roce_resize_cqe { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is overwritten when the hardware writes the CQE. + * The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; + u32 rsvd : 31; +#else + u32 rsvd : 31; + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is overwritten when the hardware writes the CQE. + * The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration + * roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + u32 rsvd : 26; +#else + u32 rsvd : 26; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration + * roce_cqe_opcode. + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + /* DW2~7 */ + u32 rsvd[6]; + + u32 common_rsvd[8]; +}; + +struct roce_err_cqe { + /* DW0 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. This bit is overwritten when + * the hardware writes the CQE. The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; + u32 rsvd : 11; /* For RoCE, this field is reserved. */ + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; +#else + /* + * Local QPN, which is used in all cases. + * The driver finds the software QPC based on the QPN. + */ + u32 qpn : 20; + u32 rsvd : 11; /* For roce, this field is reserved. */ + /* + * Owner bit. During initialization, 0 indicates all hardware, + * and 1 indicates all software. + * This bit is overwritten when the hardware writes the CQE. + * The meaning of the queue owner + * bit is reversed every time the queue owner is traversed. + */ + u32 owner : 1; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The operation type is the same as that of SQ WQE. + * For details, see roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates the SQ CQE or RQ CQE. 1-Send Completion; + * 0-Receive Completion + */ + u32 s_r : 1; + u32 inline_r : 1; + u32 flush_op : 1; + u32 fake : 1; + u32 rsvd : 3; + u32 wqebb_cnt : 20; /* The WQEBB index and SQ/RQ/SRQ are valid. */ +#else + u32 wqebb_cnt : 20; /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 rsvd : 3; + u32 fake : 1; + /* + * Indicates whether this CQE is a fake cqe or not.when fake = 1, + * optype & syndronme should be 0 + */ + u32 flush_op : 1; + u32 inline_r : 1; + /* + * Indicates the SQ CQE or RQ CQE. 1-Send Completion; + * 0-Receive Completion + */ + u32 s_r : 1; + /* + * The operation type is the same as that of SQ WQE. + * For details, see roce_cqe_opcode. + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + /* DW2~5 */ + u32 rsvd[3]; + + u32 wqe_num; + + /* DW6 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 rsvd : 8; + /* The XRC at the receive end is valid, which is XRCSRQ. */ + u32 srqn : 24; +#else + /* The XRC at the receive end is valid, which is the XRCSRQ number. */ + u32 srqn : 24; + u32 rsvd : 8; +#endif + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 wqe_cnt : 16; /* WQE index: Indicates that the SQ is valid. */ + u32 vendor_err : 8; + /* + * 0 indicates that the operation is successful. The value is + * valid when Op_type is ROCE_OPCODE_ERR. For details, see the + * enumeration roce_cqe_syndrome. + */ + u32 syndrome : 8; +#else + /* + * 0 indicates that the operation is complete. This parameter + * is valid when Op_type is set to ROCE_OPCODE_ERR. For details + * about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; + u32 vendor_err : 8; + u32 wqe_cnt : 16; /* WQE index, valid SQ */ +#endif + } bs; + u32 value; + } dw7; + + u32 timestamp_h; + u32 timestamp_l; + u32 common_rsvd[6]; +}; + + +struct roce_vbs_cqe { + /* DW0 */ + union { + struct { + u32 owner : 1; + u32 size : 2; + u32 dif_en : 1; + u32 wq_id : 4; + u32 error_code : 4; + u32 qpn : 20; + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + u32 inline_r : 1; /* Indicates whether the RQ inline;SQ ignores this bit. */ + u32 rsvd : 5; + u32 msg_fst_wqe_ci : 20; + } bs; + u32 value; + } dw1; + + /* DW2 */ + /* + * Indicates the number of transmitted bytes. This field is valid + * for the RDMA read and receive + * operations. For the recv of RDMA write imm, the value is 0. + */ + u32 byte_cnt; + + /* DW3 */ + u32 imm_invalid_rkey; /* The receiving is complete and valid. */ + + /* DW4 */ + u32 rsvd; + + /* DW5 */ + u32 wqe_num; + + /* DW6 */ + union { + struct { + /* + * The for ud only:UD receive end is valid. The 00-packet does + * not contain a VLAN ID. The + * 01-packet contains a VLAN ID. + */ + u32 vlan_pre : 2; + u32 fl : 1; /* This field is valid only for UD. Force loopback */ + u32 stp : 2; + u32 rsvd : 3; + /* + * The XRC at the receive end is valid. When the XRCSRQ at + * the receive end is received, + * the ;UD at the remote end refers to the QPN at the remote end. + */ + u32 srqn_rqpn : 24; + } bs; + u32 value; + } dw6; + + /* DW7 */ + union { + struct { + u32 wqe_cnt : 16; /* WQE index, valid SQ */ + u32 vendor_err : 8; + /* + * 0 indicates that the operation is complete. This parameter is + * valid when Op_type is set to ROCE_OPCODE_ERR. + * For details about the definition, see the enumeration + * roce_cqe_syndrome. + */ + u32 syndrome : 8; + } bs; + u32 value; + } dw7; + + u16 srq_container_idx[16]; +}; + +struct roce_nofaa_cqe { + /* DW0 */ + u32 common_dw0; + + /* DW1 */ + union { + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration + * roce_cqe_opcode. + */ + u32 op_type : 5; + /* + * Indicates whether SQ CQE or RQ CQE is used. + * 1-Send Completion; 0-Receive Completion + */ + u32 s_r : 1; + /* Indicates whether the RQ inline;SQ ignores this bit. */ + u32 inline_r : 1; + u32 flush_op : 1; + u32 fake : 1; + u32 repeat : 1; + u32 host_id : 2; + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; +#else + /* The WQEBB index and SQ/RQ/SRQ are valid. */ + u32 wqebb_cnt : 20; + u32 host_id : 2; + u32 repeat : 1; + /* + * Indicates whether this CQE is a fake cqe or not. + * when fake = 1, optype & syndronme should be 0 + */ + u32 fake : 1; + u32 flush_op : 1; + /* Indicates whether RQ inline;SQ ignores this bit. */ + u32 inline_r : 1; + /* + * Indicates the SQ CQE or RQ CQE. 1-Send Completion; + * 0-Receive Completion + */ + u32 s_r : 1; + /* + * The operation type is the same as that of SQ WQE. + * For details, see the enumeration roce_cqe_opcode. + */ + u32 op_type : 5; +#endif + } bs; + u32 value; + } dw1; + + u32 common_rsvd[13]; + + union { + struct { + u32 rsvd : 4; + u32 sw_times : 2; + u32 sw_slave_id : 6; + u32 io_time : 20; + } bs; + u32 value; + } io_status; +}; + +#endif // RDMA_XQE_FORMAT_H diff --git a/drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h new file mode 100644 index 000000000..4e5e32369 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_cdev_extension.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CDEV_EXTENSION_H +#define ROCE_CDEV_EXTENSION_H + +#include "roce.h" + +#define NOT_SUPOORT_TYPE 0xFFFFFFFF + +long ioctl_non_bonding_extend(unsigned int cmd, struct roce3_device *rdev, unsigned long arg); + +#endif diff --git a/drivers/infiniband/hw/hiroce3/include/roce_event_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_event_extension.h new file mode 100644 index 000000000..a6ea2a79c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_event_extension.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_EVENT_EXTENSION_H +#define ROCE_EVENT_EXTENSION_H + +#include "roce_event.h" + +void roce3_event_report_extend(const struct roce3_device *rdev, int event_str_index); + +int roce3_async_event_handle_extend(u8 event_type, u8 *val, struct roce3_device *rdev); + +#endif /* ROCE_EVENT_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_main_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_main_extension.h new file mode 100644 index 000000000..d55983ca1 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_main_extension.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MAIN_EXTENSION_H +#define ROCE_MAIN_EXTENSION_H + +#include <linux/pci.h> + +#include "hinic3_hw.h" +#include "hinic3_crm.h" +#include "hinic3_lld.h" +#include "hinic3_mbox.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_dfx.h" +#include "roce_cmd.h" +#include "rdma_comp.h" +#include "hmm_mr.h" +#include "roce_qp_post_send_extension.h" + +#define ROCE_MAX_WQES_COMPUTE (4 * K_UNIT - 1) +#define RDMA_MAX_SQ_DESC_SZ_COMPUTE 512 +#define ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE 448 + +struct hinic3_uld_info *roce3_info_get(void); + +void roce3_service_init_pre(void); + +void roce3_service_init_ext(void); + +void roce3_lock_rdev(void); + +void roce3_unlock_rdev(void); + +int roce3_get_rdev_by_uld(struct hinic3_lld_dev *lld_dev, void *uld_dev, struct roce3_device **rdev, + struct hinic3_event_info *event); + +void roce3_init_dev_ext_handlers(struct roce3_device *rdev); + +int roce3_board_cfg_check(struct roce3_device *rdev); + +int roce3_init_dev_ext(struct roce3_device *rdev); + +void roce3_remove_clean_res_ext(struct roce3_device *rdev); + +int roce3_set_comm_event(const struct roce3_device *rdev, const struct hinic3_event_info *event); + +bool roce3_hca_is_present(const struct roce3_device *rdev); + +int roce3_mmap_ext(struct roce3_device *rdev, struct roce3_ucontext *ucontext, + struct vm_area_struct *vma); + +int roce3_dfx_mem_alloc(struct roce3_device *rdev); + +void roce3_dfx_mem_free(struct roce3_device *rdev); + +int ib_copy_to_udata_ext(struct ib_udata *udata, struct roce3_alloc_ucontext_resp *resp); + +void *roce3_ucontext_alloc_ext(void); + +void roce3_resp_set_ext(struct roce3_device *rdev, struct roce3_alloc_ucontext_resp *resp); + +void *roce3_resp_alloc_ext(void); + +void roce3_ucontext_set_ext(struct roce3_device *rdev, struct roce3_ucontext *context); + +void *roce3_rdev_alloc_ext(void); + +void roce3_rdev_set_ext(struct roce3_device *rdev); + +void roce3_rdma_cap_ext(struct rdma_service_cap *rdma_cap); + +#endif /* ROCE_MAIN_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h new file mode 100644 index 000000000..114b03f2d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_mr_extension.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MR_EXTEND_H +#define ROCE_MR_EXTEND_H + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" + +#include "roce.h" + +int roce3_check_alloc_mr_type(enum ib_mr_type mr_type); +enum rdma_mr_type roce3_get_mrtype(enum ib_mr_type ib_mr_type); + +#endif + diff --git a/drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h new file mode 100644 index 000000000..d97d1b0d5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_netdev_extension.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_NETDEV_EXTENSION_H +#define ROCE_NETDEV_EXTENSION_H + +#include "roce_netdev.h" + +int roce3_add_real_device_mac(struct roce3_device *rdev, struct net_device *netdev); + +int roce3_add_vlan_device_mac(struct roce3_device *rdev, struct net_device *netdev); + +void roce3_del_real_device_mac(struct roce3_device *rdev); + +void roce3_del_vlan_device_mac(struct roce3_device *rdev, struct roce3_vlan_dev_list *old_list); + +void roce3_event_up_extend(struct roce3_device *rdev); + +#endif /* ROCE_NETDEV_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h new file mode 100644 index 000000000..9667039c3 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_qp_extension.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_EXTENSION_H +#define ROCE_QP_EXTENSION_H + +#include <rdma/ib_verbs.h> + +#include "roce_qp.h" +#include "roce_qp_exp.h" + +int to_roce3_qp_type(enum ib_qp_type qp_type); + +bool roce3_check_qp_modify_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, + enum ib_qp_type type, enum ib_qp_attr_mask mask, enum rdma_link_layer ll); + +int roce3_create_qp_pre_ext(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr); + +int roce3_create_qp_user_pre_ext(struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, u32 *qpn); + +int roce3_create_qp_user_post_ext(struct ib_pd *ibpd, struct roce3_device *rdev, + struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr); + +int roce3_qp_modify_cmd_ext(struct tag_cqm_cmd_buf *cqm_cmd_inbuf, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar); + +bool roce3_need_qpn_lb1_consistent_srqn(const struct roce3_qp *rqp, const struct roce3_device *rdev, + const struct ib_qp_init_attr *init_attr); + +int roce3_is_qp_normal(struct roce3_qp *rqp, struct ib_qp_init_attr *init_attr); + +void roce3_set_qp_dif_attr(struct roce3_qp *rqp, const struct ib_qp_init_attr *init_attr, + const struct roce3_device *rdev); + +int roce3_qp_modify_pre_extend(struct roce3_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata); + +#ifdef ROCE_EXTEND + +#define QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC 204800 // 200K + +#define ROCE_QP_VBS_FLAG (1U << 22) + +struct roce3_set_qp_ext_attr_cmd { + u32 qpn; + u32 attr_mask; +}; + +struct roce3_modify_qp_vbs_cmd { + u32 qpn; + u64 ci_record_addr; +}; + +struct roce3_vbs_qp { + struct tag_cqm_cmd_buf *vbs_sqpc_info; /* vbs private data */ + struct roce3_db db; /* to record ci_record_pa */ +}; + +long roce3_set_qp_ext_attr(struct roce3_device *rdev, void *buf); +long roce3_vbs_create_sqpc(struct roce3_device *rdev, void *buf); +#endif + +#endif /* ROCE_QP_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h new file mode 100644 index 000000000..966c94a6a --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_qp_post_send_extension.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_POST_SEND_EXTEND_H +#define ROCE_QP_POST_SEND_EXTEND_H + +#include <rdma/ib_verbs.h> +#include "roce_qp.h" + +int roce3_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, +const struct ib_send_wr **bad_wr); + +#endif // ROCE_QP_POST_SEND_EXTEND_H diff --git a/drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h b/drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h new file mode 100644 index 000000000..bf2d73cca --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/include/roce_srq_extension.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_SRQ_EXTENSION_H +#define ROCE_SRQ_EXTENSION_H + +#include "roce_srq.h" + +void roce3_srq_container_init(struct ib_srq_init_attr *init_attr, +struct roce3_srq *rsrq, struct roce3_device *rdev); + +void roce3_create_user_srq_update_ext(u32 *cqn, u32 srqn); + +#endif /* ROCE_SRQ_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce3/mr/roce_mr.c b/drivers/infiniband/hw/hiroce3/mr/roce_mr.c new file mode 100644 index 000000000..62ee3ac9e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/mr/roce_mr.c @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_mr.h" +#include "roce_mr_extension.h" +#include "roce_main_extension.h" + +/* + **************************************************************************** + Prototype : get_key_from_index + Description : get_key_from_index + Input : u32 mpt_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static u32 get_key_from_index(u32 mpt_index) +{ + /* Algorithm of mr key is obtained by index shift calculation */ + return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) | (mpt_index << MR_KEY_LEFT_SHIFT_OFS); +} + +/* + **************************************************************************** + Prototype : convert_ib_access + Description : convert ib access to rdma comp access + Input : int access_flag + Output : None + + 1.Date : 2015/7/16 + Modification : Created function + +**************************************************************************** +*/ +static u32 convert_ib_access(int access_flag) +{ + u32 u_access_flag = (u32)access_flag; + u32 access; + + access = (u32)((((u_access_flag & IB_ACCESS_REMOTE_ATOMIC) != 0) ? + RDMA_IB_ACCESS_REMOTE_ATOMIC : 0) | + (((u_access_flag & IB_ACCESS_REMOTE_WRITE) != 0) ? + RDMA_IB_ACCESS_REMOTE_WRITE : 0) | + (((u_access_flag & IB_ACCESS_REMOTE_READ) != 0) ? + RDMA_IB_ACCESS_REMOTE_READ : 0) | + (((u_access_flag & IB_ACCESS_LOCAL_WRITE) != 0) ? + RDMA_IB_ACCESS_LOCAL_WRITE : 0) | + (((u_access_flag & IB_ACCESS_MW_BIND) != 0) ? + RDMA_IB_ACCESS_MW_BIND : 0) | + (((u_access_flag & IB_ZERO_BASED) != 0) ? + RDMA_IB_ACCESS_ZERO_BASED : 0)); + + return access; +} + +/* + **************************************************************************** + Prototype : check_ib_access + Description : check remote access without local write + Input : int access_flag + Output : None + + 1.Date : 2017/11/18 + Modification : Created function + +**************************************************************************** +*/ +static int check_ib_access(int access_flag) +{ + /* + * Local write permission is required if remote write or + * remote atomic permission is also requested. + */ + + if (((((u32)access_flag) & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE)) != 0) && + ((((u32)access_flag) & IB_ACCESS_LOCAL_WRITE) == 0)) + return -EINVAL; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_alloc_tpt + Description : alloc mpt and mtt + Input : struct roce3_device *rdev + struct rdma_mr *mr + u32 npages + u32 page_shift + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_alloc_tpt(struct roce3_device *rdev, struct rdma_mr *mr, u32 npages, u32 page_shift) +{ + int ret = 0; + + ret = hmm_rdma_mpt_alloc(rdev->hwdev, &mr->mpt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mpt, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + return ret; + } + + mr->enabled = RDMA_MPT_EN_SW; + + /* + * npages = 0 is a legal case, when npages = 0 or npages = 1, + * MTT does not need to do address translation + */ + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, page_shift, &mr->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mtt, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + goto err_alloc_mtt; + } + + return 0; + +err_alloc_mtt: + hmm_rdma_mpt_free(rdev->hwdev, &mr->mpt); + mr->enabled = RDMA_MPT_DISABLED; + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_free_tpt + Description : free mpt and mtt + Input : struct roce3_device *rdev + struct rdma_mr *mr + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +void roce3_free_tpt(struct roce3_device *rdev, struct rdma_mr *mr) +{ + hmm_rdma_mtt_free(rdev->hwdev, &mr->mtt, SERVICE_T_ROCE); + + hmm_rdma_mpt_free(rdev->hwdev, &mr->mpt); + + mr->enabled = RDMA_MPT_DISABLED; +} + +/* + **************************************************************************** + Prototype : roce3_set_rdma_mr + Description : set the member of rdma_mr + Input : struct rdma_mr *mr + enum rdma_mr_type mr_type + u32 pdn + u64 iova + u64 size + u32 access + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +void roce3_set_rdma_mr(struct rdma_mr *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access) +{ + mr->iova = iova; + mr->size = size; + mr->pdn = pdn; + /* Convert ib permissions to rdma component permissions */ + mr->access = convert_ib_access((int)access); + /* Convert from mpt index to key */ + mr->key = get_key_from_index(mr->mpt.mpt_index); + mr->mr_type = mr_type; +} + +/* + **************************************************************************** + Prototype : roce3_set_rdma_mw + Description : set the member of rdma_mw + Input : struct rdma_mw *mw + u32 pdn + enum ib_mw_type type + Output : None + + 1.Date : 2016/6/25 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_rdma_mw(struct rdma_mw *mw, u32 pdn, enum ib_mw_type type) +{ + mw->enabled = RDMA_MPT_EN_SW; + mw->pdn = pdn; + mw->key = get_key_from_index(mw->mpt.mpt_index); + + if (type == IB_MW_TYPE_1) + mw->type = RDMA_MW_TYPE_1; + else + mw->type = RDMA_MW_TYPE_2; +} + +/* + **************************************************************************** + Prototype : roce3_get_dma_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + int access + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +struct ib_mr *roce3_get_dma_mr(struct ib_pd *ibpd, int access) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + struct roce3_pd *pd = NULL; + struct roce3_device *rdev = NULL; + + if ((ibpd == NULL) || (check_ib_access(access) != 0)) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: Invalid Param.p1:%d\n", __func__, access); + goto err_out; + } + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + ret = -EPERM; + goto err_out; + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc memory for dma mr, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + ret = roce3_alloc_tpt(rdev, &mr->rdmamr, 0, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mpt and mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_tpt; + } + + /* Set the content in rdma_mr */ + roce3_set_rdma_mr(&mr->rdmamr, RDMA_DMA_MR, pd->pdn, 0ULL, ROCE_DMA_MR_SIZE, (u32)access); + + ret = hmm_rdma_enable_mr_mpt(rdev->hwdev, &(mr->rdmamr), HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to enable mpt of DMA mr, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_enable_mpt; + } + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_enable_mpt: + roce3_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce3_alloc_priv_pages(struct ib_device *ibdev, struct roce3_mr *rmr, u32 max_pages) +{ + int ret; + +#ifndef __PC_LINT__ + rmr->page_map_size = roundup(max_pages * sizeof(u64), ROCE3_MR_PAGES_ALIGN); +#endif + rmr->pages = (__be64 *)(void *)(uintptr_t)get_zeroed_page(GFP_KERNEL); + if (rmr->pages == NULL) { + pr_err("[ROCE] %s: Failed to alloc rmr->pages\n", __func__); + ret = -ENOMEM; + goto err_out; + } + + rmr->page_map = dma_map_single(ibdev->dev.parent, rmr->pages, + rmr->page_map_size, DMA_TO_DEVICE); + if (dma_mapping_error(ibdev->dev.parent, rmr->page_map) != 0) { + pr_err("[ROCE] %s: Failed to do dma mapping\n", __func__); + ret = -ENOMEM; + goto err_free_pages; + } + + return 0; + +err_free_pages: + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + +err_out: + return ret; +} + +static void roce3_free_priv_pages(struct roce3_mr *rmr) +{ + struct ib_device *ib_dev = rmr->ibmr.device; + + if (rmr->pages) { + ib_dev = rmr->ibmr.device; + dma_unmap_single(ib_dev->dev.parent, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + rmr->pages = NULL; + } +} + +static int roce3_alloc_mr_param_validate(const struct ib_pd *ibpd, + enum ib_mr_type mr_type, u32 max_num_sg) +{ + int ret = 0; + + if (ibpd == NULL) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: Ibpd is null\n", __func__); + goto err_out; + } + + /*lint -e746*/ + ret = roce3_check_alloc_mr_type(mr_type); + /*lint +e746*/ + if (ret != 0) { + ret = -EINVAL; + pr_err("[ROCE, ERR] %s: mr_type is invalid\n", __func__); + goto err_out; + } + + if (max_num_sg >= ROCE_FRMR_MAX_PAGES) { + pr_err("[ROCE] %s: Invalid max_num_sg(%d)\n", __func__, max_num_sg); + ret = -EINVAL; + goto err_out; + } + + return 0; + +err_out: + return ret; +} + +static int roce3_alloc_mr_param_check(struct roce3_device *rdev, + struct roce3_mr **mr, u32 max_num_sg) +{ + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + + *mr = kzalloc(sizeof(struct roce3_mr), GFP_KERNEL); + if ((*mr) == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mr memory, func(%d)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + + (*mr)->max_pages = max_num_sg; + return 0; +} +/* + **************************************************************************** + Prototype : roce3_alloc_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + enum ib_mr_type mr_type + u32 max_num_sg + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +struct ib_mr *roce3_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + struct roce3_pd *pd = NULL; + struct roce3_device *rdev = NULL; + + ret = roce3_alloc_mr_param_validate(ibpd, mr_type, max_num_sg); + if (ret != 0) + goto err_out; + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + + ret = roce3_alloc_mr_param_check(rdev, &mr, max_num_sg); + if (ret != 0) + goto err_out; + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + ret = roce3_alloc_tpt(rdev, &mr->rdmamr, max_num_sg, 0); // alloc mptc and mtt + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc tpt, func(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_tpt; + } + + ret = roce3_alloc_priv_pages(ibpd->device, mr, max_num_sg); // alloc memory to store pa + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: Failed to alloc memory, func(%u)\n", + __func__, rdev->glb_func_id); + goto err_alloc_priv_pages; + } + + /*lint -e746*/ + roce3_set_rdma_mr(&mr->rdmamr, RDMA_FRMR, pd->pdn, 0ULL, 0, 0); + /*lint +e746*/ + /* send to chip by cmdq */ + ret = hmm_rdma_enable_mr_mpt(rdev->hwdev, &(mr->rdmamr), HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to enable mpt, func(%d)\n", + __func__, rdev->glb_func_id); + goto err_enale_mpt; + } + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_enale_mpt: + roce3_free_priv_pages(mr); + +err_alloc_priv_pages: + roce3_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce3_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct roce3_mr *rmr = NULL; + + if (ibmr == NULL) { + pr_err("[ROCE] %s: Ibmr is null\n", __func__); + return -EINVAL; + } + + rmr = to_roce3_mr(ibmr); + if (ROCE_UNLIKELY(rmr->npages >= rmr->max_pages)) { + pr_err("[ROCE] %s: Invalid npages(0x%x), can't set page\n", __func__, rmr->npages); + return -ENOMEM; + } + + rmr->pages[rmr->npages] = cpu_to_be64(addr | 0x1); + rmr->npages++; + + return 0; +} + +int roce3_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + int ret; + struct roce3_mr *rmr = NULL; + + /* sg_offset can be null */ + if ((ibmr == NULL) || (sg == NULL)) { + pr_err("[ROCE] %s: Ibmr or sg is null\n", __func__); + return -EINVAL; + } + + rmr = to_roce3_mr(ibmr); + rmr->npages = 0; + + if (ROCE_UNLIKELY(((u32)sg_nents) >= rmr->max_pages)) { + pr_err("[ROCE] %s: Invalid sg_nents(0x%x), ,max(0x%x)\n", + __func__, sg_nents, rmr->max_pages); + return -EINVAL; + } + + ib_dma_sync_single_for_cpu(ibmr->device, rmr->page_map, rmr->page_map_size, DMA_TO_DEVICE); + + ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, roce3_set_page); + + ib_dma_sync_single_for_device(ibmr->device, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + + return ret; +} + +static int roce3_umem_write_mtt_check(const struct roce3_device *rdev, const struct rdma_mtt *mtt, + const struct ib_umem *umem) +{ + if ((rdev == NULL) || (mtt == NULL) || (umem == NULL)) { + pr_err("[ROCE, ERR] %s: Rdev or mtt or umem is null\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int roce3_umem_write_mtt_update(struct roce3_device *rdev, + struct rdma_mtt *mtt, struct ib_umem *umem, u64 *page_list) +{ + int ret = 0; + int i = 0; + u32 j = 0; + u32 pages_in_chunk = 0; /* The number of pages of a single memory block in umem_chunk */ + u32 npages = 0; /* The number of recorded pages */ + u32 start_index = 0; /* page need to be written to mtt */ + struct scatterlist *sg = NULL; + u64 page_size = BIT(mtt->buf_page_shift); + u64 page_mask = ~(page_size - 1); + u64 dma_addr; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + /* + * Calculate the number of pages in a memory block, + * the page size is 1 << page_shift + */ + dma_addr = sg_dma_address(sg) & page_mask; + pages_in_chunk = DIV_ROUND_UP(sg_dma_len(sg), page_size); + for (j = 0; j < pages_in_chunk; ++j) { + page_list[npages] = dma_addr + (page_size * j); + npages++; + /* + * The size of the memory pointed to by page_list is one page, + * which can store up to PAGE_SIZE / sizeof(u64) pas. After a page is full, + * mtt needs to be written, else continue + */ + if (npages != PAGE_SIZE / sizeof(u64)) + continue; + + ret = hmm_rdma_write_mtt(rdev->hwdev, mtt, start_index, + npages, page_list, SERVICE_T_ROCE); + if (ret) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + start_index += npages; + npages = 0; + } + } + + if (npages != 0) { + ret = hmm_rdma_write_mtt(rdev->hwdev, mtt, start_index, npages, + page_list, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, ret(%d), start_index(%u), func_id(%u)\n", + __func__, ret, start_index, rdev->glb_func_id); + } + } + + return ret; +} + +int roce3_umem_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct ib_umem *umem) +{ + int ret = 0; + u64 *page_list = NULL; /* page_list to write to mtt */ + + ret = roce3_umem_write_mtt_check(rdev, mtt, umem); + if (ret != 0) + return ret; + + page_list = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) + return -ENOMEM; + + ret = roce3_umem_write_mtt_update(rdev, mtt, umem, page_list); + kfree(page_list); + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_buf_write_mtt + Description : roce3_buf_write_mtt + Input : struct roce3_device *rdev + struct rdma_mtt *mtt + struct tag_cqm_buf *buf + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +int roce3_buf_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct tag_cqm_buf *buf) +{ + u64 *page_list = NULL; + int ret = 0; + u32 i = 0; + + if ((rdev == NULL) || (mtt == NULL) || (buf == NULL)) { + pr_err("[ROCE, ERR] %s: Rdev or mtt or buf is null\n", __func__); + return (-EINVAL); + } + + page_list = kzalloc(buf->buf_number * sizeof(*page_list), GFP_KERNEL); + if (page_list == NULL) + return (-ENOMEM); + + /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */ + for (i = 0; i < buf->buf_number; ++i) + page_list[i] = buf->buf_list[i].pa; + + ret = hmm_rdma_write_mtt(rdev->hwdev, mtt, 0, buf->buf_number, page_list, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + } + + kfree(page_list); + + return ret; +} + +int roce3_user_mr_reg(struct roce3_device *rdev, struct roce3_mr *mr, u32 pdn, u64 virt_addr, + u64 length, int access) +{ + int ret = 0; + u32 npages = 0; + u32 page_shift = PAGE_SHIFT; + struct hinic3_hwdev *hwdev = (struct hinic3_hwdev *)rdev->hwdev; + + if (hwdev == NULL) { + pr_err("[HMM, ERR] %s(%d): hwdev is null\n", __func__, __LINE__); + return 0; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + npages = (u32)ib_umem_num_pages(mr->umem); +#ifdef CONFIG_CENTRALIZED_STORAGE + if (mr->umem->is_umm_mem) { + page_shift = PAGE_SHIFT_2M; + npages = ib_umem_num_dma_blocks(mr->umem, BIT(page_shift)); + } +#endif + ret = roce3_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[ROCE, ERR] %s(%d): Failed to alloc mpt and mtt, func_id(%u)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + roce3_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length, (u32)access); + + ret = roce3_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[ROCE, ERR] %s(%d): Failed to write mtt, func_id(%u)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + ret = hmm_rdma_enable_mr_mpt(hwdev, &mr->rdmamr, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(hwdev->dev_hdl, + "[ROCE, ERR] %s(%d): Failed to enable mpt of user mr, func_id(%u)\n", + __func__, __LINE__, hinic3_global_func_id(hwdev)); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + roce3_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + return ret; +} + +static int roce3_check_mr_param(struct ib_pd *ibpd) +{ + struct roce3_device *rdev = NULL; + + if (ibpd == NULL) { + pr_err("[ROCE, ERR] %s: Ibpd is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibpd->device); + if (roce3_hca_is_present(rdev) == 0) { + pr_err("[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_reg_user_mr + Description : register MR for user + Input : struct ib_pd *ibpd + u64 start + u64 length + u64 virt_addr + int access + struct ib_udata *udata + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** +*/ +struct ib_mr *roce3_reg_user_mr(struct ib_pd *ibpd, u64 start, + u64 length, u64 virt_addr, int access, struct ib_udata *udata) +{ + int ret; + struct roce3_mr *mr = NULL; + struct roce3_device *rdev = NULL; + struct roce3_pd *pd; + + ret = roce3_check_mr_param(ibpd); + if (ret != 0) + goto err_out; + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc memory for roce mr, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + mr->umem = ib_umem_get(&rdev->ib_dev, start, (size_t)length, access); + if (IS_ERR(mr->umem)) { + ret = (int)PTR_ERR(mr->umem); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get ib umem, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_get_umem; + } + + ret = roce3_user_mr_reg(rdev, mr, pd->pdn, virt_addr, length, access); + if (ret != 0) + goto err_mr_update; + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_mr_update: + ib_umem_release(mr->umem); + +err_get_umem: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +int roce3_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + struct roce3_device *rdev = NULL; + + if (ibmr == NULL) { + pr_err("[ROCE, ERR] %s: Ibmr is null\n", __func__); + return -EINVAL; + } + + mr = to_roce3_mr(ibmr); + rdev = to_roce3_dev(ibmr->device); + + roce3_free_priv_pages(mr); + + ret = hmm_rdma_disable_mr_mpt(rdev->hwdev, &mr->rdmamr, SERVICE_T_ROCE, + HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to disable mpt of mr, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Mr may has mw bind on it, mr_key(%#x), func_id(%d)\n", + __func__, mr->rdmamr.key, rdev->glb_func_id); + if (ret == (-RDMA_CMDQ_TIMEOUT)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return ret; + } + + roce3_free_tpt(rdev, &mr->rdmamr); + + if (mr->umem) + ib_umem_release(mr->umem); + + return 0; +} + +static void roce3_err_enable_mpt_handler(void *hwdev, struct roce3_mw *mw) +{ + hmm_rdma_mpt_free(hwdev, &mw->rdmamw.mpt); + mw->rdmamw.enabled = RDMA_MPT_DISABLED; +} + +int roce3_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) +{ + int ret; + struct roce3_mw *mw = to_roce3_mw(ibmw); + struct roce3_pd *pd = to_roce3_pd(ibmw->pd); + struct roce3_device *rdev = to_roce3_dev(ibmw->device); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = hmm_rdma_mpt_alloc(rdev->hwdev, &mw->rdmamw.mpt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc mpt, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err; + } + + roce3_set_rdma_mw(&mw->rdmamw, pd->pdn, ibmw->type); + ret = roce3_rdma_enable_mw_mpt(rdev->hwdev, &mw->rdmamw, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to enable mpt of mw, func_id(%u)\n", + __func__, rdev->glb_func_id); + if (ret == (-RDMA_CMDQ_TIMEOUT)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + goto err_enable_mpt; + } + + mw->ibmw.rkey = mw->rdmamw.key; + return 0; +err_enable_mpt: + roce3_err_enable_mpt_handler(rdev->hwdev, mw); +err: + return ret; +} + +int roce3_dealloc_mw(struct ib_mw *ibmw) +{ + int ret = 0; + struct roce3_mw *mw = NULL; + struct roce3_device *rdev = NULL; + + if (ibmw == NULL) { + pr_err("[ROCE, ERR] %s: Ibmw is null\n", __func__); + return -EINVAL; + } + + mw = to_roce3_mw(ibmw); + rdev = to_roce3_dev(ibmw->device); + ret = roce3_rdma_disable_mw_mpt(rdev->hwdev, &mw->rdmamw, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to disable mpt of mw, func_id(%d)\n", + __func__, rdev->glb_func_id); + if (ret == (-RDMA_CMDQ_TIMEOUT)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return ret; + } + + hmm_rdma_mpt_free(rdev->hwdev, &mw->rdmamw.mpt); + + mw->rdmamw.enabled = RDMA_MPT_DISABLED; + + return 0; +} + +int roce3_check_mr_status(struct ib_mr *ibmr, u32 check_mask, struct ib_mr_status *mr_status) +{ + int ret = 0; + struct roce3_mr *mr = NULL; + + if ((ibmr == NULL) || (mr_status == NULL)) { + pr_err("[ROCE, ERR] ibmr or mr_status is NULL ptr.\n"); + return -EINVAL; + } + + mr = to_roce3_mr(ibmr); + if ((check_mask & (~IB_MR_CHECK_SIG_STATUS)) != 0) { + pr_err("[ROCE, ERR] check_mask(%#x) invalid\n", check_mask); + return -EINVAL; + } + + if (!mr->signature_en) { + pr_err("[ROCE, ERR] mr not configure dif info\n"); + ret = -EINVAL; + return ret; + } + + mr->sig.sig_status_checked = true; + mr_status->fail_status = 0; + + /* If err does not exist, return 0 */ + if (!mr->sig.sig_err_exists) + return 0; + + memcpy((void *)&mr_status->sig_err, (void *)&mr->sig.sig_err_item, + sizeof(struct ib_sig_err)); + mr->sig.sig_err_exists = false; + mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/mr/roce_mr.h b/drivers/infiniband/hw/hiroce3/mr/roce_mr.h new file mode 100644 index 000000000..8f894b22c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/mr/roce_mr.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MR_H +#define ROCE_MR_H + +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/vmalloc.h> + +#include "hinic3_rdma.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_pd.h" +#include "roce_user.h" + +#include "hmm_mr.h" +#include "hmm_comp.h" +#include "hinic3_hmm.h" + +#define ROCE3_MR_DEFAULT_ACCESS \ + (IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE | \ + IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_ATOMIC) + +#define ROCE3_MR_PAGES_ALIGN 64 +#define ROCE3_MIN_PAGE 2 + +#define FRMR_PAGE_SIZE 4096 + +struct roce3_sig_ctx { + /* recorded when submitting WR */ + struct ib_sig_attrs sig_attrs; + /* Get information configuration from CQE when polling CQE */ + struct ib_sig_err sig_err_item; + /* Indicates whether the status is checked */ + bool sig_status_checked; + /* Indicates if an error exists */ + bool sig_err_exists; + /* Count the number of wrong CQEs */ + u32 sig_err_count; +}; + +struct roce3_mr { + /* mr structure provided by ofed */ + struct ib_mr ibmr; + /* Maintain the structure where the user-mode allocated buffer */ + struct ib_umem *umem; + /* mr structure provided by rdma component */ + struct rdma_mr rdmamr; + /* Record signature related information */ + struct roce3_sig_ctx sig; + /* Whether the logo contains signature information */ + bool signature_en; + __be64 *pages; + u32 npages; + u32 max_pages; + size_t page_map_size; + dma_addr_t page_map; +}; + +struct roce3_mw { + /* mw structure provided by ofed */ + struct ib_mw ibmw; + /* mw structure provided by rdma component */ + struct rdma_mw rdmamw; +}; + +/* through ibmroce3_e_mr */ +static inline struct roce3_mr *to_roce3_mr(const struct ib_mr *ibmr) +{ + return container_of(ibmr, struct roce3_mr, ibmr); +} + +/* through ibmroce3_e_mw */ +static inline struct roce3_mw *to_roce3_mw(const struct ib_mw *ibmw) +{ + return container_of(ibmw, struct roce3_mw, ibmw); +} + +/* through rdmam find roce3_mr */ +static inline struct roce3_mr *rdma_mr_to_roce3_mr(const struct rdma_mr *rdmamr) +{ + return container_of(rdmamr, struct roce3_mr, rdmamr); +} + +int roce3_alloc_tpt(struct roce3_device *rdev, struct rdma_mr *mr, u32 npages, u32 page_shift); +void roce3_free_tpt(struct roce3_device *rdev, struct rdma_mr *mr); +void roce3_set_rdma_mr(struct rdma_mr *mr, enum rdma_mr_type mr_type, + u32 pdn, u64 iova, u64 size, u32 access); + +int roce3_user_mr_reg(struct roce3_device *rdev, struct roce3_mr *mr, u32 pdn, u64 virt_addr, + u64 length, int access); + +#endif // __ROCE_MR_H_ diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_post.h b/drivers/infiniband/hw/hiroce3/qp/roce_post.h new file mode 100644 index 000000000..c1b622cee --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_post.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_POST_H +#define ROCE_POST_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "roce_wqe_format.h" +#include "roce_xqe_format.h" + +#define ROCE_SQ_DB_TYPE 2 +#define ROCE_UD_MTU_SHIFT 3 /* 4K mtu */ +#define ROCE_IMM_EXT_LEN 4 +#define ROCE_TASK_SEG_ALIGN 8 + +#define ROCE_ATOMIC_WR atomic_wr +#define ROCE_RDMA_WR rdma_wr +#define ROCE_REG_WR reg_wr +#define ROCE_UD_WR ud_wr + +enum roce_tsl_8_byte_aligned_size_e { + ROCE_SEND_LOCAL_WQE_TSL = 2, + ROCE_RDMA_WQE_TSL = 4, + ROCE_ATOMIC_CWP_WQE_TSL = 6, + ROCE_UD_WQE_COM_TSL = 8 +}; + +enum { + ROCE_WQE_OPCODE_SEND = 0x00, + ROCE_WQE_OPCODE_SEND_INVAL = 0x01, + ROCE_WQE_OPCODE_SEND_IMM = 0x02, + + ROCE_WQE_OPCODE_RDMA_WRITE = 0x04, + ROCE_WQE_OPCODE_RDMA_WRITE_IMM = 0x05, + + ROCE_WQE_OPCODE_RDMA_READ = 0x08, + + ROCE_WQE_OPCODE_ATOMIC_CMP_SWP = 0x0c, + ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD = 0x0d, + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP = 0x0e, + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD = 0x0f, + + ROCE_WQE_OPCODE_FRMR = 0x10, + ROCE_WQE_OPCODE_LOCAL_INVAL = 0x11, + ROCE_WQE_OPCODE_BIND_MW = 0x12, + ROCE_WQE_OPCODE_REG_SIG_MR = 0x13 /* Extended for further local opreation */ +}; + +enum { + ROCE_DWQE_DB_SUBTYPE_SEND = 0x1, + ROCE_DWQE_DB_SUBTYPE_SEND_IMM = 0x2, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE = 0x3, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE_IMM = 0x4, + ROCE_DWQE_DB_SUBTYPE_RDMA_READ = 0x5, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_CMP_SWP = 0x6, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_FETCH_ADD = 0x7 +}; + +/* UD send WQE task seg1 */ +struct roce3_wqe_ud_tsk_seg_cycle1 { + union roce3_wqe_tsk_com_seg common; + + /* DW0 */ + u32 data_len; + + /* DW1 */ + u32 immdata_invkey; + + /* DW2 */ +/* + * 0: No limit on the static rate (100% port speed) + * 1-6: reserved + * 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s. + * 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. + */ + union { + struct { + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; + u32 rsvd1 : 3; + u32 fl : 1; + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { + u32 hop_limit : 8; + u32 sgid_idx : 7; + u32 rsvd0 : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tc : 8; + } bs; + u32 value; + } dw3; + + /* DW4 */ + union { + struct { + u32 flow_label : 20; + u32 rsvd0 : 4; + u32 smac_index : 3; + u32 rsvd1 : 5; + } bs; + u32 value; + } dw4; + + /* DW5~8 */ + u8 dgid[16]; + + /* DW9 */ + union { + struct { + u32 dst_qp : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw9; + + /* DW10 */ + u32 qkey; +}; + +/* UD send WQE task seg2; */ +struct roce3_wqe_ud_tsk_seg_cycle2 { + /* DW0 */ + union { + struct { + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 cfi : 1; + u32 vlan_prio : 3; + } bs; + u32 value; + } dw0; + + /* DW1 */ + u32 dmac_l32; +}; + +struct roce3_post_send_normal_param { + struct roce3_wqe_ctrl_seg *ctrl_seg; + union roce3_wqe_tsk_com_seg *tsk_com_seg; + u32 wqe_size; + u8 *wqe; + struct roce3_wqe_data_seg *dseg; + union roce_sq_db sq_db; + u32 wr_num; /* record posted WR numbers */ + u32 index; /* WQEBB id */ + int inline_flag; /* Inline flag */ + u32 data_len; + u32 *data_len_addr; + u32 sq_rmd_size; + s32 opcode; + s32 cycle; + struct roce3_wqe_ctrl_seg ctrl_seg_tmp; + + unsigned long flags; +}; + +#endif /* ROCE_POST_H */ diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp.h b/drivers/infiniband/hw/hiroce3/qp/roce_qp.h new file mode 100644 index 000000000..a76a464a5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_H +#define ROCE_QP_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" +#include "hinic3_cqm.h" + +#include "roce.h" +#include "roce_cq.h" +#include "roce_db.h" +#include "roce_pd.h" + +#include "rdma_context_format.h" + +#define ROCE_RQ_MIN_SGE 4 +#define ROCE_RQ_MID_SGE 8 +#define ROCE_RQ_MAX_SGE 16 + +#define ROCE_WQE_NEXT_SGE_INVALID (1UL << 31) + +#define ROCE_QP_TIMER_CHECK_VALUE 0xce +#define ROCE_QP_DESTROY_CHECK_VALUE 0xffffffff +#define ROCE_RDMARC_DESTROY_CHECK_VALUE 0xcece5a5a +#define ROCE_CACHE_LINE_SIZE 256 +#define RDMARC_NUM_PER_CACHELINE 8 +#define RDMARC_TABLE_ENTRY_SIZE 32 +#define ROCE_STATIC_RDMARC_NUM 65536 +#define ROCE_QP_MAX_PFETCH_MTT_LAYER 3 +#define ROCE_QP_MAX_TIMER_NUM 5 + +#define ROCE_QP_SMI_QP_NUM 0x0 +#define ROCE_QP_GSI_QP_NUM 0x1 +#define ROCE_QP_INVLID_QP_NUM CQM_INDEX_INVALID + +#define ROCE_QP_MAX_DWQE_SIZE 256 +#define ROCE_QP_DEFAULT_WQE_SHIFT 6 + +#define ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE 512 + +#define ROCE_QP_STATE_MEM_INIT 0xa + +#define ROCE_UD_MAX_INLINE_LEN_SUB (40) + +#define ROCE_QP_GPA_SIG_LEN 3 +#define ROCE_WR_MIN_NUM 2 + +#define ROCE_QPC_PMTU_TRANSLATE(pmtu) ((1U << ((pmtu) - 1)) - 1) + +#define ROCE_QP_OPTPAR_ALT_ADDR_PATH (1 << 0) +#define ROCE_QP_OPTPAR_RRE (1 << 1) +#define ROCE_QP_OPTPAR_RAE (1 << 2) +#define ROCE_QP_OPTPAR_RWE (1 << 3) +#define ROCE_QP_OPTPAR_PKEY_INDEX (1 << 4) +#define ROCE_QP_OPTPAR_Q_KEY (1 << 5) +#define ROCE_QP_OPTPAR_RNR_TIMEOUT (1 << 6) +#define ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH (1 << 7) +#define ROCE_QP_OPTPAR_SRA_MAX (1 << 8) +#define ROCE_QP_OPTPAR_RRA_MAX (1 << 9) +#define ROCE_QP_OPTPAR_PM_STATE (1 << 10) +#define ROCE_QP_OPTPAR_RETRY_COUNT (1 << 11) +#define ROCE_QP_OPTPAR_RNR_RETRY (1 << 12) +#define ROCE_QP_OPTPAR_ACK_TIMEOUT (1 << 13) +#define ROCE_QP_OPTPAR_SCHED_QUEUE (1 << 14) +#define ROCE_QP_OPTPAR_COUNTER_INDEX (1 << 15) + +#define ROCE3_LB1_MASK 0x3 +#define ROCE3_QPN_BIT_INDEX 29U +#define ROCE3_QPN_CTRL_BIT_NUM 3U + +#define ROCE_VLAN_DIS 0xFFFF + +enum roce_qp_state { + ROCE_QP_STATE_RST = 0, + ROCE_QP_STATE_INIT = 1, + ROCE_QP_STATE_RTR = 2, + ROCE_QP_STATE_RTS = 3, + ROCE_QP_STATE_SQER = 4, + ROCE_QP_STATE_SQD = 5, + ROCE_QP_STATE_ERR = 6, + ROCE_QP_STATE_SQ_DRAINING = 7, + ROCE_QP_STATE_NUM +}; + +enum roce_fake_vf_start_e { + ROCE_VLD_PF_NUM_10 = 10, + ROCE_VLD_PF_NUM_11 = 11, + ROCE_VLD_PF_NUM_12 = 12 +}; + +enum { + ROCE_QP_PM_MIGRATED = 0x3, + ROCE_QP_PM_ARMED = 0x0, + ROCE_QP_PM_REARM = 0x1 +}; + +enum { + ROCE_QP_ST_RC = 0x0, /* 000 */ + ROCE_QP_ST_UC = 0x1, /* 001 */ + ROCE_QP_ST_RD = 0x2, /* 010 */ + ROCE_QP_ST_UD = 0x3, /* 011 */ + ROCE_QP_ST_XRC = 0x6, /* 110 */ + ROCE_QP_ST_PRIV = 0x7 /* 111 */ +}; + +enum { + ROCE_QP_NO_SRQ, + ROCE_QP_HAS_SRQ +}; + +enum { + ROCE_QP = 0, + ROCE_QP_EXT +}; + +struct roce3_wq { + u64 *wrid; + spinlock_t lock; + u32 wqebb_cnt; + u32 max_post; + u32 max_sge; + u32 offset; + u32 wqe_shift; + u32 head; + u32 tail; +}; + +struct roce3_qp { + struct ib_qp ibqp; + u32 qpn; + struct roce3_db db; + struct tag_cqm_queue *qp_buf_info; + struct tag_cqm_qpc_mpt *qpc_info; + struct roce3_wq rq; + u32 sq_signal_bits; + unsigned int sq_next_wqe; + u32 sq_max_wqes_per_wr; + struct roce3_wq sq; + struct ib_umem *umem; + struct rdma_mtt mtt; + struct roce3_buf buf; + int buf_size; + struct mutex mutex; + u32 xrcdn; + u8 port; + u8 atomic_rd_en; + u8 qp_state; + u8 qp_ext; + u32 qp_type; + u32 max_inline_data; + int max_dwqe_size; + struct rdma_rdmarc rdmarc; + u16 rsp_depth; + u8 sl; + bool has_rq; + u8 *sq_head_addr; + u8 *sq_tail_addr; + bool signature_en; + u8 double_sgl_mode; + u8 ext_mtu; + u8 ext_mtu_mode; + + u8 db_sgid_index; + u8 db_path_mtu; + + u32 qid; + u32 local_comm_id; /* used by NOF AA to reuse qpn when disconnection happens */ + u32 remote_comm_id; + + /* hot-plug record */ + struct list_head qps_list; + struct list_head cq_recv_list; + struct list_head cq_send_list; +#if defined(ROCE_VBS_EN) || defined(ROCE_EXTEND) + void *vbs_qp_ptr; +#endif +#ifdef ROCE_BONDING_EN + u32 tx_hash_value; +#endif +}; + +struct roce3_sqp { + struct roce3_qp qp; + int pkey_index; + u32 qkey; + u32 send_psn; +}; + +static inline struct roce3_qp *to_roce3_qp(const struct ib_qp *ibqp) +{ + return container_of(ibqp, struct roce3_qp, ibqp); +} + +static inline struct roce3_qp *cqmobj_to_roce_qp(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info; + + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct roce3_qp *)qpc_info->priv; +} + +static inline struct roce3_sqp *to_roce3_sqp(const struct roce3_qp *rqp) +{ + return container_of(rqp, struct roce3_sqp, qp); +} + +struct roce3_qp_query_outbuf { + struct roce_qp_context qpc; +}; + +void roce3_qp_async_event(struct roce3_device *rdev, struct roce3_qp *qp, int type); +u8 roce3_get_db_cos_from_vlan_pri(struct roce3_device *rdev, u8 vlan_pri); + +void roce3_free_opt_rdmarc(struct roce3_qp *rqp); +void roce3_get_cqs(struct roce3_qp *rqp, struct roce3_cq **send_cq, struct roce3_cq **recv_cq); +int roce3_qp_modify_2rst_cmd(struct roce3_device *rdev, u32 qpn); +int roce3_qp_cache_out_cmd(struct roce3_device *rdev, struct roce3_qp *rqp); +struct roce3_pd *roce3_get_pd(struct roce3_qp *rqp); +int roce3_sqp_check(const struct roce3_qp *qp); +void *roce3_get_wqe(struct roce3_qp *rqp, u32 offset); +int roce3_wq_overflow(struct roce3_wq *wq, u32 wr_num, struct ib_cq *ibcq); +void roce3_set_data_seg(struct roce3_wqe_data_seg *dseg, struct ib_sge *sge); +void roce3_qpc_to_be(struct tag_roce_verbs_qp_attr *qp_attr, struct roce3_qp *rqp, u32 *be_ctx); +void roce3_qp_rst2init(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr); +struct ib_qp *roce3_create_qp_common(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int qp_ext); +int roce3_send_qp_lb_cmd(u32 qpn, struct roce3_device *rdev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, u32 timeout); + +int roce3_qp_query(struct roce3_device *rdev, u32 qpn, u32 *context, int qpc_size); +int roce3_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +void qpc_seg_to_le32(struct roce_qp_context *be_ctx, struct roce_qp_context *le_ctx, u32 srq_vld); +void roce3_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size); + +#define IB_QP_CREATE_SIGNATURE_EN IB_QP_CREATE_INTEGRITY_EN + +#endif // ROCE_QP_H diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c new file mode 100644 index 000000000..b88d4a568 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_create.c @@ -0,0 +1,1239 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_pub_cmd.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" + +#include "hinic3_hmm.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/* + **************************************************************************** + Prototype : roce3_check_rq_size + Description : roce3_check_rq_size + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_qp_cap *cap + bool has_rq + Output : None + + 1.Date : 2017/4/27 + Modification : Created function +**************************************************************************** +*/ +static int roce3_check_rq_size(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr, bool has_rq) +{ + u32 max_sge_num; + struct ib_qp_cap *cap = &init_attr->cap; + + max_sge_num = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg); + if ((cap->max_recv_wr > rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) || + (cap->max_recv_sge > max_sge_num)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check rq size, over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (has_rq && ((cap->max_recv_wr == 0) || (cap->max_recv_sge == 0))) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Has rq, max_recv_wr(%d), max_recv_sge(%d), func_id(%d)\n", + __func__, cap->max_recv_wr, cap->max_recv_sge, rdev->glb_func_id); + return -EINVAL; + } + + if ((!has_rq) && (cap->max_recv_wr != 0)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Not has rq, cap->max_recv_wr(%d), func_id(%d)\n", + __func__, cap->max_recv_wr, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_set_rq_size + Description : roce3_set_rq_size + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_qp_cap *cap + bool is_user + bool has_rq + Output : None + + 1.Date : 2015/4/29 + Modification : Created function +**************************************************************************** +*/ +/* + **************************************************************************** + * + * 1 there is SQ or RQ + * SQ/RQ depth must be power of 2 + * if max_send_wr=0, no need to alloc buf for SQ/RQ + + * RQ WQE SIZE equals 64 or128 + * SQ WQE SIZE equals multi of wqebb size + + * SQ max_gs, max 8 and min 0 + * RQ max_gs, can be 4 or 8, but can not be 0. + + * 2 there is no SQ or RQ + * all related params should be 0 + * + **************************************************************************** + */ +static int roce3_set_rq_size(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_init_attr *init_attr, bool is_user, bool has_rq) +{ + int ret; + u32 wqebb_num = 0; + u32 sge_total_len = 0; + u32 sge_num = 0; + struct ib_qp_cap *cap = &init_attr->cap; + + ret = roce3_check_rq_size(rdev, rqp, init_attr, has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + if (has_rq) { + wqebb_num = ROCE_MAX(2U, cap->max_recv_wr); + /*lint -e587*/ + rqp->rq.wqebb_cnt = (u32)(ROCE_ROUNDUP_POW_OF_TWO(wqebb_num) & 0xffffffff); + /*lint +e587*/ + + /* RQ SGE range [0,4] select 4,[5,8] select 8 */ + rqp->rq.max_sge = (cap->max_recv_sge <= ROCE_RQ_MIN_SGE) ? ROCE_RQ_MIN_SGE : + ((cap->max_recv_sge <= ROCE_RQ_MID_SGE) ? + ROCE_RQ_MID_SGE : ROCE_RQ_MAX_SGE); + + sge_total_len = (u32)((u32)rqp->rq.max_sge * sizeof(struct roce3_wqe_data_seg)); + rqp->rq.wqe_shift = (u32)ROCE_ILOG2(sge_total_len); + + if ((u32)((u32)rqp->rq.wqebb_cnt << (unsigned int)rqp->rq.wqe_shift) < PAGE_SIZE) + rqp->rq.wqebb_cnt = (PAGE_SIZE >> (unsigned int)rqp->rq.wqe_shift); + + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + rqp->rq.max_post = ROCE_MIN( + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + cap->max_recv_sge = (u32)rqp->rq.max_sge; + } else { + rqp->rq.max_post = ROCE_MIN( + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + sge_num = ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg); + cap->max_recv_sge = ROCE_MIN((u32)rqp->rq.max_sge, sge_num); + } + } else { + rqp->rq.wqebb_cnt = 0; + rqp->rq.max_sge = 0; + rqp->rq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->rq.max_post = 0; + cap->max_recv_sge = 0; + } + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + return 0; +} + +static int roce3_set_user_sq_size_check(struct roce3_device *rdev, const struct ib_qp_cap *cap, + const struct roce3_qp *rqp) +{ + u32 ud_max_inline_size; + + if (cap->max_send_wr > (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Log_sq_bb_count over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (rqp->qp_type == IB_QPT_UD) { + ud_max_inline_size = + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz > + ROCE_UD_MAX_INLINE_LEN_SUB) ? + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz - + ROCE_UD_MAX_INLINE_LEN_SUB) : + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + } else { + ud_max_inline_size = + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + } + + /* RC:192B inline data, UD:176B */ + if (cap->max_inline_data > ud_max_inline_size) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ max inline data over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} +/* + **************************************************************************** + Prototype : roce3_set_user_sq_size + Description : roce3_set_user_sq_size + Input : struct roce3_device *rdev + struct ib_qp_cap *cap + struct roce3_qp *rqp + struct roce3_create_qp *ucmd + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_set_user_sq_size(struct roce3_device *rdev, struct ib_qp_cap *cap, + struct roce3_qp *rqp, struct create_qp_cmd *ucmd) +{ + u32 tmp_max_sq_wqe_sz = 0; + u32 sq_buf_sz = 0; + int ret = 0; + + ret = roce3_set_user_sq_size_check(rdev, cap, rqp); + if (ret != 0) + return ret; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + /* between max and min */ + /*lint -e587*/ + tmp_max_sq_wqe_sz = (u32)( + ROCE_ROUNDUP_POW_OF_TWO(rdev->rdma_cap.max_sq_desc_sz) & 0xffffffff); + /*lint +e587*/ + if ((ucmd->log_sq_stride > ROCE_ILOG2(tmp_max_sq_wqe_sz)) || + (ucmd->log_sq_stride < ROCE_ILOG2(rdev->rdma_cap.wqebb_size))) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: WQE err, func_id(%d) log_sq:%d wqebb:%d max:%d\n", + __func__, rdev->glb_func_id, ucmd->log_sq_stride, + rdev->rdma_cap.wqebb_size, tmp_max_sq_wqe_sz); + return -EINVAL; + } + + rqp->sq.wqebb_cnt = (u32)(1U << ucmd->log_sq_bb_count); + rqp->sq.wqe_shift = ucmd->log_sq_stride; + + if ((u32)((u64)rqp->sq.wqebb_cnt << (unsigned int)rqp->sq.wqe_shift) < PAGE_SIZE) + rqp->sq.wqebb_cnt = (PAGE_SIZE >> (unsigned int)rqp->sq.wqe_shift); + + sq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->sq.wqebb_cnt << + (unsigned int)rqp->sq.wqe_shift, PAGE_SIZE); + rqp->buf_size = (int)ROCE_ALIGN( + ((unsigned int)sq_buf_sz + ((unsigned int)rqp->rq.wqebb_cnt << + (unsigned int)rqp->rq.wqe_shift)), PAGE_SIZE); + + rqp->sq.offset = 0; + rqp->rq.offset = sq_buf_sz; + + rqp->max_inline_data = cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ucmd->log_sq_stride; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_send_wqe_overhead + Description : roce3_send_wqe_overhead + Input : enum ib_qp_type qp_type + Output : None + + 1.Date : 2015/5/7 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_send_wqe_overhead(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return sizeof(struct roce3_wqe_ctrl_seg) + + sizeof(struct roce3_wqe_mask_atomic_tsk_seg); + + case IB_QPT_UC: + return sizeof(struct roce3_wqe_ctrl_seg) + sizeof(struct roce3_wqe_send_tsk_seg); + + case IB_QPT_UD: + return sizeof(struct roce3_wqe_ctrl_seg) + sizeof(struct roce3_wqe_ud_tsk_seg); + + case IB_QPT_GSI: + return sizeof(struct roce3_wqe_ctrl_seg) + sizeof(struct roce3_wqe_ud_tsk_seg); + + case IB_QPT_XRC_TGT: + return 0; + + case IB_QPT_XRC_INI: + return sizeof(struct roce3_wqe_ctrl_seg) + + sizeof(struct roce3_wqe_mask_atomic_tsk_seg); + + default: + pr_err("[ROCE, ERR] %s: Not supported this qp_type\n", __func__); + return -1; + } +} + +static int roce3_set_kernel_sq_size_check(const struct roce3_device *rdev, + const struct ib_qp_cap *cap, int *sq_max_wqe_size, enum ib_qp_type qp_type) +{ + int wqe_overhead = 0; + int ret = 0; + + if (cap->max_send_wr > (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ WR over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_send_sge > + (u32)ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ SGE over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_inline_data > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ max inline data over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + wqe_overhead = roce3_send_wqe_overhead(qp_type); + if (wqe_overhead < 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Invalid service type(%d), func_id(%d)\n", + __func__, (int)qp_type, rdev->glb_func_id); + return -EINVAL; + } + + *sq_max_wqe_size = + ROCE_MAX((int)(cap->max_send_sge * sizeof(struct roce3_wqe_data_seg)), + (int)cap->max_inline_data) + wqe_overhead; + if (*sq_max_wqe_size > (int)rdev->rdma_cap.max_sq_desc_sz) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SQ WQE size over range, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + return ret; +} +/* + **************************************************************************** + Prototype : roce3_set_kernel_sq_size + Description : roce3_set_kernel_sq_size + Input : struct roce3_device *rdev + struct ib_qp_cap *cap + enum roce3_qp_type qp_type + struct roce3_qp *rqp + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_set_kernel_sq_size(struct roce3_device *rdev, + struct ib_qp_cap *cap, enum ib_qp_type qp_type, struct roce3_qp *rqp) + +{ + u32 sq_buf_sz = 0; + int sq_max_wqe_size = 0; + int ret = 0; + + ret = roce3_set_kernel_sq_size_check(rdev, cap, &sq_max_wqe_size, qp_type); + if (ret != 0) + return ret; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + rqp->sq.wqe_shift = (u32)ROCE_ILOG2(rdev->rdma_cap.wqebb_size); + rqp->sq_max_wqes_per_wr = + (u32)ROCE_DIV_ROUND_UP((unsigned int)sq_max_wqe_size, + (u32)(1UL << (unsigned int)rqp->sq.wqe_shift)); + +/* + * As one wqebb ought to be reserved for invalidate wqebb, 1 is added to sq.wqebb_cnt here + * to satisfy the actual IB_MAD_QP_SEND_SIZE set by applications + */ + rqp->sq.wqebb_cnt = (u32)((int)cap->max_send_wr * rqp->sq_max_wqes_per_wr + 1); + /*lint -e587*/ + rqp->sq.wqebb_cnt = (u32)(ROCE_ROUNDUP_POW_OF_TWO(rqp->sq.wqebb_cnt) & 0xffffffff); + /*lint +e587*/ + + if ((u32)((u64)rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift) < PAGE_SIZE) + rqp->sq.wqebb_cnt = (PAGE_SIZE >> ((u32)rqp->sq.wqe_shift)); + + rqp->sq.max_sge = (u32)(((u64)rqp->sq_max_wqes_per_wr << + (unsigned int)rqp->sq.wqe_shift) - + (unsigned int)roce3_send_wqe_overhead(qp_type)); + rqp->sq.max_sge = + (u32)ROCE_MIN(rdev->rdma_cap.max_sq_desc_sz, rqp->sq.max_sge) / + (int)sizeof(struct roce3_wqe_data_seg); + + sq_buf_sz = (u32)ALIGN((u64)rqp->sq.wqebb_cnt << + (unsigned int)rqp->sq.wqe_shift, PAGE_SIZE); + rqp->buf_size = (int)ALIGN(((unsigned int)sq_buf_sz + + ((unsigned int)rqp->rq.wqebb_cnt << (unsigned int)rqp->rq.wqe_shift)), + PAGE_SIZE); + + rqp->sq.offset = 0u; + rqp->rq.offset = sq_buf_sz; + + if (((rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr) == 1) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s:Only one WR,please check,func_id(%d)\n", + __func__, rdev->glb_func_id); + rqp->sq.max_post = ROCE_WR_MIN_NUM; + cap->max_send_wr = rqp->sq.max_post; + } else { + rqp->sq.max_post = (rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr; + cap->max_send_wr = (u32)rqp->sq.max_post; + } + + cap->max_send_sge = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg); + cap->max_send_sge = (u32)ROCE_MIN(rqp->sq.max_sge, cap->max_send_sge); + rqp->max_inline_data = cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_qp_async_event + Description : roce3_qp_async_event + Input : struct roce3_device *rdev + struct roce3_qp *qp + int type + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +void roce3_qp_async_event(struct roce3_device *rdev, struct roce3_qp *qp, int type) +{ + struct ib_event event; + struct ib_qp *ibqp = &qp->ibqp; + + memset(&event, 0, sizeof(event)); + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + + switch (type) { + case ROCE_EVENT_TYPE_COMM_EST: + event.event = IB_EVENT_COMM_EST; + break; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + break; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + + default: + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static void *roce3_buf_offset(struct roce3_buf *buf, u32 offset) +{ + return (void *)((char *)buf->direct.buf + offset); +} + +void *roce3_get_wqe(struct roce3_qp *rqp, u32 offset) +{ + return roce3_buf_offset(&rqp->buf, offset); +} + +static void *roce3_get_send_wqe_head(struct roce3_qp *rqp) +{ + return roce3_get_wqe(rqp, rqp->sq.offset); +} + +/* + **************************************************************************** + Prototype : roce3_get_send_wqe_tail + Description : roce3_get_send_wqe_tail + Input : struct roce3_qp *rqp + Output : None + + 1.Date : 2015/8/8 + Modification : Created function + +**************************************************************************** +*/ +static void *roce3_get_send_wqe_tail(struct roce3_qp *rqp) +{ + return roce3_get_wqe(rqp, rqp->sq.offset + (rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift)); +} + +/* + **************************************************************************** + Prototype : roce3_sq_buf_init + Description : roce3_sq_buf_init + Input : struct roce3_qp *qp + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_sq_buf_init(struct roce3_qp *qp) +{ + u32 entries = qp->rq.offset / (1U << qp->sq.wqe_shift); + u8 *wqe = NULL; + struct roce3_wqe_ctrl_seg *wqe_ctrl = NULL; + u32 i; + + for (i = 0; i < entries; i++) { + wqe = (u8 *)qp->buf.direct.buf + (int)(i * (int)(1U << ((u32)qp->sq.wqe_shift))); + wqe_ctrl = (struct roce3_wqe_ctrl_seg *)((void *)wqe); + wqe_ctrl->dw0.value = be32_to_cpu(wqe_ctrl->dw0.value); + wqe_ctrl->dw0.bs.owner = 1; + wqe_ctrl->dw0.value = cpu_to_be32(wqe_ctrl->dw0.value); + } +} + +static int roce3_check_and_get_umem(struct roce3_device *rdev, + struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct roce3_qp *rqp, struct create_qp_cmd *ucmd) +{ + int ret = 0; + + if ((udata == NULL) || (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0)) { + ret = -EFAULT; + return ret; + } + + ret = roce3_set_user_sq_size(rdev, &init_attr->cap, rqp, ucmd); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set user sq_size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + rqp->umem = ib_umem_get(&rdev->ib_dev, ucmd->buf_addr, (size_t)rqp->buf_size, 0); + if (IS_ERR(rqp->umem)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get ib_umem, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = (int)PTR_ERR(rqp->umem); + return ret; + } + + return ret; +} + +static int roce3_alloc_qpc(struct roce3_qp *rqp, struct roce3_device *rdev, + u32 qpn, struct ib_qp_init_attr *init_attr) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + int ret; + struct roce3_srq *rsrq; + u32 qpn_tmp = qpn; + bool low2bit_align_en = false; + + if (roce3_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + rsrq = to_roce3_srq(init_attr->srq); + qpn_tmp = ((rsrq->srqn & ROCE3_LB1_MASK) << ROCE3_QPN_BIT_INDEX) | + qpn_tmp >> ROCE3_QPN_CTRL_BIT_NUM; + low2bit_align_en = true; + } + + qpc_info = cqm_object_qpc_mpt_create(rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, rqp, qpn_tmp, + low2bit_align_en); + if (qpc_info == NULL) { + ret = -ENOMEM; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qpc mpt, func_id(%d), qpn(%u)\n", + __func__, rdev->glb_func_id, qpn_tmp); + return ret; + } + + rqp->qpc_info = qpc_info; + rqp->qpn = rqp->qpc_info->xid; + if (roce3_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + rsrq = to_roce3_srq(init_attr->srq); + if ((rsrq->srqn & ROCE3_LB1_MASK) != (rqp->qpn & ROCE3_LB1_MASK)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Wrong srqn, func_id(%d), qpn(%u) srqn(%u)\n", + __func__, rdev->glb_func_id, rqp->qpn, rsrq->srqn); + } + } + return 0; +} + +static int roce3_create_qp_user_mtt(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + u32 npages; + int ret; + + npages = (u32)ib_umem_num_pages(rqp->umem); + rqp->buf.page_shift = PAGE_SHIFT; + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, (u32)rqp->buf.page_shift, + &rqp->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_umem_write_mtt(rdev, &rqp->mtt, rqp->umem); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + return ret; +} + +static void roce3_destroy_qp_user_mtt(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_user + Description : create qp resource for user space qp + Input : struct roce3_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + struct ib_udata *udata + struct roce3_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_qp_user(struct roce3_device *rdev, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, struct ib_udata *udata, struct roce3_qp *rqp) +{ + int ret = 0; + struct roce3_ucontext *roce3_uctx = NULL; + struct create_qp_cmd ucmd = { 0 }; + u32 qpn = 0; + + ret = roce3_create_qp_user_pre_ext(init_attr, rqp, &qpn); + if (ret != 0) + return ret; + + roce3_uctx = to_roce3_ucontext(ibpd->uobject->context); + + ret = roce3_check_and_get_umem(rdev, ibpd, init_attr, udata, rqp, &ucmd); + if (ret != 0) + return ret; + + ret = roce3_create_qp_user_mtt(rdev, rqp); + if (ret != 0) + goto err_create_mtt; + + if (init_attr->qp_type != IB_QPT_XRC_TGT) { + ret = roce3_db_map_user(roce3_uctx, ucmd.db_addr, &rqp->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to map db page to user, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_map_db; + } + } + + ret = roce3_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) + goto err_alloc_qpc; + + ret = roce3_create_qp_user_post_ext(ibpd, rdev, rqp, init_attr); + if (ret != 0) + goto err_extend_post; + + return 0; + +err_extend_post: + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qpc_info->object)); + +err_alloc_qpc: + if (init_attr->qp_type != IB_QPT_XRC_TGT) + roce3_db_unmap_user(roce3_uctx, &rqp->db); + +err_map_db: + roce3_destroy_qp_user_mtt(rdev, rqp); + +err_create_mtt: + ib_umem_release(rqp->umem); + + return ret; +} + +static void roce3_free_wrid(struct roce3_qp *rqp) +{ + if (rqp->rq.wrid) + kvfree(rqp->rq.wrid); + + if (rqp->sq.wrid) + kvfree(rqp->sq.wrid); +} + +static int roce3_get_wrid(struct roce3_qp *rqp, struct roce3_device *rdev) +{ + int ret = 0; + + rqp->sq.wrid = kcalloc((unsigned int)rqp->sq.wqebb_cnt, sizeof(u64), GFP_KERNEL); + if (rqp->sq.wrid == NULL) + rqp->sq.wrid = vzalloc((size_t)(rqp->sq.wqebb_cnt * sizeof(u64))); + + rqp->rq.wrid = kcalloc((unsigned int)rqp->rq.wqebb_cnt, sizeof(u64), GFP_KERNEL); + if (rqp->rq.wrid == NULL) + rqp->rq.wrid = vzalloc((size_t)(rqp->rq.wqebb_cnt * sizeof(u64))); + + if ((rqp->sq.wrid == NULL) || (rqp->rq.wrid == NULL)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc wrid of sq or rq, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -ENOMEM; + } + + return ret; +} + +static void roce3_create_qp_kernel_set_attr(struct roce3_qp *rqp, + struct tag_cqm_queue *qp_buf_info) +{ + memset(qp_buf_info->q_room_buf_1.direct.va, 0, + (size_t)((unsigned int)rqp->buf_size)); + rqp->buf.direct.buf = qp_buf_info->q_room_buf_1.direct.va; + rqp->buf.direct.map = qp_buf_info->q_room_buf_1.direct.pa; + rqp->buf.page_shift = ROCE_ILOG2(qp_buf_info->q_room_buf_1.buf_size); + + roce3_sq_buf_init(rqp); + + rqp->db.db_record = (__be32 *)(void *)(&qp_buf_info->q_header_vaddr->doorbell_record); + rqp->db.dma = qp_buf_info->q_header_paddr; + + qp_buf_info->q_header_vaddr->doorbell_record = 0; + + rqp->qp_buf_info = qp_buf_info; + + rqp->sq_head_addr = (u8 *)roce3_get_send_wqe_head(rqp); + rqp->sq_tail_addr = (u8 *)roce3_get_send_wqe_tail(rqp); + rqp->max_dwqe_size = 0; + + if (rqp->max_inline_data != 0) + rqp->max_dwqe_size = ROCE_QP_MAX_DWQE_SIZE; +} + +static int roce3_create_qp_kernel_normal(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret; + struct tag_cqm_queue *qp_buf_info = NULL; + + qp_buf_info = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_QP, (u32)rqp->buf_size, rqp, true, ROCE_QP_INVLID_QP_NUM); + if (qp_buf_info == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma queue by cqm object, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -1; + } + + roce3_create_qp_kernel_set_attr(rqp, qp_buf_info); + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, qp_buf_info->q_room_buf_1.buf_number, + (u32)rqp->buf.page_shift, &rqp->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_mtt; + } + ret = roce3_buf_write_mtt(rdev, &rqp->mtt, &rqp->qp_buf_info->q_room_buf_1); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write rdma mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_write_mtt; + } + + ret = roce3_get_wrid(rqp, rdev); + if (ret != 0) + goto err_alloc_wrid; + + return 0; + +err_alloc_wrid: + roce3_free_wrid(rqp); + +err_write_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + +err_alloc_mtt: + hiudk_cqm_object_delete(rdev->hwdev, &rqp->qp_buf_info->object); + + return ret; +} + +static void roce3_destroy_qp_kernel_normal(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + roce3_free_wrid(rqp); + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + hiudk_cqm_object_delete(rdev->hwdev, &rqp->qp_buf_info->object); +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_kernel + Description : create qp resource for kernel space qp + Input : struct roce3_device *rdev + struct ib_qp_init_attr *init_attr + struct roce3_qp *rqp + u32 qpn + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_qp_kernel(struct roce3_device *rdev, + struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, u32 qpn) +{ + int ret = 0; + + ret = roce3_set_kernel_sq_size(rdev, &init_attr->cap, (enum ib_qp_type)rqp->qp_type, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set sq buffer size in kernel, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + ret = roce3_create_qp_kernel_normal(rdev, rqp); + if (ret != 0) + return ret; + } + + ret = roce3_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) + goto err_alloc_qpc; + + return 0; + +err_alloc_qpc: + if (rqp->qp_type != IB_QPT_XRC_TGT) + roce3_destroy_qp_kernel_normal(rdev, rqp); + + return ret; +} + +static void roce3_qp_add_cq_list(struct ib_qp_init_attr *init_attr, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + struct roce3_cq *send_rcq = to_roce3_cq(init_attr->send_cq); + struct roce3_cq *recv_rcq = to_roce3_cq(init_attr->recv_cq); + unsigned long flags; + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce3_lock_cqs(send_rcq, recv_rcq); + list_add_tail(&rqp->qps_list, &rdev->qp_list); + list_add_tail(&rqp->cq_send_list, &send_rcq->send_qp_list); + list_add_tail(&rqp->cq_recv_list, &recv_rcq->recv_qp_list); + roce3_unlock_cqs(send_rcq, recv_rcq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); +} + +static void roce3_init_qp_state(struct roce3_qp *rqp, struct roce3_device *rdev) +{ + struct roce_qp_context *qpc = (struct roce_qp_context *)((void *)rqp->qpc_info->vaddr); + + if (!cqm_need_secure_mem(rdev->hwdev)) { + qpc->sw_seg.drv_seg.dw0.bs.state = ROCE_QP_STATE_MEM_INIT; + qpc->sw_seg.drv_seg.dw0.value = cpu_to_be32(qpc->sw_seg.drv_seg.dw0.value); + } +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_common + Description : roce3_create_qp_common + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + struct ib_udata *udata + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_do_create_qp(struct roce3_device *rdev, struct roce3_qp *rqp, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, struct ib_udata *udata) +{ + int ret = 0; + + mutex_init(&rqp->mutex); + /*lint -e708 -e413 */ + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + /*lint +e708 +e413 */ + + rqp->qp_state = IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + + if (ibpd->uobject) { + ret = roce3_set_rq_size(rdev, rqp, init_attr, true, rqp->has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_create_qp_pre_ext(rdev, rqp, init_attr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to handle qp create pre extension, func_id(%d)\n", + rdev->glb_func_id); + return ret; + } + + ret = roce3_create_qp_user(rdev, ibpd, init_attr, udata, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to create qp user, func_id(%d)\n", + rdev->glb_func_id); + return ret; + } + } else { + ret = roce3_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_INVLID_QP_NUM); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp kernel, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + roce3_qp_add_cq_list(init_attr, rqp, rdev); + } + + if (roce3_is_qp_normal(rqp, init_attr) != 0) + roce3_init_qp_state(rqp, rdev); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_create_qp_gsi + Description : roce3_create_qp_gsi + Input : struct roce3_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** +*/ +static struct roce3_qp *roce3_create_qp_gsi(struct roce3_device *rdev, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr) +{ + int ret; + struct roce3_qp *rqp = NULL; + struct roce3_sqp *sqp = NULL; + + sqp = kzalloc(sizeof(struct roce3_sqp), GFP_KERNEL); + if (sqp == NULL) + return (struct roce3_qp *)ERR_PTR((long)-ENOMEM); + + rqp = &sqp->qp; + + mutex_init(&rqp->mutex); + /*lint -e708*/ + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + /*lint +e708*/ + rqp->qp_state = (u8)IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + rqp->has_rq = !init_attr->srq; + + ret = roce3_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set rq size, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + ret = roce3_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_GSI_QP_NUM); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp kernel, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + + rqp->qpn = ROCE_QP_GSI_QP_NUM; + + roce3_init_qp_state(rqp, rdev); + roce3_qp_add_cq_list(init_attr, rqp, rdev); + + return rqp; + +err_out: + kfree(sqp); + + return (struct roce3_qp *)ERR_PTR((long)ret); +} + +static int roce3_create_check_init_attr(const struct ib_pd *ibpd, + const struct ib_qp_init_attr *init_attr, const struct ib_udata *udata) +{ + if (init_attr == NULL) { + pr_err("[ROCE, ERR] %s: init_attr is null\n", __func__); + return (-EINVAL); + } + + if ((ibpd == NULL) && (init_attr->qp_type != IB_QPT_XRC_TGT)) { + pr_err("[ROCE, ERR] %s: Ibpd is null and qp_type is not IB_QPT_XRC_TGT\n", + __func__); + return (-EINVAL); + } + + if ((udata) && (init_attr->qp_type == IB_QPT_GSI)) { + pr_err("[ROCE, ERR] %s: Udata is not null and qp_type is IB_QPT_GSI\n", __func__); + return (-EINVAL); + } + + return 0; +} + +static int roce3_set_qp_attr(struct ib_qp_init_attr *init_attr, struct roce3_qp *rqp, + struct roce3_device *rdev, struct ib_pd **ibpd, int qp_ext) +{ + u16 xrcdn = 0; + + rqp->qp_ext = (u8)qp_ext; + if (init_attr->qp_type == IB_QPT_XRC_TGT) { + *ibpd = to_roce3_xrcd(init_attr->xrcd)->pd; + xrcdn = (u16)to_roce3_xrcd(init_attr->xrcd)->xrcdn; + init_attr->send_cq = to_roce3_xrcd(init_attr->xrcd)->cq; + init_attr->recv_cq = init_attr->send_cq; + + rqp->has_rq = false; + rqp->xrcdn = xrcdn; + } else if ((init_attr->qp_type == IB_QPT_XRC_INI) && + roce3_is_roceaa(rdev->cfg_info.scence_id)) { + init_attr->recv_cq = init_attr->send_cq; + rqp->has_rq = false; + } else if (init_attr->qp_type == IB_QPT_XRC_INI) { + init_attr->recv_cq = init_attr->send_cq; + rqp->has_rq = false; + if (rdev->is_vroce) + rqp->sl = rdev->group_xrc_cos; + } else if ((init_attr->qp_type == IB_QPT_RC) || (init_attr->qp_type == IB_QPT_UC) || + (init_attr->qp_type == IB_QPT_UD)) { + rqp->has_rq = !init_attr->srq; + if (init_attr->qp_type == IB_QPT_RC && rdev->is_vroce) + rqp->sl = rdev->group_rc_cos; + else if (rdev->is_vroce) + rqp->sl = rdev->group_ud_cos; + } else { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Unknown QP Type(%u), func_id(%d)\n", + __func__, init_attr->qp_type, rdev->glb_func_id); + return (-EINVAL); + } + return 0; +} + +static struct ib_qp *roce3_create_gsi_qp_result(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, struct roce3_device *rdev) +{ + int ret; + struct roce3_qp *rqp = NULL; + + rqp = roce3_create_qp_gsi(rdev, ibpd, init_attr); + if (IS_ERR(rqp)) { + ret = (int)PTR_ERR(rqp); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp gsi, func_id(%d)\n", + __func__, rdev->glb_func_id); + return (struct ib_qp *)ERR_PTR((long)ret); + } + + rqp->ibqp.qp_num = rqp->qpn; + return &rqp->ibqp; +} + +struct ib_qp *roce3_create_qp_common(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int qp_ext) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct ib_device *ibdev = NULL; + struct roce3_device *rdev = NULL; + + ret = roce3_create_check_init_attr(ibpd, init_attr, udata); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check init attr\n", __func__); + return (struct ib_qp *)ERR_PTR((long)ret); + } + + ibdev = ibpd ? ibpd->device : init_attr->xrcd->device; + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return (struct ib_qp *)ERR_PTR((long)-EPERM); + } + + if (init_attr->qp_type == IB_QPT_GSI) + return roce3_create_gsi_qp_result(ibpd, init_attr, rdev); + + rqp = kzalloc(sizeof(*rqp), GFP_KERNEL); + if (rqp == NULL) + return (struct ib_qp *)ERR_PTR((long)-ENOMEM); + + ret = roce3_set_qp_attr(init_attr, rqp, rdev, &ibpd, qp_ext); + if (ret != 0) + goto err_create_qp; + + ret = roce3_do_create_qp(rdev, rqp, ibpd, init_attr, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create qp in common process, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_qp; + } + + roce3_set_qp_dif_attr(rqp, init_attr, rdev); + rqp->ibqp.qp_num = rqp->qpn; + rqp->rsp_depth = 0; + + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.alloc_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + + return &rqp->ibqp; + +err_create_qp: + kfree(rqp); + + return (struct ib_qp *)ERR_PTR((long)ret); +} + +struct ib_qp *roce3_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + return roce3_create_qp_common(ibpd, init_attr, udata, ROCE_QP); +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c new file mode 100644 index 000000000..be000e744 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_destroy.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif +#include "hinic3_hmm.h" + +#ifdef ROCE_VBS_EN +#include "roce_vbs_qp.h" +#endif + +static void roce3_clean_qp_user(struct roce3_device *rdev, struct roce3_qp *rqp, + struct roce3_ucontext *ucontext) +{ + roce3_free_opt_rdmarc(rqp); + + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qpc_info->object)); + +#ifdef ROCE_VBS_EN + if (rqp->vbs_qp_ptr) + roce3_vbs_destroy_sqpc(rdev->hwdev, rqp); +#endif + + if (rqp->qp_type != IB_QPT_XRC_TGT) + roce3_db_unmap_user(ucontext, &rqp->db); + + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + + ib_umem_release(rqp->umem); +} + +/* + **************************************************************************** + Prototype : roce3_cq_clean_process + Description : roce3_cq_clean_process + Input : struct roce3_cq *cq + u32 qpn + struct roce3_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_clean_process(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq) +{ + u32 prod_index = 0; + int nfreed = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe *dest = NULL; + u8 owner_bit = 0; + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->cons_index; roce3_get_sw_cqe(cq, prod_index); ++prod_index) { + if (prod_index == (u32)((int)cq->cons_index + cq->ibcq.cqe)) + break; + } + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + /* + * xcqe:qpn's cqe. + * CI + * +--------+------+------+------+----------+ + * b4_clean: | | cqe1 | xcqe2| cqe3 | | + * +--------+------+------+------+----------+ + * \ + * \ + * +---------------+------+------+----------+ + * af_clean: | | cqe1 | cqe3 | | + * +---------------+------+------+----------+ + */ + while ((int) --prod_index - (int)cq->cons_index >= 0) { + cqe = (struct roce_cqe *)roce3_get_cqe(cq, prod_index & (u32)cq->ibcq.cqe); + + cqe->dw0.value = roce3_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_cpu32(cqe->dw1.value); + cqe->dw7.value = roce3_convert_cpu32(cqe->dw7.value); + + if (cqe->dw0.bs.qpn == qpn) { + if (srq && (cqe->dw1.bs.s_r == ROCE_CQE_RECV_COMP)) + roce3_free_srq_wqe(srq, cqe->dw7.bs.wqe_cnt); + + ++nfreed; + } else if (nfreed != 0) { + dest = (struct roce_cqe *)roce3_get_cqe(cq, + (unsigned int)((int)prod_index + nfreed) & + (unsigned int)cq->ibcq.cqe); + dest->dw0.value = roce3_convert_cpu32(dest->dw0.value); + owner_bit = dest->dw0.bs.owner; + memcpy((void *)dest, (void *)cqe, sizeof(struct roce_cqe)); + dest->dw0.bs.owner = owner_bit; + dest->dw0.value = roce3_convert_be32(dest->dw0.value); + dest->dw1.value = roce3_convert_be32(dest->dw1.value); + dest->dw7.value = roce3_convert_be32(dest->dw7.value); + } else { + pr_info("[ROCE] %s: Nothing need to do\n", __func__); + } + + cqe->dw0.value = roce3_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce3_convert_be32(cqe->dw1.value); + cqe->dw7.value = roce3_convert_be32(cqe->dw7.value); + } + + if (nfreed != 0) { + cq->cons_index += (u32)nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + + roce3_cq_set_ci(cq); + } +} + +/* + **************************************************************************** + Prototype : roce3_clean_qp_kernel + Description : roce3_clean_qp_kernel + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_clean_qp_kernel(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + struct roce3_cq *send_cq = NULL; + struct roce3_cq *recv_cq = NULL; + unsigned long flags; + + roce3_free_opt_rdmarc(rqp); + + roce3_get_cqs(rqp, &send_cq, &recv_cq); + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce3_lock_cqs(send_cq, recv_cq); + /* del from lists under both locks above to protect reset flow paths */ + list_del(&rqp->qps_list); + list_del(&rqp->cq_send_list); + list_del(&rqp->cq_recv_list); + + roce3_cq_clean_process(recv_cq, rqp->qpn, rqp->ibqp.srq ? + to_roce3_srq(rqp->ibqp.srq) : NULL); + + if (send_cq != recv_cq) + roce3_cq_clean_process(send_cq, rqp->qpn, NULL); + + roce3_unlock_cqs(send_cq, recv_cq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); + + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qpc_info->object)); + + if (rqp->sq.wrid) { + kvfree(rqp->sq.wrid); + rqp->sq.wrid = NULL; + } + + if (rqp->rq.wrid) { + kvfree(rqp->rq.wrid); + rqp->rq.wrid = NULL; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) + hiudk_cqm_object_delete(rdev->hwdev, &(rqp->qp_buf_info->object)); + + hmm_rdma_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +} + +static int roce3_qp_destroy(struct roce3_qp *rqp, struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify QP(0x%06x) to RESET, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce3_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: QP(0x%06x) cache invalid, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce3_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + struct roce3_ucontext *ucontext = NULL; + + if (ibqp == NULL) { + pr_err("[ROCE, ERR] %s: Ibqp is null\n", __func__); + return -EINVAL; + } + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + + if (rqp->qp_state != IB_QPS_RESET) { + ret = roce3_qp_destroy(rqp, rdev); + if (ret != 0) + return ret; + + rqp->qp_state = IB_QPS_RESET; + } + + if (ibqp->uobject) { + ucontext = rdma_udata_to_drv_context(udata, struct roce3_ucontext, ibucontext); + roce3_clean_qp_user(rdev, rqp, ucontext); + goto out; + } + + roce3_clean_qp_kernel(rdev, rqp); +out: + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.del_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h b/drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h new file mode 100644 index 000000000..d477e03ca --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_exp.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_QP_EXP_H +#define ROCE_QP_EXP_H + +#include <linux/io-mapping.h> +#include <linux/list.h> +#include <linux/cdev.h> + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" +#include "hinic3_cqm.h" + +#include "rdma_context_format.h" + +#include "roce.h" +#include "roce_db.h" +#include "roce_cq.h" +#include "roce_qp.h" + +#define EXP_OK (0) +#define EXP_PARAM_ERR (1) +#define EXP_FUNC_ERR (2) + +#define QPC_MIN_SIZE (512) +#define CONTX_SZ (sizeof(struct roce_qp_context) / QPC_MIN_SIZE) + +#define BUFFER_SZ (sizeof(struct roce_qp_context) / sizeof(u32)) + +#define TYPE_LEN (8) +#define DEV_NUM (32) +#define QPN_MAX (0xFFFFF) +#define QPN_MIN (0) +#define IB_QP_EXP_CMD (1 << 23) +#define ROCE_OPTPAR_EXP (1 << 16) +#define VA_OFFSET (8) + +struct roce3_device_list { + struct ib_device *ib_dev[DEV_NUM]; + int ib_dev_num; + struct mutex mutex; +}; + +struct ib_qp_info { + dma_addr_t sq_pg_base_addr[64]; + dma_addr_t hw_doorbell_addr; + dma_addr_t sw_doorbell_addr; + + u32 sq_depth; + u32 qpn; + u8 sq_pg_cnt; + u8 qp_cos; + u8 cntx_sz; + u8 sq_wqe_size; +}; + +struct ib_qp_attr_data { + u32 data_type; // plog:1 + u32 data_len; + u8 *data_buf; +}; + +struct ib_qp_attr_exp { + struct ib_qp_attr qp_attr; + struct ib_qp_attr_data qp_attr_data; +}; + +struct roce3_device_list *roce3_get_plog_device_info(void); + +int ib_get_qp_info(char *dev_name, int qpn, struct ib_qp_info *qp_info); +struct ib_qp *ib_get_qp(const char *dev_name, int qpn); +int ib_put_qp(struct ib_qp *ibqp); +int roce3_qp_modify_exp(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state, u16 vlan_id); +int roce3_modify_extend_qp(struct ib_qp_attr *attr, int attr_mask, struct ib_qp *ibqp); + +#endif // ROCE_QP_EXP_H diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c new file mode 100644 index 000000000..c40f508c1 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_modify.c @@ -0,0 +1,2243 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "roce_compat.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_qp_exp.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" +#include "roce_verbs_attr.h" +#include "roce_verbs_ext_attr.h" +#include "rdma_context_format.h" + +#ifdef ROCE_VBS_EN +#include "roce_vbs_qp.h" +#endif + +#ifdef ROCE_ANTI_ATTACK +#include "roce_anti_attack.h" +#include "roce_cmd.h" +#endif + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#define ROCE_MTT_GPA_VALID 0x1 +#define ROCE_WQE_PREFETCH_MAXNUM 4 +#define ROCE_SQ_WQEBB_SIZE_SHIFT 4 +#define ROCE_SQ_WQEBB_SIZE_MASK 0x7 +#define ROCE_SQ_PD_MASK 0x3ffff +#define ROCE_SQ_PAGE_SIZE_MASK 0xf +#define ROCE_SQ_WQECNT_LTH 4 +#define ROCE_SQ_PREFETCH_MINNUM 2 +#define ROCE_SQ_PREFETCH_MAXNUM 4 +#define ROCE_SQ_SIZE_MASK 0x1f +#define ROCE_SQ_CQN_MASK 0xfffff +#define ROCE_RQ_WQECNT_LTH 0xe +#define ROCE_RQ_MTT_PREFETCH_MAXLEN0 3 +#define ROCE_RQ_MTT_PREFETCH_MAXWQE 7 +#define ROCE_RQ_WQE_PREFETCH_MAXNUM 7 +#define ROCE_SRQ_INIT_STATE 0xf +#define ROCE_MAX_PI_ON_CHIP_QPN 1024 + +enum { + ROCE_IPSUTX_CHANNEL_5 = 0x0, /* 00 */ + ROCE_IPSUTX_CHANNEL_10 = 0x1, /* 01 */ + ROCE_IPSUTX_CHANNEL_15 = 0x2, /* 10 */ + ROCE_IPSUTX_CHANNEL_20 = 0x3, /* 01 */ +}; + +enum { + BIT_LENGTH_IB_MTU_256 = 8, + BIT_LENGTH_IB_MTU_512, + BIT_LENGTH_IB_MTU_1024, + BIT_LENGTH_IB_MTU_2048, + BIT_LENGTH_IB_MTU_4096 +}; + +enum { + CACHE_INVLAID_ALLOC, + CACHE_INVLAID_CMDQ, + CACHE_INVLAID_RDMARC_CHECK, + CACHE_INVLAID_QP_CHECK, + MODIFY_2RST_CMDQ, + CMDQ_SDKCALL_BUTT +}; + +#define ROCE_CMDQ_TIMEOUT 1000 +#define ROCE_CMDQ_TIMEOUT_10MS 10 + + +#ifdef ROCE_COMPUTE +static int g_sq_cqn_lb; +module_param(g_sq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_sq_cqn_lb, "default: 0"); + +static int g_rq_cqn_lb; +module_param(g_rq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_rq_cqn_lb, "default: 0"); +#else +static int g_sq_cqn_lb = 1; +module_param(g_sq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_sq_cqn_lb, "default: 1"); + +static int g_rq_cqn_lb = 1; +module_param(g_rq_cqn_lb, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_rq_cqn_lb, "default: 1"); +#endif + +static void roce3_timeout_check(struct roce3_device *rdev, const struct timespec64 *tv_start, + int func_id, int times) +{ + uint64_t cost; + struct timespec64 delta; + struct timespec64 tv_end = { 0 }; + + ktime_get_ts64(&tv_end); + delta = timespec64_sub(tv_end, *tv_start); + cost = delta.tv_sec * USEC_PER_SEC + delta.tv_nsec / NSEC_PER_USEC; + + if (cost >= ROCE_CMDQ_TIMEOUT) { + dev_warn_ratelimited(rdev->hwdev_hdl, + "[ROCE, WARN] %s:destroy qp exec too long, func_id(%d), cost(%llu), times:%d\n", + __func__, func_id, cost, times); + } +} + +#ifdef ROCE_SIGN_EN +enum mtt_check_type_e { + MTT_CHECK_TYPE_0 = 0, + MTT_CHECK_TYPE_1 +}; + +#define ROCE_QP_QPCC_SIGN_WIDTH 5 +#define ROCE_QP_QPCC_SIGN_SPLITNUM 4 +#define ROCE_QP_QPCC_SIGN_CHECKBITS (ROCE_QP_QPCC_SIGN_WIDTH * ROCE_QP_QPCC_SIGN_SPLITNUM) + +#define ROCE_QP_SUB_SIGN_WIDTH 3 +#define ROCE_QP_SUB_SIGN_SPLITNUM 5 +#define ROCE_QP_SUB_SIGN_CHECKBITS (ROCE_QP_SUB_SIGN_WIDTH * ROCE_QP_SUB_SIGN_SPLITNUM) + +#define ROCE_QP_GPA_SIGN_WIDTH 3 +#define ROCE_QP_GPA_SIGN_SPLITNUM 11 +#define ROCE_QP_GPA_SIGN_CHECKBITS (ROCE_QP_GPA_SIGN_WIDTH * ROCE_QP_GPA_SIGN_SPLITNUM) + +#define ROCE_SQ_WQE_SIGN_WIDTH 8 +#define ROCE_SQ_WQE_SIGN_SPLITNUM 9 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_1 32 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_2 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_12 (ROCE_SQ_WQE_SIGN_CHECKBITS_2 + ROCE_SQ_WQE_SIGN_CHECKBITS_1) +#define ROCE_SQ_WQE_SIGN_CHECKBITS_3 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_123 \ + (ROCE_SQ_WQE_SIGN_CHECKBITS_3 + ROCE_SQ_WQE_SIGN_CHECKBITS_12) +#define ROCE_SQ_WQE_SIGN_CHECKBITS (ROCE_SQ_WQE_SIGN_WIDTH * ROCE_SQ_WQE_SIGN_SPLITNUM) + +#define ROCE_MTT_SIGN_WIDTH 11 +#define ROCE_MTT_SIGN_SPLITNUM 5 +#define ROCE_MTT_SIGN_CHECKBITS_1 22 +#define ROCE_MTT_SIGN_CHECKBITS_2 33 +#define ROCE_MTT_SIGN_CHECKBITS_12 \ + (ROCE_MTT_SIGN_CHECKBITS_1 + ROCE_MTT_SIGN_CHECKBITS_2) +#define ROCE_MTT_SIGN_CHECKBITS (ROCE_MTT_SIGN_WIDTH * ROCE_MTT_SIGN_SPLITNUM) + +#define ROCE_MTT_BLOCK_SIGN_WIDTH 5 +#define ROCE_MTT_BLOCK_SIGN_SPLITNUM 11 + +/* + **************************************************************************** + Prototype : roce3_calculate_sign_bit + Description : roce3_calculate_signature + Input : u32 *check_bit + u32 sign_width + u32 sign_split + u32 *sign_bit + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_calculate_sign_bit(u32 *check_bit, u32 sign_width, u32 sign_split, u32 *sign_bit) +{ + u32 i = 0; + u32 j = 0; + + for (i = 0; i < sign_width; i++) { + sign_bit[i] = 0; + for (j = 0; j < sign_split; j++) + sign_bit[i] = sign_bit[i] ^ check_bit[i + sign_width * j]; + + sign_bit[i] = (~sign_bit[i]) & 0x1; + } +} + +/* + **************************************************************************** + Prototype : roce3_calculate_signature + Description : roce3_calculate_signature + Input : u32 *sign_bit + u32 sign_width + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_calculate_signature(u32 *sign_bit, u32 sign_width) +{ + u32 i = 0; + u32 signature = 0; + + for (i = 0; i < sign_width; i++) + signature |= sign_bit[i] << i; + + return (signature & ((1U << sign_width) - 1)); +} + +/* + **************************************************************************** + Prototype : roce3_gen_sq_rq_gpa_sign + Description : Signature 3bit + CheckedBits = sq_rq_l0mtt_gpa[32:00] + Input : u64 sq_rq_l0mtt_gpa + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_sq_rq_gpa_sign(u64 sq_rq_l0mtt_gpa) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_GPA_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_GPA_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_GPA_SIGN_CHECKBITS; i++) { + if ((sq_rq_l0mtt_gpa >> ROCE_QP_GPA_SIG_LEN) & (1ULL << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_GPA_SIGN_WIDTH, + ROCE_QP_GPA_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_GPA_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_qpcc_sign + Description : Signature 5bit + CheckedBits = QPn[19:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_qpcc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_QPCC_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_QPCC_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_QPCC_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_QPCC_SIGN_WIDTH, + ROCE_QP_QPCC_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_QPCC_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_rcc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_rcc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_sqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_sqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_sqac_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_sqac_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_rqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_rqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_qp_rrwc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_qp_rrwc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* + **************************************************************************** + Prototype : roce3_gen_sq_wqe_sign + Description : Signature 8bit + CheckedBits = {QPn[19:0], WQECI[19:0], WQE.Header[31:0]} + Input : u32 qpn + u32 wqe_ci + u32 wqe_header + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** +*/ +static u32 roce3_gen_sq_wqe_sign(u32 qpn, u32 wqe_ci, u32 wqe_header) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_SQ_WQE_SIGN_WIDTH] = {0}; + u32 check_bit[ROCE_SQ_WQE_SIGN_CHECKBITS] = {0}; + + for (i = 0; i < ROCE_SQ_WQE_SIGN_CHECKBITS_1; i++) { + if (wqe_header & (1U << i)) + check_bit[i] = 1; + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_1; i < ROCE_SQ_WQE_SIGN_CHECKBITS_12; i++) { + if (wqe_ci & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_1))) + check_bit[i] = 1; + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_12; i < ROCE_SQ_WQE_SIGN_CHECKBITS_123; i++) { + if (qpn & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_12))) + check_bit[i] = 1; + } + + roce3_calculate_sign_bit(check_bit, ROCE_SQ_WQE_SIGN_WIDTH, + ROCE_SQ_WQE_SIGN_SPLITNUM, sign_bit); + + signature = roce3_calculate_signature(sign_bit, ROCE_SQ_WQE_SIGN_WIDTH); + + return signature; +} +#endif + +static void roce3_qp_set_path(const struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + struct tag_roce_verbs_qpc_attr_path *path_info) +{ + u8 *dmac = ah_attr->roce.dmac; + + path_info->dw7.bs.hoplmt = (u8)ah_attr->grh.hop_limit; + path_info->dw6.bs.tclass = (u8)(ah_attr->grh.traffic_class | 0x2); + path_info->dw6.bs.flow_label = ah_attr->grh.flow_label & 0xfffff; /* flow_label: 20bit */ + path_info->dw7.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f; /* sgid_index: 7bit */ + + path_info->dw7.bs.base_sgid_n = (path_info->dw7.bs.sgid_index != ROCE_BASE_GID_IDX); + + memcpy((void *)path_info->dgid, (void *)ah_attr->grh.dgid.raw, + sizeof(path_info->dgid)); + + memcpy((void *)&path_info->dmac_l32, (void *)&dmac[ROCE_RAH_DMAC_L32_START], + sizeof(path_info->dmac_l32)); + path_info->dmac_l32 = cpu_to_be32(path_info->dmac_l32); + // shift dmac[0] left by 8 bits + path_info->dw0.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | dmac[1]; + path_info->dw7.bs.sl = (ah_attr->sl & 0x7); /* sl: 3bit */ +} + +struct roce3_pd *roce3_get_pd(struct roce3_qp *rqp) +{ + if (rqp->qp_type == IB_QPT_XRC_TGT) + return to_roce3_pd(to_roce3_xrcd(rqp->ibqp.xrcd)->pd); + else + return to_roce3_pd(rqp->ibqp.pd); +} + +void roce3_get_cqs(struct roce3_qp *rqp, struct roce3_cq **send_cq, struct roce3_cq **recv_cq) +{ + switch (rqp->qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = to_roce3_cq(to_roce3_xrcd(rqp->ibqp.xrcd)->cq); + *recv_cq = *send_cq; + break; + + case IB_QPT_XRC_INI: + *send_cq = to_roce3_cq(rqp->ibqp.send_cq); + *recv_cq = *send_cq; + break; + + default: + *send_cq = to_roce3_cq(rqp->ibqp.send_cq); + *recv_cq = to_roce3_cq(rqp->ibqp.recv_cq); + break; + } +} + +/* + **************************************************************************** + Prototype : roce3_set_access_flags + Description : roce3_set_access_flags + Input : struct roce3_qp *rqp + struct tag_roce_verbs_qp_attr *qp_attr + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + 2.Date : 2016/2/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_access_flags(struct roce3_qp *rqp, struct tag_roce_verbs_qp_attr *qp_attr, + struct ib_qp_attr *attr, int attr_mask) +{ + u16 dest_rd_atomic = 0; + u32 access_flags = 0; + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) + dest_rd_atomic = attr->max_dest_rd_atomic; + else + dest_rd_atomic = rqp->rsp_depth; + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) + access_flags = (u32)attr->qp_access_flags; + else + access_flags = rqp->atomic_rd_en; + + if (dest_rd_atomic == 0) + access_flags &= IB_ACCESS_REMOTE_WRITE; + + if ((access_flags & IB_ACCESS_REMOTE_READ) != 0) + qp_attr->com_info.dw0.bs.rre = 1; /* rc_rre: 1bits */ + + if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) != 0) + qp_attr->com_info.dw0.bs.rae = 1; /* rc_rae: 1bits */ + + if ((access_flags & IB_ACCESS_REMOTE_WRITE) != 0) + qp_attr->com_info.dw0.bs.rwe = 1; /* rrw_rwe: 1bits */ +} + +int roce3_sqp_check(const struct roce3_qp *qp) +{ + if (((qp->qp_type == IB_QPT_GSI) && (qp->qpn == ROCE_QP_SMI_QP_NUM)) || + (qp->qpn == ROCE_QP_GSI_QP_NUM)) + return 1; + + return 0; +} + +/* + **************************************************************************** + Prototype : store_sqp_attrs + Description : save the special + Input : struct roce3_sqp *sqp + const struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2015/7/24 + Modification : Created function + +**************************************************************************** +*/ +static void store_sqp_attrs(struct roce3_sqp *sqp, const struct ib_qp_attr *attr, int attr_mask) +{ + if (((u32)attr_mask & IB_QP_PKEY_INDEX) != 0) + sqp->pkey_index = attr->pkey_index; + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) + sqp->qkey = attr->qkey; + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) + sqp->send_psn = attr->sq_psn; +} + +/* + **************************************************************************** + Prototype : qpc_seg_to_be + Input : struct roce_qp_context *be_ctx + struct roce_qp_context *le_ctx + struct roce3_qp *rqp + Output : None + + 1.Date : 2015/7/29 + Modification : Created function + +**************************************************************************** +*/ +static void qpc_seg_to_be(struct tag_roce_verbs_qp_attr *be_ctx, + struct tag_roce_verbs_qp_attr *le_ctx, struct roce3_qp *rqp) +{ + /* DRV Seg */ + memcpy((void *)be_ctx->path_info.dgid, (void *)le_ctx->path_info.dgid, + sizeof(le_ctx->path_info.dgid)); + + /* CHIP Seg */ + be_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa = cpu_to_be64(le_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa); + + be_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + cpu_to_be64(le_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num); +} + +/* + **************************************************************************** + Prototype : roce3_qpc_to_be + Description : translate big endian + Input : struct tag_roce_verbs_qp_attr *qp_attr + struct roce3_qp *rqp + u32 *be_ctx + Output : None + + 1.Date : 2015/7/29 + Modification : Created function + +**************************************************************************** +*/ +void roce3_qpc_to_be(struct tag_roce_verbs_qp_attr *qp_attr, struct roce3_qp *rqp, u32 *be_ctx) +{ + u32 *ctx = NULL; + u32 *ctx1 = NULL; + u32 i = 0; + u32 ctx_size = 0; + + ctx = be_ctx; + ctx1 = (u32 *)qp_attr; + ctx_size = sizeof(struct tag_roce_verbs_qp_attr) / sizeof(u32); + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) + *ctx = cpu_to_be32(*ctx1); + + qpc_seg_to_be((struct tag_roce_verbs_qp_attr *)((void *)be_ctx), qp_attr, rqp); +} + +/* + **************************************************************************** + Prototype : roce3_alloc_opt_rdmarc + Description : alloc rdmarc when modify qp + Input : struct roce3_qp *rqp + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_alloc_opt_rdmarc(struct roce3_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + u32 response_depth = 0; + + rdev = to_roce3_dev(rqp->ibqp.device); + /*lint -e587*/ + response_depth = (attr->max_dest_rd_atomic == 0) ? + 1 : (u32)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_dest_rd_atomic); + /*lint +e587*/ + if (rqp->rsp_depth != response_depth) { + if (rqp->rsp_depth > 0) + roce3_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + + ret = roce3_rdma_rdmarc_alloc(rdev->hwdev, response_depth, &rqp->rdmarc); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma rdmarc, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + rqp->rsp_depth = (u16)response_depth; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_free_opt_rdmarc + Description : free rdmarc when modify qp failed or destroy qp + Input : struct roce3_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +void roce3_free_opt_rdmarc(struct roce3_qp *rqp) +{ + struct roce3_device *rdev = NULL; + + rdev = to_roce3_dev(rqp->ibqp.device); + + if (rqp->rsp_depth > 0) + roce3_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + + rqp->rsp_depth = 0; +} + +/* + **************************************************************************** + Prototype : roce3_set_opt_rdmarc + Description : set opt rdmarc field + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_opt_rdmarc(struct roce3_device *rdev, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + qp_attr->chip_seg.dw10.bs.rc_page_gpa_h = + (rqp->rdmarc.dma_addr >> 40) & 0xffffff; /* high:24bit, 64-24= 40 */ + qp_attr->chip_seg.rc_page_gpa_l = (u32)(rqp->rdmarc.dma_addr >> 8); // bit[39:8] + qp_attr->chip_seg.dw4.bs.rc_size = rqp->rdmarc.order; + qp_attr->com_info.dw1.bs.rra_max = ((u8)fls(rqp->rsp_depth - 1)) & 0x7; /* rra_max:3bit */ + qp_attr->chip_seg.dw4.bs.rc_max_size = rqp->rdmarc.ext_order - 3; +} + +static void roce3_set_opt_mtu(struct tag_roce_verbs_qp_attr *qp_attr, const struct ib_qp_attr *attr) +{ + u32 pmtu = (u32)attr->path_mtu & 0x7; + + qp_attr->com_info.dw3.bs.pmtu = pmtu; /* 1,2,3,4,5 */ + qp_attr->com_info.dw5.bs.mtu_code = + ROCE_QPC_PMTU_TRANSLATE(pmtu); /* 0,1,3,7,15 */ + if (qp_attr->com_info.dw3.bs.pmtu == IB_MTU_1024) + qp_attr->com_info.dw3.bs.base_mtu_n = 0; + else + qp_attr->com_info.dw3.bs.base_mtu_n = 1; +} + +/* + **************************************************************************** + Prototype : roce3_set_opt_sra + Description : set opt sra + Input : struct tag_roce_verbs_qp_attr *qp_attr + struct ib_qp_attr *attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_opt_sra(struct tag_roce_verbs_qp_attr *qp_attr, struct ib_qp_attr *attr) +{ + u8 initiator_depth; + + /*lint -e587*/ + initiator_depth = (attr->max_rd_atomic == 0) ? 1 : + (u8)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_rd_atomic); + /*lint +e587*/ + qp_attr->com_info.dw1.bs.sra_max = + ((u32)ROCE_FLS(initiator_depth - 1)) & 0x7; /* sra_max:3bit */ +} + +u8 roce3_get_db_cos_from_vlan_pri(struct roce3_device *rdev, u8 vlan_pri) +{ + u8 db_cos; + int ret; + + ret = hinic3_get_cos_by_pri(rdev->hwdev, vlan_pri, &db_cos); + if (ret != 0) { + pr_info("%s, ret:%d vlan_pri:%u\n", __func__, ret, vlan_pri); + db_cos = 0; + } + + return db_cos; +} + +static void roce3_set_opt_attr_av(struct roce3_qp *rqp, struct roce3_device *rdev, + struct ib_qp_attr *attr, struct tag_roce_verbs_qp_attr *qp_attr, u32 *optpar) +{ + u8 cos; + struct roce3_get_cos_inbuf inbuf = { 0 }; + + roce3_qp_set_path(rdev, &attr->ah_attr, &qp_attr->path_info); + qp_attr->com_info.dw3.bs.ext_mtu = rqp->ext_mtu & 0x1; + qp_attr->com_info.dw3.bs.ext_md = rqp->ext_mtu_mode & 0x1; + qp_attr->com_info.dw2.bs.vroce_en = (u32)rdev->is_vroce; + *optpar |= ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH; + + inbuf.sl = attr->ah_attr.sl; + inbuf.sgid_index = attr->ah_attr.grh.sgid_index; + inbuf.port_num = attr->ah_attr.port_num; + inbuf.traffic_class = attr->ah_attr.grh.traffic_class; + (void)roce3_get_dcb_cfg_cos(rdev, &inbuf, &cos); + if (rdev->is_vroce) + cos = rqp->sl; + + qp_attr->com_info.dw5.bs.cos = cos & 0x7; + + rqp->db_sgid_index = attr->ah_attr.grh.sgid_index & 0x7f; +} + +static void roce3_set_opt_field_attr(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + int attr_mask, struct tag_roce_verbs_qp_attr *qp_attr, u32 *optpar) +{ + /* RTR2RTS/SQD2SQD RETRY_CNT */ + if ((((u32)attr_mask) & IB_QP_RETRY_CNT) != 0) { + qp_attr->com_info.dw1.bs.to_retry_limit = + attr->retry_cnt & 0x7; /* to_retry_limit:3 */ + *optpar |= ROCE_QP_OPTPAR_RETRY_COUNT; + } + + /* RTR2RTS/SQD2SQD RNR_RETRY */ + if ((((u32)attr_mask) & IB_QP_RNR_RETRY) != 0) { + qp_attr->com_info.dw1.bs.rnr_retry_limit = + attr->rnr_retry & 0x7; /* rnr_retry_limit:3bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_RETRY; + } + + /* INIT2RTR/SQD2SQD/RTR2RTS/RTS2RTS MIN_RNR_TIMER */ + if ((((u32)attr_mask) & IB_QP_MIN_RNR_TIMER) != 0) { + qp_attr->com_info.dw2.bs.min_rnr_nak = + attr->min_rnr_timer & 0x1f; /* min_rnr_nak:5bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) { + qp_attr->com_info.dw8.bs.next_send_psn = + attr->sq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_RQ_PSN) != 0) { + qp_attr->com_info.dw9.bs.next_rcv_psn = + attr->rq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_DEST_QPN) != 0) { + qp_attr->com_info.dw0.bs.dest_qp = + attr->dest_qp_num & 0xffffff; /* dest_qp:24bit */ + } + +#ifdef ROCE_VBS_EN + if ((((u32)attr_mask) & ROCE_QP_VBS_FLAG) != 0) { /* IBV_QP_VBS_OSD_FLAG */ + roce3_vbs_set_attr(rqp, qp_attr, attr_mask); + } +#endif +} + +static s32 roce3_set_db_path_mtu(const struct ib_qp_attr *attr, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + switch (attr->path_mtu) { + case IB_MTU_256: + case IB_MTU_512: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s:Only support mtu larger than 1K.func_id(%d) mtu:%u\n", + __func__, rdev->glb_func_id, attr->path_mtu); + return -EINVAL; + case IB_MTU_1024: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_1024; + break; + case IB_MTU_2048: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_2048; + break; + case IB_MTU_4096: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_4096; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int roce3_set_opt_field(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask, + struct tag_roce_verbs_qp_attr *qp_attr, u32 *optpar) +{ + struct roce3_device *rdev = to_roce3_dev(rqp->ibqp.device); + int ret = 0; + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) { + qp_attr->com_info.q_key = attr->qkey; + *optpar |= ROCE_QP_OPTPAR_Q_KEY; + } + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + if (rdev->is_vroce) { + if (rqp->qp_type == (u32)IB_QPT_RC) { + attr->ah_attr.sl = rdev->group_rc_cos; + rqp->sl = rdev->group_rc_cos; + } else if (rqp->qp_type == (u32)IB_QPT_UD) { + attr->ah_attr.sl = rdev->group_ud_cos; + rqp->sl = rdev->group_ud_cos; + } else { + attr->ah_attr.sl = rdev->group_xrc_cos; + rqp->sl = rdev->group_xrc_cos; + } + } + roce3_set_opt_attr_av(rqp, rdev, attr, qp_attr, optpar); + } + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) { + qp_attr->com_info.dw5.bs.port = attr->port_num & 0x7; /* port: 3bits */ + *optpar |= ROCE_QP_OPTPAR_SCHED_QUEUE; + } + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + roce3_set_opt_rdmarc(rdev, rqp, qp_attr); + *optpar |= ROCE_QP_OPTPAR_RRA_MAX; + } + + /* RTR2RTS/SQD2SQD SRA_MAX */ + if ((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) { + roce3_set_opt_sra(qp_attr, attr); + *optpar |= ROCE_QP_OPTPAR_SRA_MAX; + } + + /* RWE/RRE/RAE,RST2INIT */ + if ((((u32)attr_mask) & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) != 0) { + roce3_set_access_flags(rqp, qp_attr, attr, attr_mask); + *optpar |= ROCE_QP_OPTPAR_RWE | ROCE_QP_OPTPAR_RRE | ROCE_QP_OPTPAR_RAE; + } + + /* RTR2RTS/SQD2SQD ACK TIMEOUT */ + if ((((u32)attr_mask) & IB_QP_TIMEOUT) != 0) { + qp_attr->com_info.dw2.bs.ack_to = attr->timeout & 0x1f; /* ack_to:5bits */ + *optpar |= ROCE_QP_OPTPAR_ACK_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) { + ret = roce3_set_db_path_mtu(attr, rqp, rdev); + if (ret != 0) + return ret; + + roce3_set_opt_mtu(qp_attr, attr); + } + + roce3_set_opt_field_attr(rqp, attr, attr_mask, qp_attr, optpar); + return ret; +} + +static void roce3_set_qp_srq_attr(struct roce3_qp *rqp, struct tag_roce_verbs_qp_attr *qp_attr, + struct roce3_pd *pd, const struct roce3_cq *recv_cq) +{ + struct roce3_srq *rsrq = to_roce3_srq(rqp->ibqp.srq); + struct roce_srq_context temp; + + if (rsrq->container_flag != 0) { + struct roce_srq_context *srqc = (struct roce_srq_context *) + ((void *)rsrq->cqm_srq->q_ctx_vaddr); + + temp.dw0.value = be32_to_cpu(srqc->dw0.value); + qp_attr->chip_seg.dw12.bs.srq_pd = temp.dw0.bs.pdn & 0x3ffff; /* srq_pd:18bits */ + qp_attr->chip_seg.dw12.bs.srq_wqebb_size = temp.dw0.bs.wqebb_size; + qp_attr->chip_seg.dw12.bs.srq_page_size = temp.dw0.bs.page_size; + qp_attr->chip_seg.dw12.bs.srq_size = temp.dw0.bs.size; + + temp.dw1.value = be32_to_cpu(srqc->dw1.value); + qp_attr->com_info.dw5.bs.so_ro = temp.dw1.bs_c.so_ro; + qp_attr->com_info.dw5.bs.dma_attr_idx = temp.dw1.bs_c.dma_attr_idx; + + qp_attr->chip_seg.dw9.bs.container_sz = + MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode; + qp_attr->chip_seg.dw9.bs.container_en = 1; + + qp_attr->com_info.dw3.bs.srq_container = 1; + + if (rsrq->rqe_cnt_th > 0) { + qp_attr->chip_seg.dw9.bs.srq_warth_flag = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = + rsrq->rqe_cnt_th & 0xf; /* srq_rqecnt_th:4bits */ + } + + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & 0x3ffff; /* srqn:18bits */ + } else { + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & 0x3ffff; /* srqn:18bits */ + } + + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw0.bs.rkey_en = 1; + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & 0xfffff; /* rq_cqn:20bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; +} + +static void roce3_set_qp_rst2init_attr(struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, struct roce3_pd *pd, struct roce3_cq *recv_cq) +{ + qp_attr->chip_seg.dw4.bs.rq_base_ci = + (u32)(ROCE_ILOG2(rqp->rq.offset >> rqp->rq.wqe_shift)); + qp_attr->chip_seg.dw19.bs.qp_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + qp_attr->com_info.dw3.bs.dsgl_en = rqp->double_sgl_mode & 0x1; + + if (rqp->ibqp.srq) { + roce3_set_qp_srq_attr(rqp, qp_attr, pd, recv_cq); + } else { + qp_attr->chip_seg.dw15.bs.rq_size = (rqp->rq.wqebb_cnt > 0) ? + (((u32)ROCE_ILOG2(rqp->rq.wqebb_cnt)) & 0x1f) : 0; + qp_attr->chip_seg.dw15.bs.rq_page_size = + ((u32)((u32)rqp->buf.page_shift - PAGE_SHIFT_4K)) & ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw15.bs.rq_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw8.bs.rq_wqebb_size = + (u32)(rqp->rq.wqe_shift - BYTES_TO_16B_SHIFT) & 0x7; + qp_attr->com_info.dw7.bs.rq_cqn = recv_cq->cqn & 0xfffff; /* rq_cqn:20bits */ + qp_attr->com_info.dw7.bs.rq_cqn_lb = g_rq_cqn_lb; + qp_attr->chip_seg.dw15.bs.rq_rkey_en = 1; + + qp_attr->chip_seg.dw8.bs.rq_pi_on_chip = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_lth = ROCE_RQ_WQECNT_LTH; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen0 = ROCE_RQ_MTT_PREFETCH_MAXLEN0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen1 = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxwqe = ROCE_RQ_MTT_PREFETCH_MAXWQE; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_minnum = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_maxnum = ROCE_RQ_WQE_PREFETCH_MAXNUM; + } +} + +static void roce3_qp_rst2init_set_qpcc(struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_qp *rqp, const struct roce3_device *rdev) +{ + qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = rqp->mtt.mtt_paddr; + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->db.dma & (~0x3ULL); /* bits:63-2 */ + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num |= + rqp->mtt.mtt_layers & 0x3; /* bits:1-0 */ + if (rqp->qp_type == IB_QPT_XRC_TGT) { + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->qpc_info->paddr & (~0x3ULL); /* bits:63-2 */ + } + + qp_attr->chip_seg.dw4.bs.sq_rq_mtt_page_size = rqp->mtt.mtt_page_shift - PAGE_SHIFT_4K; + /* rc_entry_size:2bits */ + qp_attr->chip_seg.dw10.bs.rc_entry_size = + ((u32)ROCE_ILOG2(rdev->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz >> + BYTES_TO_16B_SHIFT) & 0x3); + qp_attr->chip_seg.dw14.bs.qp_rkey_en = 1; + qp_attr->chip_seg.dw10.bs.rc_entry_prefetch_maxnum = 0; +} + +static void roce3_qp_rst2init_set_sqc_sqac(struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_qp *rqp, const struct roce3_pd *pd, const struct roce3_cq *send_cq) +{ +#ifdef ROCE_COMPUTE + u8 sq_pi_on_chip = (u8)((rqp->qpn <= ROCE_MAX_PI_ON_CHIP_QPN) && + (rqp->qp_type != IB_QPT_UD)); + + qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = sq_pi_on_chip; +#else + qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = 0; +#endif + qp_attr->chip_seg.dw16.bs.sq_wqebb_size = + ((u32)(rqp->sq.wqe_shift - ROCE_SQ_WQEBB_SIZE_SHIFT)) & ROCE_SQ_WQEBB_SIZE_MASK; + qp_attr->chip_seg.dw16.bs.sq_pd = ((u32)pd->pdn) & 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw16.bs.sq_page_size = + ((u32)(rqp->buf.page_shift - PAGE_SHIFT_4K)) & ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw5.bs.sq_inline_en = rqp->max_inline_data > 0 ? 1 : 0; + + qp_attr->chip_seg.dw16.bs.sq_rkey_en = 1; + + qp_attr->chip_seg.dw5.bs.sq_wqecnt_lth = ROCE_SQ_WQECNT_LTH; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_minnum = ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_maxnum = ROCE_SQ_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw6.bs.sq_wqe_prefetch_mode = 0; + qp_attr->chip_seg.dw6.bs.sq_mtt_prefetch_maxlen = 0; + + if (rqp->sq.wqebb_cnt != 0) { + qp_attr->chip_seg.dw5.bs.sq_size = + (u32)(ROCE_ILOG2((u32)rqp->sq.wqebb_cnt)) & ROCE_SQ_SIZE_MASK; + } + + qp_attr->com_info.dw7.bs.sq_cqn_lb = g_sq_cqn_lb; + qp_attr->chip_seg.dw17.bs.sqa_cqn = send_cq->cqn & ROCE_SQ_CQN_MASK; + + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_minnum = ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_maxnum = ROCE_WQE_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw6.bs.sqa_mtt_prefetch_maxlen = 0; + qp_attr->chip_seg.dw5.bs.sqa_wqe_cache_thd_sel = 0; +} + +static void roce3_qp_rst2init_set_drv(struct tag_roce_verbs_qp_attr *qp_attr, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + u16 host_oqid = 0; + u8 host_id = 0; + + host_id = hinic3_host_id(rdev->hwdev); + if (host_id == ROCE_SPU_HOST_ID) { + host_id = ROCE_SPU_OQID_HOST_ID; + host_oqid = (u16)ROCE_GET_SPU_HOST_OQID(host_id, rdev->glb_func_id, rqp->qpn); + } else { + host_oqid = (u16)ROCE_GET_HOST_OQID(host_id, rqp->qpn); + } + + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce3_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7; + if (rdev->is_vroce) { + if (qp_attr->com_info.dw0.bs.service_type == ROCE_QP_ST_RC) { + qp_attr->path_info.dw7.bs.sl = rdev->group_rc_cos; + rqp->sl = rdev->group_rc_cos; + } else if (qp_attr->com_info.dw0.bs.service_type == ROCE_QP_ST_UD) { + qp_attr->path_info.dw7.bs.sl = rdev->group_ud_cos; + rqp->sl = rdev->group_ud_cos; + } else { + qp_attr->path_info.dw7.bs.sl = rdev->group_xrc_cos; + rqp->sl = rdev->group_xrc_cos; + } + } + qp_attr->com_info.dw3.bs.dif_en = (u32)rqp->signature_en; + qp_attr->com_info.dw2.bs.vroce_en = (u32)rdev->is_vroce; + + if (!rqp->ibqp.uobject) + qp_attr->com_info.dw0.bs.fre = 1; + + qp_attr->com_info.dw3.bs.xrc_vld = (u32)(rqp->qp_type == IB_QPT_XRC_INI); + qp_attr->com_info.dw1.bs.local_qp = rqp->qpn & 0xffffff; /* local_qp:24bits */ + qp_attr->com_info.dw5.bs.ep = rdev->hw_info.ep_id; + if (rdev->is_vroce) + qp_attr->com_info.dw2.bs.host_oqid = (host_oqid & 0xfffc) | ROCE_IPSUTX_CHANNEL_15; + else + qp_attr->com_info.dw2.bs.host_oqid = host_oqid & 0xffff; /* host_oqid:16bits */ + + + qp_attr->chip_seg.dw4.bs.dsgl = rqp->double_sgl_mode & 0x1; + qp_attr->path_info.dw7.bs.udp_src_port = (qp_attr->com_info.dw1.bs.local_qp & 0xff); +} + +static void roce3_qp_rst2init_set_common(struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_pd *pd) +{ + qp_attr->com_info.dw12.bs.ccf_app_id = ROCE_CC_DISABLE; // STUB + qp_attr->chip_seg.dw18.bs.ud_pd = pd->pdn & 0x3ffff; /* pd:18bits */ +} + +static void roce3_qp_rst2init_set_dcqcn(struct tag_roce_verbs_qp_attr *qp_attr) +{ +/* ecn ver + * b300 nic do not support the func + * context->sw_seg.ucode_seg.common.dw1.bs.ecn_ver = rdev->ecn_ctx.ecn_ver & 0xf; + */ +} + +static void roce3_qp_rst2init_set_fake_vf(const struct roce3_device *rdev, + const struct roce3_pd *pd, struct tag_roce_verbs_qp_attr *qp_attr) +{ + u32 fake_vf_id = 0; + struct roce3_ucontext *roce3_uctx = NULL; + u32 db_offset; + + fake_vf_id = ((u32)rdev->glb_func_id) << rdev->cfg_info.pf_start_bit; + if (rdev->cfg_info.pf_end_bit < ROCE_VLD_PF_NUM_10) + fake_vf_id |= 0x1C00; /* 12 11 10 bit 1 */ + else if (rdev->cfg_info.pf_end_bit < ROCE_VLD_PF_NUM_11) + fake_vf_id |= 0x1800; /* 12 11 bit 1 */ + else if (rdev->cfg_info.pf_end_bit < ROCE_VLD_PF_NUM_12) + fake_vf_id |= 0x1000; /* 12 bit 1 */ + + if (pd->ibpd.uobject) { + roce3_uctx = to_roce3_ucontext(pd->ibpd.uobject->context); + db_offset = (u32)((roce3_uctx->db_dma_addr >> (12UL + rdev->cfg_info.page_bit)) & + ((1UL << rdev->cfg_info.pf_start_bit) - 1)); + qp_attr->com_info.dw11.bs.vf_id = fake_vf_id | db_offset; + pr_info("[ROCE] func_id:%u The fake_id:%u ,start:%u end:%u page_bit:%u roce3_uctx->db_dma_addr:%llu\n", + rdev->glb_func_id, qp_attr->com_info.dw11.bs.vf_id, + rdev->cfg_info.pf_start_bit, rdev->cfg_info.pf_end_bit, + rdev->cfg_info.page_bit, roce3_uctx->db_dma_addr); + qp_attr->com_info.dw11.bs.fake = 1; + } else { + pr_info("[ROCE] kernel not support fake_vf currently. func:%u\n", + rdev->glb_func_id); + } +} + +static void roce3_qp_rst2init_set_xrc_attr(const struct roce3_qp *rqp, + const struct ib_qp_attr *attr, struct tag_roce_verbs_qp_attr *qp_attr, + const struct roce3_device *rdev, const struct roce3_cq *recv_cq) +{ + qp_attr->com_info.dw2.bs.xrc_srq_en = 1; + + if ((((u32)attr->qp_access_flags & XRC_CONTAINER_FLAG) != 0) || + (rdev->cfg_info.srq_container_en != 0)) { + qp_attr->chip_seg.dw9.bs.container_en = 1; + qp_attr->com_info.dw3.bs.srq_container = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & 0xffff; /* xrcd:16bits */ + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw13.bs.srq_state = ROCE_SRQ_INIT_STATE; + } else { + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & 0xffff; /* xrcd:16bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw14.bs.srq_xrcd = rqp->xrcdn & 0xffff; /* xrcd:16bits */ + } + if (recv_cq != NULL) + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & 0xfffff; /* rq_cqn:20bits */ +} + +/* + **************************************************************************** + Prototype : roce3_qp_rst2init + Description : parameter set when QP is from RESET to INIT + Input : struct roce3_qp *rqp + const struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2015/8/6 + Modification : Created function + +**************************************************************************** +*/ +void roce3_qp_rst2init(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + struct roce3_pd *pd = NULL; + struct roce3_cq *send_cq = NULL; + struct roce3_cq *recv_cq = NULL; + struct roce3_device *rdev = NULL; + + rdev = to_roce3_dev(rqp->ibqp.device); + pd = roce3_get_pd(rqp); + roce3_get_cqs(rqp, &send_cq, &recv_cq); + + /* CHIP_SEG: QPCC */ + roce3_qp_rst2init_set_qpcc(qp_attr, rqp, rdev); + + /* CHIP_SEG: SQC & SQAC */ + roce3_qp_rst2init_set_sqc_sqac(qp_attr, rqp, pd, send_cq); + + /* RRWC */ + qp_attr->chip_seg.dw4.bs.rrw_mtt_prefetch_maxlen = 0; + + /* RCC */ + qp_attr->chip_seg.dw10.bs.rc_mtt_prefetch_maxlen = ROCE_QP_MAX_PFETCH_MTT_LAYER; + + /* TIMER_SEG */ + qp_attr->com_info.dw3.bs.tss_timer_num = ROCE_QP_MAX_TIMER_NUM; + + /* DRV_SEG */ + roce3_qp_rst2init_set_drv(qp_attr, rqp, rdev); + + /* UCODE COM_SEG */ + roce3_qp_rst2init_set_common(qp_attr, pd); + + /* EXT DCQCN SEG */ + roce3_qp_rst2init_set_dcqcn(qp_attr); + + if (rdev->cfg_info.fake_en != 0) + roce3_qp_rst2init_set_fake_vf(rdev, pd, qp_attr); + + if (rqp->qp_type == IB_QPT_XRC_TGT) + roce3_qp_rst2init_set_xrc_attr(rqp, attr, qp_attr, rdev, recv_cq); + else + roce3_set_qp_rst2init_attr(rqp, qp_attr, pd, recv_cq); + +#ifdef ROCE_BONDING_EN + roce3_bond_rr_set_flow(rdev, rqp, qp_attr); +#endif +} + +/* + **************************************************************************** + Prototype : roce3_qp_init2rtr + Description : init2rtr attributes modify + Input : struct roce3_qp *rqp + const struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2015/8/1 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_qp_init2rtr(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + if (rqp->ibqp.srq || (rqp->qp_type == IB_QPT_XRC_TGT) || + (rqp->qp_type == IB_QPT_XRC_INI) || (rqp->qp_type == IB_QPT_UD) || + (rqp->qp_type == IB_QPT_UC) || (rqp->qp_type == IB_QPT_GSI)) { + qp_attr->com_info.dw3.bs.invalid_credit = 1; + } + + if ((rqp->qp_type == IB_QPT_GSI) || (rqp->qp_type == IB_QPT_UD)) { + qp_attr->com_info.dw3.bs.pmtu = IB_MTU_4096 & 0x7; /* for UD, mtu = 4096 */ + qp_attr->com_info.dw5.bs.mtu_code = ROCE_MTU_CODE_4K; + } + + qp_attr->com_info.dw2.bs.ack_to = 0xf; /* ack_to:5bits */ + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce3_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7; /* service_type: 3bit */ +} + +/* + **************************************************************************** + Prototype : roce3_qp_rtr2rts + Description : roce3_qp_rtr2rts + Input : struct roce3_qp *rqp + const struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + Output : None + + 1.Date : 2015/8/2 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_qp_rtr2rts(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ +} + +static void roce3_qp_2err(struct roce3_qp *rqp, const struct ib_qp_attr *attr, + struct tag_roce_verbs_qp_attr *qp_attr) +{ +} + +/* + **************************************************************************** + Prototype : roce3_set_abs_field + Description : set abs field to qpc + Input : struct roce3_qp *rqp + struct ib_qp_attr *attr + struct tag_roce_verbs_qp_attr *qp_attr + enum ib_qp_state cur_state + enum ib_qp_state new_state + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_abs_field(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask, + struct tag_roce_verbs_qp_attr *qp_attr) +{ + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) + roce3_qp_rst2init(rqp, attr, qp_attr); + + if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR)) + roce3_qp_init2rtr(rqp, attr, qp_attr); + + if ((cur_state == IB_QPS_RTR) && (new_state == IB_QPS_RTS)) + roce3_qp_rtr2rts(rqp, attr, qp_attr); + + if (new_state == IB_QPS_ERR) + roce3_qp_2err(rqp, attr, qp_attr); +} + +/* + **************************************************************************** + Prototype : roce3_record_opt_field + Description : set opt field to roce3_qp after send cmd modify_qp + Input : struct roce3_qp *rqp + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_record_opt_field(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask) +{ + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + rqp->qp_state = new_state; + + if ((((u32)attr_mask) & IB_QP_AV) != 0) + rqp->sl = attr->ah_attr.sl & 0x7; + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) + rqp->atomic_rd_en = (u8)attr->qp_access_flags; + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) + rqp->port = attr->port_num; + + if (roce3_sqp_check(rqp) != 0) + store_sqp_attrs(to_roce3_sqp(rqp), attr, attr_mask); +} + +/* + **************************************************************************** + Prototype : roce3_qp_modify_2rst_cmd + Description : cache out qpc mtt + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2016/6/17 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** +*/ +int roce3_qp_modify_2rst_cmd(struct roce3_device *rdev, u32 qpn) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_uni_cmd_qp_modify2rst *qp_moidfy2rst_inbuf = NULL; + struct timespec64 tv_start; + + ktime_get_ts64(&tv_start); + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_qp_modify2rst), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + qp_moidfy2rst_inbuf = (struct tag_roce_uni_cmd_qp_modify2rst *)cqm_cmd_inbuf->buf; + qp_moidfy2rst_inbuf->com.index = cpu_to_be32(qpn); + if (!roce3_is_roceaa(rdev->cfg_info.scence_id)) { + qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + } else { + qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_NOFAA_BITMASK); //lint !e778 + } + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_2RST_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send 2RST_QP command, qpn(0x%x), func_id(%d), ret(0x%x)\n", + __func__, qpn, rdev->glb_func_id, ret); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(2RST_QP), qpn(0x%x), func_id(%u)\n", + __func__, qpn, rdev->glb_func_id); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return -1; + } + } + roce3_timeout_check(rdev, &tv_start, MODIFY_2RST_CMDQ, 0); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_qp_check_rdmarc + Description : roce3_qp_check_rdmarc + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Create function + +**************************************************************************** +*/ +static int roce3_qp_check_rdmarc(struct roce3_device *rdev, struct roce3_qp *rqp, int *cost) +{ + int times = 0; + u32 rc_num = 0; + u32 rc_check_index = 0; + void *rdmarc_vaddr = NULL; + dma_addr_t rdmarc_paddr = 0; + + rc_num = (u32)(((u32)1 << rqp->rdmarc.order) / RDMARC_NUM_PER_CACHELINE); + rdmarc_vaddr = rqp->rdmarc.vaddr; + rdmarc_paddr = rqp->rdmarc.dma_addr; + + times = rdev->try_times; + do { + if (roce3_hca_is_present(rdev) == 0) + return 0; + + if ((*((u32 *)rdmarc_vaddr) == ROCE_RDMARC_DESTROY_CHECK_VALUE) && + (cpu_to_be32(*((u32 *)rdmarc_vaddr + 1)) == rqp->qpn) && + (cpu_to_be64(*((u64 *)rdmarc_vaddr + 1)) == + ((u64)rdmarc_paddr + ((u64)ROCE_CACHE_LINE_SIZE * (u64)rc_check_index)))) { + rdmarc_vaddr = (void *)((u8 *)rdmarc_vaddr + ROCE_CACHE_LINE_SIZE); + rc_check_index++; + if (rc_num == rc_check_index) + break; + } else { + ROCE_UDELAY(US_PERF_DELAY); + times--; + } + } while (times != 0); + + if (rc_num != rc_check_index) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read rdmarc field back after try %d times, func_id(%d) rc_check_index:%u rc_num:%u\n", + __func__, ((rdev->try_times - times) + 1), + rdev->glb_func_id, rc_check_index, rc_num); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static int roce3_qp_check_state(struct roce3_device *rdev, + const struct roce3_qp *rqp, const u32 *wb_va, int *cost) +{ + int times = 0; + int read_back_flag = 0; + + times = rdev->try_times; + while ((times--) != 0) { + if (roce3_hca_is_present(rdev) == 0) + return 0; + + if (*(u32 *)wb_va == ROCE_QP_DESTROY_CHECK_VALUE) { + read_back_flag = 1; + break; + } + ROCE_UDELAY(US_PERF_DELAY); + } + + if (read_back_flag == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read QP field back after try %d times,qpn(0x%x), wb_data(0x%x), func_id(%d)\n", + __func__, ((rdev->try_times - times) + 1), rqp->qpn, + *(u32 *)wb_va, rdev->glb_func_id); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static void roce3_qp_fill_cacheout_inbuf(struct roce3_device *rdev, + struct roce3_qp *rqp, dma_addr_t wb_dma, struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct tag_roce_uni_cmd_qp_cache_invalid *qp_cacheout_inbuf = NULL; + struct rdma_service_cap *rdma_cap = NULL; + u16 host_oqid = 0; + u8 host_id = 0; + + host_id = hinic3_host_id(rdev->hwdev); + if (host_id == ROCE_SPU_HOST_ID) { + host_id = ROCE_SPU_OQID_HOST_ID; + host_oqid = (u16)ROCE_GET_SPU_HOST_OQID(host_id, rdev->glb_func_id, rqp->qpn); + } else { + host_oqid = (u16)ROCE_GET_HOST_OQID(host_id, rqp->qpn); + } + + qp_cacheout_inbuf = (struct tag_roce_uni_cmd_qp_cache_invalid *)(cqm_cmd_inbuf)->buf; + rdma_cap = &rdev->rdma_cap; + + if (!roce3_is_roceaa(rdev->cfg_info.scence_id)) { + qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + } else { + qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_NOFAA_BITMASK); //lint !e778 + } + + qp_cacheout_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_cacheout_inbuf->qp_cache.sq_buf_len = + cpu_to_be32(ALIGN(((u32)rqp->sq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), PAGE_SIZE)); + qp_cacheout_inbuf->qp_cache.rq_buf_len = + cpu_to_be32(ALIGN(((u32)rqp->rq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), PAGE_SIZE)); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_flags = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_num = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + qp_cacheout_inbuf->qp_cache.wb_gpa = cpu_to_be64(wb_dma); + qp_cacheout_inbuf->qp_cache.dw9.bs.host_oqid = cpu_to_be16(host_oqid); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_flags = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_num = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); +} + +static int roce3_qp_send_cache_out_cmd(struct roce3_device *rdev, + struct roce3_qp *rqp, void *wb_va, dma_addr_t wb_dma, int *cost) +{ + unsigned long end; + int times = 0; + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_qp_cache_invalid), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce3_qp_fill_cacheout_inbuf(rdev, rqp, wb_dma, cqm_cmd_inbuf); + + end = jiffies + (ROCE_CMDQ_TIMEOUT_10MS * HZ); + while (time_before(jiffies, end)) { + times++; + ret = roce3_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_MISC_CACHE_INVLD, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_C); + if (ret <= 0) { /* return success or exception happened */ + break; + } + + /*lint -e160 -e522*/ + cond_resched(); + /*lint +e160 +e522*/ + }; + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed MISC_CACHE_INVLD command after try %d times, ret(%d), func_id(%d) qpn(%d)\n", + __func__, times, ret, rdev->glb_func_id, rqp->qpn); + /* Card not present need to return OK */ + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return ok), QPN(0x%x), ret(%d), func_id(%d)\n", + __func__, rqp->qpn, ret, rdev->glb_func_id); + goto out; + } + + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: CMDq timeout, func_id(%d)\n", + __func__, rdev->glb_func_id); + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = times; + return (-EINVAL); + } + +out: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = (rdev->try_times - times) + 1; + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_qp_cache_out_cmd + Description : cache out qpc mtt + Input : struct roce3_device *rdev + struct roce3_qp *rqp + Output : None + + 1.Date : 2016/9/24 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** +*/ +int roce3_qp_cache_out_cmd(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret = 0; + int cost = 0; + void *wb_va = NULL; + dma_addr_t wb_dma = 0; + struct timespec64 tv_start; + + ktime_get_ts64(&tv_start); + wb_va = dma_alloc_coherent(&rdev->pdev->dev, (size_t)4UL, &wb_dma, GFP_KERNEL); + if (wb_va == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc wb_va, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_ALLOC, cost); + + ret = roce3_qp_send_cache_out_cmd(rdev, rqp, wb_va, wb_dma, &cost); + if (ret != 0) + goto err; + + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_CMDQ, cost); + if (rqp->rsp_depth > 0) { + ret = roce3_qp_check_rdmarc(rdev, rqp, &cost); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check qp rdmarc, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err; + } + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_RDMARC_CHECK, cost); + } + ret = roce3_qp_check_state(rdev, rqp, (u32 *)wb_va, &cost); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to check qp state, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err; + } + roce3_timeout_check(rdev, &tv_start, CACHE_INVLAID_QP_CHECK, cost); + +err: + dma_free_coherent(&rdev->pdev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static u8 roce3_get_qp_cmdq_cmd(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_attr *attr, int attr_mask) +{ + static u16 op[ROCE_QP_STATE_NUM][ROCE_QP_STATE_NUM]; + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + if ((cur_state > IB_QPS_ERR) || (new_state > IB_QPS_ERR)) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: QP state modify invalid, cur_state(%d), new_state(%d), func_id(%d)\n", + __func__, cur_state, new_state, rdev->glb_func_id); + return 0; + } + op[IB_QPS_RESET][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RESET][IB_QPS_INIT] = ROCE_CMD_RST2INIT_QP; + + op[IB_QPS_INIT][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_INIT][IB_QPS_INIT] = ROCE_CMD_INIT2INIT_QP; + op[IB_QPS_INIT][IB_QPS_RTR] = ROCE_CMD_INIT2RTR_QP; + + op[IB_QPS_RTR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTR][IB_QPS_RTS] = ROCE_CMD_RTR2RTS_QP; + + op[IB_QPS_RTS][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTS][IB_QPS_RTS] = ROCE_CMD_RTS2RTS_QP; + op[IB_QPS_RTS][IB_QPS_SQD] = ROCE_CMD_RTS2SQD_QP; + + op[IB_QPS_SQD][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQD][IB_QPS_RTS] = ROCE_CMD_SQD2RTS_QP; + op[IB_QPS_SQD][IB_QPS_SQD] = ROCE_CMD_SQD2SQD_QP; + + op[IB_QPS_SQE][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQE][IB_QPS_RTS] = ROCE_CMD_SQERR2RTS_QP; + + op[IB_QPS_ERR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + + return (u8)op[cur_state][new_state]; +} + +static int roce3_qp_rts2sqd_cmd(struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + int sqd_event = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_uni_cmd_qp_modify_rts2sqd *qp_rts2sqd_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_qp_modify_rts2sqd), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + sqd_event = ((((u32)attr_mask) & IB_QP_EN_SQD_ASYNC_NOTIFY) != 0) && + (attr->en_sqd_async_notify != 0); + qp_rts2sqd_inbuf = (struct tag_roce_uni_cmd_qp_modify_rts2sqd *)cqm_cmd_inbuf->buf; + qp_rts2sqd_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_rts2sqd_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + qp_rts2sqd_inbuf->sqd_event_en = cpu_to_be32((u32)(sqd_event != 0)); + + ret = roce3_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_RTS2SQD_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send RTS2SQD_QP cmd, func_id(%d), ret(0x%x)\n", + __func__, rdev->glb_func_id, ret); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +static void roce3_qp_fill_modify_inbuf(struct tag_cqm_cmd_buf *cqm_cmd_inbuf, struct roce3_qp *rqp, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar) +{ + struct tag_roce_uni_cmd_modify_qpc *qp_modify_inbuf = NULL; + + qp_modify_inbuf = (struct tag_roce_uni_cmd_modify_qpc *)cqm_cmd_inbuf->buf; + roce3_qpc_to_be(qp_attr, rqp, (u32 *)&qp_modify_inbuf->qp_attr); + + qp_modify_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + qp_modify_inbuf->com.opt = cpu_to_be32(optpar); + +#ifdef ROCE_VBS_EN + if (rqp->vbs_qp_ptr) { // vbs_en, transmit record addr to ucode + roce3_vbs_transmit_sqpc_ci_record_addr(rqp, cqm_cmd_inbuf->buf); + } +#endif + +} + +static int roce3_qp_handle_err(struct roce3_device *rdev, const struct roce3_qp *rqp, int ret) +{ + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send MODIFY_QP command, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(MODIFY_QP), qpn(0x%x), func_id(%u)\n", + __func__, rqp->qpn, rdev->glb_func_id); + + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + return -1; + } + + return 0; +} + +static int roce3_qp_modify_cmd(struct roce3_device *rdev, struct roce3_qp *rqp, u8 cmd, + struct tag_roce_verbs_qp_attr *qp_attr, u32 optpar) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_modify_qpc), + &cqm_cmd_outbuf, ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return (-ENOMEM); + } + + if (!roce3_is_roceaa(rdev->cfg_info.scence_id)) + roce3_qp_fill_modify_inbuf(cqm_cmd_inbuf, rqp, qp_attr, optpar); + else + roce3_qp_modify_cmd_ext(cqm_cmd_inbuf, rqp, qp_attr, optpar); + + ret = roce3_send_qp_lb_cmd(rqp->qpn, rdev, cmd, cqm_cmd_inbuf, + cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) + ret = roce3_qp_handle_err(rdev, rqp, ret); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +void roce3_cq_clean(struct roce3_cq *cq, u32 qpn, struct roce3_srq *srq) +{ + spin_lock_irq(&cq->lock); + roce3_cq_clean_process(cq, qpn, srq); + spin_unlock_irq(&cq->lock); +} + +static int roce3_qp_clean_kernel_res(struct roce3_qp *rqp) +{ + struct roce3_cq *send_cq = NULL; + struct roce3_cq *recv_cq = NULL; + + roce3_get_cqs(rqp, &send_cq, &recv_cq); + + roce3_cq_clean(recv_cq, rqp->qpn, rqp->ibqp.srq ? to_roce3_srq(rqp->ibqp.srq) : NULL); + if (send_cq != recv_cq) + roce3_cq_clean(send_cq, rqp->qpn, NULL); + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + rqp->sq_next_wqe = 0; + + *(rqp->db.db_record) = 0; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + if (rqp->has_rq && rqp->qp_buf_info->q_header_vaddr) + memset(rqp->qp_buf_info->q_header_vaddr, 0, sizeof(u64)); + } + + return 0; +} + +static int roce3_qp_modify_2rst(struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret = 0; + + ret = roce3_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify QP(0x%06x) to RESET, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce3_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: QP(0x%06x) cache invalid failed, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + + rqp->qp_state = IB_QPS_RESET; + + if (rqp->ibqp.uobject == NULL) { + ret = roce3_qp_clean_kernel_res(rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Clean kernel QP(0x%06x) failed, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static int roce3_qp_modify(struct roce3_device *rdev, struct roce3_qp *rqp, + struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + u32 optpar = 0; + struct tag_roce_verbs_qp_attr qp_attr; + u8 cmd = 0; + + memset(&qp_attr, 0, sizeof(qp_attr)); + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + ret = roce3_alloc_opt_rdmarc(rqp, attr, attr_mask); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdmarc, func(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + ret = roce3_set_opt_field(rqp, attr, attr_mask, &qp_attr, &optpar); + if (ret != 0) + goto err_out; + + roce3_set_abs_field(rqp, attr, attr_mask, &qp_attr); + + cmd = roce3_get_qp_cmdq_cmd(rdev, rqp, attr, attr_mask); + if (cmd == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Invalid cmd, func_id(%d), qpn(%d), cmd(%d)\n", + __func__, rdev->glb_func_id, rqp->qpn, cmd); + ret = (-EINVAL); + goto err_out; + } + if (cmd == ROCE_CMD_RTS2SQD_QP) + ret = roce3_qp_rts2sqd_cmd(rdev, rqp, attr, attr_mask); + else + ret = roce3_qp_modify_cmd(rdev, rqp, cmd, &qp_attr, optpar); + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send modify cmd(%d), func(%d), qpn(%d), ret(%d)\n", + __func__, cmd, rdev->glb_func_id, rqp->qpn, ret); + goto err_out; + } + + roce3_record_opt_field(rqp, attr, attr_mask); + return 0; + +err_out: + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) + roce3_free_opt_rdmarc(rqp); + + return ret; +} + +static int roce3_qp_get_path(struct roce3_device *rdev, struct roce3_qp *rqp, + struct rdma_ah_attr *ah_attr) +{ + return 0; +} + +static int roce3_qp_modify_check_rd_and_mtu(const struct roce3_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask, const struct rdma_service_cap *rdma_cap) +{ + enum ib_qp_state cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + enum ib_qp_state new_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ? + attr->qp_state : cur_state); + + if (((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) && + (attr->max_rd_atomic > rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma)) { + pr_err( + "[ROCE, ERR] %s: Qpn(0x%x), max_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->max_rd_atomic, cur_state, + new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) && + (attr->max_dest_rd_atomic > rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma)) { + pr_err("[ROCE, ERR] %s: Qpn(0x%x), max_dest_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->max_dest_rd_atomic, + cur_state, new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) && + ((attr->path_mtu < IB_MTU_256) || (attr->path_mtu > IB_MTU_4096))) { + pr_err("[ROCE, ERR] %s: Input path MTU(%u) is invalid\n", __func__, attr->path_mtu); + return -EINVAL; + } + + return 0; +} + +static int roce3_qp_modify_check(struct roce3_qp *rqp, struct ib_qp_attr *attr, int attr_mask, + struct rdma_service_cap *rdma_cap) +{ + int ret; + enum ib_qp_state cur_state, new_state; + + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ? + attr->qp_state : cur_state); + + if (!roce3_check_qp_modify_ok(cur_state, new_state, (enum ib_qp_type)(rqp->ibqp.qp_type), + (enum ib_qp_attr_mask)attr_mask, IB_LINK_LAYER_ETHERNET)) { + pr_err( + "[ROCE, ERR] %s: Qpn(0x%x), invalid attribute mask specified for transition %d to %d, qp_type(%d), attr_mask(0x%x)\n", + __func__, rqp->ibqp.qp_num, cur_state, new_state, + rqp->ibqp.qp_type, attr_mask); + return (-EINVAL); + } + + if (((((u32)attr_mask) & IB_QP_PORT) != 0) && (attr->port_num != ROCE_DEFAULT_PORT_NUM)) { + pr_err("[ROCE, ERR] %s: Qpn(0x%x), invalid port number(%d) specified for transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->port_num, cur_state, + new_state, rqp->ibqp.qp_type); + return (-EINVAL); + } + + if ((((u32)attr_mask) & IB_QP_PKEY_INDEX) != 0) { + if (attr->pkey_index > 0) { + pr_err( + "[ROCE, ERR] %s: Qpn(0x%x), invalid pkey index (%d) specified for transition %d to %d. qp_type(%d)\n", + __func__, rqp->ibqp.qp_num, attr->pkey_index, cur_state, + new_state, rqp->ibqp.qp_type); + return (-EINVAL); + } + } + + ret = roce3_qp_modify_check_rd_and_mtu(rqp, attr, attr_mask, rdma_cap); + + return ret; +} + +#define ROCE3_QP_HANDLE 0 +#define ROCE3_QP_CONTINUE 1 +static int roce3_modify_qp_pre(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + int ret; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + enum ib_qp_state cur_state, new_state; + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + cur_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : cur_state; + + ret = roce3_qp_modify_check(rqp, attr, attr_mask, &rdev->rdma_cap); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed qp modify check, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + if (new_state == IB_QPS_RESET) { + if (cur_state != new_state) { + ret = roce3_qp_modify_2rst(rdev, rqp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify 2rst, func_id(%d)\n", + __func__, rdev->glb_func_id); + } + } + return ROCE3_QP_HANDLE; + } + + ret = roce3_qp_modify_pre_extend(rqp, attr, attr_mask, udata); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify extend qp, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return ROCE3_QP_CONTINUE; +} + +int roce3_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + + mutex_lock(&rqp->mutex); + + ret = roce3_modify_qp_pre(ibqp, attr, attr_mask, udata); + if ((ret == ROCE3_QP_HANDLE) || (ret < 0)) + goto out; + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + ret = roce3_qp_get_path(rdev, rqp, &attr->ah_attr); + if (ret != 0) + goto out; + } + + ret = roce3_qp_modify(rdev, rqp, attr, attr_mask); + if (ret != 0) + goto out; + +out: + mutex_unlock(&rqp->mutex); + return ret; +} + +int roce3_send_qp_lb_cmd(u32 qpn, struct roce3_device *rdev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, struct tag_cqm_cmd_buf *buf_out, u32 timeout) +{ + int ret = 0; + struct timespec64 tv_start; + + ktime_get_ts64(&tv_start); + if ((rdev->cfg_info.lb_en != 0) && (rdev->cfg_info.lb_mode == ROCE_LB_MODE_1)) { + u8 cos = qpn & 0x3; + + ret = cqm_lb_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, cmd, cos, + buf_in, buf_out, NULL, timeout, HINIC3_CHANNEL_ROCE); + } else { + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, cmd, buf_in, + buf_out, NULL, timeout, HINIC3_CHANNEL_ROCE); + } + + roce3_timeout_check(rdev, &tv_start, CMDQ_SDKCALL_BUTT, 0); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c new file mode 100644 index 000000000..d449da4d2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_recv.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_main_extension.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/* + **************************************************************************** + Prototype : roce3_get_recv_wqe + Description : roce3_get_recv_wqe + Input : struct roce3_qp *rqp + int n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void *roce3_get_recv_wqe(struct roce3_qp *rqp, u32 n) +{ + return roce3_get_wqe(rqp, rqp->rq.offset + (n << (unsigned int)rqp->rq.wqe_shift)); +} + +static int roce3_post_recv_check_qp(const struct roce3_qp *qp, const struct roce3_device *rdev, + const struct ib_recv_wr **bad_wr, const struct ib_recv_wr *wr) +{ + if (ROCE_UNLIKELY((qp->qp_type == IB_QPT_XRC_INI) || (qp->qp_type == IB_QPT_XRC_TGT))) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't post WQE when TGT XRC QP, func_id(%d)\n", + __func__, rdev->glb_func_id); + *bad_wr = wr; + return -EINVAL; + } + + if (ROCE_UNLIKELY(qp->qp_state == IB_QPS_RESET)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't post WQE when QP is RST state, func_id(%d)\n", + __func__, rdev->glb_func_id); + *bad_wr = wr; + return -EINVAL; + } + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + return 0; +} + +static int roce3_post_recv_check_wr(struct roce3_qp *qp, const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, u32 wr_num) +{ + if (roce3_wq_overflow(&qp->rq, wr_num, qp->ibqp.recv_cq) != 0) { + /*lint -e160 -e522*/ + pr_err_once("[ROCE, ERR] %s: SQ is full head:%x, tali:%x, wr_num:%d\n", + __func__, (&qp->rq)->head, (&qp->rq)->tail, wr_num); + /*lint +e160 +e522*/ + *bad_wr = wr; + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > qp->rq.max_sge)) { + *bad_wr = wr; + return -EINVAL; + } + + return 0; +} + +static int roce3_post_recv_check_length(struct roce3_device *rdev, const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, int i) +{ + u32 data_len = 0; + + /* single SGEָ signs data len should less than (2G-1)B */ + if (ROCE_UNLIKELY(wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - 1)) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Sge data len is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, rdev->glb_func_id); + return -EINVAL; + } + + data_len += wr->sg_list[i].length; + /* all SGEָ signs data len should less than 2GB */ + if (ROCE_UNLIKELY(data_len > rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data_len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)\n", + __func__, data_len, rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +static int roce3_post_recv_set_data_seg(const struct ib_recv_wr *wr, struct roce3_device *rdev, + const struct ib_recv_wr **bad_wr, struct roce3_qp *qp, u32 index) +{ + int i = 0; + int ret = 0; + struct roce3_wqe_data_seg *dseg = NULL; + + dseg = (struct roce3_wqe_data_seg *)roce3_get_recv_wqe(qp, index); + for (i = 0; i < wr->num_sge; ++i) { + ret = roce3_post_recv_check_length(rdev, bad_wr, wr, i); + if (ret != 0) + return ret; + + roce3_set_data_seg(dseg, wr->sg_list + i); + dseg++; + } + + if (wr->num_sge != 0) { + dseg--; + dseg->key = cpu_to_be32((u32)(wr->sg_list[wr->num_sge - 1].lkey | + ROCE_WQE_NEXT_SGE_INVALID)); + } else { + dseg->key = ROCE_WQE_NEXT_SGE_INVALID; + dseg->key = cpu_to_be32(dseg->key); + dseg->dw2.bs.len = 0; + } + + return 0; +} +/* + **************************************************************************** + Prototype : roce3_post_recv + Description : roce3_post_recv + Input : struct ib_qp *ibqp + struct ib_recv_wr *wr + struct ib_recv_wr **bad_wr + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/4/29 + Author : + Modification : Created function + +**************************************************************************** +*/ +int roce3_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct roce3_qp *qp = to_roce3_qp(ibqp); + + struct roce3_device *rdev = to_roce3_dev(ibqp->device); + unsigned long flags = 0; + int ret = 0; + u32 wr_num = 0; + u32 index = 0; + const struct ib_recv_wr *wr_tmp = wr; + + ret = roce3_post_recv_check_qp(qp, rdev, bad_wr, wr_tmp); + if (ret != 0) + return ret; + + spin_lock_irqsave(&qp->rq.lock, flags); + + index = (unsigned int)qp->rq.head & ((unsigned int)qp->rq.wqebb_cnt - 1); + + for (wr_num = 0; wr_tmp != NULL; ++wr_num, wr_tmp = wr_tmp->next) { + ret = roce3_post_recv_check_wr(qp, bad_wr, wr_tmp, wr_num); + if (ret != 0) + goto out; + + ret = roce3_post_recv_set_data_seg(wr_tmp, rdev, bad_wr, qp, index); + if (ret != 0) + goto out; + + qp->rq.wrid[index] = wr_tmp->wr_id; + + index = (index + 1) & (qp->rq.wqebb_cnt - 1); + } + +out: + if (ROCE_LIKELY(wr_num != 0)) { + qp->rq.head += (u32)wr_num; + + wmb(); /* Memory barrier before write db */ + + /* SQ need head 4B,RQ need tail 4B */ + *(qp->db.db_record + 1) = cpu_to_be32(qp->rq.head & 0xffff); + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return ret; +} + +void roce3_drain_rq(struct ib_qp *ibqp) +{ + +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c new file mode 100644 index 000000000..2e4f2ee0d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_post_send.c @@ -0,0 +1,1315 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_post.h" +#include "roce_dif_format.h" +#include "roce_main_extension.h" + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#define GOTO_LOCAL 1 + +#define GOTO_OUT 2 + +#define ROCE_NEED_JUMP 1 + +#define BYTE_MASK 255 + +#define ROCE_CTRL_SEG_SL 2 +/* + **************************************************************************** + Prototype : roce3_get_send_wqe + Description : roce3_get_send_wqe + Input : struct roce3_qp *rqp + u32 n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void *roce3_get_send_wqe(struct roce3_qp *rqp, u32 n) +{ + u32 wqebb_index = (n & (((u32)rqp->sq.wqebb_cnt) - 1)); + u32 wqebb_pos = wqebb_index << rqp->sq.wqe_shift; + + return roce3_get_wqe(rqp, rqp->sq.offset + wqebb_pos); +} + +/* + **************************************************************************** + Prototype : roce3_wq_overflow + Description : roce3_wq_overflow + Input : struct roce3_wq *wq + int wr_num + struct ib_cq *ibcq + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +int roce3_wq_overflow(struct roce3_wq *wq, u32 wr_num, struct ib_cq *ibcq) +{ + unsigned int cur = 0; + struct roce3_cq *cq = NULL; + + cur = wq->head - wq->tail; + if (ROCE_LIKELY((cur + wr_num) < (unsigned int)wq->max_post)) + return 0; + + cq = to_roce3_cq(ibcq); + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return ((cur + (unsigned int)wr_num) >= (unsigned int)wq->max_post); +} + +/* + **************************************************************************** + Prototype : roce3_ib_opcode + Description : + Input : + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/7/30 + Author : + Modification : Created function + +**************************************************************************** +*/ +static const u32 roce3_ib_opcode[] = { + ROCE_WQE_OPCODE_RDMA_WRITE, /* [IB_WR_RDMA_WRITE] : 0 */ + ROCE_WQE_OPCODE_RDMA_WRITE_IMM, /* [IB_WR_RDMA_WRITE_WITH_IMM] : 1 */ + ROCE_WQE_OPCODE_SEND, /* [IB_WR_SEND] : 2 */ + ROCE_WQE_OPCODE_SEND_IMM, /* [IB_WR_SEND_WITH_IMM] : 3 */ + ROCE_WQE_OPCODE_RDMA_READ, /* [IB_WR_RDMA_READ] : 4 */ + ROCE_WQE_OPCODE_ATOMIC_CMP_SWP, /* [IB_WR_ATOMIC_CMP_AND_SWP] : 5 */ + ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD, /* [IB_WR_ATOMIC_FETCH_AND_ADD] : 6 */ + 0, /* [IB_WR_LSO] : 7 NO SUPPORT */ + ROCE_WQE_OPCODE_SEND_INVAL, /* [IB_WR_SEND_WITH_INV] : 8 */ + 0, /* [IB_WR_RDMA_READ_WITH_INV] : 9 */ + ROCE_WQE_OPCODE_LOCAL_INVAL, /* [IB_WR_LOCAL_INV] : 10 */ + ROCE_WQE_OPCODE_FRMR, /* [IB_WR_REG_MR] : 11 */ + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP, /* [IBV_EXP_WR_EXT_MASKED_ATOMIC_CMP_AND_SWP] : 12 */ + /* [IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD] : 13 */ + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD, + ROCE_WQE_OPCODE_REG_SIG_MR, /* [IB_WR_REG_SIG_MR] : 14 */ + + /* 15-64 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* IBV_EXP_WR_SEND_ENABLE 96 */ + 0, /* IBV_EXP_WR_RECV_ENABLE 97 */ + 0, /* IBV_EXP_WR_CQE_WAIT 98 */ + + /* ex */ + ROCE_WQE_OPCODE_FRMR, /* [IB_WR_FAST_REG_MR] : 11 */ +}; + +/* + **************************************************************************** + Prototype : roce3_set_send_seg + Description : roce3_set_send_seg + Input : struct roce3_wqe_snd_tsk_seg *snd_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2017/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_send_seg(struct roce3_wqe_send_tsk_seg *snd_tsk_seg, + const struct ib_send_wr *wr) +{ + if (wr->opcode == IB_WR_SEND_WITH_IMM) + snd_tsk_seg->immdata_invkey = wr->ex.imm_data; + + if (wr->opcode == IB_WR_SEND_WITH_INV) + snd_tsk_seg->immdata_invkey = cpu_to_be32(wr->ex.invalidate_rkey); +} + +/* + **************************************************************************** + Prototype : roce3_set_atomic_seg + Description : roce3_set_atomic_seg + Input : struct roce3_wqe_atomic_tsk_seg *atomic_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ + +static void roce3_set_atomic_seg(struct roce3_wqe_atomic_tsk_seg *atomic_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_atomic_wr *atomic = ROCE_ATOMIC_WR(wr); + + atomic_tsk_seg->key = cpu_to_be32(atomic->rkey); + atomic_tsk_seg->va = cpu_to_be64(atomic->remote_addr); + + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->swap); + atomic_tsk_seg->cmp_data = cpu_to_be64(atomic->compare_add); + } else if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { + atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->cmp_data = 0; + } else { + /* IB_WR_MASKED_ATOMIC_FETCH_AND_ADD */ + atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->cmp_data = cpu_to_be64(atomic->compare_add_mask); + } +} + +/* + **************************************************************************** + Prototype : roce3_set_masked_atomic_seg + Description : roce3_set_masked_atomic_seg + Input : struct roce3_wqe_mask_atomic_tsk_seg *mask_atomic_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_masked_atomic_seg(struct roce3_wqe_mask_atomic_tsk_seg *mask_atomic_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_atomic_wr *atomic = ROCE_ATOMIC_WR(wr); + + mask_atomic_tsk_seg->rkey = cpu_to_be32(atomic->rkey); + mask_atomic_tsk_seg->va = cpu_to_be64(atomic->remote_addr); + + mask_atomic_tsk_seg->swap_add_data = cpu_to_be64(atomic->swap); + mask_atomic_tsk_seg->swap_msk = cpu_to_be64(atomic->swap_mask); + mask_atomic_tsk_seg->cmp_data = cpu_to_be64(atomic->compare_add); + mask_atomic_tsk_seg->cmp_msk = cpu_to_be64(atomic->compare_add_mask); +} + +/* + **************************************************************************** + Prototype : roce3_set_rdma_seg + Description : roce3_set_rdma_seg + Input : struct roce3_wqe_rdma_tsk_seg *rdma_tsk_seg + struct ib_send_wr *wr + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/26 + Author : + Modification : Created function + +**************************************************************************** +*/ +static __always_inline void roce3_set_rdma_seg(struct roce3_wqe_rdma_tsk_seg *rdma_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_rdma_wr *rdma = ROCE_RDMA_WR(wr); + + rdma_tsk_seg->va = cpu_to_be64(rdma->remote_addr); + rdma_tsk_seg->rkey = cpu_to_be32(rdma->rkey); + + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) + rdma_tsk_seg->imm_data = wr->ex.imm_data; +} + +/* + **************************************************************************** + Prototype : roce3_set_local_inv_seg + Description : roce3_set_local_inv_seg + Input : struct roce3_wqe_local_inv_tsk_seg *inv_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_local_inv_seg(struct roce3_wqe_local_inv_tsk_seg *inv_tsk_seg, + const struct ib_send_wr *wr) +{ + inv_tsk_seg->inv_key = cpu_to_be32(wr->ex.invalidate_rkey); +} + +/* + **************************************************************************** + Prototype : roce3_set_ud_seg_cycle1 + Description : set ud type task segment before cycled + Input : struct roce3_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1 + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/8/8 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_ud_seg_cycle1(struct roce3_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ROCE_UD_WR(wr); + struct roce3_ah *rah = to_roce3_ah(ud->ah); + + if (wr->opcode == IB_WR_SEND_WITH_IMM) + ud_tsk_seg_cycle1->immdata_invkey = wr->ex.imm_data; + + ud_tsk_seg_cycle1->dw2.value = rah->priv_ah.dw0.value; + ud_tsk_seg_cycle1->dw3.value = rah->priv_ah.dw1.value; + ud_tsk_seg_cycle1->dw4.value = rah->priv_ah.dw2.value; + memcpy((void *)ud_tsk_seg_cycle1->dgid, (void *)rah->priv_ah.dgid, ROCE_GID_LEN); + ud_tsk_seg_cycle1->dw9.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg_cycle1->qkey = cpu_to_be32(ud->remote_qkey); +} + +/* + **************************************************************************** + Prototype : roce3_set_ud_seg_cycle2 + Description : set ud type task segment after cycled + Input : struct roce3_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2 + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_ud_seg_cycle2(struct roce3_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ROCE_UD_WR(wr); + struct roce3_ah *rah = to_roce3_ah(ud->ah); + + ud_tsk_seg_cycle2->dw0.value = rah->priv_ah.dw7.value; + ud_tsk_seg_cycle2->dmac_l32 = rah->priv_ah.dmac_l32; +} + +/* + **************************************************************************** + Prototype : roce3_set_ud_seg + Description : roce3_set_ud_seg + Input : struct roce3_wqe_ud_tsk_seg *ud_tsk_seg + struct ib_send_wr *wr + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_ud_seg(struct roce3_wqe_ud_tsk_seg *ud_tsk_seg, const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ROCE_UD_WR(wr); + struct roce3_ah *rah = to_roce3_ah(ud->ah); + + if (wr->opcode == IB_WR_SEND_WITH_IMM) + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + memcpy((void *)ud_tsk_seg->dgid, (void *)rah->priv_ah.dgid, + sizeof(ud_tsk_seg->dgid)); + ud_tsk_seg->dw10.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32(ud->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; +} + +/* + **************************************************************************** + Prototype : roce3_set_data_seg + Description : roce3_set_data_seg + Input : struct roce3_wqe_data_seg *dseg + struct ib_sge *sge + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +void roce3_set_data_seg(struct roce3_wqe_data_seg *dseg, struct ib_sge *sge) +{ + dseg->addr = cpu_to_be64(sge->addr); + + dseg->dw2.bs.len = sge->length & 0x7fffffff; + dseg->dw2.bs.rsvd = 0; + + dseg->dw2.value = cpu_to_be32(dseg->dw2.value); + + dseg->key = cpu_to_be32(sge->lkey); +} + +static void roce3_set_dwqe_db(struct roce3_qp *qp, struct roce3_wqe_ctrl_seg *ctrl_seg, + unsigned int index) +{ + struct roce3_device *rdev = to_roce3_dev(qp->ibqp.device); + + ctrl_seg->dw2.bs.qpn = qp->qpn & 0xfffff; + ctrl_seg->dw2.bs.ctx_size = 1; /* 512B */ + ctrl_seg->dw2.bs.cp_flag = 0; + ctrl_seg->dw2.bs.cos = roce3_get_db_cos_from_vlan_pri(rdev, qp->sl & 0x7); + ctrl_seg->dw2.bs.type = ROCE_SQ_DB_TYPE; + + ctrl_seg->dw3.bs.pi = index & 0xffff; + ctrl_seg->dw3.bs.sub_type = 0; + + ctrl_seg->dw3.bs.mtu_shift = qp->db_path_mtu; + ctrl_seg->dw3.bs.sgid_index = qp->db_sgid_index; + if (qp->qp_type == IB_QPT_RC) { + // ctrl_seg->dw3.bs.rc_flag = 1; + } else if (qp->qp_type == IB_QPT_UD) { + // ctrl_seg->dw3.bs.ud_vld = 1; + ctrl_seg->dw3.bs.mtu_shift = 3; + } else if (qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) { + ctrl_seg->dw3.bs.xrc_vld = 1; + // ctrl_seg->dw3.bs.rc_flag = 1; + } + // ctrl_seg->dw3.bs.local_trans = 0; + + ctrl_seg->dw1.value = cpu_to_be32(ctrl_seg->dw1.value); + ctrl_seg->dw2.value = cpu_to_be32(ctrl_seg->dw2.value); + ctrl_seg->dw3.value = cpu_to_be32(ctrl_seg->dw3.value); +} + +static void roce3_set_sq_db(struct roce3_qp *qp, struct roce_sq_db_seg *db, unsigned int index) +{ + struct roce3_device *rdev = to_roce3_dev(qp->ibqp.device); + + db->dw0.bs.qpn = qp->qpn & 0xfffff; + db->dw0.bs.ctx_size = 1; /* 512B */ + db->dw0.bs.cp_flag = 0; + db->dw0.bs.cos = roce3_get_db_cos_from_vlan_pri(rdev, qp->sl & 0x7); + db->dw0.bs.type = ROCE_SQ_DB_TYPE; + + db->dw1.bs.pi = ((index & 0xffff) >> 8) & 0xff; /* 8bits */ + db->dw1.bs.sub_type = 0; + db->dw1.bs.mtu_shift = qp->db_path_mtu; + db->dw1.bs.sgid_index = qp->db_sgid_index; + if (qp->qp_type != IB_QPT_RC) { + if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) + db->dw1.bs.mtu_shift = ROCE_UD_MTU_SHIFT; + else if (qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) + db->dw1.bs.xrc_vld = 1; + } + + db->dw0.value = roce3_convert_be32(db->dw0.value); + db->dw1.value = roce3_convert_be32(db->dw1.value); +} + +static void roce3_set_inline_data(void *start_addr, struct ib_sge *sge) +{ + memcpy(start_addr, (void *)(uintptr_t)sge->addr, sge->length); +} + +static void roce3_set_inline_data_cycle1(void *start_addr, struct ib_sge *sge, u32 size) +{ + memcpy(start_addr, (void *)(uintptr_t)sge->addr, size); +} + +static void roce3_set_inline_data_cycle2(void *start_addr, struct ib_sge *sge, u32 size) +{ + memcpy(start_addr, (void *)((u8 *)(void *)(uintptr_t)sge->addr + size), size); +} + +/* + **************************************************************************** + Prototype : roce3_dwqe_copy + Description : roce3_dwqe_copy + Input : unsigned long *dst + unsigned long *src + unsigned int bytecnt + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +/* + * Avoid using memcpy() to copy to DWQE Page, since memcpy() + * implementations may use move-string-buffer assembler instructions, + * which do not guarantee order of copying. + * Assume that the DWQE page header address is VA, the WQE size is size, and the WQE address is wqe + size=64, iowrite64_copy(va + 256, wqe, size) + size=128, iowrite64_copy(va + 256*2, wqe, size) + size=192, iowrite64_copy(va + 256*3, wqe, size) + The current chip supports this function, but the software wqe_size is aligned by + the power of 2. Therefore, this function is not used and can be ignored. + size=256, __iowrite64_copy(va, wqe, size) + + formula: + __iowrite64_copy(va + ((size & 255) << 2), wqe, size) + */ +static void roce3_dwqe_copy(unsigned long *dst, unsigned long *src, unsigned int bytecnt) +{ + unsigned long *src_tmp = src; + unsigned int bytecnt_tmp = bytecnt; + unsigned long *dst_tmp = (unsigned long *)( + (u64)dst + ((bytecnt_tmp & BYTE_MASK) << 2)); // 2 is addr offset + + while ((int)bytecnt_tmp > 0) { + wmb(); /* Sequential write memory barrier */ + *dst_tmp++ = *src_tmp++; + bytecnt_tmp -= (unsigned int)sizeof(u64); + } +} + +/* + **************************************************************************** + Prototype : roce3_set_gsi_seg + Input : struct roce3_sqp *sqp + struct ib_send_wr *wr + void *wqe + Output : None + + 1.Date : 2015/8/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_set_gsi_seg(struct roce3_sqp *sqp, const struct ib_send_wr *wr, void *wqe) +{ + struct roce3_wqe_ud_tsk_seg *ud_tsk_seg = (struct roce3_wqe_ud_tsk_seg *)wqe; + struct ib_ud_wr *ud_wr = (struct ib_ud_wr *)(void *)wr; + struct roce3_ah *rah = to_roce3_ah(ud_wr->ah); + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + ud_tsk_seg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + ud_tsk_seg->dw2.value = cpu_to_be32(ud_tsk_seg->dw2.value); + } + + ud_tsk_seg->common.bs.c = 1; + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + memcpy((void *)ud_tsk_seg->dgid, (void *)rah->priv_ah.dgid, + sizeof(ud_tsk_seg->dgid)); + ud_tsk_seg->dw10.value = cpu_to_be32(ud_wr->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32(((ud_wr->remote_qkey & 0x80000000) != 0) ? + sqp->qkey : ud_wr->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; +} + +/* + **************************************************************************** + Prototype : write_invalid_wqe + Description : set the next wqe invalidate + Input : struct roce3_qp *qp + u32 index + Output : None + + 1.Date : 2015/9/8 + Modification : Created function + +**************************************************************************** +*/ +static void write_invalid_wqe(struct roce3_qp *qp, u32 index) +{ + u32 invalid_wqe_dw0 = 0; + struct roce3_wqe_ctrl_seg *ctrl_seg = NULL; + + ctrl_seg = (struct roce3_wqe_ctrl_seg *)roce3_get_send_wqe(qp, index); + invalid_wqe_dw0 = ((index & (u32)qp->sq.wqebb_cnt) == 0) ? 0xff000000 : 0x7f000000; + ctrl_seg->dw0.value = cpu_to_be32(invalid_wqe_dw0); +} + +/* + **************************************************************************** + Prototype : roce3_validate_wr + Description : roce3_validate_wr + Input : struct roce3_qp *rqp + struct ib_send_wr *wr + int wr_num + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_validate_wr(struct roce3_qp *rqp, const struct ib_send_wr *wr, u32 wr_num) +{ +#define IB_WR_REG_SIG_MR IB_WR_REG_MR_INTEGRITY + if (roce3_wq_overflow(&rqp->sq, wr_num, rqp->ibqp.send_cq) != 0) { + pr_err("[ROCE] %s: SQ is full\n", __func__); + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > rqp->sq.max_sge)) { + pr_err("[ROCE, ERR] %s: Sge num is invalid, wr->num_sge(%d), rqp->sq.max_sge(%d)\n", + __func__, wr->num_sge, rqp->sq.max_sge); + return -EINVAL; + } + + if (ROCE_UNLIKELY((wr->opcode == IB_WR_LSO) || (wr->opcode > IB_WR_REG_SIG_MR))) { + pr_err("[ROCE, ERR] %s: wr->opcode(%d)\n", __func__, wr->opcode); + return -EINVAL; + } + + return 0; +} + +static void roce3_set_rc_wqe_for_optype_send(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_send_seg((struct roce3_wqe_send_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_send_tsk_seg); +} + +static void roce3_set_rc_wqe_for_optype_atomic(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_atomic_seg((struct roce3_wqe_atomic_tsk_seg *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_atomic_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_atomic_tsk_seg); +} + +static void roce3_set_rc_wqe_for_optype_cmp_and_swap(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_ATOMIC_CWP_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_masked_atomic_seg((struct roce3_wqe_mask_atomic_tsk_seg *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_mask_atomic_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_atomic_tsk_seg); +} + +static void roce3_set_rc_wqe_for_optype_write(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_rdma_seg((struct roce3_wqe_rdma_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)(sizeof(struct roce3_wqe_rdma_tsk_seg)); +} + +static void roce3_set_rc_wqe_for_optype_local(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce3_set_local_inv_seg((struct roce3_wqe_local_inv_tsk_seg *)param->wqe, wr); + param->wqe_size += (u32)sizeof(struct roce3_wqe_local_inv_tsk_seg); + param->inline_flag = 0; + param->ctrl_seg_tmp.dw0.bs.df = 0; +} + +static void roce3_set_wqe_last_len(u8 *wqe) +{ + struct roce3_wqe_send_tsk_seg *snd_task = (struct roce3_wqe_send_tsk_seg *)(void *)wqe; + + snd_task->dw3.value = 0; + snd_task->dw3.bs.last_ext_len = ROCE_IMM_EXT_LEN; + snd_task->dw3.value = cpu_to_be32(snd_task->dw3.value); +} + +static inline u32 roce3_log2(u32 x) +{ + u32 shift; + + for (shift = 0; (1U << shift) < x; ++shift) + ; + + return shift; +} + +static int set_frmr_wqe_section(const struct ib_send_wr *wr, + struct roce3_post_send_normal_param *param) +{ + u64 iova; + u32 fbo; + u32 page_num; + struct ib_reg_wr *frmr_reg_wr = NULL; + struct roce3_mr *roce3_kernel_frmr = NULL; + struct roce3_wqe_frmr_tsk_seg *frmr_wqe_task = NULL; + + frmr_reg_wr = (struct ib_reg_wr *)reg_wr(wr); //lint !e605 + + roce3_kernel_frmr = to_roce3_mr(frmr_reg_wr->mr); + page_num = (u32)((frmr_reg_wr->mr->length + + frmr_reg_wr->mr->page_size - 1) / frmr_reg_wr->mr->page_size); + if (page_num > ROCE_FRMR_MAX_PAGES) + return -EINVAL; + + /* base section set */ + param->wqe_size += sizeof(struct roce3_wqe_frmr_tsk_seg); + + frmr_wqe_task = (struct roce3_wqe_frmr_tsk_seg *)((void *)param->tsk_com_seg); + + param->dseg = (struct roce3_wqe_data_seg *) + (((u8 *)param->tsk_com_seg) + sizeof(struct roce3_wqe_frmr_tsk_seg)); + + param->ctrl_seg_tmp.dw0.bs.tsl = + sizeof(struct roce3_wqe_frmr_tsk_seg) / ROCE_TASK_SEG_ALIGN; + /* fill wqe ctrl head section:cl bit */ + param->ctrl_seg_tmp.dw1.bs.cl = 1; //lint !e572 !e778 !e845 + + /* local invalidate associated section set */ + param->tsk_com_seg->bs.so = 1; + + iova = (u64)(void *)(frmr_reg_wr->mr->iova); + fbo = (u32)(iova & (FRMR_PAGE_SIZE - 1)); + frmr_wqe_task->dw2.bs.fbo = fbo & 0x3FFFFF; + frmr_wqe_task->dw2.value = cpu_to_be32(frmr_wqe_task->dw2.value); + + frmr_wqe_task->dw5.va = ((u32)frmr_reg_wr->access & IB_ZERO_BASED) ? 0 : cpu_to_be64(iova); + + /* 填写访问权限 */ + frmr_wqe_task->dw3.bs.zbva = (u32)(!!((u32)frmr_reg_wr->access & IB_ZERO_BASED)); + /* 绑定操作使能 */ + frmr_wqe_task->dw3.bs.be = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_MW_BIND)); + /* 本地读使能 */ + frmr_wqe_task->dw3.bs.lre = 1; + /* 本地写使能 */ + frmr_wqe_task->dw3.bs.lwe = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_LOCAL_WRITE)); + /* 远端读使能 */ + frmr_wqe_task->dw3.bs.rre = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_READ)); + /* 远端写使能 */ + frmr_wqe_task->dw3.bs.rwe = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_WRITE)); + /* 远端Atomic使能 */ + frmr_wqe_task->dw3.bs.rae = (u32)(!!((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_ATOMIC)); + frmr_wqe_task->dw3.bs.block = 1; + frmr_wqe_task->dw3.bs.rsvd = 0; + frmr_wqe_task->dw3.bs.pa_num = page_num; + /* MR内存页大小。equals to (2^page_size)*4KB */ + frmr_wqe_task->dw3.bs.page_size = roce3_log2(frmr_reg_wr->mr->page_size) - PAGE_4K_SHIFT; + frmr_wqe_task->dw3.value = cpu_to_be32(frmr_wqe_task->dw3.value); + + frmr_wqe_task->m_key = cpu_to_be32(frmr_reg_wr->key); + frmr_wqe_task->dw7.len = cpu_to_be64(frmr_reg_wr->mr->length); + frmr_wqe_task->dw9.pbl_addr = cpu_to_be64(roce3_kernel_frmr->page_map); + /* set inline and direct wqe handle section */ + frmr_wqe_task->rsvd[0] = 0; + frmr_wqe_task->rsvd[1] = 0; + + param->inline_flag = 0; + + return 0; +} + + +static int roce3_post_send_rc(struct roce3_post_send_normal_param *param, struct roce3_device *rdev, + const struct ib_send_wr *wr) +{ + int ret = 0; + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND_WITH_INV: + roce3_set_wqe_last_len(param->wqe); + roce3_set_rc_wqe_for_optype_send(wr, param); + break; + case IB_WR_SEND: + roce3_set_rc_wqe_for_optype_send(wr, param); + break; + + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: + roce3_set_rc_wqe_for_optype_atomic(wr, param); + break; + + case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: + roce3_set_rc_wqe_for_optype_cmp_and_swap(wr, param); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + roce3_set_wqe_last_len(param->wqe); + roce3_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + roce3_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_LOCAL_INV: + roce3_set_rc_wqe_for_optype_local(wr, param); + ret = ROCE_NEED_JUMP; + break; + + case IB_WR_REG_MR: + set_frmr_wqe_section(wr, param); + ret = ROCE_NEED_JUMP; + break; + + default: + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Invalid opcode, func_id(%d)\n", + __func__, rdev->glb_func_id); + break; + } + + return ret; +} + +static int roce3_post_send_uc(struct roce3_post_send_normal_param *param, struct roce3_device *rdev, + const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND: + if (wr->opcode == IB_WR_SEND_WITH_IMM) + roce3_set_wqe_last_len(param->wqe); + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_send_seg((struct roce3_wqe_send_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_send_tsk_seg); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_WRITE: + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) + roce3_set_wqe_last_len(param->wqe); + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce3_set_rdma_seg((struct roce3_wqe_rdma_tsk_seg *)param->wqe, wr); + param->data_len_addr = &((struct roce3_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_rdma_tsk_seg); + break; + + case IB_WR_LOCAL_INV: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce3_set_local_inv_seg((struct roce3_wqe_local_inv_tsk_seg *)param->wqe, wr); + param->wqe_size += (u32)sizeof(struct roce3_wqe_local_inv_tsk_seg); + return ROCE_NEED_JUMP; + + default: + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Invalid opcode, func_id(%d)\n", + __func__, rdev->glb_func_id); + break; + } + + return 0; +} + +static int roce3_post_send_ud(struct roce3_post_send_normal_param *param, struct roce3_device *rdev, + struct roce3_qp *rqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + struct roce3_wqe_ud_tsk_seg *tsk_sg = (struct roce3_wqe_ud_tsk_seg *)param->wqe; + + if (ROCE_UNLIKELY((wr->opcode != IB_WR_SEND) && (wr->opcode != IB_WR_SEND_WITH_IMM))) { + // ret = -EINVAL; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Wr->opcode(%d), func_id(%d)\n", + __func__, wr->opcode, rdev->glb_func_id); + *bad_wr = wr; + return ROCE_NEED_JUMP; + } + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + tsk_sg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + tsk_sg->dw2.value = cpu_to_be32(tsk_sg->dw2.value); + } + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->data_len_addr = &((struct roce3_wqe_ud_tsk_seg *)param->wqe)->data_len; + + if (param->sq_rmd_size > (1UL << (unsigned int)rqp->sq.wqe_shift)) { + roce3_set_ud_seg((struct roce3_wqe_ud_tsk_seg *)(param->wqe), wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg); + } else { + roce3_set_ud_seg_cycle1((struct roce3_wqe_ud_tsk_seg_cycle1 *)param->wqe, wr); + param->wqe = rqp->sq_head_addr; + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg_cycle1); + roce3_set_ud_seg_cycle2((struct roce3_wqe_ud_tsk_seg_cycle2 *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_ud_tsk_seg_cycle2)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg_cycle2); + } + + return 0; +} + +static void roce3_post_send_set_normal_data_seg(const struct ib_send_wr *wr, + struct roce3_wqe_data_seg **dseg, struct roce3_qp *rqp) +{ + if (wr->num_sge != 0) { + if ((u8 *)(*dseg) == rqp->sq_head_addr) { + *dseg = (struct roce3_wqe_data_seg *)( + (void *)(rqp->sq_tail_addr - sizeof(struct roce3_wqe_data_seg))); + } else { + (*dseg)--; + } + (*dseg)->key = (u32)wr->sg_list[wr->num_sge - 1].lkey | ROCE_WQE_NEXT_SGE_INVALID; + (*dseg)->key = cpu_to_be32((*dseg)->key); + } +} + +static int roce3_post_send_set_normal_sge(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int i; + + param->data_len = 0; + param->dseg = (struct roce3_wqe_data_seg *)param->wqe; + param->wqe_size += (u32)(wr->num_sge * (int)sizeof(struct roce3_wqe_data_seg)); + for (i = 0; i < wr->num_sge; ++i) { + if (param->sq_rmd_size < sizeof(struct roce3_wqe_data_seg)) + param->dseg = (struct roce3_wqe_data_seg *)((void *)rqp->sq_head_addr); + + if (ROCE_UNLIKELY(wr->sg_list[i].length > + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz))) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Sge_data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + param->data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(param->data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)\n", + __func__, param->data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + roce3_set_data_seg(param->dseg, wr->sg_list + i); + + (param->dseg)++; + param->sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param->dseg); + } + + return 0; +} + +static void roce3_copy_inline_data(struct roce3_post_send_normal_param *param, + const struct ib_send_wr *wr, struct roce3_qp *rqp, int i) +{ + if (param->sq_rmd_size >= wr->sg_list[i].length) { + roce3_set_inline_data(param->wqe, wr->sg_list + i); + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length); + if ((u8 *)param->wqe == rqp->sq_tail_addr) + param->wqe = rqp->sq_head_addr; + } else { + roce3_set_inline_data_cycle1(param->wqe, wr->sg_list + i, param->sq_rmd_size); + param->wqe = rqp->sq_head_addr; + + roce3_set_inline_data_cycle2(param->wqe, wr->sg_list + i, + wr->sg_list[i].length - param->sq_rmd_size); + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length - param->sq_rmd_size); + + param->cycle = 1; + } +} + +static int roce3_post_send_set_data_seg(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int i; + int ret; + + if ((((unsigned int)wr->send_flags & (unsigned int)IB_SEND_INLINE) != 0) && + (wr->num_sge != 0) && (param->inline_flag != 0)) { + param->data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + param->wqe_size += wr->sg_list[i].length; + param->data_len += wr->sg_list[i].length; + if (param->data_len > rqp->max_inline_data) { + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data len is too big, data_len(%d), max_inline_data(%d), func_id(%d)\n", + __func__, param->data_len, + rqp->max_inline_data, rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + if ((u8 *)param->wqe == rqp->sq_head_addr) + param->cycle = 1; + + roce3_copy_inline_data(param, wr, rqp, i); + + param->sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.bdsl = + (ALIGN((u32)(param->data_len), ROCE_TASK_SEG_ALIGN) / + ROCE_TASK_SEG_ALIGN) & 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 1; + param->inline_flag = 1; + } else { + ret = roce3_post_send_set_normal_sge(param, rdev, rqp, wr, bad_wr); + if (ret != 0) + return ret; + + roce3_post_send_set_normal_data_seg(wr, ¶m->dseg, rqp); + + param->ctrl_seg_tmp.dw0.bs.bdsl = + (u32)((wr->num_sge * (int)sizeof(struct roce3_wqe_data_seg)) / + ROCE_TASK_SEG_ALIGN) & 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 0; + } + + return 0; +} + +static int roce3_post_send_ring_db(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp) +{ + int ret = 0; + + if ((rdev->kernel_dwqe_map != NULL) && (param->wr_num == 1) && (param->inline_flag != 0) && + (param->wqe_size > (u32)sizeof(struct roce3_wqe_ctrl_seg)) && + ((int)param->wqe_size <= rqp->max_dwqe_size) && + (param->cycle == 0)) { + wmb(); /* Ring db memory barrier */ + + param->ctrl_seg->dw1.value = be32_to_cpu(param->ctrl_seg->dw1.value); + param->ctrl_seg->dw1.bs.mask_pi = + (u32)((param->index & ((unsigned int)rqp->sq.wqebb_cnt - 1)) & 0xfffff); + + roce3_set_dwqe_db(rqp, param->ctrl_seg, param->index); + + param->wqe_size = ALIGN((u32)param->wqe_size, ROCE_SQ_WQEBB_SIZE); + + *(rqp->db.db_record) = cpu_to_be32(param->index); + + wmb(); /* Ring db memory barrier */ + + ++rqp->sq.head; + roce3_dwqe_copy((unsigned long *)rdev->kernel_dwqe_map, + (unsigned long *)param->ctrl_seg, (u32)param->wqe_size); + wc_wmb(); /* Ring db memory barrier */ + } else if (param->wr_num != 0) { + rqp->sq.head += param->wr_num; + + wmb(); /* Ring db memory barrier */ + + memset(&(param->sq_db.sq_db_val), 0, sizeof(u64)); + roce3_set_sq_db(rqp, &(param->sq_db.sq_db_seg), param->index); + + *(rqp->db.db_record) = cpu_to_be32(param->index & 0xffff); + + wmb(); /* Ring db memory barrier */ + ret = cqm_ring_hardware_db(rdev->hwdev, SERVICE_T_ROCE, + param->index & 0xff, param->sq_db.sq_db_val); + wmb(); /* Ring db memory barrier */ + } + + return ret; +} + +static void roce3_fill_task_data_len(u32 *data_len_addr, u32 data_len) +{ + if (data_len_addr) + *data_len_addr = cpu_to_be32(data_len); +} + +static int roce3_construct_wqe(struct roce3_post_send_normal_param *param, + struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + + switch (rqp->qp_type) { + case IB_QPT_RC: + need_goto = roce3_post_send_rc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) + return GOTO_LOCAL; + break; + + case IB_QPT_UC: + need_goto = roce3_post_send_uc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) + return GOTO_LOCAL; + break; + + case IB_QPT_UD: + need_goto = roce3_post_send_ud(param, rdev, rqp, wr, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + // ret = -EINVAL; + return GOTO_OUT; + } + break; + + case IB_QPT_GSI: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + + roce3_set_gsi_seg(to_roce3_sqp(rqp), wr, param->wqe); + + param->data_len_addr = &((struct roce3_wqe_ud_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce3_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce3_wqe_ud_tsk_seg); + break; + + default: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Unknown qp type, func_id(%d)\n", + __func__, rdev->glb_func_id); + break; + } + + return 0; +} + +static void roce3_init_ctrl_seg(struct roce3_post_send_normal_param *param, + const struct ib_send_wr *wr, const struct roce3_qp *rqp) +{ + param->ctrl_seg_tmp.dw0.bs.owner = ((param->index & (u32)rqp->sq.wqebb_cnt) != 0) ? 1 : 0; + param->ctrl_seg_tmp.dw0.bs.drvsl = 0; + param->ctrl_seg_tmp.dw0.bs.wf = 0; + param->ctrl_seg_tmp.dw0.bs.cf = 0; + param->ctrl_seg_tmp.dw0.bs.va = 1; + param->ctrl_seg_tmp.dw0.bs.df = 0; /* va:1bits, 0-sge,1-inline */ + param->ctrl_seg_tmp.dw0.bs.cr = + (u32)(!(((((u32)wr->send_flags) & IB_SEND_SIGNALED) | rqp->sq_signal_bits) == 0)); + param->ctrl_seg_tmp.dw0.bs.difsl = 0; + param->ctrl_seg_tmp.dw0.bs.csl = 0; + param->ctrl_seg_tmp.dw0.bs.ctrlsl = ROCE_CTRL_SEG_SL; + param->ctrl_seg_tmp.dw0.bs.bdsl = 0; +} + +static void roce3_init_task_com_seg(union roce3_wqe_tsk_com_seg **tsk_com_seg, u8 *wqe, + const struct ib_send_wr *wr, const struct roce3_qp *rqp) +{ + *tsk_com_seg = (union roce3_wqe_tsk_com_seg *)(void *)wqe; + + (*tsk_com_seg)->bs.c = + (u32)(!((((unsigned int)wr->send_flags & + (unsigned int)IB_SEND_SIGNALED) | rqp->sq_signal_bits) == 0)); + (*tsk_com_seg)->bs.se = + (u32)(!(((unsigned int)wr->send_flags & (unsigned int)IB_SEND_SOLICITED) == 0)); + (*tsk_com_seg)->bs.f = (u32)(!(((unsigned int)wr->send_flags & + (unsigned int)IB_SEND_FENCE) == 0)); + (*tsk_com_seg)->bs.op_type = roce3_ib_opcode[wr->opcode] & 0x1f; + if (wr->opcode == IB_WR_REG_MR) + (*tsk_com_seg)->bs.op_type = ROCE_WQE_OPCODE_FRMR; +} + +static void roce3_post_send_local_operation(struct roce3_post_send_normal_param *param, + struct roce3_qp *rqp, struct roce3_device *rdev) +{ + param->opcode = param->tsk_com_seg->bs.op_type; + param->tsk_com_seg->value = cpu_to_be32(param->tsk_com_seg->value); + param->ctrl_seg->dw1.value = cpu_to_be32(param->ctrl_seg_tmp.dw1.value); + param->index += (u32)DIV_ROUND_UP(param->wqe_size, 1U << (u32)rqp->sq.wqe_shift); + + write_invalid_wqe(rqp, param->index); + + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + wmb(); + + param->ctrl_seg->dw0.value = cpu_to_be32(param->ctrl_seg_tmp.dw0.value); +} +/* + **************************************************************************** + Prototype : roce3_post_send_normal + Description : roce3_post_send_normal + Input : struct roce3_device *rdev + struct roce3_qp *rqp + struct ib_send_wr *wr + struct ib_send_wr **bad_wr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + + 2.Date : 2015/8/8 + Modification : Modify function + +**************************************************************************** +*/ +static int roce3_post_send_normal(struct roce3_device *rdev, struct roce3_qp *rqp, + const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + int ret = 0; + struct roce3_post_send_normal_param param = { 0 }; + const struct ib_send_wr *wr_tmp = wr; + + spin_lock_irqsave(&rqp->sq.lock, param.flags); + + param.index = rqp->sq_next_wqe; + + for (param.wr_num = 0; wr_tmp != NULL; ++(param.wr_num), wr_tmp = wr_tmp->next) { + ret = roce3_validate_wr(rqp, wr_tmp, param.wr_num); + if (ret != 0) { + *bad_wr = wr_tmp; + pr_err("[ROCE, ERR] %s: Failed to validate wr, func(%d) qpn(%u)\n", + __func__, rdev->glb_func_id, rqp->qpn); + goto out; + } + + param.wqe = (u8 *)roce3_get_send_wqe(rqp, param.index); + param.ctrl_seg = (struct roce3_wqe_ctrl_seg *)param.wqe; + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + rqp->sq.wrid[(rqp->sq.head + param.wr_num) & + (u32)(rqp->sq.wqebb_cnt - 1)] = wr_tmp->wr_id; + + roce3_init_ctrl_seg(¶m, wr_tmp, rqp); + + param.wqe = ((u8 *)param.wqe + sizeof(*(param.ctrl_seg))); + param.wqe_size = (u32)sizeof(*param.ctrl_seg); + roce3_init_task_com_seg(¶m.tsk_com_seg, param.wqe, wr_tmp, rqp); + + ret = roce3_construct_wqe(¶m, rdev, rqp, wr_tmp, bad_wr); + if (ret == GOTO_LOCAL) { + ret = 0; + goto local; + } else if (ret == GOTO_OUT) { + ret = -EINVAL; + goto out; + } + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + need_goto = roce3_post_send_set_data_seg(¶m, rdev, rqp, wr_tmp, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + ret = -EINVAL; + goto out; + } + + roce3_fill_task_data_len(param.data_len_addr, param.data_len); + +local: + roce3_post_send_local_operation(¶m, rqp, rdev); + } + +out: + + ret = roce3_post_send_ring_db(¶m, rdev, rqp); + + if (ROCE_LIKELY(param.wr_num != 0)) + rqp->sq_next_wqe = param.index; + + spin_unlock_irqrestore(&rqp->sq.lock, param.flags); + + return ret; +} + +static int roce3_post_send_check_qp_type(const struct roce3_qp *rqp) +{ + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_TGT)) { + pr_err("[ROCE, ERR] %s: Can't post WQE when TGT XRC QP\n", __func__); + return -EINVAL; + } + + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_INI)) { + pr_err("[ROCE, ERR] %s: not support xrc in kernel space\n", __func__); + return -EINVAL; + } + + if (ROCE_UNLIKELY((rqp->qp_state == IB_QPS_RESET) || (rqp->qp_state == IB_QPS_INIT) || + (rqp->qp_state == IB_QPS_RTR))) { + pr_err("[ROCE, ERR] %s: Can't post WQE when QP is RST/INIT/RTR state\n", __func__); + return -EINVAL; + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_post_send + Description : roce3_post_send + Input : struct ib_qp *ibqp + struct ib_send_wr *wr + const struct ib_send_wr **bad_wr + Output : None +**************************************************************************** +*/ +int roce3_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int ret = 0; + struct roce3_qp *rqp = to_roce3_qp(ibqp); + struct roce3_device *rdev = to_roce3_dev(ibqp->device); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_post_send_check_qp_type(rqp); + if (ret != 0) { + *bad_wr = wr; + pr_err("[ROCE, ERR] %s: Failed to check qp.ret(%d)\n", __func__, ret); + return ret; + } + + ret = roce3_post_send_normal(rdev, rqp, wr, bad_wr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to post send normal wr, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c b/drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c new file mode 100644 index 000000000..f73cb0cbf --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/qp/roce_qp_query.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "hinic3_crm.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_pub_cmd.h" +#include "roce_main_extension.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +static enum ib_mig_state to_ib_mig_state(int roce3_mig_state) +{ + switch (roce3_mig_state) { + case ROCE_QP_PM_ARMED: + return IB_MIG_ARMED; + + case ROCE_QP_PM_REARM: + return IB_MIG_REARM; + + case ROCE_QP_PM_MIGRATED: + return IB_MIG_MIGRATED; + + default: + return (enum ib_mig_state)(-1); + } +} + +static int to_ib_qp_access_flags(const struct roce_qp_context *context) +{ + unsigned int ib_flags = 0; + u32 tmp = 0; + + /* RRE */ + if (context->chip_seg.qpcc.dw5.bs.qp_rre != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_READ; + ib_flags = tmp; + } + + /* RWE */ + if (context->chip_seg.qpcc.dw5.bs.qp_rwe != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_WRITE; + ib_flags = tmp; + } + + /* RAE */ + if (context->chip_seg.qpcc.dw5.bs.qp_rae != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_ATOMIC; + ib_flags = tmp; + } + + return (int)ib_flags; +} + +/* + **************************************************************************** + Prototype : to_ib_ah_attr + Description : to_ib_ah_attr + Input : struct roce3_device *rdev + struct ib_ah_attr *ah_attr(struct rdma_ah_attr *ah_attr) + struct roce_qp_context *context + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/26 + Author : + Modification : Created function + +**************************************************************************** +*/ +static void to_ib_ah_attr(struct roce3_device *rdev, + struct rdma_ah_attr *ah_attr, struct roce_qp_context *context) +{ + struct drv_path_info *path_seg = &context->sw_seg.path_seg; + + memset(ah_attr, 0, sizeof(*ah_attr)); + + ah_attr->port_num = context->sw_seg.ucode_seg.common.dw0.bs.port; + + ah_attr->sl = 0x7 - path_seg->dw7.bs.sl; + + ah_attr->static_rate = 0; + + ah_attr->ah_flags = IB_AH_GRH; + + ah_attr->grh.sgid_index = path_seg->dw7.bs.sgid_index; + ah_attr->grh.hop_limit = path_seg->dw7.bs.hoplmt; + ah_attr->grh.traffic_class = path_seg->dw6.bs.tclass; + ah_attr->grh.flow_label = path_seg->dw6.bs.flow_label; + + memcpy((void *)ah_attr->grh.dgid.raw, (void *)path_seg->dgid, + sizeof(ah_attr->grh.dgid.raw)); +} + +/* + **************************************************************************** + Prototype : roce3_qp_query + Description : roce3_qp_query + Input : struct roce3_device *rdev + struct roce3_qp *qp + struct roce_qp_context *context + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** +*/ +int roce3_qp_query(struct roce3_device *rdev, u32 qpn, u32 *context, int qpc_size) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_qp_query *qp_query_inbuf = NULL; + struct roce3_qp_query_outbuf *qp_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_qp_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_qp_query_outbuf)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + qp_query_inbuf = (struct tag_roce_cmd_qp_query *)cqm_cmd_inbuf->buf; + qp_query_outbuf = (struct roce3_qp_query_outbuf *)cqm_cmd_outbuf->buf; + qp_query_inbuf->com.index = cpu_to_be32(qpn); + qp_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); //lint !e778 + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_QUERY_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_B); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send QUERY_QP command, func_id(%d)\n", + __func__, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(QUERY_QP), qpn(0x%x), func_id(%u)\n", + __func__, qpn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto free_cqm_buf; + } + + memcpy((void *)context, (void *)&qp_query_outbuf->qpc, (size_t)qpc_size); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; + +free_cqm_buf: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + **************************************************************************** + Prototype : qpc_seg_to_le32 + Input : struct roce_qp_context *le_ctx + struct roce_qp_context *be_ctx + struct roce3_qp *rqp + Output : None + + 1.Date : 2015/8/15 + Modification : Created function + +**************************************************************************** +*/ +void qpc_seg_to_le32(struct roce_qp_context *be_ctx, struct roce_qp_context *le_ctx, u32 srq_vld) +{ + /* DRV Seg */ + memcpy((void *)le_ctx->sw_seg.path_seg.dgid, + (void *)be_ctx->sw_seg.path_seg.dgid, sizeof(be_ctx->sw_seg.path_seg.dgid)); + + /* CHIP Seg */ + le_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa = be64_to_cpu(be_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa); + + le_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num = + cpu_to_be64(be_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num); + + le_ctx->chip_seg.rcc.rc_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.rcc.rc_curt_sge_va); + + le_ctx->chip_seg.sqc.sq_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.sqc.sq_curt_sge_va); + + le_ctx->chip_seg.sqac.sqa_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.sqac.sqa_curt_sge_va); + + if (srq_vld != 0) { + le_ctx->chip_seg.srqc.srq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.srqc.srq_curt_sge_va); + } else { + le_ctx->chip_seg.rqc.rq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rqc.rq_curt_sge_va); + } + + le_ctx->chip_seg.rrwc.rrw_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va); + + le_ctx->chip_seg.rrwc.rrw_curt_sge_va = be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va); +} + +void roce3_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size) +{ + u32 *ctx = NULL, *ctx1 = NULL; + u32 i = 0; + + ctx = le_ctx; + ctx1 = (u32 *)context; + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) + *ctx = be32_to_cpu(*ctx1); +} + +static void roce3_get_ah_attr(const struct roce3_qp *rqp, struct ib_qp_attr *qp_attr, + struct roce3_device *rdev, struct roce_qp_context *context) +{ + if ((rqp->qp_type == IB_QPT_RC) || (rqp->qp_type == IB_QPT_UC)) + to_ib_ah_attr(rdev, &qp_attr->ah_attr, context); +} + +static void roce3_query_set_common_attr(struct ib_qp_attr *qp_attr, + const struct ib_qp *ibqp, struct roce3_qp *rqp, struct ib_qp_init_attr *qp_init_attr) +{ + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = (u32)rqp->rq.wqebb_cnt; + qp_attr->cap.max_recv_sge = (u32)rqp->rq.max_sge; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = (u32)rqp->sq.max_post; + qp_attr->cap.max_send_sge = (u32)rqp->sq.max_sge; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_attr->cap.max_inline_data = rqp->max_inline_data; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = (enum ib_qp_create_flags)0; + + qp_init_attr->sq_sig_type = (rqp->sq_signal_bits == 1) ? + IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; +} + +static void roce3_query_set_attr(struct ib_qp_attr *qp_attr, u8 tmp_qp_state, struct roce3_qp *rqp, + struct roce_qp_context context, struct roce3_device *rdev) +{ + qp_attr->qp_state = (enum ib_qp_state)rqp->qp_state; + qp_attr->path_mtu = (enum ib_mtu)context.sw_seg.drv_seg.dw2.bs.pmtu; + qp_attr->path_mig_state = (enum ib_mig_state)to_ib_mig_state(ROCE_QP_PM_MIGRATED); + qp_attr->qkey = context.sw_seg.ucode_seg.common.dw2.qkey; + qp_attr->rq_psn = context.sw_seg.ucode_seg.rq_ctx.dw20.bs.next_rcv_psn; + qp_attr->sq_psn = context.sw_seg.ucode_seg.sq_ctx.dw8.bs.next_send_psn; + qp_attr->dest_qp_num = context.sw_seg.drv_seg.dw0.bs.dest_qp; + qp_attr->qp_access_flags = to_ib_qp_access_flags(&context); + + roce3_get_ah_attr(rqp, qp_attr, rdev, &context); + + qp_attr->pkey_index = 0; + + if (qp_attr->qp_state == IB_QPS_INIT) + qp_attr->port_num = rqp->port; + else + qp_attr->port_num = context.sw_seg.ucode_seg.common.dw0.bs.port; + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + + /* SQ Draining FLAG */ + qp_attr->sq_draining = (tmp_qp_state == ROCE_QP_STATE_SQ_DRAINING); + + qp_attr->max_rd_atomic = (1 << context.sw_seg.drv_seg.dw3.bs.sra_max); + qp_attr->max_dest_rd_atomic = (1 << context.sw_seg.drv_seg.dw3.bs.rra_max); + + qp_attr->min_rnr_timer = context.sw_seg.drv_seg.dw2.bs.min_rnr_nak; + qp_attr->timeout = context.sw_seg.drv_seg.dw2.bs.ack_to; + qp_attr->retry_cnt = context.sw_seg.drv_seg.dw2.bs.to_retry_limit; + qp_attr->rnr_retry = context.sw_seg.drv_seg.dw2.bs.rnr_retry_limit; +} + +static int roce3_query_qp_struct_clean(struct ib_qp_attr *qp_attr, + struct ib_qp_init_attr *qp_init_attr, struct roce_qp_context *be_ctx, + struct roce_qp_context *context) +{ + memset(qp_attr, 0, sizeof(struct ib_qp_attr)); + memset(qp_init_attr, 0, sizeof(struct ib_qp_init_attr)); + memset(be_ctx, 0, sizeof(struct roce_qp_context)); + memset(context, 0, sizeof(struct roce_qp_context)); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_query_qp + Description : roce3_query_qp + Input : struct ib_qp *ibqp + struct ib_qp_attr *qp_attr + int qp_attr_mask + struct ib_qp_init_attr *qp_init_attr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + int ret = 0; + struct roce3_qp *rqp = NULL; + struct roce3_device *rdev = NULL; + struct roce_qp_context be_ctx; /* BE context structure */ + struct roce_qp_context context; /* LE context structure */ + u32 srq_vld; + u8 tmp_qp_state = 0; + + ret = roce3_query_qp_struct_clean(qp_attr, qp_init_attr, &be_ctx, &context); + if (ret != 0) + return ret; + + rqp = to_roce3_qp(ibqp); + rdev = to_roce3_dev(ibqp->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + ret = roce3_qp_query(rdev, rqp->qpn, (u32 *)(void *)(&be_ctx), sizeof(be_ctx)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to query qp, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + mutex_unlock(&rqp->mutex); + return ret; + } + + srq_vld = (!(rqp->ibqp.srq)) ? ROCE_QP_NO_SRQ : ROCE_QP_HAS_SRQ; + roce3_be32_2_le32(&be_ctx, (u32 *)(void *)&context, + sizeof(struct roce_qp_context) / sizeof(u32)); + + qpc_seg_to_le32(&be_ctx, &context, srq_vld); + + rqp->qp_state = context.sw_seg.drv_seg.dw0.bs.state; /* state:4bits */ + + tmp_qp_state = rqp->qp_state; + if (rqp->qp_state == ROCE_QP_STATE_SQ_DRAINING) + rqp->qp_state = IB_QPS_SQD; + + roce3_query_set_attr(qp_attr, tmp_qp_state, rqp, context, rdev); + +done: + roce3_query_set_common_attr(qp_attr, ibqp, rqp, qp_init_attr); + + mutex_unlock(&rqp->mutex); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c new file mode 100644 index 000000000..92988da70 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "rdma_comp.h" +#include "rdma_bitmap.h" + +u32 rdma_bitmap_alloc(struct rdma_bitmap *bitmap) +{ + u32 index = 0; + + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return RDMA_INVALID_INDEX; + } + + spin_lock(&bitmap->lock); + + index = (u32)find_next_zero_bit(bitmap->table, + (unsigned long)bitmap->max_num, (unsigned long)bitmap->last); + if (index >= bitmap->max_num) { + bitmap->top = (bitmap->top + bitmap->max_num + bitmap->reserved_top) & bitmap->mask; + index = (u32)find_first_zero_bit(bitmap->table, (unsigned long)bitmap->max_num); + } + + if (index < bitmap->max_num) { + set_bit(index, bitmap->table); + bitmap->last = index + 1; + if (bitmap->last == bitmap->max_num) + bitmap->last = 0; + + index |= bitmap->top; + --bitmap->avail; + } else { + pr_err("%s: Get a invalid index\n", __func__); + spin_unlock(&bitmap->lock); + return RDMA_INVALID_INDEX; + } + + spin_unlock(&bitmap->lock); + + return index; +} + +void rdma_bitmap_free(struct rdma_bitmap *bitmap, u32 index) +{ + u32 index_tmp = index; + + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return; + } + + if (index_tmp >= bitmap->max_num) { + pr_err("%s: Index(%d) is bigger or equal than max(%d)\n", + __func__, index_tmp, bitmap->max_num); + return; + } + + index_tmp &= bitmap->max_num + bitmap->reserved_top - 1; + + spin_lock(&bitmap->lock); + + bitmap->last = min(bitmap->last, index_tmp); + bitmap->top = (bitmap->top + bitmap->max_num + bitmap->reserved_top) & bitmap->mask; + + bitmap_clear(bitmap->table, (int)index_tmp, 1); + ++bitmap->avail; + spin_unlock(&bitmap->lock); +} + +int rdma_bitmap_init(struct rdma_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 reserved_top) +{ + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return -EINVAL; + } + + /*lint -e587 */ + if (num != (u32)(ROCE_BITMAP_ROUNDUP_POW_OF_TWO(num) & 0xffffffff)) { + pr_err("%s: Num(%d) isn't pow of two, err(%d)\n", __func__, num, -EINVAL); + return -EINVAL; + } + /*lint +e587 */ + + if (num <= (reserved_bot + reserved_top)) { + pr_err("%s: Reserved num is bigger than total num, err(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + bitmap->last = 0; + bitmap->top = 0; + bitmap->max_num = num - reserved_top; + bitmap->mask = mask; + bitmap->reserved_top = reserved_top; + bitmap->avail = (num - reserved_top) - reserved_bot; + + /*lint -e708*/ + spin_lock_init(&bitmap->lock); + /*lint +e708*/ + bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max_num), sizeof(long), GFP_KERNEL); + if (bitmap->table == NULL) { + bitmap->table = vzalloc((size_t)(BITS_TO_LONGS(bitmap->max_num) * sizeof(long))); + if (bitmap->table == NULL) + return -ENOMEM; + } + + bitmap_set(bitmap->table, 0, (int)reserved_bot); + + return 0; +} + +void rdma_bitmap_cleanup(struct rdma_bitmap *bitmap) +{ + if (bitmap == NULL) { + pr_err("%s: Bitmap is null\n", __func__); + return; + } + + if (is_vmalloc_addr(bitmap->table)) + vfree(bitmap->table); + else + kfree(bitmap->table); + + bitmap->table = NULL; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h new file mode 100644 index 000000000..8072dd458 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_bitmap.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_BITMAP_H +#define RDMA_BITMAP_H + +#include <linux/spinlock.h> + +#ifndef RDMA_INVALID_INDEX +#define RDMA_INVALID_INDEX 0xFFFFFFFF +#endif + +#define ROCE_BITMAP_ROUNDUP_POW_OF_TWO(n) (roundup_pow_of_two(n)) + +struct rdma_bitmap { + u32 last; /* bottom of available id */ + u32 top; /* top value of non zone of id */ + u32 max_num; /* max id num */ + u32 reserved_top; /* unavailable top num */ + u32 mask; /* mask of id */ + u32 avail; /* num of available id */ + spinlock_t lock; /* spinlock of bitmap */ + unsigned long *table; /* memory of bitmap */ +}; + +u32 rdma_bitmap_alloc(struct rdma_bitmap *bitmap); + +void rdma_bitmap_free(struct rdma_bitmap *bitmap, u32 index); + +int rdma_bitmap_init(struct rdma_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 reserved_top); + +void rdma_bitmap_cleanup(struct rdma_bitmap *bitmap); + +#endif // __RDMA_BITMAP_H__ + diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c new file mode 100644 index 000000000..c2b4d06a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" + + +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev) +{ + struct rdma_comp_priv *comp_private = NULL; + + comp_private = (struct rdma_comp_priv *)hinic3_get_service_adapter(hwdev, SERVICE_T_ROCE); + return comp_private; +} + +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->pd_bitmap); +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h new file mode 100644 index 000000000..d21486033 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef RDMA_COMP_H +#define RDMA_COMP_H + +/* x86 */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#include <linux/delay.h> +#include <linux/types.h> + +#include "rdma_context_format.h" +#include "hinic3_crm.h" +#include "hinic3_cqm.h" +#include "hinic3_rdma.h" + +#include "roce_verbs_cmd.h" +#include "roce_verbs_ulp_format.h" + +#include "rdma_bitmap.h" +#include "hmm_buddy.h" +#include "hmm_em.h" +#include "hmm_comp.h" + +#ifndef pr_fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": [RDMA]" fmt +#endif + +#define RDMA_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define RDMA_KEY_MASK 0xFFFFFF00 + +#define RDMA_INVALID_GUID 0 + +#define RDMA_SERVICE_TYPE_ROCE SERVICE_T_ROCE + +#define RDMA_BIT_SHIFT_1 1 +#define RDMA_BIT_SHIFT_2 2 +#define RDMA_BIT_SHIFT_4 4 +#define RDMA_BIT_SHIFT_8 8 +#define RDMA_BIT_SHIFT_16 16 + +#define ROCE3_RDMARC_MIN_DEPTH 1 +#define ROCE3_RDMARC_MAX_DEPTH 512 +#define ROCE3_RDMARC_MIN_ENTRY 8 /* min entry aligned required by engine */ +#define ROCE3_RDMARC_EXT_ENTRY 384 // 384 /* ext tbl 12K. 12k/32 = 384 */ + +extern u32 g_mtt_page_size; + +#define RDMA_EM_MIN_ORDER 2 /* rdma rc extend table at least 4 page */ + +#define ROCE_MTT_PAGE_SIZE_4K 4096 +#define ROCE_MTT_PAGE_SIZE_64K (64 * 1024) +#define ROCE_MTT_PAGE_SIZE_2M (2 * 1024 * 1024) +#define ROCE_MTT_PAGE_SIZE_4K_SHIFT 12 +#define ROCE_MTT_PAGE_SIZE_64K_SHIFT 16 +#define ROCE_MTT_PAGE_SIZE_2M_SHIFT 21 + +enum { + ROCE3_RDMA_MTT_PAGE_SIZE_4K = 0, + ROCE3_RDMA_MTT_PAGE_SIZE_64K = 1, + ROCE3_RDMA_MTT_PAGE_SIZE_2M = 2 +}; + +enum { + ROCE3_RDMA_CMD_TIME_OUT_A = 30000, + ROCE3_RDMA_CMD_TIME_OUT_B = 40000, + ROCE3_RDMA_CMD_TIME_OUT_C = 50000 +}; + +enum rdma_roce3_cmd { + RDMA_ROCE_CMD_UPDATE_GID = 0x60, + RDMA_ROCE_CMD_QUERY_GID = 0x61, + RDMA_ROCE_CMD_CLEAR_GID = 0x62 +}; + +struct rdma_comp_priv { + struct rdma_comp_resource rdma_comp_res; /* gid & guid */ + + struct hmm_buddy mtt_buddy; + struct hmm_em_table mtt_em_table; + void *hwdev; + struct pci_dev *pdev; + struct rdma_mr rsvd_lkey; + struct rdma_mr fixed_mr; + u32 mtt_page_size; /* 4K, 8K, 16K, 32K */ + u32 mtt_page_shift; /* 12, 13, 14, 15 */ + struct rdma_service_cap rdma_cap; + + struct rdma_bitmap pd_bitmap; + struct rdma_bitmap xrcd_bitmap; + + struct hmm_buddy rdmarc_buddy; + struct hmm_em_table rdmarc_em_table; +}; + +struct rdma_gid_update_inbuf { + struct tag_roce_verbs_cmd_com com; + __be32 port; + __be32 rsvd; + struct rdma_gid_entry gid_entry; +}; + +struct rdma_gid_clear_inbuf { + __be32 port; + __be32 gid_num; +}; + +/* for llt to set stub */ +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv); + +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev); + +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, struct rdma_gid_entry *gid_entry); + +int roce3_rdma_init_rsvd_lkey(void *hwdev); + +void roce3_rdma_free_rsvd_lkey(void *hwdev); + +#endif // RDMA_COMP_H diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c new file mode 100644 index 000000000..89cb35194 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_gid.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "roce.h" +#include "roce_cqm_cmd.h" + +static int rdma_update_gid(struct rdma_comp_priv *comp_priv, + struct rdma_gid_entry *new_gid_entry, int port, int gid_index) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct rdma_gid_update_inbuf *gid_update_inbuf = NULL; + int ret; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(comp_priv->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct rdma_gid_update_inbuf), NULL, 0); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + gid_update_inbuf = (struct rdma_gid_update_inbuf *)cqm_cmd_inbuf->buf; + gid_update_inbuf->port = cpu_to_be32((u32)port); + gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index); + gid_update_inbuf->gid_entry.global.subnet_prefix = new_gid_entry->global.subnet_prefix; + gid_update_inbuf->gid_entry.global.interface_id = new_gid_entry->global.interface_id; + gid_update_inbuf->gid_entry.dw4.value = cpu_to_be32(new_gid_entry->dw4.value); + gid_update_inbuf->gid_entry.dw6_h.value = cpu_to_be16(new_gid_entry->dw6_h.value); + gid_update_inbuf->gid_entry.hdr_len_value = cpu_to_be32(new_gid_entry->hdr_len_value); + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); //lint !e778 + + memcpy(&gid_update_inbuf->gid_entry.smac[0], &new_gid_entry->smac[0], ETH_ALEN); + + pr_info("[ROCE] %s:gid_index:0x%x gid:0x%x %x %x %x %x %x %x %x\n", + __func__, gid_update_inbuf->com.index, + *((u32 *)(void *)new_gid_entry + 0), + *((u32 *)(void *)new_gid_entry + 1), // 0 1 is gid array idx + *((u32 *)(void *)new_gid_entry + 2), + *((u32 *)(void *)new_gid_entry + 3), // 2 3 is gid array idx + *((u32 *)(void *)new_gid_entry + 4), + *((u32 *)(void *)new_gid_entry + 5), // 4 5 is gid array idx + *((u32 *)(void *)new_gid_entry + 6), + *((u32 *)(void *)new_gid_entry + 7)); // 6 7 is gid array idx + ret = cqm_send_cmd_box(comp_priv->hwdev, HINIC3_MOD_ROCE, + RDMA_ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE3_RDMA_CMD_TIME_OUT_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Send cmd update_gid failed, ret(%d)\n", __func__, ret); + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(comp_priv->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, struct rdma_gid_entry *gid_entry) +{ + struct rdma_gid_entry entry1; + struct rdma_gid_entry entry2; + + memcpy(&entry1, gid_tbl_entry, sizeof(struct rdma_gid_entry)); + memcpy(&entry2, gid_entry, sizeof(struct rdma_gid_entry)); + + /* compare ip+vlan, compare after clear smac */ + memset((void *)entry1.smac, 0, sizeof(entry1.smac)); + memset((void *)entry2.smac, 0, sizeof(entry2.smac)); + return memcmp((void *)&entry1, (void *)&entry2, sizeof(struct rdma_gid_entry)); +} + +static int rdma_reset_gid(struct rdma_comp_priv *comp_priv, u32 port, u32 gid_num) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_clear_gid *gid_clear_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(comp_priv->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_clear_gid), NULL, 0); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + gid_clear_inbuf = (struct tag_roce_clear_gid *)cqm_cmd_inbuf->buf; + gid_clear_inbuf->port = cpu_to_be32(port); + gid_clear_inbuf->com.index = 0; // cpu_to_be32(gid_num); + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); //lint !e778 + + ret = cqm_send_cmd_box(comp_priv->hwdev, HINIC3_MOD_ROCE, + RDMA_ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE3_RDMA_CMD_TIME_OUT_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Send cmd clear_gid failed, ret(%d)\n", __func__, ret); + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(comp_priv->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int roce3_rdma_update_gid_mac(void *hwdev, u32 port, struct rdma_gid_entry *gid_entry) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + u32 gid_index = 0; + int update_index = -1; /* -1 is initvalue, indicating no need to update */ + int ret = 0; + + if ((hwdev == NULL) || (gid_entry == NULL)) { + pr_err("%s: Hwdev or gid_tbl is null\n", __func__); + return -EINVAL; + } + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + /* port num index from zero */ + if (port >= comp_priv->rdma_cap.num_ports) { + pr_err("%s: Input port(%d) is invalid\n", __func__, port); + return -EINVAL; + } + /* invalid gid cause of with the head fe80:: of IPV6 */ + if (cpu_to_be64(gid_entry->global.subnet_prefix) == RDMA_DEFAULT_GID_SUBNET_PREFIX) + return 0; + + gid_table = comp_priv->rdma_comp_res.gid_table; + mutex_lock(&comp_priv->rdma_comp_res.mutex); + /* loop all gid, ensure all gid cleared not exist */ + for (gid_index = 1; gid_index < comp_priv->rdma_cap.max_gid_per_port; gid_index++) { + if (rdma_gid_entry_cmp(&gid_table[port][gid_index], gid_entry) == 0) { + update_index = (int)gid_index; + break; + } + } + if (update_index > 0) { + ret = rdma_update_gid(comp_priv, gid_entry, (int)port, update_index); + if (ret != 0) { + pr_err("%s: Update gid tbl failed, ret(%d)\n", __func__, ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + /* update gid table after CMDQ */ + memcpy((void *)&gid_table[port][update_index], (void *)gid_entry, + sizeof(*gid_entry)); + + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; + } + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; +} + +int roce3_rdma_update_gid(void *hwdev, u32 port, u32 update_index, struct rdma_gid_entry *gid_entry) +{ + int ret = 0; + u32 port_num = 0; + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table; + + if ((hwdev == NULL) || (gid_entry == NULL)) { + pr_err("%s: Hwdev or gid_tbl is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + + /* port start from zero */ + port_num = comp_priv->rdma_cap.num_ports; + if (port >= port_num) { + pr_err("%s: Input port(%d) is invalid\n", __func__, port); + return -EINVAL; + } + + mutex_lock(&comp_priv->rdma_comp_res.mutex); + ret = rdma_update_gid(comp_priv, gid_entry, (int)port, (int)update_index); + if (ret != 0) { + pr_err("%s: Rdma_update_gid failed\n", __func__); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + + memcpy((void *)&gid_table[port][update_index], (void *)gid_entry, sizeof(*gid_entry)); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + + return 0; +} + +int roce3_rdma_reset_gid_table(void *hwdev, u32 port) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + int ret = 0; + u32 gids_per_port = 0; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + pr_err("%s: Gid_table is null\n", __func__); + return -EINVAL; + } + + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + if (gids_per_port == 0) { + pr_err("%s: Gids_per_port(%d) is invalid\n", __func__, gids_per_port); + return -EINVAL; + } + + memset(&gid_table[port][0], 0, + gids_per_port * sizeof(struct rdma_gid_entry)); + ret = rdma_reset_gid(comp_priv, port, gids_per_port); + if (ret != 0) { + pr_err("%s: Reset gid table failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +int roce3_rdma_get_gid(void *hwdev, u32 port, u32 gid_index, struct rdma_gid_entry *gid) +{ + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + if (gid == NULL) { + pr_err("%s: Gid is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + if (port < 1 || port > rdma_cap->num_ports) { + pr_err("%s: Port(%d) invalid\n", __func__, port); + return -EINVAL; + } + + if (gid_index >= rdma_cap->max_gid_per_port) { + pr_err("%s: gid_index(%d) invalid\n", __func__, gid_index); + return -EINVAL; + } + + memcpy((void *)gid, (void *)&comp_priv->rdma_comp_res.gid_table[port - 1][gid_index], + sizeof(*gid)); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c new file mode 100644 index 000000000..895bb5b78 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_init.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "hinic3_hmm.h" + +#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 12K */ + +static int rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.num_pds; + reserved_bot = comp_priv->rdma_cap.reserved_pds; + + ret = rdma_bitmap_init(&comp_priv->pd_bitmap, num, num - 1, reserved_bot, 0); + if (ret != 0) { + pr_err("%s: Can't initialize pd's bitmap, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + int i = 0; + int port_num = 0; + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + pr_err("%s: Gid_table is null\n", __func__); + return; + } + + port_num = (int)comp_priv->rdma_cap.num_ports; + for (i = 0; i < port_num; i++) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; +} + +static int rdma_init_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + u32 i = 0; + u32 port_num = 0; + u32 gids_per_port = 0; + + port_num = comp_priv->rdma_cap.num_ports; + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + if ((port_num == 0) || (gids_per_port == 0)) { + pr_err("%s: Alloc memory for gid_tbl failed, port_num(%d), gids_per_ports(%d)\n", + __func__, port_num, gids_per_port); + return -EINVAL; + } + + gid_table = kcalloc(port_num, sizeof(struct rdma_gid_entry *), GFP_KERNEL); + if (gid_table == NULL) + return -ENOMEM; + + comp_priv->rdma_comp_res.gid_table = gid_table; + + for (i = 0; i < port_num; i++) { + gid_table[i] = kcalloc(gids_per_port, sizeof(struct rdma_gid_entry), GFP_KERNEL); + if (gid_table[i] == NULL) + goto err_out; + } + return 0; +err_out: + for (i = 0; i < port_num; i++) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; + + return -ENOMEM; +} + +static int rdma_init_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.max_xrcds; + reserved_bot = comp_priv->rdma_cap.reserved_xrcds; + + ret = rdma_bitmap_init(&comp_priv->xrcd_bitmap, num, num - 1, reserved_bot, 0); + if (ret != 0) { + pr_err("%s: Can't initialize xrcd's bitmap!, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->xrcd_bitmap); +} + +static int rdma_init_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 i = 0; + u32 max_order = 0; /* rdmarc buddy max priority num */ + u32 qp_num = 0; + u32 rdmarc_per_qp = 0; /* rdmarc num per qp */ + u32 rdmarc_size = 0; + u32 log_rdmarc_per_seg = 0; /* min num of entry, 2^log_rdmarc_per_seg */ + u32 rdmarc_pow_of_two = 0; + + qp_num = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qps; + rdmarc_per_qp = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qp_dest_rdma + + ROCE_MAX_RDMA_RC_EXTEND; + rdmarc_size = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz; + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + for (i = 1; i < qp_num * rdmarc_per_qp; i <<= 1) + max_order++; + + max_order = (max_order > log_rdmarc_per_seg) ? (max_order - log_rdmarc_per_seg) : 0; + + ret = hmm_buddy_init(&comp_priv->rdmarc_buddy, max_order); + if (ret != 0) { + pr_err("%s: Initialize rdmarc's buddy failed, ret(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + /*lint -e587*/ + rdmarc_pow_of_two = (u32)(HMM_EM_ROUNDUP_POW_OF_TWO( + (u32)(qp_num * rdmarc_per_qp)) & 0xffffffff); + /*lint +e587*/ + ret = hmm_em_init_table(comp_priv->pdev, &comp_priv->rdmarc_em_table, + rdmarc_size, rdmarc_pow_of_two, 0, RDMA_EM_MIN_ORDER); + if (ret != 0) { + pr_err("%s: Initialize rdmarc's em_table failed, ret(%d)\n", __func__, ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); + + return -ENOMEM; +} + +static int roce3_rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = rdma_init_pd_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize pd's table failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_rdma_init_mtt_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = hmm_init_mtt_table((struct hmm_comp_priv *)(void *)comp_priv); + if (ret != 0) { + pr_err("%s: Initialize mtt's table failed, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static int roce3_rdma_init_bitmap(struct rdma_comp_priv *comp_priv) +{ + int ret; + + ret = roce3_rdma_init_pd_table(comp_priv); + if (ret != 0) + return ret; + + ret = roce3_rdma_init_mtt_table(comp_priv); + if (ret != 0) { + rdma_cleanup_pd_table(comp_priv); + return ret; + } + + return 0; +} + +static void roce3_rdma_unit_bitmap(struct rdma_comp_priv *comp_priv) +{ + hmm_cleanup_mtt_table((struct hmm_comp_priv *)(void *)comp_priv); + rdma_cleanup_pd_table(comp_priv); +} + +static int roce3_rdma_init_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = roce3_rdma_init_bitmap(comp_priv); + if (ret != 0) + return ret; + + if (hinic3_support_roce(hwdev, NULL)) { + ret = rdma_init_gid_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize gid table failed, ret(%d)\n", __func__, ret); + goto err_init_gid_table; + } + ret = rdma_init_xrcd_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize xrcd's table failed, ret(%d)\n", __func__, ret); + goto err_init_xrcd; + } + ret = rdma_init_rdmarc_table(comp_priv); + if (ret != 0) { + pr_err("%s: Initialize rdmarc's table failed, ret(%d)\n", + __func__, ret); + goto err_init_rdmarc_table; + } + } + + pr_info("%s: Rdma init resource successful\n", __func__); + return 0; +err_init_rdmarc_table: + rdma_cleanup_xrcd_table(comp_priv); +err_init_xrcd: + rdma_cleanup_gid_table(comp_priv); +err_init_gid_table: + roce3_rdma_unit_bitmap(comp_priv); + return ret; +} + +static void rdma_cleanup_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->pdev, &comp_priv->rdmarc_em_table); + + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); +} + +static void roce3_rdma_unit_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + if (hinic3_support_roce(hwdev, NULL)) { + rdma_cleanup_rdmarc_table(comp_priv); + rdma_cleanup_xrcd_table(comp_priv); + rdma_cleanup_gid_table(comp_priv); + } + + roce3_rdma_unit_bitmap(comp_priv); +} + +void roce3_rdma_cleanup_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + if (!hinic3_support_rdma(hwdev, NULL)) { + pr_err("%s: Not support rdma service\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + roce3_rdma_unit_table(hwdev, comp_priv); + + kfree(comp_priv); + + hinic3_unregister_service_adapter((void *)hwdev, SERVICE_T_ROCE); + + pr_info("%s: Rdma cleanup resource successful", __func__); +} + +static int roce3_rdma_init_comp_priv(struct rdma_comp_priv *comp_priv, + void *hwdev, struct rdma_service_cap *rdma_cap) +{ + int ret; + + mutex_init(&comp_priv->rdma_comp_res.mutex); + comp_priv->hwdev = hwdev; + comp_priv->pdev = (struct pci_dev *)((struct hinic3_hwdev *)hwdev)->pcidev_hdl; + memcpy((void *)&comp_priv->rdma_cap, (void *)rdma_cap, + sizeof(struct rdma_service_cap)); + // to adapt hmm struct + comp_priv->rdma_cap.dmtt_cl_start = rdma_cap->dev_rdma_cap.roce_own_cap.dmtt_cl_start; + comp_priv->rdma_cap.dmtt_cl_end = rdma_cap->dev_rdma_cap.roce_own_cap.dmtt_cl_end; + comp_priv->rdma_cap.dmtt_cl_sz = rdma_cap->dev_rdma_cap.roce_own_cap.dmtt_cl_sz; + + switch (g_mtt_page_size) { + case ROCE3_RDMA_MTT_PAGE_SIZE_4K: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + case ROCE3_RDMA_MTT_PAGE_SIZE_64K: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_64K; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_64K_SHIFT; + break; + case ROCE3_RDMA_MTT_PAGE_SIZE_2M: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_2M; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_2M_SHIFT; + break; + default: + comp_priv->mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->mtt_page_shift = ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + } + ret = roce3_rdma_init_table(hwdev, comp_priv); + if (ret != 0) + return ret; + + ret = hinic3_register_service_adapter((void *)hwdev, (void *)comp_priv, SERVICE_T_ROCE); + if (ret != 0) { + roce3_rdma_unit_table(hwdev, comp_priv); + pr_err("%s: put rdma_comp_res failed, ret(%d)\n", __func__, ret); + return ret; + } + return ret; +} + +int roce3_rdma_init_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + if (!hinic3_support_rdma(hwdev, &rdma_cap)) { + pr_info("%s: Neither ROCE nor IWARP\n", __func__); + return 0; + } + + comp_priv = kzalloc(sizeof(struct rdma_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) + return -ENOMEM; + + ret = roce3_rdma_init_comp_priv(comp_priv, hwdev, &rdma_cap); + if (ret != 0) { + kfree(comp_priv); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c new file mode 100644 index 000000000..4701a8ca4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_mw_mr.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "roce_cqm_cmd.h" +#include "hmm_mr.h" + +static void rdma_roce3_mpt_to_big_endian(struct roce_mpt_context *mpt_ctx) +{ + mpt_ctx->dw0.value = cpu_to_be32(mpt_ctx->dw0.value); + mpt_ctx->dw1.value = cpu_to_be32(mpt_ctx->dw1.value); + mpt_ctx->dw2.value = cpu_to_be32(mpt_ctx->dw2.value); + mpt_ctx->dw3.value = cpu_to_be32(mpt_ctx->dw3.value); + mpt_ctx->iova = cpu_to_be64(mpt_ctx->iova); + mpt_ctx->length = cpu_to_be64(mpt_ctx->length); + mpt_ctx->mtt_base_addr = cpu_to_be64(mpt_ctx->mtt_base_addr); + mpt_ctx->mtt_sz = cpu_to_be32(mpt_ctx->mtt_sz); +} + +static void rdma_set_roce3_mw_cmd_buf(struct roce_mpt_context *mpt_ctx, struct rdma_mw *mw) +{ + /* fill mpt_entry */ + mpt_ctx->dw0.bs.bpd = 1; + mpt_ctx->dw0.bs.r_w = RDMA_MPT_MW; + mpt_ctx->dw0.bs.access_lr = 1; + mpt_ctx->dw0.bs.access_lw = 1; + + mpt_ctx->dw1.bs.dma_attr_idx = RDMA_MPT_DMA_ATTR_IDX; + mpt_ctx->dw1.bs.so_ro = RDMA_MPT_SO_RO; + + mpt_ctx->dw2.bs.pdn = mw->pdn; + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_VALID; + + mpt_ctx->dw3.bs.mkey = mw->key & 0xFF; + + /* only type2 mw binds with QP and suppoort invalid operation, init value is FREE */ + if (mw->type == RDMA_MW_TYPE_2) { + mpt_ctx->dw0.bs.invalid_en = 1; + mpt_ctx->dw0.bs.remote_invalid_en = 1; + mpt_ctx->dw0.bs.bqp = 1; + + mpt_ctx->dw2.bs.status = RDMA_MPT_STATUS_FREE; + } + + rdma_roce3_mpt_to_big_endian(mpt_ctx); +} + +int roce3_rdma_enable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type) +{ + int ret = 0; + struct rdma_comp_priv *comp_priv = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct roce_mpt_context *mpt_entry = NULL; + struct tag_roce_sw2hw_mpt *mpt_sw2hw_inbuf = NULL; + + if ((hwdev == NULL) || (mw == NULL)) { + pr_err("%s: Hwdev or mw is null\n", __func__); + return -EINVAL; + } + + if (service_type != RDMA_SERVICE_TYPE_ROCE) { + pr_err("%s: service_type not support\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + ret = roce3_cqm_cmd_zalloc_inoutbuf(hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_sw2hw_mpt), NULL, 0); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + /* cmd_buf: mpt_index + mpt_entry */ + mpt_sw2hw_inbuf = (struct tag_roce_sw2hw_mpt *)cqm_cmd_inbuf->buf; + mpt_sw2hw_inbuf->com.index = cpu_to_be32(mw->mpt.mpt_index); + mpt_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_MR_BITMASK); //lint !e778 + mpt_entry = &mpt_sw2hw_inbuf->mpt_entry; + rdma_set_roce3_mw_cmd_buf(mpt_entry, mw); + ret = hmm_enable_roce_mpt(hwdev, cqm_cmd_inbuf, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Enable mr's mpt failed, ret(%d)\n", __func__, ret); + goto out; + } + mw->enabled = RDMA_MPT_EN_HW; +out: + roce3_cqm_cmd_free_inoutbuf(hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int roce3_rdma_disable_mw_mpt(void *hwdev, struct rdma_mw *mw, u32 service_type) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if ((hwdev == NULL) || (mw == NULL)) { + pr_err("%s: Hwdev or mw is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (mw->enabled == RDMA_MPT_EN_HW) { + if (service_type == RDMA_SERVICE_TYPE_ROCE) { + ret = hmm_disable_roce_mpt((struct hmm_comp_priv *)(void *)comp_priv, + &mw->mpt, HINIC3_CHANNEL_ROCE); + } else { + pr_err("%s: service_type not support\n", __func__); + return -EINVAL; + } + + if (ret != 0) { + pr_err("%s: Disable mw's mpt failed, ret(%d)\n", __func__, ret); + return ret; + } + + mw->enabled = RDMA_MPT_EN_SW; + } + + return 0; +} + +static int rdma_mpt_alloc_rsvd_lkey(struct rdma_comp_priv *comp_priv, struct rdma_mpt *mpt) +{ + struct rdma_mpt_entry *mpt_entry = NULL; + u32 mpt_entry_size = 0; + + mpt_entry_size = comp_priv->rdma_cap.mpt_entry_sz; + + /* 调用cqm接口分配mpt，并将mpt_index和vaddr保存在rdma_mpt结构 */ + mpt->mpt_object = (void *)cqm_object_qpc_mpt_create(comp_priv->hwdev, + RDMA_SERVICE_TYPE_ROCE, CQM_OBJECT_MPT, + mpt_entry_size, mpt, mpt->mpt_index, false); + if (mpt->mpt_object == NULL) { + pr_err("[ROCE, ERR]%s: Alloc mpt_object failed, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + mpt->vaddr = (void *)((struct tag_cqm_qpc_mpt *)mpt->mpt_object)->vaddr; + if (!cqm_need_secure_mem(comp_priv->hwdev)) { + memset(mpt->vaddr, 0, sizeof(struct rdma_mpt_entry)); + + mpt_entry = (struct rdma_mpt_entry *)mpt->vaddr; + mpt_entry->roce_mpt_ctx.dw2.bs.status = RDMA_MPT_STATUS_MEM_INIT; + mpt_entry->roce_mpt_ctx.dw2.value = cpu_to_be32(mpt_entry->roce_mpt_ctx.dw2.value); + } + + return 0; +} + +int roce3_rdma_init_rsvd_lkey(void *hwdev) +{ + struct rdma_mr *mr = NULL; + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR]%s: Hwdev is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("[ROCE, ERR]%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + mr = &comp_priv->rsvd_lkey; + mr->mpt.mpt_index = rdma_cap->reserved_lkey >> MR_KEY_LEFT_SHIFT_OFS; + if (mr->mpt.mpt_index >= rdma_cap->reserved_mrws) { + pr_err("[ROCE, ERR]%s: Cfg err, reserved_lkey(0x%x), reserved_mrws(0x%x)\n", + __func__, rdma_cap->reserved_lkey, rdma_cap->reserved_mrws); + return -EINVAL; + } + /* Alloc MPT */ + ret = rdma_mpt_alloc_rsvd_lkey(comp_priv, &mr->mpt); + if (ret != 0) { + pr_err("[ROCE, ERR]%s: Rdma_mpt_alloc_rsvd_lkey failed, ret(%d)\n", + __func__, ret); + return ret; + } + mr->mtt.mtt_layers = 0; + mr->iova = 0; + mr->size = ~0ULL; + mr->key = comp_priv->rdma_cap.reserved_lkey; + mr->mr_type = RDMA_RSVD_LKEY; + mr->access = RDMA_IB_ACCESS_LOCAL_WRITE; + /* Enable MPT */ + ret = hmm_rdma_enable_mr_mpt(hwdev, mr, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR]%s: Rdma_enable_mr_mpt failed, ret(%d)\n", __func__, ret); + goto err_out; + } + return 0; +err_out: + hmm_rdma_mpt_free(hwdev, &mr->mpt); + return ret; +} + +void roce3_rdma_free_rsvd_lkey(void *hwdev) +{ + struct rdma_mr *mr = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR]%s: Hwdev is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("[ROCE, ERR]%s: Comp_priv is null\n", __func__); + return; + } + + mr = &comp_priv->rsvd_lkey; + ret = hmm_rdma_disable_mr_mpt(hwdev, mr, RDMA_SERVICE_TYPE_ROCE, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR]%s: Disable mpt of mr failed, ret(%d)\n", __func__, ret); + return; + } + + hmm_rdma_mpt_free(hwdev, &mr->mpt); +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c new file mode 100644 index 000000000..6aba6ea1b --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_pd.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" + +int roce3_rdma_pd_alloc(void *hwdev, u32 *pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (pdn == NULL)) { + pr_err("%s: Hwdev or pdn is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + *pdn = rdma_bitmap_alloc(&comp_priv->pd_bitmap); + if (*pdn == RDMA_INVALID_INDEX) { + pr_err("%s: Can't get valid pdn, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +void roce3_rdma_pd_free(void *hwdev, u32 pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + rdma_bitmap_free(&comp_priv->pd_bitmap, pdn); +} diff --git a/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c new file mode 100644 index 000000000..23b42a369 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/rdma/rdma_comp_res.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic3_hw.h" +#include "rdma_comp.h" +#include "hmm_em.h" +#include "hmm_buddy.h" + +static int rdma_check_map_param(const void *hwdev, const struct rdma_fmr *fmr, + const u64 *page_list, int npages, + u64 iova) +{ + u32 page_mask = 0; + + if ((hwdev == NULL) || (fmr == NULL) || (page_list == NULL)) { + pr_err("%s: Hwdev or fmr or page_list is null\n", __func__); + return -EINVAL; + } + + if ((u32)npages > fmr->max_pages) { + pr_err("%s: Npages is bigger than fmr->max_pages, ret(%d)\n", + __func__, -EINVAL); + return -EINVAL; + } + + page_mask = (1U << fmr->page_shift) - 1; + if ((iova & (u64)page_mask) != 0) { + pr_err("%s: Iova isn't page aligned, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + if (fmr->maps >= fmr->max_maps) { + pr_err("%s: Maps over range(fmr->max_maps), ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + return 0; +} + +int roce3_rdma_map_phys_fmr(void *hwdev, struct rdma_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 service_type) +{ + __be64 *mtt_vaddr = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + int i = 0; + u32 mpt_index = 0; + u64 length = 0; + u32 new_key = 0; + u64 iova_tmp = iova; + u16 sign_val = 0; + + ret = rdma_check_map_param(hwdev, fmr, page_list, npages, iova_tmp); + if (ret != 0) { + pr_err("%s: Rdma check map param failed\n", __func__); + return ret; + } + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + fmr->mr.enabled = RDMA_MPT_EN_SW; + mpt_index = fmr->mr.mpt.mpt_index; + length = (u64)((unsigned int)npages * (1UL << fmr->page_shift)); + if ((fmr->mr.access & RDMA_IB_ACCESS_ZERO_BASED) != 0) + iova_tmp = 0; + + new_key = (fmr->mr.key & (~0xFF)) | ((fmr->mr.key + 1) & 0xFF); + ret = hmm_modify_roce_mpt(hwdev, mpt_index, new_key, length, iova_tmp, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("%s: Modify mpt failed, ret(%d), new_key(%d)\n", __func__, ret, new_key); + return ret; + } +#ifdef RDMA_SIGN_MTT_EN + sign_val = hmm_gen_mtt_sign(fmr->mr.mtt.mtt_paddr, fmr->mr.mtt.mtt_type); +#endif + mtt_vaddr = fmr->mr.mtt.mtt_vaddr; + for (i = 0; i < npages; i++) + mtt_vaddr[i] = cpu_to_be64(page_list[i] | RDMA_MTT_PA_VALID | (sign_val << 1)); + + fmr->maps++; + fmr->mr.key = new_key; + fmr->mr.enabled = RDMA_MPT_EN_HW; + return 0; +} + +int roce3_rdma_unmap_fmr(void *hwdev, struct rdma_fmr *fmr, u32 service_type) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + u32 new_key = 0; + + if ((hwdev == NULL) || (fmr == NULL)) { + pr_err("%s: Hwdev or fmr is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + if (fmr->maps == 0) + return 0; + + new_key = fmr->mr.key & RDMA_KEY_MASK; + + if (service_type == RDMA_SERVICE_TYPE_ROCE) { + ret = hmm_modify_roce_mpt(hwdev, fmr->mr.mpt.mpt_index, new_key, 0ULL, + 0ULL, HINIC3_CHANNEL_ROCE); + } else { + pr_err("%s: service_type not support\n", __func__); + return -EINVAL; + } + + if (ret != 0) { + pr_err("%s: Modify mpt failed, ret(%d)\n", __func__, ret); + return ret; + } + + fmr->maps = 0; + fmr->mr.key = new_key; + + return 0; +} + +static int roce3_rdma_rdmarc_alloc_check(struct rdma_comp_priv **comp_priv, void *hwdev, + const struct rdma_rdmarc *rdmarc, u32 *log_rdmarc_per_seg, u32 num) +{ + if ((hwdev == NULL) || (rdmarc == NULL)) { + pr_err("%s: Hwdev or rdmarc is null\n", __func__); + return -EINVAL; + } + + *comp_priv = get_rdma_comp_priv(hwdev); + if (*comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + *log_rdmarc_per_seg = (*comp_priv)->rdma_cap.log_rdmarc_seg; + + if ((num < ROCE3_RDMARC_MIN_DEPTH) || (num > ROCE3_RDMARC_MAX_DEPTH)) { + pr_err("%s: Num is invalid, ret(%d)\n", __func__, -EINVAL); + return -EINVAL; + } + + return 0; +} + +static void roce3_rdma_rdmarc_order_set(struct rdma_rdmarc *rdmarc, u32 order, + u32 ext_order, u32 log_rdmarc_per_seg) +{ + rdmarc->order = order; + rdmarc->ext_order = ext_order + log_rdmarc_per_seg; +} + +int roce3_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + u32 i = 0; + u32 order = 0; + u32 ext_order = 0; + u32 ext_num = num + ROCE3_RDMARC_EXT_ENTRY; + u32 offset = 0; + void *vaddr = NULL; + u32 log_rdmarc_per_seg = 0; + u32 chip_num = (num < ROCE3_RDMARC_MIN_ENTRY) ? ROCE3_RDMARC_MIN_ENTRY : num; + + ret = roce3_rdma_rdmarc_alloc_check(&comp_priv, hwdev, rdmarc, &log_rdmarc_per_seg, num); + if (ret != 0) + return ret; + + for (i = 1; i < chip_num; i <<= 1) + order++; + + for (i = 1; i < ext_num; i <<= 1) + ext_order++; + + ext_order = ext_order > log_rdmarc_per_seg ? (ext_order - log_rdmarc_per_seg) : 0; + roce3_rdma_rdmarc_order_set(rdmarc, order, ext_order, log_rdmarc_per_seg); + offset = hmm_buddy_alloc(&comp_priv->rdmarc_buddy, ext_order); + if (offset == RDMA_INVALID_INDEX) { + pr_err("%s: Alloc rdmarc index failed, ret(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + rdmarc->offset = offset << log_rdmarc_per_seg; + ret = hmm_em_table_get_range(comp_priv->pdev, &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); + if (ret != 0) { + pr_err("%s: Alloc rdmarc entry failed, ret(%d)\n", __func__, -ENOMEM); + goto err_table_get; + } + vaddr = hmm_em_table_find(&comp_priv->rdmarc_em_table, rdmarc->offset, &rdmarc->dma_addr); + if (vaddr == NULL) { + ret = -ENOMEM; + pr_err("%s: Can't find va and pa of rdmarc entry, ret(%d)\n", + __func__, -ENOMEM); + goto err_rdmarc_find; + } + rdmarc->vaddr = vaddr; + return 0; +err_rdmarc_find: + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); +err_table_get: + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, ext_order); + return ret; +} + +void roce3_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + u32 order = 0; + u32 offset = 0; + u32 log_rdmarc_per_seg = 0; + + if ((hwdev == NULL) || (rdmarc == NULL)) { + pr_err("%s: Hwdev or rdmarc is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + hmm_em_table_put_range(comp_priv->pdev, &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); + + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + + order = rdmarc->ext_order - log_rdmarc_per_seg; + offset = rdmarc->offset >> log_rdmarc_per_seg; + + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, order); +} diff --git a/drivers/infiniband/hw/hiroce3/roce.h b/drivers/infiniband/hw/hiroce3/roce.h new file mode 100644 index 000000000..436d3f836 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce.h @@ -0,0 +1,574 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_H +#define ROCE_H + +#include <linux/types.h> +#include <linux/io-mapping.h> +#include <linux/list.h> +#include <linux/cdev.h> +#include <linux/version.h> + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> + +#include "hinic3_hw.h" +#include "hinic3_crm.h" +#include "hinic3_hw_cfg.h" +#include "hinic3_lld.h" +#include "hinic3_cqm.h" +#include "hinic3_rdma.h" + +#include "hinic3_mgmt_interface.h" + +#include "roce_db.h" +#include "roce_sysfs.h" + +#include "roce_verbs_cmd.h" +#include "roce_verbs_format.h" +#include "roce_verbs_ulp_format.h" + +#define HIROCE3_DRV_NAME "roce3_drv" +#define HIROCE3_DRV_AUTHOR "Huawei Technologies CO., Ltd" +#define HIROCE3_DRV_DESC "Huawei(R) Intelligent Network Interface Card, RoCE Driver" +#define HIROCE3_DRV_VERSION "" + +#define ROCE_IB_UVERBS_ABI_VERSION 1 +#define ROCE_ULD_DEV_NAME_LEN 16 + +#define MAX_CEQ_NEED 256 +#define MS_DELAY 5 +#define US_PERF_DELAY 100 + +#define DEV_ADDR_FIRST_BYTE_VAL_MASK 2 + +#define ROCE_NODE_DESC_LEN 5 + +#define ROCE_SQ_WQEBB_SIZE 64 + +#define ROCE_GID_LEN 16 + +#define ROCE_PCI_CFG_REGS_BAR0 0 +#define ROCE_PCI_CFG_REGS_BAR3 3 + +#define DEFAULT_ROCE_DEV_NODE_PRI 0640 + +#define ALPHA_THREADHOLD_UNIT_SHIFT 3 + +#define PAGE_4K_SHIFT 12 + +#define ROCE_MAX_PORT_NUM 8 + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#define MAX_ROCE_DEV (28 * 4) + +enum { + ROCE3_2_PORT_NUM = 2, + ROCE3_4_PORT_NUM = 4 +}; + +enum { + ROCE3_25G_PORT_SPEED = 25, + ROCE3_100G_PORT_SPEED = 100 +}; + +enum { + ROCE3_INVALID_HCA = -1, + ROCE3_2_100G_HCA = 0, + ROCE3_4_25G_HCA = 1, + ROCE3_2_25G_HCA = 2 +}; + +enum ROCE3_100G_BW_PARAM_E { + ROCE3_100G_CIR = 46500000, + ROCE3_100G_PIR = 52500000, + ROCE3_100G_CNP = 100 +}; + +enum ROCE3_25G_BW_PARAM_E { + ROCE3_25G_CIR = 23200000, + ROCE3_25G_PIR = 25500000, + ROCE3_25G_CNP = 3 +}; + +enum roce_bitshift_e { + BYTES_TO_2B_SHIFT = 1, + BYTES_TO_4B_SHIFT, + BYTES_TO_8B_SHIFT, + BYTES_TO_16B_SHIFT, + BYTES_TO_32B_SHIFT +}; + +#define roce3_pr_err_once pr_err_once + +/* BIG/LITTLE ENGIAN switch */ +#ifdef HW_CONVERT_ENDIAN +#define roce3_convert_be32(val) (val) +#define roce3_convert_cpu32(val) (val) +#define roce3_more_be32(val) cpu_to_be32(val) +#else +#define roce3_convert_be32(val) cpu_to_be32(val) +#define roce3_convert_cpu32(val) be32_to_cpu(val) +#define roce3_more_be32(val) (val) +#endif + +enum roce3_aeq_type { + /* ofed err */ + OFED_ET_PATH_MIG = 0, + OFED_ET_COMM_EST, + OFED_ET_SQ_DRAINED, + OFED_ET_SRQ_QP_LAST_WQE, + OFED_ET_WQ_CATAS_ERR, + + OFED_ET_PATH_MIG_FAILED = 5, + OFED_ET_WQ_INVAL_REQ_ERR, + OFED_ET_WQ_ACCESS_ERR, + OFED_ET_CQ_ERR, + OFED_ET_SRQ_LIMIT, + OFED_ET_SRQ_CATAS_ERR, + + /* non ofed err */ + NON_OFED_ET_QPC_LOOKUP_ERR = 11, + NON_OFED_ET_OTHER_TYPE_ERR, + + /* NOF AA err */ + OFED_NOF_AA_QP_DISCONNECT = 64, + OFED_NOF_AA_MASTER_CHANGE, + + INVAL_ET_ERR +}; + +enum { + ROCE_CMD_TIME_CLASS_A = 3000, + ROCE_CMD_TIME_CLASS_B = 4000, + ROCE_CMD_TIME_CLASS_C = 5000 +}; + +enum roce3_sgl_mode { + ROCE_DOUBLE_SGL = 0, + ROCE_SINGLE_SGL +}; + +enum roce3_qpc_mtucode { + ROCE_MTU_CODE_256 = 0x0, + ROCE_MTU_CODE_512 = 0x1, + ROCE_MTU_CODE_1K = 0x3, + ROCE_MTU_CODE_2K = 0x7, + ROCE_MTU_CODE_4K = 0xf +}; + +enum roce3_ctrl_status { + ROCE3_PORT_EVENT = BIT(0) +}; + +#define ROCE_DEFAULT_PORT_NUM 1 + +#if defined(ROCE_VBS_EN) || defined(ROCE_CHIP_TEST) +#define ROCE_UVERBS_CMD_MASK \ + ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \ + (1ULL << IB_USER_VERBS_CMD_REG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_ARM_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_MR) | \ + (1ULL << IB_USER_VERBS_CMD_REG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_MAP_FRMR_SG) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_QP)) +#else +#define ROCE_UVERBS_CMD_MASK \ + ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \ + (1ULL << IB_USER_VERBS_CMD_REG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_REREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_ATTACH_MCAST) | \ + (1ULL << IB_USER_VERBS_CMD_DETACH_MCAST) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_CLOSE_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_QP)) +#endif + +#define ROCE_UVERBS_EXT_CMD_MASK (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ) + +enum roce3_load_balance_mode_e { + ROCE_LB_MODE_0 = 0, + ROCE_LB_MODE_1, + ROCE_LB_MODE_2, + ROCE_LB_MODE_N, +}; + +enum push_ofed_device_status { + ROCE_DEV_STATUS_NORMAL = 0, + ROCE_DEV_STATUS_CMDQ_TIMEOUT +}; + +enum roce3_func_state { + ROCE_FUNC_DISABLE = 0, + ROCE_FUNC_ENABLE +}; + +enum { + ROCE_Cl_TYPE_QPC = 0x0, /* cl_start: 0x0, cl_end: 0x7f, cl_size: 0 */ + ROCE_CL_TYPE_MPT = 0x1, /* cl_start: 0x150, cl_end: 0x15f, cl_size: 0 */ + ROCE_CL_TYPE_SQ_WQE = 0x2, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_RQ_WQE = 0x3, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_CQC_SRQC = 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_RDMARC = 0x5, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_CMTT = 0x6, /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ + ROCE_CL_TYPE_DMTT = 0x7 /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ +}; + +#define XRC_CONTAINER_FLAG ((int)(1L << 20)) + +struct roce3_notifier { + struct notifier_block nb; + struct notifier_block nb_inet; + struct notifier_block nb_inet6; +}; + +struct roce3_buf_list { + void *buf; + dma_addr_t map; +}; + +struct roce3_buf { + struct roce3_buf_list direct; + struct roce3_buf_list *page_list; + int nbufs; + + int npages; + int page_shift; +}; + +struct roce3_ucontext { + struct ib_ucontext ibucontext; + void __iomem *db_map; + void __iomem *dwqe_map; + u64 db_dma_addr; + u64 dwqe_dma_addr; + struct list_head db_page_list; + struct mutex db_page_mutex; +}; + +struct roce3_qp_cnt { + struct mutex cur_qps_mutex; + u32 alloc_qp_cnt; + u32 del_qp_cnt; +}; + +struct roce3_cdev_file { + struct roce3_cdev *cdev; +}; + +struct roce3_cdev { + struct cdev cdev; + /*lint -e104 -e808*/ + struct class *cdev_class; + /*lint +e104 +e808*/ + struct device *dev; + int dev_num; + dev_t dev_major; +}; + +struct roce3_netlink { + int dev_num; +}; + +struct roce3_dev_hw_info { + int config_num_ports; /* Number of ports from configuration file */ + int hca_type; /* HCA version: 4x25G or 2x100G */ + u8 phy_port; + u8 ep_id; /* EP ID */ + u8 cpu_endian; + u8 rsvd; + + bool is_vf; +}; + +bool roce3_is_roceaa(u8 scence_id); + +struct roce3_dev_cfg_info { + u8 scence_id; /* load scenes ID as aa_en */ + u8 lb_en; /* load balance enable */ + u8 lb_mode; /* load balance mode */ + u8 rsvd; + + u8 srq_container_en; + u8 srq_container_mode; + u8 xrc_srq_container_mode; + u8 warn_th; + + u8 fake_en; + u8 page_bit; + u8 pf_start_bit; + u8 pf_end_bit; + + u8 port_num; /* cfg data port num */ + u8 host_num; /* cfg data host num */ + u8 master_func; /* nofaa master func */ + u8 rsvd1; +}; + +struct roce3_device { + struct ib_device ib_dev; + struct hinic3_lld_dev *lld_dev; + struct net_device *ndev; + struct pci_dev *pdev; + void *hwdev; + void *hwdev_hdl; + // struct dev_version_info dev_ver; /*version info */ + struct hinic3_board_info board_info; + struct roce3_cdev cdev; + struct roce3_netlink netlink_dev; + + struct roce3_notifier notifier; + void __iomem *kernel_db_map; + void __iomem *kernel_dwqe_map; + spinlock_t node_desc_lock; + struct mutex cap_mask_mutex; + struct rdma_service_cap rdma_cap; + + u8 mac[6]; /* Mac addr. */ + u16 glb_func_id; + unsigned long status; + int ceq_num; + int ceqn[MAX_CEQ_NEED]; + int try_times; + bool ib_active; + + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; + + struct roce3_dev_hw_info hw_info; /* Hw info read from nic/pcie */ + struct roce3_dev_cfg_info cfg_info; /* Cfg data info read from MPU */ + +#ifdef ROCE_BONDING_EN + int want_bond_slave_cnt; + int want_bond_slave_bits[2]; /* The maximum number of bonds supported is 2 */ + enum ib_port_state port_state; + struct hinic3_dcb_state dcb_info; + char *sdi_bond_name; + struct roce3_bond_device *bond_dev; +#endif + + struct mutex mac_vlan_mutex; + struct list_head mac_vlan_list_head; + struct net_device **gid_dev; + + struct roce3_ecn_ctx ecn_ctx; + struct roce3_dfx_ctx dfx_ctx; + + struct roce3_qp_cnt qp_cnt; + + struct srcu_struct mr_srcu; + atomic_t num_prefetch; + struct completion comp_prefetch; + enum push_ofed_device_status dev_status_to_ofed; + + spinlock_t reset_flow_resource_lock; + struct list_head qp_list; + + void *fake_data_buf; + dma_addr_t fake_data_page_addr; + // mailbox, ppf store all pf's roce device, to improve the effiency. + void *pri_roce_dev[ROCE_MAX_PORT_NUM]; + bool is_vroce; +}; + +static inline struct roce3_device *to_roce3_dev(const struct ib_device *ibdev) +{ + return container_of(ibdev, struct roce3_device, ib_dev); +} + +static inline struct roce3_ucontext *to_roce3_ucontext(const struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct roce3_ucontext, ibucontext); +} + +int roce3_db_map_user(struct roce3_ucontext *context, unsigned long virt, struct roce3_db *db); +void roce3_db_unmap_user(struct roce3_ucontext *context, struct roce3_db *db); + +int roce3_buf_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct tag_cqm_buf *buf); +int roce3_umem_write_mtt(struct roce3_device *rdev, struct rdma_mtt *mtt, struct ib_umem *umem); + +int roce3_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw); + +int roce3_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props); + +int roce3_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma); + +int roce3_create_cq(struct ib_cq *cq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); + +int roce3_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); + +int roce3_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); + +int roce3_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); + +int roce3_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); + +int roce3_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); + +int roce3_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata); + +void roce3_dealloc_ucontext(struct ib_ucontext *ibcontext); + +int roce3_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int roce3_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int roce3_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata); + +int roce3_destroy_ah(struct ib_ah *ibah, u32 flags); + +int roce3_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); + +int roce3_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); + +int roce3_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); + +int roce3_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); + + +int roce3_query_port(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); + +int roce3_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid); + +int roce3_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props); + +int roce3_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); + +struct net_device *roce3_ib_get_netdev(struct ib_device *ibdev, u8 port_num); + +enum rdma_link_layer roce3_port_link_layer(struct ib_device *ibdev, u8 port_num); + +int roce3_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey); + +struct ib_qp *roce3_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); + + +int roce3_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period); + +int roce3_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); + +int roce3_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); + +int roce3_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); + + +int roce3_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata); + +int roce3_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr); + +int roce3_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +int roce3_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + +void roce3_drain_rq(struct ib_qp *ibqp); + +int roce3_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); + +int roce3_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); + +struct ib_mr *roce3_get_dma_mr(struct ib_pd *ibpd, int access); +struct ib_mr *roce3_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg); +struct ib_mr *roce3_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, + u64 virt_addr, int access, struct ib_udata *udata); + +int roce3_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); + +int roce3_dealloc_mw(struct ib_mw *ibmw); + +int roce3_query_device_status(struct ib_device *ibdev, int *dev_status); +int roce3_init_cdev(struct roce3_device *rdev); +void roce3_remove_cdev(struct roce3_device *rdev); +int roce3_bond_get_dcb_info(struct roce3_device *rdev); + +int roce3_init_sysfs(struct roce3_device *rdev); +void roce3_remove_sysfs(struct roce3_device *rdev); + +int roce3_is_eth_port_of_netdev(struct net_device *rdma_ndev, struct net_device *cookie); + +void roce3_async_event(void *svc_hd, u8 event_type, u8 *val); +u8 roce3_async_event_level(void *svc_hd, u8 event_type, u8 *val); +void roce3_cq_completion(void *svc_hd, u32 cqn, void *cq_handler); +void roce3_unregister_netdev_event(struct roce3_device *rdev); +int roce3_ifconfig_up_down_event_report(struct roce3_device *rdev, u8 net_event); +int roce3_register_netdev_event(struct roce3_device *rdev); +void roce3_clean_vlan_device_mac(struct roce3_device *rdev); +void roce3_clean_real_device_mac(struct roce3_device *rdev); + +int roce3_ib_add_gid(const struct ib_gid_attr *attr, __always_unused void **context); + +int roce3_ib_del_gid(const struct ib_gid_attr *attr, __always_unused void **context); + +void roce3_remove_dev_file(struct roce3_device *rdev); +int roce3_init_dev_file(struct roce3_device *rdev); + +#endif // ROCE_H diff --git a/drivers/infiniband/hw/hiroce3/roce_cdev.c b/drivers/infiniband/hw/hiroce3/roce_cdev.c new file mode 100644 index 000000000..93a6cfe86 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cdev.c @@ -0,0 +1,1259 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/io.h> +#include <linux/sched.h> +#include <linux/uaccess.h> +#include <linux/atomic.h> +#include <linux/module.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/errno.h> +#include <linux/mm.h> +#include <linux/init.h> +#include <linux/cdev.h> +#include <linux/timer.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/if_ether.h> + +#include "hinic3_hw.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_mix.h" +#include "roce_netdev.h" +#include "roce_k_ioctl.h" +#include "roce_cqm_cmd.h" +#include "roce_pub_cmd.h" +#include "roce_cdev_extension.h" +#include "roce_verbs_cmd.h" +#include "roce_cmd.h" +#include "roce_srq.h" +#include "roce_qp.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifdef ROCE_EXTEND +#include "roce_qp_extension.h" +#endif + +#define P2PCOS_MAX_VALUE 2 +#define MAX_COS_NUM 0x7 + +static long roce3_cdev_create_ah(struct roce3_device *rdev, void *buf) +{ + long ret; + struct rdma_ah_attr attr; + union roce3_create_ah_buf ah_buf; + struct ib_udata udata; + + ret = (long)copy_from_user(&ah_buf, buf, sizeof(ah_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + attr.sl = ah_buf.cmd.attr.sl; + attr.static_rate = ah_buf.cmd.attr.static_rate; + attr.ah_flags = (ah_buf.cmd.attr.is_global != 0) ? IB_AH_GRH : 0; + attr.port_num = ah_buf.cmd.attr.port_num; + attr.grh.flow_label = ah_buf.cmd.attr.grh.flow_label; + attr.grh.sgid_index = ah_buf.cmd.attr.grh.sgid_index; + attr.grh.hop_limit = ah_buf.cmd.attr.grh.hop_limit; + attr.grh.traffic_class = ah_buf.cmd.attr.grh.traffic_class; + memset(attr.roce.dmac, 0, sizeof(attr.roce.dmac)); + memcpy(attr.grh.dgid.raw, ah_buf.cmd.attr.grh.dgid, ROCE_GID_LEN); + + memset(&udata, 0, sizeof(struct ib_udata)); + if (roce3_resolve_grh(rdev, &attr, &ah_buf.resp.vlan_id, &udata) != 0) { + pr_err("[ROCE, ERR] %s: Failed to resolve grh\n", __func__); + return -EINVAL; + } + + memcpy(ah_buf.resp.dmac, attr.roce.dmac, ETH_ALEN); + + ret = (long)copy_to_user((void __user *)buf, &ah_buf, sizeof(ah_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +#ifdef ROCE_BONDING_EN + +struct roce3_get_rx_port { + __be32 rx_port; +}; + +struct roce3_get_func_table { + __be32 func_table_val; +}; + +struct roce3_get_udp_src_port { + __be32 udp_src_port; +}; + +static struct roce3_qp *roce3_cdev_lookup_rqp(struct roce3_device *rdev, u32 qpn) +{ + struct tag_cqm_object *cqm_obj_qp = NULL; + struct roce3_qp *rqp = NULL; + + cqm_obj_qp = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_qp == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't find rqp according to qpn(0x%x), func_id(%d)\n", + __func__, qpn, rdev->glb_func_id); + return NULL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_qp); + + return rqp; +} + +static int roce3_get_qp_func_table(struct roce3_device *rdev, u32 qpn, u32 *func_table_val) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_get_qp_func_table *get_func_tbl_inbuf = NULL; + struct roce3_get_func_table *get_func_tbl_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, + &cqm_cmd_inbuf, (u16)sizeof(struct tag_roce_get_qp_func_table), + &cqm_cmd_outbuf, (u16)sizeof(struct tag_roce_get_qp_func_table)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_func_tbl_inbuf = (struct tag_roce_get_qp_func_table *)cqm_cmd_inbuf->buf; + get_func_tbl_inbuf->com.index = cpu_to_be32(qpn); + get_func_tbl_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_QP_FUNC_TABLE, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send ROCE_CMD_GET_QP_FUNC_TABLE command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return -1; + } + get_func_tbl_outbuf = (struct roce3_get_func_table *)cqm_cmd_outbuf->buf; + *func_table_val = be32_to_cpu(get_func_tbl_outbuf->func_table_val); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; +} + +static long roce3_cdev_query_qp_tx_port(struct roce3_device *rdev, void *buf) +{ + long ret; + u32 slave_cnt; + struct roce3_qp *rqp = NULL; + struct roce3_qp_port_buf qp_port_buf; + u32 func_tbl_value = 0; + + memset(&qp_port_buf, 0, sizeof(qp_port_buf)); + ret = (long)copy_from_user(&qp_port_buf, buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + rqp = roce3_cdev_lookup_rqp(rdev, qp_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + if (!roce3_bond_is_active(rdev)) { + qp_port_buf.port = 0; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; + } + + ret = (long)roce3_get_qp_func_table(rdev, qp_port_buf.qpn, &func_tbl_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get func_tbl, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + slave_cnt = func_tbl_value & 0xf; + if (slave_cnt == 0) { + pr_err("[ROCE, ERR] %s: slave_cnt is 0\n", __func__); + return -EINVAL; + } + + qp_port_buf.port = (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +static int roce3_modify_bond_hash_value(struct roce3_device *rdev, u32 qpn, u32 new_hash_value) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_modify_hash_value *modify_hash_value = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_modify_hash_value), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + modify_hash_value = (struct tag_roce_modify_hash_value *)cqm_cmd_inbuf->buf; + + modify_hash_value->com.index = cpu_to_be32(qpn); + modify_hash_value->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + modify_hash_value->hash_value = cpu_to_be32(new_hash_value); + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_HASH_VALUE_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +static long roce3_cdev_set_qp_tx_port(struct roce3_device *rdev, const void *buf) +{ + long ret; + u32 new_tx_hash_value; + u32 slave_cnt; + u32 target_hash_port; + u32 old_hash_port; + struct roce3_qp *rqp = NULL; + struct roce3_qp_port_buf qp_port_buf; + u32 func_tbl_value = 0; + + ret = (long)copy_from_user(&qp_port_buf, buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + /* check qp exist */ + rqp = roce3_cdev_lookup_rqp(rdev, qp_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + if (!roce3_bond_is_active(rdev)) + return 0; + + ret = (long)roce3_get_qp_func_table(rdev, qp_port_buf.qpn, &func_tbl_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get func_tbl, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + slave_cnt = func_tbl_value & 0xf; + if (slave_cnt == 0) { + pr_err("[ROCE, ERR] %s: slave_cnt is 0\n", __func__); + return -EINVAL; + } + target_hash_port = qp_port_buf.port & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + old_hash_port = (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + if (target_hash_port == old_hash_port) + return 0; + + new_tx_hash_value = rqp->tx_hash_value + (((slave_cnt - old_hash_port) + + target_hash_port) % slave_cnt); + + ret = (long)roce3_modify_bond_hash_value(rdev, rqp->qpn, new_tx_hash_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to modify QP(0x%06x) hash value, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return -EIO; + } + + rqp->tx_hash_value = new_tx_hash_value; + + return 0; +} + +static int roce3_get_qp_rx_port(struct roce3_device *rdev, u32 qpn, u32 *rx_port) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_get_qp_rx_port *get_rx_port_inbuf = NULL; + struct roce3_get_rx_port *get_rx_port_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, + &cqm_cmd_inbuf, (u16)sizeof(struct tag_roce_get_qp_rx_port), + &cqm_cmd_outbuf, (u16)sizeof(struct tag_roce_get_qp_rx_port)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_rx_port_inbuf = (struct tag_roce_get_qp_rx_port *)cqm_cmd_inbuf->buf; + get_rx_port_inbuf->com.index = cpu_to_be32(qpn); + get_rx_port_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_QP_RX_PORT, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send ROCE_CMD_GET_QP_RX_PORT command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return -1; + } + get_rx_port_outbuf = (struct roce3_get_rx_port *)cqm_cmd_outbuf->buf; + *rx_port = be32_to_cpu(get_rx_port_outbuf->rx_port); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return 0; +} + +static long roce3_cdev_query_qp_rx_port(struct roce3_device *rdev, void *buf) +{ + long ret; + u32 rx_port = 0; + struct roce3_qp *rqp = NULL; + struct roce3_qp_port_buf qp_port_buf; + + memset(&qp_port_buf, 0, sizeof(qp_port_buf)); + ret = (long)copy_from_user(&qp_port_buf, buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + rqp = roce3_cdev_lookup_rqp(rdev, qp_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + if (rqp->qp_type != IB_QPT_RC) { + pr_err("[ROCE, ERR] %s: not support qp type(%d), only support RC.\n", + __func__, rqp->qp_type); + return -EINVAL; + } + + if (!roce3_bond_is_active(rdev)) { + qp_port_buf.port = 0; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; + } + + ret = (long)roce3_get_qp_rx_port(rdev, qp_port_buf.qpn, &rx_port); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get QP(0x%06x) rx port\n", + __func__, qp_port_buf.qpn); + return ret; + } + + qp_port_buf.port = rx_port; + + ret = (long)copy_to_user((void __user *)buf, &qp_port_buf, sizeof(qp_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +static long roce3_cdev_query_next_wqe_idx(struct roce3_device *rdev, void *buf) +{ + long ret; + int ret_adm; + struct roce3_dfx_query_inbuf dfx_query_inbuf = {0}; + u32 out_size = (u32)sizeof(union roce3_dfx_query_outbuf); + union roce3_dfx_query_outbuf dfx_query_outbuf; + int next_wqe_idx; + + ret = (long)copy_from_user(&dfx_query_inbuf, buf, sizeof(dfx_query_inbuf)); + if (ret != 0) { + (void)pr_err("[ROCE] %s: Failed to copy data from user, ret(%lux)\n", + __func__, ret); + return ret; + } + + dfx_query_inbuf.cmd_type = ROCE_CMD_GET_SRQC_FROM_CACHE; + ret_adm = roce3_adm_dfx_query(rdev, &dfx_query_inbuf, sizeof(dfx_query_inbuf), + &dfx_query_outbuf, &out_size); + if (ret_adm != 0) { + (void)pr_err("[ROCE] %s: Failed to do roce_adm_dfx_query, ret(%d)\n", + __func__, ret_adm); + return -EINVAL; + } + + dfx_query_outbuf.srq_ctx.dw2.value = be32_to_cpu(dfx_query_outbuf.srq_ctx.dw2.value); + dfx_query_outbuf.srq_ctx.dw3.value = be32_to_cpu(dfx_query_outbuf.srq_ctx.dw3.value); + dfx_query_outbuf.srq_ctx.dw5.value = be32_to_cpu(dfx_query_outbuf.srq_ctx.dw5.value); + + if (dfx_query_outbuf.srq_ctx.dw2.bs.container_en == 0) { + next_wqe_idx = (int)(dfx_query_outbuf.srq_ctx.dw5.bs.next_wqe_idx); + } else { + u32 container_mode = MAX_SUPPORT_CONTAINER_MODE - + dfx_query_outbuf.srq_ctx.dw2.bs_c.container_size; + u32 container_size = roce3_get_container_sz(container_mode); + + next_wqe_idx = (int)(container_size * + dfx_query_outbuf.srq_ctx.dw3.bs_c.head_index); + } + ret = (long)copy_to_user((void __user *)buf, &next_wqe_idx, sizeof(int)); + if (ret != 0) { + (void)pr_err( + "[ROCE] %s: Failed to copy next_wqe_idx to user, ret(0x%lx)\n", + __func__, ret); + return ret; + } + + return 0; +} + +static void roce3_cdev_set_bond_port_info(struct roce3_bond_device *bond_dev, + u32 func_tbl_value, struct roce3_bond_port_info_buf *bond_port_info_buf) +{ + u32 i; + + bond_port_info_buf->original_port_num = bond_dev->slave_cnt; + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) + bond_port_info_buf->original_port[i] = bond_dev->slaves[i].func_id; + + mutex_unlock(&bond_dev->slave_lock); + + bond_port_info_buf->alive_port_num = func_tbl_value & 0xf; + for (i = 0; i < bond_port_info_buf->alive_port_num; i++) { + bond_port_info_buf->alive_port[i] = + (func_tbl_value >> (ROCE_BOND_FWD_ID_TBL_ALL_BITS - ((i + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + } +} + +static long roce3_cdev_query_bond_port_info(struct roce3_device *rdev, void *buf) +{ + long ret; + struct roce3_bond_device *bond_dev = NULL; + struct roce3_bond_port_info_buf bond_port_info_buf; + u32 func_tbl_value = 0; + + memset(&bond_port_info_buf, 0, sizeof(bond_port_info_buf)); + + if (!roce3_bond_is_active(rdev)) { + ret = (long)copy_to_user((void __user *)buf, &bond_port_info_buf, + sizeof(bond_port_info_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; + } + + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Can't find bond_dev, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + ret = roce3_get_func_table(rdev->hwdev, rdev->glb_func_id, &func_tbl_value); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get func_tbl, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + roce3_cdev_set_bond_port_info(bond_dev, func_tbl_value, &bond_port_info_buf); + + ret = (long)copy_to_user((void __user *)buf, &bond_port_info_buf, + sizeof(bond_port_info_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +static int roce3_modify_udp_src_port(struct roce3_device *rdev, u32 qpn, u32 new_udp_src_port) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_modify_udp_src_port *modify_udp_src_port = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_modify_udp_src_port), + NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + modify_udp_src_port = (struct tag_roce_modify_udp_src_port *)cqm_cmd_inbuf->buf; + modify_udp_src_port->com.index = cpu_to_be32(qpn); + modify_udp_src_port->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + modify_udp_src_port->udp_src_port = cpu_to_be32(new_udp_src_port); + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +static long roce3_cdev_set_qp_udp_src_port(struct roce3_device *rdev, const void *buf) +{ + long ret; + struct roce3_qp *rqp = NULL; + struct roce3_qp_udp_src_port_buf qp_udp_src_port_buf; + + ret = (long)copy_from_user(&qp_udp_src_port_buf, buf, sizeof(qp_udp_src_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + /* check qp exist */ + rqp = roce3_cdev_lookup_rqp(rdev, qp_udp_src_port_buf.qpn); + if (rqp == NULL) { + pr_err("[ROCE, ERR] %s: Failed to look up rqp\n", __func__); + return -EINVAL; + } + + ret = (long)roce3_modify_udp_src_port(rdev, rqp->qpn, qp_udp_src_port_buf.udp_src_port); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to modify QP(0x%06x) hash value, func_id(%d)\n", + __func__, rqp->qpn, rdev->glb_func_id); + return -EIO; + } + + return 0; +} + +static int roce3_get_qp_udp_src_port(struct roce3_device *rdev, u32 qpn, u32 *udp_src_port) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct roce_get_qp_udp_src_port *get_udp_src_port = NULL; + struct roce3_get_udp_src_port *get_udp_src_port_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, + &cqm_cmd_inbuf, (u16)sizeof(struct roce_get_qp_udp_src_port), + &cqm_cmd_outbuf, (u16)sizeof(struct roce_get_qp_udp_src_port)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_udp_src_port = (struct roce_get_qp_udp_src_port *)cqm_cmd_inbuf->buf; + get_udp_src_port->com.index = cpu_to_be32(qpn); + get_udp_src_port->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + + ret = roce3_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send ROCE_CMD_GET_QP_RX_PORT command, func_id(%d)\n", + __func__, rdev->glb_func_id); + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return -1; + } + + get_udp_src_port_outbuf = (struct roce3_get_udp_src_port *)cqm_cmd_outbuf->buf; + *udp_src_port = be32_to_cpu(get_udp_src_port_outbuf->udp_src_port); + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return 0; +} + +static long roce3_cdev_query_qp_udp_src_port(struct roce3_device *rdev, void *buf) +{ + long ret; + u32 udp_src_port = 0; + struct roce3_qp_udp_src_port_buf qp_udp_src_port_buf; + + memset(&qp_udp_src_port_buf, 0, sizeof(qp_udp_src_port_buf)); + ret = (long)copy_from_user(&qp_udp_src_port_buf, buf, sizeof(qp_udp_src_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + + ret = (long)roce3_get_qp_udp_src_port(rdev, qp_udp_src_port_buf.qpn, &udp_src_port); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get QP(0x%06x) udp src port\n", + __func__, qp_udp_src_port_buf.qpn); + return ret; + } + + qp_udp_src_port_buf.udp_src_port = udp_src_port; + + ret = (long)copy_to_user((void __user *)buf, + &qp_udp_src_port_buf, sizeof(qp_udp_src_port_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + + return 0; +} + +#endif + +static int roce3_cdev_open(struct inode *inode, struct file *filp) +{ + struct roce3_cdev *dev = NULL; + struct roce3_cdev_file *file = NULL; + + file = kzalloc(sizeof(*file), GFP_KERNEL); + if (file == NULL) + return -ENOMEM; + + dev = container_of(inode->i_cdev, struct roce3_cdev, cdev); + file->cdev = dev; + filp->private_data = file; + + return 0; +} + +static int roce3_cdev_close(struct inode *inode, struct file *filp) +{ + struct roce3_cdev_file *file = filp->private_data; + + kfree(file); + + return 0; +} + +static ssize_t roce3_cdev_read(struct file *fp, char __user *buf, size_t size, loff_t *pos) +{ + (void)(fp); + (void)(buf); + (void)(size); + (void)(pos); + return -EOPNOTSUPP; +} + +static ssize_t roce3_cdev_write(struct file *fp, const char __user *buf, size_t size, loff_t *pos) +{ + (void)(fp); + (void)(buf); + (void)(size); + (void)(pos); + return -EOPNOTSUPP; +} + +#ifdef ROCE_BONDING_EN +static long roce3_cdev_bonding_ioctl_part_1(unsigned int cmd, + unsigned long arg, struct roce3_device *rdev) +{ + long ret = 0; + + switch (cmd) { + case ROCE_CMD_QUERY_QP_TX_PORT: + ret = roce3_cdev_query_qp_tx_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_tx_port failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_SET_QP_TX_PORT: + ret = roce3_cdev_set_qp_tx_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_set_qp_tx_port failed.\n", __func__); + return ret; + } + break; + case ROCE_CMD_QUERY_QP_RX_PORT: + ret = roce3_cdev_query_qp_rx_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_rx_port failed.\n", __func__); + return ret; + } + break; + default: + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + ret = -1; + return ret; + } + + return ret; +} + +static long roce3_cdev_bonding_ioctl_part_2(unsigned int cmd, + unsigned long arg, struct roce3_device *rdev) +{ + long ret = 0; + + switch (cmd) { + case ROCE_CMD_QUERY_BOND_PORT_INFO: + ret = roce3_cdev_query_bond_port_info(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_tx_port failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_SET_QP_UDP_SRC_PORT: + ret = roce3_cdev_set_qp_udp_src_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_set_qp_tx_port failed.\n", __func__); + return ret; + } + break; + case ROCE_CMD_QUERY_QP_UDP_SRC_PORT: + ret = roce3_cdev_query_qp_udp_src_port(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_qp_rx_port failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_QUERY_NEXT_WQE_IDX: + ret = roce3_cdev_query_next_wqe_idx(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_next_wqe_idx failed.\n", __func__); + return ret; + } + break; + + default: + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + ret = -1; + return ret; + } + + return ret; +} +static long roce3_cdev_ioctl_bonding(unsigned int cmd, unsigned long arg, struct roce3_device *rdev) +{ + long ret = 0; + + if (cmd < ROCE_CMD_QUERY_BOND_PORT_INFO) { + ret = roce3_cdev_bonding_ioctl_part_1(cmd, arg, rdev); + if (ret != 0) + return ret; + } else { + ret = roce3_cdev_bonding_ioctl_part_2(cmd, arg, rdev); + if (ret != 0) + return ret; + } + + return ret; +} + +static int roce3_bond_get_dcb_info_from_buddy(struct roce3_device *rdev, + struct roce3_bond_device *bond_dev) +{ + int i; + int ret; + struct roce3_bond_slave *slave = NULL; + struct hinic3_lld_dev *lld_dev = NULL; + struct hinic3_dcb_state dcb_info, buddy_dcb_info; + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) + continue; + + if ((bond_dev->attr.active_slaves & (1U << slave->func_id)) == 0) + continue; + + memset(&buddy_dcb_info, 0, sizeof(buddy_dcb_info)); + + lld_dev = hinic3_get_lld_dev_by_netdev(slave->netdev); + if (!lld_dev) { + mutex_unlock(&bond_dev->slave_lock); + return -ENODEV; + } + ret = hinic3_get_dcb_state(lld_dev->hwdev, &buddy_dcb_info); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to get dcb state, ret(%d)\n", + __func__, ret); + return ret; + } + + memset(&dcb_info, 0, sizeof(dcb_info)); + + ret = hinic3_get_dcb_state(rdev->hwdev, &dcb_info); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + pr_err("[ROCE, ERR] %s: Failed to get dcb state, ret(%d)\n", + __func__, ret); + return ret; + } + + buddy_dcb_info.default_cos = dcb_info.default_cos; + for (i = 0; i < NIC_DCB_UP_MAX; i++) { + if (buddy_dcb_info.pcp2cos[i] > P2PCOS_MAX_VALUE) + buddy_dcb_info.pcp2cos[i] = dcb_info.default_cos; + } + + memcpy(&rdev->dcb_info, &buddy_dcb_info, sizeof(buddy_dcb_info)); + + mutex_unlock(&bond_dev->slave_lock); + return ret; + } + + mutex_unlock(&bond_dev->slave_lock); + return 0; +} + +int roce3_bond_get_dcb_info(struct roce3_device *rdev) +{ + bool bond_dev_up; + int ret; + struct roce3_bond_device *bond_dev = rdev->bond_dev; + + if (!roce3_bond_is_active(rdev)) + return 0; + + bond_dev_up = (bond_dev->attr.active_slaves & (1U << rdev->glb_func_id)) != 0; + if (!bond_dev_up) { + ret = hinic3_get_dcb_state(rdev->hwdev, &rdev->dcb_info); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb state, ret(%d)\n", + __func__, ret); + return ret; + } + } else { + ret = roce3_bond_get_dcb_info_from_buddy(rdev, bond_dev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb state from buddy, ret(%d)\n", + __func__, ret); + return ret; + } + } + + return 0; +} +#endif + +static long roce3_cdev_query_dcb(struct roce3_device *rdev, void *buf) +{ + union roce3_query_dcb_buf dcb_buf; + long ret; + int get_group_id; + struct roce_group_id group_id = {0}; + u8 cos; + + ret = (long)copy_from_user(&dcb_buf, buf, sizeof(dcb_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data from user\n", __func__); + return ret; + } + +#ifdef ROCE_BONDING_EN + if (roce3_bond_get_dcb_info(rdev) != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb info, ret(%lu)\n", __func__, ret); + return (-EINVAL); + } +#endif + + ret = (long)roce3_get_dcb_cfg_cos(rdev, + (struct roce3_get_cos_inbuf *)(void *)(&dcb_buf), &cos); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get cos from dcb, ret:%ld\n", __func__, ret); + return ret; + } + + dcb_buf.resp.cos = cos; + + if (rdev->is_vroce) { + get_group_id = roce3_get_group_id(rdev->glb_func_id, rdev->hwdev, &group_id); + if (get_group_id != 0) { + pr_warn("Failed to get group id, ret(%d)", get_group_id); + } else { + rdev->group_rc_cos = group_id.group_rc_cos; + rdev->group_ud_cos = group_id.group_ud_cos; + rdev->group_xrc_cos = group_id.group_xrc_cos; + } + if (dcb_buf.cmd.dscp_type == (u8)IB_QPT_RC) + dcb_buf.resp.cos = rdev->group_rc_cos & MAX_COS_NUM; + else if (dcb_buf.cmd.dscp_type == (u8)IB_QPT_UD) + dcb_buf.resp.cos = rdev->group_ud_cos & MAX_COS_NUM; + else + dcb_buf.resp.cos = rdev->group_xrc_cos & MAX_COS_NUM; + } + + ret = (long)copy_to_user((void __user *)buf, &dcb_buf, sizeof(dcb_buf)); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to copy data to user\n", __func__); + return ret; + } + return 0; +} + +static long roce3_cdev_ioctl_non_bonding(unsigned int cmd, + struct roce3_device *rdev, unsigned long arg) +{ + long ret = 0; + + switch (cmd) { + case ROCE_CMD_QUERY_DCB: + ret = roce3_cdev_query_dcb(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_query_dcb failed.\n", __func__); + return ret; + } + break; + + case ROCE_CMD_CREATE_AH: + ret = roce3_cdev_create_ah(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: roce3_cdev_create_ah failed.\n", __func__); + return ret; + } + break; + + default: + ret = ioctl_non_bonding_extend(cmd, rdev, arg); + if (ret != NOT_SUPOORT_TYPE) + return ret; + + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + ret = -1; + return ret; + } + + return ret; +} + +static long roce3_ioctrl_version(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_1(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_2(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_3(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_4(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_5(struct roce3_device *rdev, void *buf); +static long roce3_ioctrl_reserved_6(struct roce3_device *rdev, void *buf); + +struct tag_HW_ROCE_IOCTL_ACTION { + unsigned int cmd; + long (*action_func)(struct roce3_device *rdev, void *buf); +}; + +static struct tag_HW_ROCE_IOCTL_ACTION ioctl_reserved_tbl[] = { + {HW_ROCE_CMD_VERSION, roce3_ioctrl_version}, + {HW_ROCE_CMD_RESERVED_1, roce3_ioctrl_reserved_1}, + {HW_ROCE_CMD_RESERVED_2, roce3_ioctrl_reserved_2}, + {HW_ROCE_CMD_RESERVED_3, roce3_ioctrl_reserved_3}, + {HW_ROCE_CMD_RESERVED_4, roce3_ioctrl_reserved_4}, + {HW_ROCE_CMD_RESERVED_5, roce3_ioctrl_reserved_5}, + {HW_ROCE_CMD_RESERVED_6, roce3_ioctrl_reserved_6}, +}; + +static long roce3_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long ret = 0; + struct roce3_device *rdev = NULL; + struct roce3_cdev_file *cdev_file = NULL; + unsigned int idx = 0; + + if (_IOC_TYPE(cmd) != ROCE_IOCTL_MAGIC) { + pr_err("[ROCE, ERR] %s: ROCE_IOCTL_MAGIC check failed.\n", __func__); + return -EINVAL; + } + + cdev_file = filp->private_data; + rdev = container_of(cdev_file->cdev, struct roce3_device, cdev); + + if (cmd < ROCE_CMD_QUERY_QP_TX_PORT) { + ret = roce3_cdev_ioctl_non_bonding(cmd, rdev, arg); + return ret; + } +#ifdef ROCE_BONDING_EN + else if (cmd < ROCE_CMD_MAX) { + ret = roce3_cdev_ioctl_bonding(cmd, arg, rdev); + return ret; + } +#endif +#ifdef ROCE_EXTEND + else if (cmd == HW_ROCE_EXT_CMD_SET_QP_ATTR) { + ret = (long)roce3_set_qp_ext_attr(rdev, (void *)(uintptr_t)arg); + if (ret != 0) + pr_err("[ROCE, ERR] %s: roce3_set_qp_ext_attr failed.\n", __func__); + + return ret; + } else if (cmd == HW_ROCE_EXT_CMD_CREATE_SQPC) { + ret = (long)roce3_vbs_create_sqpc(rdev, (void *)(uintptr_t)arg); + if (ret != 0) + pr_err("[ROCE, ERR] %s: roce3_vbs_create_sqpc failed.\n", __func__); + + return ret; + } +#endif + + for (idx = 0; idx < sizeof(ioctl_reserved_tbl) / sizeof((ioctl_reserved_tbl)[0]); idx++) { + if (cmd == ioctl_reserved_tbl[idx].cmd && + ioctl_reserved_tbl[idx].action_func != NULL) { + ret = ioctl_reserved_tbl[idx].action_func(rdev, (void *)(uintptr_t)arg); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: cmd 0x%x ioctl action failed ret %ld.\n", + __func__, cmd, ret); + } + return ret; // cmd excute end + } + } + // unsupport cmd + pr_err("[ROCE, ERR] %s: Not supported cmd(%d).\n", __func__, cmd); + return -EOPNOTSUPP; +} + +static const struct file_operations roce3_cdev_fops = { + .owner = THIS_MODULE, + .open = roce3_cdev_open, + .release = roce3_cdev_close, + .read = roce3_cdev_read, + .write = roce3_cdev_write, + .unlocked_ioctl = roce3_cdev_ioctl, +}; + +#define ROCE_BASE_CDEV_MAJOR 232 +#define ROCE_BASE_CDEV_MINOR 256 +#define ROCE_MAX_DEVICES 256 + +/*lint -e708*/ +static DEFINE_SPINLOCK(map_lock); +/*lint +e708*/ +static DECLARE_BITMAP(dev_map, ROCE_MAX_DEVICES); + +static char *roce3_devnode(struct device *dev, umode_t *mode) +{ + if (mode) + *mode = DEFAULT_ROCE_DEV_NODE_PRI; + + return kasprintf(GFP_KERNEL, "%s", dev_name(dev)); +} + +static int roce3_init_cdev_info(struct roce3_device *rdev) +{ + int ret; + + cdev_init(&rdev->cdev.cdev, &roce3_cdev_fops); + rdev->cdev.cdev.owner = THIS_MODULE; + rdev->cdev.cdev.ops = &roce3_cdev_fops; + + ret = cdev_add(&rdev->cdev.cdev, rdev->cdev.dev_major, 1); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Add cdev failed, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_add_cdev; + } + + /*lint -e160*/ + rdev->cdev.cdev_class = class_create(THIS_MODULE, rdev->ib_dev.name); + /*lint +e160*/ + if (IS_ERR(rdev->cdev.cdev_class)) { + ret = (int)PTR_ERR(rdev->cdev.cdev_class); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Create class failed, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_class; + } + + /*lint -e10 -e40 -e63*/ + rdev->cdev.cdev_class->devnode = roce3_devnode; + /*lint +e10 +e40 +e63*/ + + rdev->cdev.dev = device_create(rdev->cdev.cdev_class, NULL, + rdev->cdev.dev_major, NULL, "%s", rdev->ib_dev.name); + if (IS_ERR(rdev->cdev.dev)) { + ret = (int)PTR_ERR(rdev->cdev.dev); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Create device failed, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_device; + } + + return 0; + +err_create_device: + class_destroy(rdev->cdev.cdev_class); + +err_create_class: + cdev_del(&rdev->cdev.cdev); + +err_add_cdev: + return ret; +} + +int roce3_init_cdev(struct roce3_device *rdev) +{ + int ret; + + spin_lock(&map_lock); + rdev->cdev.dev_num = (int)find_first_zero_bit(dev_map, ROCE_MAX_DEVICES); + if (rdev->cdev.dev_num >= ROCE_MAX_DEVICES) { + spin_unlock(&map_lock); + return -ENOMEM; + } + + rdev->cdev.dev_major = MKDEV(ROCE_BASE_CDEV_MAJOR, + (u32)rdev->cdev.dev_num + ROCE_BASE_CDEV_MINOR); + set_bit((u32)rdev->cdev.dev_num, dev_map); + spin_unlock(&map_lock); + + ret = register_chrdev_region(rdev->cdev.dev_major, 1, rdev->ib_dev.name); + if (ret == -EBUSY) { + /* alloc dynamic cdev by OS */ + ret = alloc_chrdev_region(&(rdev->cdev.dev_major), + ROCE_BASE_CDEV_MINOR, 1, rdev->ib_dev.name); + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: alloc cdev region, major(%d), minor(%d), func_id(%d), ret(%d)\n", + __func__, MAJOR(rdev->cdev.dev_major), + MINOR(rdev->cdev.dev_major), rdev->glb_func_id, ret); + } + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Register region failed, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + goto err_register_region; + } + + ret = roce3_init_cdev_info(rdev); + if (ret != 0) + goto err_init_cdev_info; + + return 0; + +err_init_cdev_info: + unregister_chrdev_region(rdev->cdev.dev_major, 1); + +err_register_region: + clear_bit(rdev->cdev.dev_num, dev_map); + + return ret; +} + +void roce3_remove_cdev(struct roce3_device *rdev) +{ + device_destroy(rdev->cdev.cdev_class, rdev->cdev.dev_major); + + class_destroy(rdev->cdev.cdev_class); + + cdev_del(&rdev->cdev.cdev); + + unregister_chrdev_region(rdev->cdev.dev_major, 1); + + clear_bit(rdev->cdev.dev_num, dev_map); +} + +// get version +static long roce3_ioctrl_version(struct roce3_device *rdev, void *buf) +{ + char ver[VERSION_LEN] = "1.01"; + long ret = copy_to_user(buf, ver, VERSION_LEN); + return ret; +} + +static long roce3_ioctrl_reserved_1(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_2(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_3(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_4(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_5(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + +static long roce3_ioctrl_reserved_6(struct roce3_device *rdev, void *buf) +{ + (void)(rdev); + (void)(buf); + return NOT_SUPOORT_TYPE; +} + diff --git a/drivers/infiniband/hw/hiroce3/roce_cmd.c b/drivers/infiniband/hw/hiroce3/roce_cmd.c new file mode 100644 index 000000000..333351319 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cmd.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/errno.h> +#include <linux/if_ether.h> +#include <linux/moduleparam.h> + +#include "rdma_comp.h" +#include "roce_compat.h" +#include "roce_mpu_common.h" +#include "roce_cmd.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +int roce3_cfg_func_tbl_er_fwd_id_compact(void *hwdev, u32 bond_tbl_val, u16 func_id) +{ + struct roce_bond_cfg_er_fwd_id_cmd cfg_er_fwd_id_comp_info; + u16 out_size = (u16)sizeof(cfg_er_fwd_id_comp_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_er_fwd_id_comp_info, 0, sizeof(cfg_er_fwd_id_comp_info)); + cfg_er_fwd_id_comp_info.func_id = func_id; + cfg_er_fwd_id_comp_info.bond_tbl_val = bond_tbl_val; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_BOND_ER_FWD_ID_COMPACT, + &cfg_er_fwd_id_comp_info, sizeof(cfg_er_fwd_id_comp_info), + &cfg_er_fwd_id_comp_info, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_er_fwd_id_comp_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg func er_fwd_id(compact), err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_er_fwd_id_comp_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_set_port_tbl_bond_state(void *hwdev, u8 bond_state, u16 func_id) +{ + struct roce_bond_cfg_state_cmd set_bond_state_info; + u16 out_size = (u16)sizeof(set_bond_state_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&set_bond_state_info, 0, sizeof(set_bond_state_info)); + set_bond_state_info.func_id = func_id; + set_bond_state_info.bond_en = bond_state; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_BOND_SET_STATE, &set_bond_state_info, + sizeof(set_bond_state_info), &set_bond_state_info, &out_size); + if ((ret != 0) || (out_size == 0) || (set_bond_state_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg bond state, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_bond_state_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_add_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id) +{ + struct roce_cfg_mac_cmd add_mac_entry_info; + u16 out_size = (u16)sizeof(add_mac_entry_info); + int ret; + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + return -EINVAL; + } + + memset(&add_mac_entry_info, 0, sizeof(add_mac_entry_info)); + add_mac_entry_info.vni_en = 0; + add_mac_entry_info.func_id = func_id; + add_mac_entry_info.vlan_id = vlan_id; + add_mac_entry_info.er_fwd_id = er_fwd_id; + memcpy(add_mac_entry_info.mac, mac_addr, ETH_ALEN); + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_MAC, &add_mac_entry_info, + sizeof(add_mac_entry_info), &add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || (add_mac_entry_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to add mac entry, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, add_mac_entry_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_del_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id) +{ + struct roce_cfg_mac_cmd del_mac_tbl_info; + u16 out_size = (u16)sizeof(del_mac_tbl_info); + int ret; + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + return -EINVAL; + } + + memset(&del_mac_tbl_info, 0, sizeof(del_mac_tbl_info)); + del_mac_tbl_info.vni_en = 0; + del_mac_tbl_info.func_id = func_id; + del_mac_tbl_info.vlan_id = vlan_id; + del_mac_tbl_info.er_fwd_id = er_fwd_id; + memcpy(del_mac_tbl_info.mac, mac_addr, ETH_ALEN); + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_MAC, + &del_mac_tbl_info, sizeof(del_mac_tbl_info), + &del_mac_tbl_info, &out_size); + if ((ret != 0) || (out_size == 0) || (del_mac_tbl_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, del_mac_tbl_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +void roce3_add_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id) +{ + struct roce_cfg_ipsu_mac_cmd ipsu_add_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_add_mac_entry_info); + int ret; + +#ifdef ROCE_BONDING_EN + if (!roce3_get_bond_ipsurx_en()) { + pr_err("[ROCE, INFO] %s: No need to add ipsu tbl mac entry\n", __func__); + return; + } +#endif + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + + memset(&ipsu_add_mac_entry_info, 0, sizeof(ipsu_add_mac_entry_info)); + ipsu_add_mac_entry_info.func_id = func_id; + ipsu_add_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_add_mac_entry_info.vni_en = 0; + memcpy(ipsu_add_mac_entry_info.mac, mac_addr, ETH_ALEN); + ipsu_add_mac_entry_info.er_id = er_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_IPSU_MAC, &ipsu_add_mac_entry_info, + sizeof(ipsu_add_mac_entry_info), &ipsu_add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_add_mac_entry_info.head.status != 0) && + (ipsu_add_mac_entry_info.head.status != ERR_EXIST))) { + pr_err("[ROCE, ERR] %s: Failed to add mac entry to IPSU, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, ipsu_add_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_add_mac_entry_info.head.status == ERR_EXIST) + pr_info("[ROCE, INFO] %s: This mac entry has been added to IPSU\n", __func__); +} + +void roce3_del_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id) +{ + struct roce_cfg_ipsu_mac_cmd ipsu_del_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_del_mac_entry_info); + int ret; + + if ((hwdev == NULL) || (mac_addr == NULL)) { + pr_err("[ROCE, ERR] %s: Hwdev or mac_addr is null\n", __func__); + return; + } + +#ifdef ROCE_BONDING_EN + if (!roce3_get_bond_ipsurx_en()) { + pr_err("[ROCE, INFO] %s: No need to del ipsu tbl mac entry\n", __func__); + return; + } +#endif + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + pr_err("[ROCE, ERR] %s: Invalid vlan_id(%d)\n", __func__, vlan_id); + return; + } + + memset(&ipsu_del_mac_entry_info, 0, sizeof(ipsu_del_mac_entry_info)); + ipsu_del_mac_entry_info.func_id = func_id; + ipsu_del_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_del_mac_entry_info.vni_en = 0; + memcpy(ipsu_del_mac_entry_info.mac, mac_addr, ETH_ALEN); + ipsu_del_mac_entry_info.er_id = er_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_IPSU_MAC, &ipsu_del_mac_entry_info, + sizeof(ipsu_del_mac_entry_info), &ipsu_del_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_mac_entry_info.head.status != 0) && + (ipsu_del_mac_entry_info.head.status != ERR_NOT_FOUND))) { + pr_err("[ROCE, ERR] %s: Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, ipsu_del_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_del_mac_entry_info.head.status == ERR_NOT_FOUND) + pr_info("[ROCE, INFO] %s: This mac entry of ipsu has been deleted\n", __func__); +} + +int roce3_do_cache_out(void *hwdev, u8 cl_id, u16 func_id) +{ + struct roce_dfx_cache_out_cmd cache_out_info; + u16 out_size = (u16)sizeof(cache_out_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cache_out_info, 0, sizeof(cache_out_info)); + cache_out_info.func_idx = func_id; + cache_out_info.cache_index = cl_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_CACHE_OUT, + &cache_out_info, sizeof(cache_out_info), &cache_out_info, &out_size); + if ((ret != 0) || (out_size == 0) || (cache_out_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cache out, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cache_out_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_ccf_param(void *hwdev, u16 func_id, u32 *ccf_param) +{ + struct roce_cc_cfg_param_cmd cfg_ccf_param; + u16 out_size = (u16)sizeof(cfg_ccf_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_ccf_param, 0, sizeof(cfg_ccf_param)); + cfg_ccf_param.func_id = func_id; + cfg_ccf_param.param[0] = ccf_param[0]; // 0 is param array idx + cfg_ccf_param.param[1] = ccf_param[1]; // 1 is param array idx + cfg_ccf_param.param[2] = ccf_param[2]; // 2 is param array idx + cfg_ccf_param.param[3] = ccf_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_CCF_PARAM, + &cfg_ccf_param, sizeof(cfg_ccf_param), &cfg_ccf_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_ccf_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg ccf param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_ccf_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_dcqcn_param(void *hwdev, u16 func_id, u32 *dcqcn_param) +{ + struct roce_cc_cfg_param_cmd cfg_dcqcn_param; + u16 out_size = (u16)sizeof(cfg_dcqcn_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_dcqcn_param, 0, sizeof(cfg_dcqcn_param)); + cfg_dcqcn_param.func_id = func_id; + cfg_dcqcn_param.param[0] = dcqcn_param[0]; // 0 is param array idx + cfg_dcqcn_param.param[1] = dcqcn_param[1]; // 1 is param array idx + cfg_dcqcn_param.param[2] = dcqcn_param[2]; // 2 is param array idx + cfg_dcqcn_param.param[3] = dcqcn_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM, + &cfg_dcqcn_param, sizeof(cfg_dcqcn_param), &cfg_dcqcn_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_dcqcn_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg dcqcn param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_dcqcn_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_ipqcn_param(void *hwdev, u16 func_id, u32 *ipqcn_param) +{ + struct roce_cc_cfg_param_cmd cfg_ipqcn_param; + u16 out_size = (u16)sizeof(cfg_ipqcn_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_ipqcn_param, 0, sizeof(cfg_ipqcn_param)); + cfg_ipqcn_param.func_id = func_id; + cfg_ipqcn_param.param[0] = ipqcn_param[0]; // 0 is param array idx + cfg_ipqcn_param.param[1] = ipqcn_param[1]; // 1 is param array idx + cfg_ipqcn_param.param[2] = ipqcn_param[2]; // 2 is param array idx + cfg_ipqcn_param.param[3] = ipqcn_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM, + &cfg_ipqcn_param, sizeof(cfg_ipqcn_param), &cfg_ipqcn_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_ipqcn_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg ipqcn param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_ipqcn_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cfg_ldcp_param(void *hwdev, u16 func_id, u32 *ldcp_param) +{ + struct roce_cc_cfg_param_cmd cfg_ldcp_param; + u16 out_size = (u16)sizeof(cfg_ldcp_param); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&cfg_ldcp_param, 0, sizeof(cfg_ldcp_param)); + cfg_ldcp_param.func_id = func_id; + cfg_ldcp_param.param[0] = ldcp_param[0]; // 0 is param array idx + cfg_ldcp_param.param[1] = ldcp_param[1]; // 1 is param array idx + cfg_ldcp_param.param[2] = ldcp_param[2]; // 2 is param array idx + cfg_ldcp_param.param[3] = ldcp_param[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_CC_CFG_LDCP_PARAM, + &cfg_ldcp_param, sizeof(cfg_ldcp_param), &cfg_ldcp_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_ldcp_param.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to cfg ldcp param, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, cfg_ldcp_param.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_set_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg) +{ + struct roce_dfx_cfg_cap_param_cmd set_cap_cfg; + u16 out_size = (u16)sizeof(set_cap_cfg); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&set_cap_cfg, 0, sizeof(set_cap_cfg)); + set_cap_cfg.index = index; + set_cap_cfg.param[0] = cap_cfg[0]; // 0 is param array idx + set_cap_cfg.param[1] = cap_cfg[1]; // 1 is param array idx + set_cap_cfg.param[2] = cap_cfg[2]; // 2 is param array idx + set_cap_cfg.param[3] = cap_cfg[3]; // 3 is param array idx + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_SET_CAP_CFG, + &set_cap_cfg, sizeof(set_cap_cfg), &set_cap_cfg, &out_size); + if ((ret != 0) || (out_size == 0) || (set_cap_cfg.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set cap cfg, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_cap_cfg.head.status, out_size); + + return -EINVAL; + } + + return 0; +} + +int roce3_get_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg) +{ + struct roce_dfx_cfg_cap_param_cmd get_cap_cfg; + u16 out_size = (u16)sizeof(get_cap_cfg); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&get_cap_cfg, 0, sizeof(get_cap_cfg)); + get_cap_cfg.index = index; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_GET_CAP_CFG, + &get_cap_cfg, sizeof(get_cap_cfg), &get_cap_cfg, &out_size); + if ((ret != 0) || (out_size == 0) || (get_cap_cfg.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to get cap cfg, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, get_cap_cfg.head.status, out_size); + + return -EINVAL; + } + + memcpy(cap_cfg, get_cap_cfg.param, sizeof(get_cap_cfg.param)); + + return 0; +} + +int roce3_clear_cap_counter(void *hwdev, u16 index, u32 *value) +{ + struct roce_dfx_cap_ctr_cmd clear_cap_counter; + u16 out_size = (u16)sizeof(clear_cap_counter); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&clear_cap_counter, 0, sizeof(clear_cap_counter)); + clear_cap_counter.index = index; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_CLEAR_CAP_CTR, + &clear_cap_counter, sizeof(clear_cap_counter), + &clear_cap_counter, &out_size); + if ((ret != 0) || (out_size == 0) || (clear_cap_counter.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to clear cap counter, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, clear_cap_counter.head.status, out_size); + + return -EINVAL; + } + + *value = clear_cap_counter.value; + + return 0; +} + +int roce3_read_cap_counter(void *hwdev, u16 index, u32 *value) +{ + struct roce_dfx_cap_ctr_cmd read_cap_counter; + u16 out_size = (u16)sizeof(read_cap_counter); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&read_cap_counter, 0, sizeof(read_cap_counter)); + read_cap_counter.index = index; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DFX_READ_CAP_CTR, &read_cap_counter, + sizeof(read_cap_counter), &read_cap_counter, &out_size); + if ((ret != 0) || (out_size == 0) || (read_cap_counter.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to read cap counter, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, read_cap_counter.head.status, out_size); + + return -EINVAL; + } + + *value = read_cap_counter.value; + + return 0; +} + +int roce3_set_func_tbl_func_state(struct roce3_device *rdev, u8 func_state) +{ + void *hwdev = rdev->hwdev; + u16 func_id = rdev->glb_func_id; + struct roce_set_func_state_cmd set_func_state_info; + u16 out_size = (u16)sizeof(set_func_state_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&set_func_state_info, 0, sizeof(set_func_state_info)); + set_func_state_info.func_id = func_id; + set_func_state_info.func_en = func_state; + set_func_state_info.tag = 0; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_FUNC_STATE, + &set_func_state_info, sizeof(set_func_state_info), + &set_func_state_info, &out_size); + if ((ret != 0) || (out_size == 0) || (set_func_state_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set func state(%d), err(%d), status(0x%x), out size(0x%x), func_id(%d)\n", + __func__, func_state, ret, set_func_state_info.head.status, + out_size, func_id); + return (-EINVAL); + } + + return 0; +} + +int roce3_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value) +{ + struct roce_get_func_table_cmd get_func_table; + struct roce_get_func_table_rsp get_func_table_rsp; + u16 out_size = (u16)sizeof(get_func_table_rsp); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&get_func_table, 0, sizeof(get_func_table)); + memset(&get_func_table_rsp, 0, out_size); + get_func_table.func_id = func_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_FUNC_TABLE, + &get_func_table, sizeof(get_func_table), + &get_func_table_rsp, &out_size); + if ((ret != 0) || (out_size == 0) || (get_func_table_rsp.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to get func table, err(%d), status(0x%x), out size(0x%x), func_id(%d)\n", + __func__, ret, get_func_table_rsp.head.status, out_size, func_id); + return (-EINVAL); + } + + *func_tbl_value = get_func_table_rsp.func_tbl_value; + + return 0; +} + +int roce3_set_func_tbl_cpu_endian(void *hwdev, u8 cpu_endian, u16 func_id) +{ + struct roce_set_cpu_endian_cmd set_cpu_endian_info; + u16 out_size = (u16)sizeof(set_cpu_endian_info); + int ret; + + if (hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return -EINVAL; + } + + memset(&set_cpu_endian_info, 0, sizeof(set_cpu_endian_info)); + set_cpu_endian_info.func_id = func_id; + set_cpu_endian_info.cpu_endian = cpu_endian; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_CPU_ENDIAN, + &set_cpu_endian_info, sizeof(set_cpu_endian_info), + &set_cpu_endian_info, &out_size); + if ((ret != 0) || (out_size == 0) || (set_cpu_endian_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set func state, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_cpu_endian_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce3_init_cfg_info(struct roce3_device *rdev) +{ + struct roce_get_cfg_info_cmd get_cfg_info; + struct roce_get_cfg_info_resp resp; + u16 out_size = (u16)sizeof(resp); + int ret = 0; + + if (rdev->hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&get_cfg_info, 0, sizeof(get_cfg_info)); + memset(&resp, 0, sizeof(resp)); + + get_cfg_info.func_id = rdev->glb_func_id; + + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO, + &get_cfg_info, sizeof(get_cfg_info), &resp, &out_size); + if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to init cfg info, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, get_cfg_info.head.status, out_size); + return (-EINVAL); + } + + rdev->cfg_info.scence_id = resp.scence_id; + rdev->cfg_info.lb_en = resp.lb_en; + rdev->cfg_info.lb_mode = resp.lb_mode; + rdev->cfg_info.srq_container_en = (resp.container_mode != ROCE_CHIP_SRQ_MODE_N); + rdev->cfg_info.srq_container_mode = roce3_srq_mode_chip_adapt(resp.container_mode); + rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3; + rdev->cfg_info.warn_th = 0; + + rdev->cfg_info.fake_en = resp.fake_en; + rdev->cfg_info.pf_start_bit = resp.pf_start_bit; + rdev->cfg_info.pf_end_bit = resp.pf_end_bit; + rdev->cfg_info.page_bit = resp.page_bit; + + rdev->cfg_info.port_num = resp.port_num; + rdev->cfg_info.host_num = resp.host_num; + rdev->cfg_info.master_func = (u8)(resp.port_num * resp.host_num); + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: RoCE init,func_id(%d),lb_en(%d),lb_mode(%d),srq_mode(%d),aa_en(%d)\n", + __func__, rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode, + rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id); + return 0; +} + +int roce3_get_group_id(u16 func_id, void *hwdev, struct roce_group_id *group_id) +{ + struct roce_cmd_get_group_id group_id_info; + u16 out_size = (u16)sizeof(group_id_info); + int ret; + + if (hwdev == NULL) { + (void)pr_err("[ROCE] %s(%d): Hwdev is null\n", __func__, __LINE__); + return -EINVAL; + } + + /* 组命令并下发给uP */ + memset(&group_id_info, 0, sizeof(group_id_info)); + group_id_info.func_id = func_id; + ret = roce3_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_GROUP_ID, + &group_id_info, sizeof(group_id_info), &group_id_info, &out_size); + if (!!ret || (!out_size) || !!group_id_info.status) { + (void)pr_err( + "[ROCE] %s(%d): Failed to get group id, ret(%d), status(%u), out size(0x%x)\n", + __func__, __LINE__, ret, group_id_info.status, out_size); + return -EINVAL; + } + + group_id->group_rc_cos = group_id_info.group_rc_cos; + group_id->group_ud_cos = group_id_info.group_ud_cos; + group_id->group_xrc_cos = group_id_info.group_xrc_cos; + + return 0; +} + +int roce3_del_func_res(struct roce3_device *rdev) +{ + struct roce_set_func_state_cmd del_func_res_info; + u16 out_size = (u16)sizeof(del_func_res_info); + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + pr_err("[ROCE, ERR] %s: rdev/Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&del_func_res_info, 0, sizeof(del_func_res_info)); + del_func_res_info.func_id = rdev->glb_func_id; + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_FUNC_RES, + &del_func_res_info, sizeof(del_func_res_info), + &del_func_res_info, &out_size); + if ((ret != 0) || (out_size == 0) || (del_func_res_info.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set func state, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, del_func_res_info.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce3_set_bw_ctrl_state(struct roce3_device *rdev, u8 cmd, struct roce3_bw_ctrl_inbuf *inbuf) +{ + int ret = 0; + struct roce_cc_cfg_bw_ctrl_cmd set_bw_ctrl; + u16 out_size = (u16)sizeof(set_bw_ctrl); + + if (rdev->hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&set_bw_ctrl, 0, sizeof(set_bw_ctrl)); + + set_bw_ctrl.func_id = rdev->glb_func_id; + set_bw_ctrl.cmd = cmd; + set_bw_ctrl.cir = inbuf->ctrl_param.cir; + set_bw_ctrl.pir = inbuf->ctrl_param.pir; + set_bw_ctrl.cnp = inbuf->ctrl_param.cnp; + pr_info("[ROCE, DEBUG] %s: func_id:%u, cmd:%u, cir:%u, pir:%u, cnp:%u\n", + __func__, rdev->glb_func_id, set_bw_ctrl.cmd, set_bw_ctrl.cir, + set_bw_ctrl.pir, set_bw_ctrl.cnp); + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_SET_BW_CTRL, + &set_bw_ctrl, sizeof(set_bw_ctrl), &set_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (set_bw_ctrl.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set set_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, set_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce3_query_bw_ctrl_state(struct roce3_device *rdev, struct roce3_bw_ctrl_param *bw_ctrl_param) +{ + int ret = 0; + struct roce_cc_cfg_bw_ctrl_cmd query_bw_ctrl; + u16 out_size = (u16)sizeof(query_bw_ctrl); + + if (rdev->hwdev == NULL) { + pr_err("[ROCE, ERR] %s: Hwdev is null\n", __func__); + return (-EINVAL); + } + + memset(&query_bw_ctrl, 0, sizeof(query_bw_ctrl)); + + query_bw_ctrl.func_id = rdev->glb_func_id; + ret = roce3_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_QUERY_BW_CTRL, + &query_bw_ctrl, sizeof(query_bw_ctrl), + &query_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (query_bw_ctrl.head.status != 0)) { + pr_err("[ROCE, ERR] %s: Failed to set query_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, query_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + memcpy((void *)bw_ctrl_param, + (void *)((u8 *)(&query_bw_ctrl) + ROCE_CMD_DEFAULT_SIZE), + sizeof(struct roce3_bw_ctrl_param)); + + pr_info("[ROCE, DEBUG] %s: query_bw_ctrl_state:func_id:%u, cir:%u, pir:%u, cnp:%u\n", + __func__, rdev->glb_func_id, bw_ctrl_param->cir, + bw_ctrl_param->pir, bw_ctrl_param->cnp); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_cmd.h b/drivers/infiniband/hw/hiroce3/roce_cmd.h new file mode 100644 index 000000000..ce598e059 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cmd.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CMD_H +#define ROCE_CMD_H + +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" + +#define ROCE_VLAN_ID_MASK 0x7FFF +#define ROCE_VLAN_N_VID 4096 +#define ROCE_VNI_N_VID 0xFFFFFF +#define ROCE_CMD_DEFAULT_SIZE 12 + +#define ERR_EXIST 6 +#define ERR_NOT_FOUND 13 +struct roce_cmd_get_group_id { + u8 status; + u8 version; + u8 rsvd0[6]; + + u16 func_id; + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; +}; + +struct roce_group_id { + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; + u8 rsvd; +}; + +#define roce3_msg_to_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, out_size) \ + (hinic3_msg_to_mgmt_sync(hwdev, HINIC3_MOD_ROCE, cmd, buf_in, in_size, \ + buf_out, out_size, 0, HINIC3_CHANNEL_ROCE)) + +int roce3_cfg_func_tbl_er_fwd_id_compact(void *hwdev, u32 bond_tbl_val, u16 func_id); + +int roce3_set_port_tbl_bond_state(void *hwdev, u8 bond_state, u16 func_id); + +int roce3_add_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id); +int roce3_del_mac_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 er_fwd_id, u16 func_id); +typedef int (*roce3_modify_mac_tbl)(void *hwdev, u8 *mac_addr, u32 vlan_id, + u16 er_fwd_id, u16 func_id); +void roce3_add_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id); +void roce3_del_ipsu_tbl_mac_entry(void *hwdev, u8 *mac_addr, u32 vlan_id, u16 func_id, u8 er_id); +typedef void (*roce3_modify_ipsu_tbl_mac)(void *hwdev, u8 *mac_addr, + u32 vlan_id, u16 func_id, u8 er_id); + +int roce3_do_cache_out(void *hwdev, u8 cl_id, u16 func_id); + +int roce3_set_cfg_ccf_param(void *hwdev, u16 func_id, u32 *ccf_param); +int roce3_set_cfg_ipqcn_param(void *hwdev, u16 func_id, u32 *ipqcn_param); +int roce3_set_cfg_ldcp_param(void *hwdev, u16 func_id, u32 *ldcp_param); +int roce3_set_cfg_dcqcn_param(void *hwdev, u16 func_id, u32 *dcqcn_param); +int roce3_set_bw_ctrl_state(struct roce3_device *rdev, u8 cmd, struct roce3_bw_ctrl_inbuf *inbuf); +int roce3_query_bw_ctrl_state(struct roce3_device *rdev, struct roce3_bw_ctrl_param *bw_ctrl_param); + +int roce3_set_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg); +int roce3_get_cap_cfg(void *hwdev, u16 index, u32 *cap_cfg); +int roce3_clear_cap_counter(void *hwdev, u16 index, u32 *value); +int roce3_read_cap_counter(void *hwdev, u16 index, u32 *value); + +int roce3_set_func_tbl_func_state(struct roce3_device *rdev, u8 func_state); +int roce3_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value); +int roce3_set_func_tbl_cpu_endian(void *hwdev, u8 cpu_endian, u16 func_id); +int roce3_init_cfg_info(struct roce3_device *rdev); +int roce3_del_func_res(struct roce3_device *rdev); +int roce3_get_group_id(u16 func_id, void *hwdev, struct roce_group_id *group_id); + +#endif /* ROCE_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_compat.h b/drivers/infiniband/hw/hiroce3/roce_compat.h new file mode 100644 index 000000000..92e64512d --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_compat.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_COMPAT_H +#define ROCE_COMPAT_H + +#include <linux/delay.h> + +#define ROCE_LIKELY(x) /*lint -e730*/ likely(x) /*lint +e730*/ + +#define ROCE_UNLIKELY(x) /*lint -e730*/ unlikely(x) /*lint +e730*/ + +#define ROCE_ILOG2(n) /*lint -e866*/ ilog2(n) /*lint +e866*/ + +#define ROCE_ROUNDUP_POW_OF_TWO(n) /*lint -e866*/ roundup_pow_of_two(n) /*lint +e866*/ + +/*lint -e506 -e160 -e522*/ +#define ROCE_MDELAY(n) mdelay(n) +#define ROCE_UDELAY(n) udelay(n) + +#define ROCE_ALIGN(a, b) ALIGN(a, b) +#define ROCE_MAX(a, b) max(a, b) +#define ROCE_MIN(a, b) min(a, b) +#define ROCE_DIV_ROUND_UP(a, b) DIV_ROUND_UP(a, b) +#define ROCE_FLS(n) fls(n) + +#define ROCE_MEMCMP(a, b, count) memcmp(a, b, count) +#define ROCE_IO_MAPPING_MAP_WC(mapping, offset) io_mapping_map_wc(mapping, offset) +#define ROCE_IOREMAP(phys_addr, size) ioremap(phys_addr, size) +#define ROCE_IOUNMAP(addr) iounmap(addr) +#define ROCE_IO_MAPPING_UNMAP(vaddr) io_mapping_unmap(vaddr) + +#ifndef wc_wmb + +#if defined(__i386__) +static inline void wc_wmb(void) +{ + asm volatile("lock; addl $0,0(%%esp) " ::: "memory"); +} +#elif defined(__x86_64__) +static inline void wc_wmb(void) +{ + asm volatile("sfence" ::: "memory"); +} +#elif defined(__ia64__) +static inline void wc_wmb(void) +{ + asm volatile("fwb" ::: "memory"); +} +#else +static inline void wc_wmb(void) +{ + /* Write memory barrier in aarch64 */ + wmb(); +} +#endif + +#endif + +#endif // ROCE_COMPAT_H diff --git a/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c new file mode 100644 index 000000000..3e1e2d7ad --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_cqm_cmd.h" + +void roce3_cqm_cmd_free_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, + struct tag_cqm_cmd_buf *cqm_cmd_outbuf) +{ + if (cqm_cmd_outbuf != NULL) + cqm_cmd_free(ex_handle, cqm_cmd_outbuf); + + if (cqm_cmd_inbuf != NULL) + cqm_cmd_free(ex_handle, cqm_cmd_inbuf); +} + +int roce3_cqm_cmd_zalloc_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf **cqm_cmd_inbuf, + u16 inbuf_size, struct tag_cqm_cmd_buf **cqm_cmd_outbuf, + u16 outbuf_size) +{ + int ret; + + if (cqm_cmd_inbuf != NULL) { + *cqm_cmd_inbuf = cqm_cmd_alloc(ex_handle); + + if (*cqm_cmd_inbuf == NULL) { + ret = -ENOMEM; + goto err; + } + (*cqm_cmd_inbuf)->size = inbuf_size; + + memset((*cqm_cmd_inbuf)->buf, 0, inbuf_size); + } + + if (cqm_cmd_outbuf != NULL) { + *cqm_cmd_outbuf = cqm_cmd_alloc(ex_handle); + + if (*cqm_cmd_outbuf == NULL) { + ret = -ENOMEM; + goto err; + } + (*cqm_cmd_outbuf)->size = outbuf_size; + + memset((*cqm_cmd_outbuf)->buf, 0, outbuf_size); + } + + return 0; +err: + roce3_cqm_cmd_free_inoutbuf(ex_handle, *cqm_cmd_inbuf, *cqm_cmd_outbuf); + *cqm_cmd_inbuf = NULL; + *cqm_cmd_outbuf = NULL; + return ret; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h new file mode 100644 index 000000000..72932ae0f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_cqm_cmd.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_CQM_CMD_H +#define ROCE_CQM_CMD_H + +#include "hinic3_cqm.h" + +#include "roce.h" + +void roce3_cqm_cmd_free_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf *cqm_cmd_inbuf, + struct tag_cqm_cmd_buf *cqm_cmd_outbuf); +int roce3_cqm_cmd_zalloc_inoutbuf(void *ex_handle, struct tag_cqm_cmd_buf **cqm_cmd_inbuf, + u16 inbuf_size, struct tag_cqm_cmd_buf **cqm_cmd_outbuf, + u16 outbuf_size); + +#endif // ROCE_CQM_CMD_H diff --git a/drivers/infiniband/hw/hiroce3/roce_db.c b/drivers/infiniband/hw/hiroce3/roce_db.c new file mode 100644 index 000000000..0869fe0ac --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_db.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/kernel.h> + +#include "roce.h" + +/* + **************************************************************************** + Prototype : roce3_db_map_user + Description : roce3_db_map_user + Input : struct roce3_ucontext *context + unsigned long virt + struct roce3_db *db + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +int roce3_db_map_user(struct roce3_ucontext *context, unsigned long virt, struct roce3_db *db) +{ + int ret = 0; + struct roce3_db_page *db_page = NULL; + + mutex_lock(&context->db_page_mutex); + + list_for_each_entry(db_page, &context->db_page_list, list) { + if (db_page->user_virt == (virt & PAGE_MASK)) + goto found; + } + + db_page = kmalloc(sizeof(*db_page), GFP_KERNEL); + if (db_page == NULL) { + ret = -ENOMEM; + pr_err("[ROCE, ERR] %s: Failed to alloc DB page\n", __func__); + goto out; + } + + db_page->user_virt = (virt & PAGE_MASK); + db_page->refcnt = 0; + db_page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, PAGE_SIZE, 0); + if (IS_ERR(db_page->umem)) { + ret = (int)PTR_ERR(db_page->umem); + pr_err("[ROCE, ERR] %s: Failed to get ib_umem ret:%d\n", __func__, ret); + kfree(db_page); + goto out; + } + + list_add(&db_page->list, &context->db_page_list); + +found: + db->dma = sg_dma_address(db_page->umem->sg_head.sgl) + (virt & ~PAGE_MASK); + db->user_page = db_page; + ++db_page->refcnt; + +out: + mutex_unlock(&context->db_page_mutex); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_db_unmap_user + Description : roce3_db_unmap_user + Input : struct roce3_ucontext *context + struct roce3_db *db + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** +*/ +void roce3_db_unmap_user(struct roce3_ucontext *context, struct roce3_db *db) +{ + mutex_lock(&context->db_page_mutex); + + if ((--db->user_page->refcnt) == 0) { + list_del(&db->user_page->list); + ib_umem_release(db->user_page->umem); + kfree(db->user_page); + } + + mutex_unlock(&context->db_page_mutex); +} diff --git a/drivers/infiniband/hw/hiroce3/roce_db.h b/drivers/infiniband/hw/hiroce3/roce_db.h new file mode 100644 index 000000000..c4a3abb92 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_db.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_DB_H +#define ROCE_DB_H + +#include <rdma/ib_umem.h> +#include <linux/types.h> +#include <linux/list.h> + +struct roce3_db_page { + struct list_head list; + struct ib_umem *umem; + unsigned long user_virt; + int refcnt; +}; + +struct roce3_db { + __be32 *db_record; + dma_addr_t dma; + struct roce3_db_page *user_page; +}; + +static inline void roce3_write64(u32 val[2], void __iomem *dest) +{ + __raw_writeq(*(u64 *)(void *)val, dest); +} + +#endif // ROCE_DB_H diff --git a/drivers/infiniband/hw/hiroce3/roce_event.c b/drivers/infiniband/hw/hiroce3/roce_event.c new file mode 100644 index 000000000..717cb8adf --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_event.c @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_event.h" +#include "roce_event_extension.h" +#include "roce_pub_cmd.h" +#include "hinic3_mt.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +static void roce3_event_report_common_a(const struct roce3_device *rdev, int event_str_index) +{ + switch (event_str_index) { + case OFED_ET_PATH_MIG: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Path mig. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_COMM_EST: + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [ofed event type] Communication establish. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ + break; + + case OFED_ET_SQ_DRAINED: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Sq drained. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_SRQ_QP_LAST_WQE: + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [ofed event type] Srq/qp last wqe. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ + break; + + case OFED_ET_WQ_CATAS_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Wq catas error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_PATH_MIG_FAILED: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Path mig failed. function id: %u\n", + __func__, rdev->glb_func_id); + break; + default: + break; + } +} + +static void roce3_event_report_common_b(const struct roce3_device *rdev, int event_str_index) +{ + switch (event_str_index) { + case OFED_ET_WQ_INVAL_REQ_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Wq inval req error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_WQ_ACCESS_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Wq access error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_CQ_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Cq error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_SRQ_LIMIT: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Srq limit. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case OFED_ET_SRQ_CATAS_ERR: + pr_err_ratelimited("[ROCE] %s: [ofed event type] Srq catas error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case NON_OFED_ET_QPC_LOOKUP_ERR: + pr_err_ratelimited("[ROCE] %s: [non ofed event type] Qpc lookup error. function id: %u\n", + __func__, rdev->glb_func_id); + break; + + case NON_OFED_ET_OTHER_TYPE_ERR: + /*lint -e160 -e522*/ + roce3_pr_err_once("[ROCE] %s: [non ofed event type] other type error. function id: %u\n", + __func__, rdev->glb_func_id); + /*lint +e160 +e522*/ + break; + default: + break; + } +} + +static void roce3_event_report_common(const struct roce3_device *rdev, int event_str_index) +{ + if (event_str_index <= OFED_ET_PATH_MIG_FAILED) + roce3_event_report_common_a(rdev, event_str_index); + else + roce3_event_report_common_b(rdev, event_str_index); +} + +/* + **************************************************************************** + Prototype : roce3_event_report + Description : roce3_event_report + Input : struct roce3_device *rdev + int event_str_index + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_event_report(struct roce3_device *rdev, int event_str_index) +{ + if (event_str_index <= NON_OFED_ET_OTHER_TYPE_ERR) + roce3_event_report_common(rdev, event_str_index); + else + roce3_event_report_extend(rdev, event_str_index); +} + +/* + **************************************************************************** + Prototype : to_qp_event_str_index + Description : to_qp_event_str_index + Input : u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int to_qp_event_str_index(u8 event_type) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + return OFED_ET_PATH_MIG; + + case ROCE_EVENT_TYPE_COMM_EST: + return OFED_ET_COMM_EST; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + return OFED_ET_SQ_DRAINED; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + return OFED_ET_SRQ_QP_LAST_WQE; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + return OFED_ET_WQ_CATAS_ERR; + + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + return OFED_ET_PATH_MIG_FAILED; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + return OFED_ET_WQ_INVAL_REQ_ERR; + + default: + return OFED_ET_WQ_ACCESS_ERR; + } +} + +int to_unknown_event_str_index(u8 event_type, const u8 *val) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_OFED_NO_DEF: + if (*((u64 *)val) == QPC_LOOKUP_ERR_VALUE) + return NON_OFED_ET_QPC_LOOKUP_ERR; + + return NON_OFED_ET_OTHER_TYPE_ERR; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + return NON_OFED_ET_OTHER_TYPE_ERR; + + default: + return INVAL_ET_ERR; + } +} + +/* + **************************************************************************** + Prototype : roce3_handle_qp_async_event + Description : roce3_handle_qp_async_event + Input : struct roce3_device *rdev + u32 qpn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_qp_async_event(struct roce3_device *rdev, u32 qpn, u8 event_type) +{ + struct roce3_qp *rqp = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, false); + if (cqm_obj_queue == NULL) { + dev_err_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Failed to get cqm obj queue, func_id(%d), qpn:%u, event_type:%u\n", + __func__, rdev->glb_func_id, qpn, event_type); + return; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_queue); + roce3_qp_async_event(rdev, rqp, (int)event_type); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_queue); +} + +/* + **************************************************************************** + Prototype : roce3_handle_cq_async_event + Description : roce3_handle_cq_async_event + Input : struct roce3_device *rdev + u32 cqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_cq_async_event(struct roce3_device *rdev, u32 cqn, u8 event_type) +{ + struct roce3_cq *rcq = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_queue == NULL) { + dev_err_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Failed to get cqm obj queue, func_id(%d), cqn:%u, event_type:%u\n", + __func__, rdev->glb_func_id, cqn, event_type); + return; + } + + rcq = cqmobj_to_roce3_cq(cqm_obj_queue); + roce3_cq_async_event(rdev, rcq, (int)event_type); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_queue); +} + +/* + **************************************************************************** + Prototype : roce3_handle_srq_async_event + Description : roce3_handle_srq_async_event + Input : struct roce3_device *rdev + u32 srqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_handle_srq_async_event(struct roce3_device *rdev, u32 srqn, u8 event_type) +{ + struct roce3_srq *rsrq = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_queue == NULL) { + dev_err_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: Failed to get cqm obj queue, func_id(%d), srqn:%u, event_type:%u\n", + __func__, rdev->glb_func_id, srqn, event_type); + return; + } + + rsrq = cqmobj_to_roce3_srq(cqm_obj_queue); + roce3_srq_async_event(rdev, rsrq, (int)event_type); + hiudk_cqm_object_put(rdev->hwdev, cqm_obj_queue); +} + +static bool roce3_async_event_need_reset(u8 event_status) +{ + if ((event_status == API_FORMAT_ERROR) || (event_status == UNKNOWN_RESERVED_ERROR1)) + return true; + + return false; +} + +u8 roce3_async_event_level(void *svc_hd, u8 event_type, u8 *val) +{ + u8 event_level = FAULT_LEVEL_MAX; + u8 event_status; + u64 param = *((u64 *)val); + + if (svc_hd == NULL) { + pr_err("[ROCE, ERR] %s: Svc_hd is null\n", __func__); + return FAULT_LEVEL_SUGGESTION; + } + + event_status = param & 0xff; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_SRQ_LIMIT: + case ROCE_EVENT_TYPE_PORT_MNG_CHG_EVENT: + case ROCE_EVENT_TYPE_PORT_CHANGE: + case ROCE_EVENT_TYPE_ECC_DETECT: + case ROCE_EVENT_TYPE_CMD: + case ROCE_EVENT_TYPE_COMM_CHANNEL: + case ROCE_EVENT_TYPE_OP_REQUIRED: + event_level = FAULT_LEVEL_SUGGESTION; + break; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: // ROCE_AEQE_EVENT_QP_REQ_ERR + if (event_status == API_FORMAT_ERROR) + return FAULT_LEVEL_SERIOUS_RESET; + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: // ROCE_AEQE_EVENT_WQE_CATAS + if (roce3_async_event_need_reset(event_status)) + return FAULT_LEVEL_SERIOUS_RESET; + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: // CQ ERR + if (roce3_async_event_need_reset(event_status)) + return FAULT_LEVEL_SERIOUS_RESET; + break; + + default: + event_level = FAULT_LEVEL_GENERAL; + break; + } + + return event_level; +} + +static int roce3_async_event_handle_common(u8 event_type, u8 *val, struct roce3_device *rdev) +{ + u32 xid = 0; + int event_str_index = 0; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + xid = *((u32 *)val); + roce3_handle_qp_async_event(rdev, xid, event_type); + event_str_index = to_qp_event_str_index(event_type); + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: + xid = *((u32 *)val); + roce3_handle_cq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_CQ_ERR; + break; + + case ROCE_EVENT_TYPE_SRQ_LIMIT: + xid = *((u32 *)val); + roce3_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_LIMIT; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + xid = *((u32 *)val); + roce3_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_CATAS_ERR; + break; + + default: + event_str_index = to_unknown_event_str_index(event_type, val); + break; + } + + return event_str_index; +} + +/* + **************************************************************************** + Prototype : roce3_async_event + Description : roce3_async_event + Input : void *svc_hd + u8 event_type + u64 val + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_async_event(void *svc_hd, u8 event_type, u8 *val) +{ + int event_str_index = 0; + struct roce3_device *rdev = NULL; + + if (svc_hd == NULL) { + pr_err_ratelimited("[ROCE, ERR] %s: Svc_hd is null\n", __func__); + return; + } + + rdev = (struct roce3_device *)svc_hd; + if (event_type <= ROCE_EVENT_TYPE_ODP_PAGE_FAULT) + event_str_index = roce3_async_event_handle_common(event_type, val, rdev); + else + event_str_index = roce3_async_event_handle_extend(event_type, val, rdev); + + roce3_event_report(rdev, event_str_index); +} + +static void roce3_handle_sq_hot_plug(struct roce3_qp *rqp, unsigned long flags_qp, + struct list_head *cq_notify_list) +{ + struct roce3_cq *send_rcq = NULL; + unsigned long flags_cq = 0; + unsigned long flags_qp_tmp = flags_qp; + + spin_lock_irqsave(&rqp->sq.lock, flags_qp_tmp); + if (rqp->sq.tail == rqp->sq.head) + goto roce_sq_hot_plug_end; + + send_rcq = to_roce3_cq(rqp->ibqp.send_cq); + spin_lock_irqsave(&send_rcq->lock, flags_cq); + if (send_rcq->reset_flow_comp && rqp->ibqp.send_cq->comp_handler) { + if (send_rcq->reset_notify_added == 0) { + send_rcq->reset_notify_added = 1; + list_add_tail(&send_rcq->reset_notify, cq_notify_list); + } + } + spin_unlock_irqrestore(&send_rcq->lock, flags_cq); + +roce_sq_hot_plug_end: + spin_unlock_irqrestore(&rqp->sq.lock, flags_qp_tmp); +} + +static void roce3_handle_rq_hot_plug(struct roce3_qp *rqp, unsigned long flags_qp, + struct list_head *cq_notify_list) +{ + struct roce3_cq *recv_rcq = NULL; + unsigned long flags_cq = 0; + unsigned long flags_qp_tmp = flags_qp; + + spin_lock_irqsave(&rqp->rq.lock, flags_qp_tmp); + /* no handling is needed for SRQ */ + if (rqp->ibqp.srq != NULL) + goto roce_rq_hot_plug_end; + else if (rqp->rq.tail == rqp->rq.head) + goto roce_rq_hot_plug_end; + + recv_rcq = to_roce3_cq(rqp->ibqp.recv_cq); + spin_lock_irqsave(&recv_rcq->lock, flags_cq); + if (recv_rcq->reset_flow_comp && rqp->ibqp.recv_cq->comp_handler) { + if (recv_rcq->reset_notify_added == 0) { + recv_rcq->reset_notify_added = 1; + list_add_tail(&recv_rcq->reset_notify, cq_notify_list); + } + } + spin_unlock_irqrestore(&recv_rcq->lock, flags_cq); +roce_rq_hot_plug_end: + spin_unlock_irqrestore(&rqp->rq.lock, flags_qp_tmp); +} + +void roce3_handle_hotplug_arm_cq(struct roce3_device *rdev) +{ + struct roce3_qp *rqp = NULL; + struct roce3_cq *rcq = NULL; + unsigned long flags_qp = 0; + unsigned long flags = 0; + struct list_head cq_notify_list; + + pr_err("[ROCE, ERR] %s: Hotplug arm cq start:\n", __func__); + + INIT_LIST_HEAD(&cq_notify_list); + + /* Go over qp list reside on that rdev, sync with create/destroy qp */ + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + list_for_each_entry(rqp, &rdev->qp_list, qps_list) { + roce3_handle_sq_hot_plug(rqp, flags_qp, &cq_notify_list); + roce3_handle_rq_hot_plug(rqp, flags_qp, &cq_notify_list); + } + + list_for_each_entry(rcq, &cq_notify_list, reset_notify) { + rcq->reset_flow_comp(rcq); + } + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); + + pr_err("[ROCE, ERR] %s: Hotplug arm cq end!\n", __func__); +} + +void roce3_kernel_hotplug_event_trigger(struct roce3_device *rdev) +{ + int carrier_ok = 0; + + if (rdev->ndev) { + carrier_ok = netif_carrier_ok(rdev->ndev); + if (carrier_ok != 0) { + dev_info(rdev->hwdev_hdl, "[ROCE] %s: Turn off, func_id(%u), name(%s), pci(%s),\n", + __func__, rdev->glb_func_id, + rdev->ib_dev.name, pci_name(rdev->pdev)); + netif_carrier_off(rdev->ndev); + } + + ROCE_MDELAY(ROCE_M_DELAY); + return; + } + + dev_info(rdev->hwdev_hdl, "[ROCE] %s: rdev->ndev is NULL\n", __func__); +} + +#ifdef ROCE_BONDING_EN +void roce3_handle_bonded_port_state_event(struct roce3_device *rdev) +{ + struct net_device *master = netdev_master_upper_dev_get_rcu(rdev->ndev); + enum ib_port_state bonded_port_state = IB_PORT_NOP; + struct roce3_bond_device *bond_dev = NULL; + enum ib_port_state curr_port_state; + enum ib_event_type event; + struct roce3_bond_slave *slave = NULL; + int i; + + if (master == NULL) + return; + + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + pr_err("[ROCE, ERR] %s: No bond_dev found\n", __func__); + return; + } + + if (netif_running(master) != 0) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->netdev == NULL) + continue; + + curr_port_state = (netif_running(slave->netdev) && + netif_carrier_ok(slave->netdev)) ? + IB_PORT_ACTIVE : + IB_PORT_DOWN; + + bonded_port_state = (bonded_port_state != IB_PORT_ACTIVE) ? + curr_port_state : IB_PORT_ACTIVE; + } + mutex_unlock(&bond_dev->slave_lock); + } else { + bonded_port_state = IB_PORT_DOWN; + } + + if (rdev->port_state != bonded_port_state) { + event = (bonded_port_state == IB_PORT_ACTIVE) ? + IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; + if (roce3_ifconfig_up_down_event_report(rdev, event) != 0) + return; + rdev->port_state = bonded_port_state; + } +} +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_event.h b/drivers/infiniband/hw/hiroce3/roce_event.h new file mode 100644 index 000000000..c111a54fa --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_event.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_EVENT_H +#define ROCE_EVENT_H + +#include <net/ipv6.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/pci.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> + +#include "roce.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_cmd.h" + +#define ROCE_M_DELAY 100 +#define API_FORMAT_ERROR 0x2 +#define UNKNOWN_RESERVED_ERROR1 0xe +#define QPC_LOOKUP_ERR_VALUE 0x300d00016 + +int to_unknown_event_str_index(u8 event_type, const u8 *val); + +void roce3_handle_hotplug_arm_cq(struct roce3_device *rdev); +void roce3_kernel_hotplug_event_trigger(struct roce3_device *rdev); + +#endif /* ROCE_EVENT_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_k_ioctl.h b/drivers/infiniband/hw/hiroce3/roce_k_ioctl.h new file mode 100644 index 000000000..a000b9fba --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_k_ioctl.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_K_IOCTL_H +#define ROCE_K_IOCTL_H + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> + +#define ROCE_IOCTL_MAGIC 'R' +#define ROCE_CMD_QUERY_DCB _IO(ROCE_IOCTL_MAGIC, 0) +#define ROCE_CMD_CREATE_AH _IO(ROCE_IOCTL_MAGIC, 1) + +#define ROCE_CMD_QUERY_QP_TX_PORT _IO(ROCE_IOCTL_MAGIC, 3) +#define ROCE_CMD_SET_QP_TX_PORT _IO(ROCE_IOCTL_MAGIC, 4) +#define ROCE_CMD_QUERY_QP_RX_PORT _IO(ROCE_IOCTL_MAGIC, 5) +#define ROCE_CMD_QUERY_BOND_PORT_INFO _IO(ROCE_IOCTL_MAGIC, 6) +#define ROCE_CMD_SET_QP_UDP_SRC_PORT _IO(ROCE_IOCTL_MAGIC, 7) +#define ROCE_CMD_QUERY_QP_UDP_SRC_PORT _IO(ROCE_IOCTL_MAGIC, 8) +#define ROCE_CMD_QUERY_NEXT_WQE_IDX _IO(ROCE_IOCTL_MAGIC, 9) + +#define ROCE_CMD_MAX _IO(ROCE_IOCTL_MAGIC, 10) + +#ifdef ROCE_EXTEND +#define HW_ROCE_EXT_CMD_SET_QP_ATTR _IO(ROCE_IOCTL_MAGIC, 11) +#define HW_ROCE_EXT_CMD_CREATE_SQPC _IO(ROCE_IOCTL_MAGIC, 12) +#endif + +// roce3_ioctrl_version +#define VERSION_LEN (20) +#define HW_ROCE_CMD_VERSION _IO(ROCE_IOCTL_MAGIC, 20) +// reserved op code +#define HW_ROCE_CMD_RESERVED_1 _IO(ROCE_IOCTL_MAGIC, 21) +#define HW_ROCE_CMD_RESERVED_2 _IO(ROCE_IOCTL_MAGIC, 22) +#define HW_ROCE_CMD_RESERVED_3 _IO(ROCE_IOCTL_MAGIC, 23) +#define HW_ROCE_CMD_RESERVED_4 _IO(ROCE_IOCTL_MAGIC, 24) +#define HW_ROCE_CMD_RESERVED_5 _IO(ROCE_IOCTL_MAGIC, 25) +#define HW_ROCE_CMD_RESERVED_6 _IO(ROCE_IOCTL_MAGIC, 26) + +#define ROCE_DEV_NAME_MAX 64 + +union roce3_query_dcb_buf { + struct roce3_query_dcb_cmd { + u8 sl; + u8 sgid_idx; + u8 port; + u8 traffic_class; // dscp + u8 dscp_type; + u8 rsvd[3]; + } cmd; + + struct roce3_query_dcb_resp { + u8 cos; + u8 rsvd[7]; + } resp; +}; + +union roce3_create_ah_buf { + struct roce3_create_ah_cmd { + struct ib_uverbs_ah_attr attr; + } cmd; + + struct roce3_create_ah_resp { + u8 dmac[ETH_ALEN]; + u16 vlan_id; + } resp; +}; + +struct roce3_qp_port_buf { + u32 qpn; + u32 port; +}; + +struct roce3_bond_port_info_buf { + int original_port_num; + char original_port[8]; + int rsvd1; + + int alive_port_num; + char alive_port[8]; + int rsvd2; +}; + +struct roce3_qp_udp_src_port_buf { + u32 qpn; + u32 udp_src_port; +}; + +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_main.c b/drivers/infiniband/hw/hiroce3/roce_main.c new file mode 100644 index 000000000..1f252161e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_main.c @@ -0,0 +1,1609 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <net/ipv6.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_user_verbs.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include "hinic3_crm.h" +#include "hinic3_hw.h" +#include "hinic3_hwdev.h" +#include "hinic3_srv_nic.h" +#include "hinic3_rdma.h" +#include "hinic3_bond.h" +#include "hinic3_pci_id_tbl.h" + +#include "roce_event.h" +#include "roce_compat.h" +#include "roce_dfx.h" +#include "roce_mr.h" +#include "roce_main_extension.h" + +#ifdef ROCE_NETLINK_EN +#include "roce_netlink.h" +#endif + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#include "roce_pub_cmd.h" + +MODULE_AUTHOR(HIROCE3_DRV_AUTHOR); +MODULE_DESCRIPTION(HIROCE3_DRV_DESC); +MODULE_VERSION(HIROCE3_DRV_VERSION); +MODULE_LICENSE("GPL"); + +static int g_loop_times = 50000; +module_param(g_loop_times, int, 0444); //lint !e806 +MODULE_PARM_DESC(g_loop_times, "default: 50000"); + +static bool g_ppf_stateful_init; +static u8 g_vf_stateful_num; + +#ifdef ROCE_BONDING_EN +static int g_want_bond_slave_cnt = ROCE_BOND_WANT_TWO_SLAVES; +module_param(g_want_bond_slave_cnt, int, 0444); +MODULE_PARM_DESC(g_want_bond_slave_cnt, "default: 2, 2: two slaves, 3: three slaves, 4: four slaves"); + +static int g_want_bond0_slave_bits = 0x3; /* 0011 */ +module_param(g_want_bond0_slave_bits, int, 0444); +MODULE_PARM_DESC(g_want_bond0_slave_bits, "default: 0x3(PF0+PF1), 4bits"); + +static char *g_bond_name; +module_param(g_bond_name, charp, 0444); +MODULE_PARM_DESC(g_bond_name, "bond name for sdi"); +#endif + +struct roce3_func_info { + u16 func_id; + struct list_head node; +}; + +LIST_HEAD(g_roce_device_list); +static void roce3_remove_device_from_list(struct hinic3_lld_dev *lld_dev); +static int roce3_add_device_to_list(struct hinic3_lld_dev *lld_dev); +static void roce3_wait_probe(struct hinic3_lld_dev *lld_dev); +DECLARE_WAIT_QUEUE_HEAD(g_roce_probe_queue); + +/* + **************************************************************************** + Prototype : roce3_cq_completion + Description : RoCE's callback function for CQ's completion events on ARM CQs + Input : void *svc_hd + u32 cqn + void *cq_handler + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_cq_completion(void *svc_hd, u32 cqn, void *cq_handler) +{ + struct roce3_cq *cq = NULL; + struct ib_cq *ibcq = NULL; + + if (cq_handler == NULL) { + pr_err("[ROCE, ERR] %s: Cq_handler is null\n", __func__); + return; + } + + cq = (struct roce3_cq *)cq_handler; + + ++cq->arm_sn; + cq->arm_flag = 0; + + ibcq = &cq->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +/* + **************************************************************************** + Prototype : get_cpu_endian + Description : Acquire CPU's enianness + Return Value : 0: little endian; 1: big endian + + 1.Date : 2016/6/23 + Modification : Created function + +**************************************************************************** +*/ +static int get_cpu_endian(void) +{ + int cpu_mode = 0; + union { + unsigned int i; + unsigned char s[4]; + } c; + c.i = 0x12345678; + + if (c.s[0] == 0x12) { + pr_info("[ROCE] %s: CPU is be\n", __func__); + cpu_mode = 1; + } else { + pr_info("[ROCE] %s: CPU is le\n", __func__); + cpu_mode = 0; + } + + return cpu_mode; +} + +static int roce3_alloc_hw_resource(struct roce3_device *rdev) +{ + int ret; + + ret = roce3_rdma_init_rsvd_lkey(rdev->hwdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init rsvd lkey, func_id(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_rdma_reset_gid_table(rdev->hwdev, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR]: Failed to reset gid_table, func_id(%u)\n", + rdev->glb_func_id); + goto err_gid_reset; + } + + return 0; + +err_gid_reset: + roce3_rdma_free_rsvd_lkey(rdev->hwdev); + + return ret; +} + +static void roce3_dealloc_hw_resource(struct roce3_device *rdev) +{ + roce3_rdma_free_rsvd_lkey(rdev->hwdev); +} + +static struct roce3_device *roce3_rdev_alloc(struct hinic3_lld_dev *lld_dev, void **uld_dev, + const struct rdma_service_cap *rdma_cap) +{ + struct roce3_device *rdev = NULL; + + rdev = (struct roce3_device *)roce3_rdev_alloc_ext(); + if (rdev == NULL) { + pr_err("[ROCE, ERR] %s: Failed to alloc rdev\n", __func__); + return NULL; + } + + *uld_dev = (void *)rdev; + rdev->lld_dev = lld_dev; + rdev->hwdev = lld_dev->hwdev; + rdev->pdev = lld_dev->pdev; + mutex_init(&rdev->qp_cnt.cur_qps_mutex); + + rdev->hwdev_hdl = ((struct hinic3_hwdev *)(rdev->hwdev))->dev_hdl; + memcpy((void *)&rdev->rdma_cap, (void *)rdma_cap, sizeof(*rdma_cap)); + + rdev->ndev = hinic3_get_netdev_by_lld(rdev->lld_dev); + if (rdev->ndev == NULL) { + pr_err("[ROCE, ERR] %s roce add failed, netdev is null.\n", __func__); + ib_dealloc_device(&rdev->ib_dev); + return NULL; + } + + roce3_rdev_set_ext(rdev); + + return rdev; +} + +static int roce3_board_info_get(struct roce3_device *rdev) +{ + int ret = 0; + + ret = hinic3_get_board_info(rdev->hwdev, &rdev->board_info, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get board info\n", __func__); + return ret; + } + + pr_info("[ROCE] Get board info success, board_type:0x%x, port_num:0x%x, pf_num:0x%x, vf_total_num:0x%x, work_mode:0x%x, service_mode:0x%x, speed:0x%x.\n", + rdev->board_info.board_type, rdev->board_info.port_num, + rdev->board_info.pf_num, rdev->board_info.vf_total_num, + rdev->board_info.work_mode, rdev->board_info.service_mode, + rdev->board_info.port_speed); + +#ifdef ROCE_BONDING_EN + ret = roce3_bond_attach(rdev); + if (ret != 0) + return ret; +#endif + + ret = roce3_board_cfg_check(rdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check board cfg info\n", __func__); + return ret; + } + + return ret; +} + +static int roce3_init_info_get(struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_board_info_get(rdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get board info\n", __func__); + return ret; + } + + return ret; +} + +static void roce3_fix_ibdev_name(struct roce3_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + strscpy(rdev->ib_dev.name, "hrn3_bond_%d", sizeof("hrn3_bond_%d")); + return; + } +#endif + { + strscpy(rdev->ib_dev.name, (rdev->is_vroce ? "efi_%d" : "hrn3_%d"), + (rdev->is_vroce ? sizeof("efi_%d") : sizeof("hrn3_%d"))); + } +} + +static int roce3_register_template(struct roce3_device *rdev) +{ + struct tag_service_register_template svc_template; + int ret; + + memset(&svc_template, 0, sizeof(svc_template)); + + svc_template.service_type = SERVICE_T_ROCE; + svc_template.scq_ctx_size = rdev->rdma_cap.cqc_entry_sz; + svc_template.srq_ctx_size = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.srqc_entry_sz; + svc_template.service_handle = rdev; + svc_template.embedded_cq_ceq_callback = NULL; + svc_template.no_cq_ceq_callback = NULL; + + svc_template.shared_cq_ceq_callback = roce3_cq_completion; + svc_template.aeq_level_callback = roce3_async_event_level; + svc_template.aeq_callback = roce3_async_event; + + ret = cqm_service_register(rdev->hwdev, &svc_template); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to register cqm_service, func(%u)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce3_init_dev_file(struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_init_sysfs(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init sysfs, func_id(%u) ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return ret; + } + + ret = roce3_init_cdev(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init cdev, func_id(%u)\n", + __func__, rdev->glb_func_id); + roce3_remove_sysfs(rdev); + } + + return ret; +} + +void roce3_remove_dev_file(struct roce3_device *rdev) +{ + roce3_remove_cdev(rdev); + + roce3_remove_sysfs(rdev); +} + +/* Alloc CEQs and alloc CEQN */ +static int roce3_alloc_ceq(struct roce3_device *rdev) +{ + int ret; + + ret = hinic3_alloc_ceqs(rdev->hwdev, SERVICE_T_ROCE, rdev->ib_dev.num_comp_vectors, + rdev->ceqn, &rdev->ceq_num); + if (ret < 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqes, num_comp_vectors(%d), func_id(%u)\n", + __func__, rdev->ib_dev.num_comp_vectors, rdev->glb_func_id); + return ret; + } + + return 0; +} + +static void roce3_free_ceq(struct roce3_device *rdev) +{ + int i; + + for (i = 0; i < rdev->ceq_num; ++i) + hinic3_free_ceq(rdev->hwdev, SERVICE_T_ROCE, rdev->ceqn[i]); +} + +static void roce3_init_hw_info(struct roce3_device *rdev) +{ + int ret; + struct roce_group_id group_id = {0}; + + rdev->hw_info.config_num_ports = (int)rdev->rdma_cap.num_ports; + rdev->hw_info.ep_id = hinic3_ep_id(rdev->hwdev); + rdev->hw_info.phy_port = 1; + rdev->hw_info.is_vf = (hinic3_func_type(rdev->hwdev) == TYPE_VF); + rdev->hw_info.cpu_endian = (u8)get_cpu_endian(); + + if (!rdev->is_vroce) + return; + + ret = roce3_get_group_id(rdev->glb_func_id, rdev->hwdev, &group_id); + if (!!ret) { + dev_info(rdev->hwdev_hdl, "[ROCE , INFO] Failed to get group id, ret(%d)", ret); + return; + } + + rdev->group_rc_cos = group_id.group_rc_cos; + rdev->group_ud_cos = group_id.group_ud_cos; + rdev->group_xrc_cos = group_id.group_xrc_cos; + dev_info(rdev->hwdev_hdl, "[ROCE , INFO] group id rc(%u), ud(%u), xrc(%u)", + group_id.group_rc_cos, group_id.group_ud_cos, group_id.group_xrc_cos); +} + +static int roce3_init_dev_upper(struct roce3_device *rdev) +{ + int ret; + + /* Set function table to ENABLE */ + ret = roce3_set_func_tbl_func_state(rdev, ROCE_FUNC_ENABLE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to set func_tbl, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_set_func_tbl; + } + return 0; + +err_set_func_tbl: + return ret; +} + +static void roce3_deinit_dev_upper(struct roce3_device *rdev) +{ + (void)roce3_set_func_tbl_func_state(rdev, ROCE_FUNC_DISABLE); +} + +static int roce3_init_dev_info(struct roce3_device *rdev) +{ + int ret; + + ret = roce3_set_func_tbl_cpu_endian(rdev->hwdev, rdev->hw_info.cpu_endian, + rdev->glb_func_id); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR]: Failed to set func_tbl cpu_endian, func_id(%u)\n", + rdev->glb_func_id); + goto err_init_endianness; + } + + ret = roce3_init_dev_ext(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR]: Failed to init extended service for func_id(%u)\n", + rdev->glb_func_id); + goto err_init_dev_ext; + } + + ret = roce3_dfx_mem_alloc(rdev); + if (ret != 0) + goto err_init_dev_dfx; + + ret = roce3_init_dev_upper(rdev); + if (ret != 0) + goto err_init_dev_upper; + + return 0; + +err_init_dev_upper: + roce3_dfx_mem_free(rdev); +err_init_dev_dfx: + roce3_remove_clean_res_ext(rdev); +err_init_dev_ext: +err_init_endianness: + return ret; +} + +static void roce3_deinit_dev_info(struct roce3_device *rdev) +{ + roce3_deinit_dev_upper(rdev); + roce3_dfx_mem_free(rdev); + roce3_remove_clean_res_ext(rdev); +} +#ifdef ROCE_BONDING_EN +static int roce3_ib_register_bond_device(struct roce3_device *rdev) +{ + return ib_register_device(&rdev->ib_dev, "hrn3_bond_%d", &rdev->pdev->dev); +} +#endif // ROCE_BONDING_EN + +static int roce3_ib_register_unbond_device(struct roce3_device *rdev) +{ + return ib_register_device(&rdev->ib_dev, "hrn3_%d", &rdev->pdev->dev); +} + +static int roce3_ib_register_device(struct roce3_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + return roce3_ib_register_bond_device(rdev); +#endif + return roce3_ib_register_unbond_device(rdev); +} + +static int roce3_init_dev(struct roce3_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce3_init_dev_info(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init dev info, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_init_dev_info; + } + + /* Clear gid table before register for a new IB device */ + ret = roce3_alloc_hw_resource(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc hw res, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_alloc_hw_res; + } + + roce3_wait_probe(rdev->lld_dev); + + ret = roce3_ib_register_device(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to reg ibdev, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_reg_dev; + } + + roce3_remove_device_from_list(rdev->lld_dev); + + memcpy(uld_dev_name, rdev->ib_dev.name, ROCE_ULD_DEV_NAME_LEN); + + ret = roce3_register_netdev_event(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to reg netdev, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_memcpy_uld; + } + + ret = roce3_init_dev_file(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to init sysfs, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_init_dev_file; + } + rdev->ib_active = true; + dev_info(rdev->hwdev_hdl, + "[ROCE] %s: RoCE add init all ok, func_id(%u), dev_name(%s)\n", + __func__, rdev->glb_func_id, rdev->ib_dev.name); + + return 0; + +err_init_dev_file: + roce3_unregister_netdev_event(rdev); +err_memcpy_uld: + ib_unregister_device(&rdev->ib_dev); +err_reg_dev: + roce3_dealloc_hw_resource(rdev); +err_alloc_hw_res: + roce3_deinit_dev_info(rdev); +err_init_dev_info: + return ret; +} + +static int roce3_add_rdev_init(struct roce3_device *rdev) +{ + int ret = 0; + + ret = roce3_init_info_get(rdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check\n", __func__); + return ret; + } + + rdev->gid_dev = kzalloc(rdev->rdma_cap.max_gid_per_port * + sizeof(struct net_device *), GFP_KERNEL); + if (rdev->gid_dev == NULL) + return -ENOMEM; + + ret = roce3_rdma_init_resource(rdev->hwdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init rdma resources.\n", __func__); + goto err_init_resource; + } + + pr_info("[ROCE] %s: Initializing(%s)\n", __func__, pci_name(rdev->pdev)); + + rdev->glb_func_id = hinic3_global_func_id(rdev->hwdev); + + rdev->is_vroce = false; + + roce3_init_hw_info(rdev); + + ret = roce3_init_cfg_info(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR]%s: Failed to get roce cfg, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_init_cfg; + } + + return 0; + +err_init_cfg: + roce3_rdma_cleanup_resource(rdev->hwdev); + +err_init_resource: + if (rdev->gid_dev != NULL) + kfree(rdev->gid_dev); + return ret; +} + +static void roce3_add_rdev_unit(struct roce3_device *rdev) +{ + roce3_rdma_cleanup_resource(rdev->hwdev); + if (rdev->gid_dev != NULL) + kfree(rdev->gid_dev); +} + +static __be64 rdma_gen_node_guid(u8 *dev_mac) +{ + u8 guid[8]; + u8 mac_addr[6]; + __be64 node_guid = 0; + + if (dev_mac == NULL) { + pr_err("[ROCE, ERR]%s: Dev_mac is null\n", __func__); + return RDMA_INVALID_GUID; + } + + memcpy((void *)&mac_addr[0], (void *)dev_mac, sizeof(mac_addr)); +/* + * 0 is mac & guid array idx + * 1 is mac & guid array idx + * 2 is mac & guid array idx + * 3 is guid array idx + * 4 is guid array idx + * 5 is guid array idx, 3 is mac array idx + * 6 is guid array idx, 4 is mac array idx + * 7 is guid array idx, 5 is mac array idx + */ + guid[0] = mac_addr[0] ^ DEV_ADDR_FIRST_BYTE_VAL_MASK; + guid[1] = mac_addr[1]; + guid[2] = mac_addr[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac_addr[3]; + guid[6] = mac_addr[4]; + guid[7] = mac_addr[5]; + + /* node_guid is calculated by guid. */ + node_guid = ((u64)guid[0] << 56) | // 0 is guid array idx, 56 is guid offset + ((u64)guid[1] << 48) | // 1 is guid array idx, 48 is guid offset + ((u64)guid[2] << 40) | // 2 is guid array idx, 40 is guid offset + ((u64)guid[3] << 32) | // 3 is guid array idx, 32 is guid offset + ((u64)guid[4] << 24) | // 4 is guid array idx, 24 is guid offset + ((u64)guid[5] << 16) | // 5 is guid array idx, 16 is guid offset + ((u64)guid[6] << 8) | // 6 is guid array idx, 8 is guid offset + (u64)guid[7]; // 7 is guid array idx + + return (__be64)cpu_to_be64(node_guid); +} + +static __be64 roce3_rdma_init_guid(void *hwdev, struct net_device *netdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (netdev == NULL)) { + pr_err("[ROCE, ERR]%s: Hwdev or netdev is null\n", __func__); + return ~0ULL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("[ROCE, ERR]%s: Comp_priv is null\n", __func__); + return ~0ULL; + } + + comp_priv->rdma_comp_res.node_guid = rdma_gen_node_guid((u8 *)netdev->dev_addr); + + return comp_priv->rdma_comp_res.node_guid; +} + +static const struct ib_device_ops dev_ops = { + .owner = THIS_MODULE, + .uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION, + + .create_qp = roce3_create_qp, + .modify_qp = roce3_modify_qp, + .query_qp = roce3_query_qp, + .destroy_qp = roce3_destroy_qp, + .post_send = roce3_post_send, + .post_recv = roce3_post_recv, + .create_cq = roce3_create_cq, + .modify_cq = roce3_modify_cq, + .resize_cq = roce3_resize_cq, + .destroy_cq = roce3_destroy_cq, + .poll_cq = roce3_poll_cq, + .req_notify_cq = roce3_arm_cq, + .create_srq = roce3_create_srq, + .modify_srq = roce3_modify_srq, + .query_srq = roce3_query_srq, + .destroy_srq = roce3_destroy_srq, + .post_srq_recv = roce3_post_srq_recv, + .map_mr_sg = roce3_map_kernel_frmr_sg, + .get_dma_mr = roce3_get_dma_mr, + .reg_user_mr = roce3_reg_user_mr, + .dereg_mr = roce3_dereg_mr, + .alloc_mr = roce3_alloc_mr, + .alloc_mw = roce3_alloc_mw, + .dealloc_mw = roce3_dealloc_mw, + .query_device = roce3_query_device, + .query_port = roce3_query_port, + .get_link_layer = roce3_port_link_layer, + .query_gid = roce3_query_gid, + .add_gid = roce3_ib_add_gid, + .del_gid = roce3_ib_del_gid, + .query_pkey = roce3_query_pkey, + .modify_device = roce3_modify_device, + .modify_port = roce3_modify_port, + .alloc_ucontext = roce3_alloc_ucontext, + .dealloc_ucontext = roce3_dealloc_ucontext, + .mmap = roce3_mmap, + .alloc_pd = roce3_alloc_pd, + .dealloc_pd = roce3_dealloc_pd, + .create_ah = roce3_create_ah, + .query_ah = roce3_query_ah, + .destroy_ah = roce3_destroy_ah, + .alloc_xrcd = roce3_alloc_xrcd, + .dealloc_xrcd = roce3_dealloc_xrcd, + .get_port_immutable = roce3_port_immutable, + .get_netdev = roce3_ib_get_netdev, + INIT_RDMA_OBJ_SIZE(ib_ah, roce3_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, roce3_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_pd, roce3_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_srq, roce3_srq, ibsrq), + INIT_RDMA_OBJ_SIZE(ib_ucontext, roce3_ucontext, ibucontext), + INIT_RDMA_OBJ_SIZE(ib_xrcd, roce3_xrcd, ibxrcd), + INIT_RDMA_OBJ_SIZE(ib_mw, roce3_mw, ibmw), +}; + +static void roce3_add_init(struct roce3_device *rdev) +{ + struct ib_device *ib_dev = &rdev->ib_dev; + + ib_dev->local_dma_lkey = rdev->rdma_cap.reserved_lkey; + ib_dev->phys_port_cnt = (u8)rdev->rdma_cap.num_ports; + ib_dev->num_comp_vectors = + (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ? + (int)rdev->rdma_cap.num_comp_vectors : MAX_CEQ_NEED; + ib_dev->node_type = RDMA_NODE_IB_CA; + ib_dev->node_guid = roce3_rdma_init_guid(rdev->hwdev, rdev->ndev); + ib_dev->dma_device = &rdev->pdev->dev; + ib_dev->dev.parent = ib_dev->dma_device; + strscpy(ib_dev->node_desc, "hrn3", sizeof("hrn3")); + + rdev->ib_dev.uverbs_cmd_mask = ROCE_UVERBS_CMD_MASK; + ib_set_device_ops(ib_dev, &dev_ops); + roce3_init_dev_ext_handlers(rdev); +} + +static void roce3_mod_param_parse(struct roce3_device *rdev) +{ + rdev->try_times = g_loop_times; + +#ifdef ROCE_BONDING_EN + if (g_bond_name != NULL) { + rdev->want_bond_slave_cnt = SDI_BOND_SUPPORT_ROCE_FUNC_CNT; + rdev->want_bond_slave_bits[0] = SDI_BOND_SUPPORT_ROCE_FUNC_BIT; + rdev->want_bond_slave_bits[1] = 0; + rdev->sdi_bond_name = g_bond_name; + return; + } + rdev->want_bond_slave_cnt = g_want_bond_slave_cnt; + rdev->want_bond_slave_bits[0] = g_want_bond0_slave_bits; + rdev->want_bond_slave_bits[1] = 0; + rdev->sdi_bond_name = NULL; +#endif + +} + +static void *roce3_get_ppf_lld_dev(struct roce3_device *rdev) +{ + struct hinic3_lld_dev *ppf_lld_dev = NULL; + + ppf_lld_dev = hinic3_get_ppf_lld_dev_unsafe(rdev->lld_dev); + if (!ppf_lld_dev) { + pr_err("[ROCE, ERR] %s: Failed to get ppf lld_dev\n", __func__); + return ERR_PTR(-EINVAL); + } + + return ppf_lld_dev->hwdev; +} + +static int roce3_rdev_init(struct roce3_device *rdev) +{ + int ret; + void *ppf_hwdev = NULL; + + if (!hinic3_is_vm_slave_host(rdev->hwdev)) { + if ((hinic3_func_type(rdev->hwdev) == TYPE_VF) && + (g_ppf_stateful_init == false) && (g_vf_stateful_num == 0)) { + ppf_hwdev = roce3_get_ppf_lld_dev(rdev); + ret = hinic3_stateful_init(ppf_hwdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init ppf stateful resource\n", + __func__); + return ret; + } + } + + if (hinic3_func_type(rdev->hwdev) == TYPE_PPF) + g_ppf_stateful_init = true; + } + + // BM:When the device is PPF, stateful_init is performed only when g_vf_stateful_num is 0. + if (hinic3_is_vm_slave_host(rdev->hwdev) || + (hinic3_func_type(rdev->hwdev) != TYPE_PPF) || !g_vf_stateful_num) { + ret = hinic3_stateful_init(rdev->hwdev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init stateful resource\n", __func__); + goto err_stateful_init; + } + } + + ret = roce3_add_rdev_init(rdev); + if (ret != 0) + goto err_rdev_init; + + /*lint -e708*/ + spin_lock_init(&rdev->node_desc_lock); + /*lint +e708*/ + + mutex_init(&rdev->cap_mask_mutex); + INIT_LIST_HEAD(&rdev->mac_vlan_list_head); + mutex_init(&rdev->mac_vlan_mutex); + /*lint -e708*/ + spin_lock_init(&rdev->reset_flow_resource_lock); + /*lint +e708*/ + INIT_LIST_HEAD(&rdev->qp_list); + roce3_mod_param_parse(rdev); + + roce3_fix_ibdev_name(rdev); + + if (hinic3_func_type(rdev->hwdev) == TYPE_VF) + g_vf_stateful_num++; + + return 0; + +err_rdev_init: + hinic3_stateful_deinit(rdev->hwdev); +err_stateful_init: + if ((g_vf_stateful_num == 0) && (g_ppf_stateful_init == false)) + hinic3_stateful_deinit(ppf_hwdev); + return ret; +} + +static void roce3_stateful_unit(struct roce3_device *rdev) +{ + void *ppf_hwdev = NULL; + + if (!hinic3_is_vm_slave_host(rdev->hwdev)) { + if (hinic3_func_type(rdev->hwdev) == TYPE_VF) { + hinic3_stateful_deinit(rdev->hwdev); + g_vf_stateful_num--; + // Delete the last VF when no PF is added + if ((g_vf_stateful_num == 0) && (g_ppf_stateful_init == false)) { + ppf_hwdev = roce3_get_ppf_lld_dev(rdev); + hinic3_stateful_deinit(ppf_hwdev); + } + } else { + if (g_vf_stateful_num == 0) + hinic3_stateful_deinit(rdev->hwdev); + } + } else { + hinic3_stateful_deinit(rdev->hwdev); + } +} + +#ifdef ROCE_NETLINK_EN +static void roce3_adapt_unit(struct roce3_device *rdev) +{ + struct hiroce_netlink_dev *adp_dev; + int offset = 0; + + offset = get_instance_of_func_id(rdev->glb_func_id); + if (offset >= MAX_FUNCTION_NUM) { + pr_err("[ROCE, ERR] %s: offset is over size\n", __func__); + return; + } + adp_dev = hiroce_get_adp(); + mutex_lock(&adp_dev->mutex_dev); + adp_dev->used_dev_num--; + + if (adp_dev->used_dev_num <= 0 && adp_dev->netlink) + kfree(adp_dev->netlink); + mutex_unlock(&adp_dev->mutex_dev); +} +#endif + +static void roce3_rdev_unit(struct roce3_device *rdev) +{ + /* FLR by MPU when hotplug, don't need deinit anymore */ + if (hinic3_func_type(rdev->hwdev) == TYPE_PPF) + g_ppf_stateful_init = false; + + if (roce3_hca_is_present(rdev) != 0) + roce3_stateful_unit(rdev); + +#ifdef ROCE_NETLINK_EN + roce3_netlink_unit(); + roce3_adapt_unit(rdev); +#endif + roce3_add_rdev_unit(rdev); +} + +#ifdef ROCE_NETLINK_EN +static int roce3_adapt_init(struct roce3_device *rdev) +{ + int offset = 0; + struct hiroce_netlink_dev *adp_dev = NULL; + + offset = get_instance_of_func_id(rdev->glb_func_id); + if (offset >= ROCE_MAX_FUNCTION) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get offset , func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + adp_dev = hiroce_get_adp(); + mutex_lock(&adp_dev->mutex_dev); + if (adp_dev->used_dev_num == 0 && adp_dev->netlink == NULL) { + adp_dev->netlink = kzalloc(sizeof(struct netlink_devk_dev), GFP_KERNEL); + if (adp_dev->netlink == NULL) { + mutex_unlock(&adp_dev->mutex_dev); + return -EINVAL; + } + } + + adp_dev->used_dev_num++; + mutex_unlock(&adp_dev->mutex_dev); + adp_dev->netlink->rdev[offset] = rdev; + + return 0; +} +#endif + +static int roce3_add_do_init(struct roce3_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce3_rdev_init(rdev); + if (ret != 0) + goto err_init_rdev; + + ret = roce3_register_template(rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to register cqm_service, func_id(%u) ret(%d)\n", + __func__, rdev->glb_func_id, ret); + goto err_cqm_register; + } + + ret = hinic3_alloc_db_addr(rdev->hwdev, &rdev->kernel_db_map, &rdev->kernel_dwqe_map); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc db or dwqe, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_alloc_db; + } + + roce3_add_init(rdev); + + ret = roce3_alloc_ceq(rdev); + if (ret < 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR]: Failed to alloc ceqs, func_id(%u)\n", + rdev->glb_func_id); + goto err_alloc_ceq; + } + + ret = roce3_init_dev(rdev, uld_dev_name); + if (ret != 0) + goto err_init_dev; + +#ifdef ROCE_NETLINK_EN + ret = roce3_adapt_init(rdev); + if (ret != 0) + goto err_init_adp; + + roce3_netlink_init(); + + return 0; +err_init_adp: + roce3_adapt_unit(rdev); +#else + return 0; +#endif + +err_init_dev: + roce3_free_ceq(rdev); + +err_alloc_ceq: + hinic3_free_db_addr(rdev->hwdev, rdev->kernel_db_map, rdev->kernel_dwqe_map); + +err_alloc_db: + cqm_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + +err_cqm_register: + roce3_rdev_unit(rdev); + +err_init_rdev: + roce3_remove_device_from_list(rdev->lld_dev); + return ret; +} + +static bool is_device_v100(const struct hinic3_lld_dev *lld_dev) +{ + struct pci_dev *pdev = lld_dev->pdev; + unsigned short ssdid = pdev->subsystem_device; + + return (ssdid == HINIC3_DEV_SSID_2X25G) || (ssdid == HINIC3_DEV_SSID_4X25G) || + (ssdid == HINIC3_DEV_SSID_2X100G); +} + +static int roce3_add_check(const struct hinic3_lld_dev *lld_dev) +{ + int ret = 0; + u16 func_id; + u8 enable_roce = false; + bool is_slave_func = false; + + if (lld_dev == NULL) { + pr_err("[ROCE, ERR] %s: Lld_dev is null\n", __func__); + return (-EINVAL); + } + + if (!is_device_v100(lld_dev)) { + pr_err("[ROCE, ERR] %s: ssdid 0x%x is NOT standard Card\n", + __func__, lld_dev->pdev->subsystem_device); + return -ENXIO; + } + + ret = hinic3_is_slave_func(lld_dev->hwdev, &is_slave_func); + if (ret != 0) + pr_err("[ROCE, ERR] %s: Failed to get slave_func.\n", __func__); + if (!is_slave_func) + return 0; + + func_id = hinic3_global_func_id(lld_dev->hwdev); + ret = hinic3_get_func_vroce_enable(lld_dev->hwdev, func_id, &enable_roce); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get roce state.\n", __func__); + return ret; + } + if (!enable_roce) { + pr_warn("[ROCE] %s: %s RoCE dev is not enable, func: %u\n", __func__, + pci_name(lld_dev->pdev), func_id); + return (-EPERM); + } + + return 0; +} + +static void roce3_remove_device_from_list(struct hinic3_lld_dev *lld_dev) +{ + struct roce3_func_info *info; + struct roce3_func_info *tmp; + + if (list_empty(&g_roce_device_list)) + return; + + list_for_each_entry_safe(info, tmp, &g_roce_device_list, node) { + if (info->func_id == hinic3_global_func_id(lld_dev->hwdev)) { + list_del(&info->node); + kfree(info); + } + } + + wake_up(&g_roce_probe_queue); +} + +static int roce3_add_device_to_list(struct hinic3_lld_dev *lld_dev) +{ + struct roce3_func_info *info = kzalloc( + sizeof(struct roce3_func_info), GFP_ATOMIC); + + if (info == NULL) { + pr_err("[ROCE, ERR] %s: no memory\n", __func__); + return -ENOMEM; + } + + info->func_id = hinic3_global_func_id(lld_dev->hwdev); + list_add_tail(&info->node, &g_roce_device_list); + + return 0; +} + +static void roce3_wait_probe(struct hinic3_lld_dev *lld_dev) +{ + struct roce3_func_info *info = NULL; + bool wait_flg = false; + DECLARE_WAITQUEUE(wait_queue, current); + + add_wait_queue(&g_roce_probe_queue, &wait_queue); + pr_info("[ROCE] %s func %u start to wait\n", __func__, + hinic3_global_func_id(lld_dev->hwdev)); + + do { + might_sleep(); + info = list_first_entry(&g_roce_device_list, struct roce3_func_info, node); + wait_flg = (info->func_id == hinic3_global_func_id(lld_dev->hwdev)) ? true : false; + if (!wait_flg) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + if (signal_pending(current)) { /* if alarmed by signal */ + goto out; + } + } + } while (!wait_flg); + + set_current_state(TASK_RUNNING); + remove_wait_queue(&g_roce_probe_queue, &wait_queue); + + pr_info("[ROCE] %s func %u wait finished\n", + __func__, hinic3_global_func_id(lld_dev->hwdev)); + return; +out: + pr_info("[ROCE] %s func %u wait fail\n", __func__, hinic3_global_func_id(lld_dev->hwdev)); + remove_wait_queue(&g_roce_probe_queue, &wait_queue); + set_current_state(TASK_RUNNING); +} + +/* + **************************************************************************** + Prototype : roce3_add + Description : roce3_add + Input : struct hinic_lld_dev *lld_dev + void **uld_dev + char *uld_dev_name + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_add(struct hinic3_lld_dev *lld_dev, void **uld_dev, char *uld_dev_name) +{ + struct roce3_device *rdev = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + + ret = roce3_add_check(lld_dev); + if (ret != 0) + goto err_check; + + pr_info("[ROCE] %s: Initializing pci(%s)\n", __func__, pci_name(lld_dev->pdev)); + + /* return 0 if the rdev don't support ROCE, make sure it probe success */ + if (!hinic3_support_roce(lld_dev->hwdev, &rdma_cap)) { + pr_err("[ROCE, ERR] %s: %s Not support RoCE, func: %u\n", + __func__, pci_name(lld_dev->pdev), hinic3_global_func_id(lld_dev->hwdev)); + goto err_check; + } + + /* make sure roce device probe in order */ + ret = roce3_add_device_to_list(lld_dev); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add device to list, ret: %d, func: %u\n", + __func__, ret, hinic3_global_func_id(lld_dev->hwdev)); + return ret; + } + + roce3_rdma_cap_ext(&rdma_cap); + + rdev = roce3_rdev_alloc(lld_dev, uld_dev, &rdma_cap); + if (rdev == NULL) { + roce3_remove_device_from_list(lld_dev); + pr_err("[ROCE, ERR] %s: Failed to alloc rdev, func: %u\n", + __func__, hinic3_global_func_id(lld_dev->hwdev)); + return -EINVAL; + } + + ret = roce3_add_do_init(rdev, uld_dev_name); + if (ret != 0) + goto err_do_init; + + return 0; + +err_do_init: + ib_dealloc_device(&rdev->ib_dev); +err_check: + *uld_dev = NULL; + return ret; +} + +static void roce3_do_remove(struct roce3_device *rdev, u16 glb_func_id, const char *dev_name) +{ + roce3_remove_dev_file(rdev); + + roce3_unregister_netdev_event(rdev); + + if (roce3_hca_is_present(rdev) == 0) { + roce3_handle_hotplug_arm_cq(rdev); + roce3_kernel_hotplug_event_trigger(rdev); + } + + ib_unregister_device(&rdev->ib_dev); + pr_info("[ROCE] %s: Unregister IB device ok, func_id(%u), name(%s), pci(%s)\n", + __func__, glb_func_id, dev_name, pci_name(rdev->pdev)); + + roce3_dealloc_hw_resource(rdev); + + roce3_deinit_dev_info(rdev); + +#ifdef ROCE_BONDING_EN + roce3_set_bond_ipsurx_en(true); +#endif + + roce3_clean_vlan_device_mac(rdev); + roce3_clean_real_device_mac(rdev); + + roce3_free_ceq(rdev); + + hinic3_free_db_addr(rdev->hwdev, rdev->kernel_db_map, rdev->kernel_dwqe_map); + + cqm_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + + roce3_del_func_res(rdev); + pr_info("[ROCE] %s: Function level resource clear ok, func_id(%u), name(%s), pci(%s)\n", + __func__, glb_func_id, dev_name, pci_name(rdev->pdev)); + + roce3_do_cache_out(rdev->hwdev, ROCE_CL_TYPE_CQC_SRQC, rdev->glb_func_id); + + roce3_rdev_unit(rdev); + pr_info("[ROCE] %s: RoCE rdev uninit ok, func_id(%u), name(%s)\n", + __func__, glb_func_id, dev_name); + + ib_dealloc_device(&rdev->ib_dev); +} + +static int roce3_remove_check(const struct hinic3_lld_dev *lld_dev, const void *uld_dev) +{ + if ((uld_dev == NULL) || (lld_dev == NULL)) { + pr_err("[ROCE, ERR] %s: input param is null\n", __func__); + return (-EINVAL); + } + + return 0; +} + +static void roce3_remove(struct hinic3_lld_dev *lld_dev, void *uld_dev) +{ + struct roce3_device *rdev = NULL; + char *dev_name = NULL; + u16 glb_func_id; + int ret; + + ret = roce3_remove_check(lld_dev, uld_dev); + if (ret != 0) + return; + + rdev = (struct roce3_device *)uld_dev; + rdev->ib_active = false; + dev_name = rdev->ib_dev.name; + glb_func_id = rdev->glb_func_id; + + if (!hinic3_support_roce(rdev->hwdev, NULL)) { + pr_err("[ROCE, ERR] %s: Not support RoCE\n", __func__); + return; + } + + dev_info(rdev->hwdev_hdl, + "[ROCE] %s: RoCE remove start, func_id(%u), name(%s), pci(%s)\n", __func__, + rdev->glb_func_id, dev_name, pci_name(rdev->pdev)); + + roce3_do_remove(rdev, glb_func_id, dev_name); + + pr_info("[ROCE] %s: RoCE remove end, func_id(%u), name(%s)\n", + __func__, glb_func_id, dev_name); +} + +bool roce3_need_proc_link_event(void *hwdev) +{ + int ret = 0; + u16 func_id; + u8 roce_enable = false; + bool is_slave_func = false; + struct hinic3_hw_bond_infos hw_bond_infos = {0}; + + ret = hinic3_is_slave_func(hwdev, &is_slave_func); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: lld_dev is null\n", __func__); + return true; + } + + if (!is_slave_func) + return true; + + func_id = hinic3_global_func_id(hwdev); + ret = hinic3_get_func_vroce_enable(hwdev, func_id, &roce_enable); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get vroce info\n", __func__); + return true; + } + if (!roce_enable) + return true; + + hw_bond_infos.bond_id = HINIC_OVS_BOND_DEFAULT_ID; + + ret = hinic3_get_hw_bond_infos(hwdev, &hw_bond_infos, HINIC3_CHANNEL_COMM); + if (ret != 0) { + pr_err("[ROCE, ERR] Get chipf bond info failed (%d)\n", ret); + return true; + } + + if (!hw_bond_infos.valid) + return true; + + return false; +} + +bool roce3_need_proc_bond_event(void *hwdev) +{ + return !roce3_need_proc_link_event(hwdev); +} + +static int roce3_proc_bond_status_change(struct roce3_device *rdev, + const struct hinic3_event_info *event) +{ + switch (event->type) { + case EVENT_NIC_BOND_UP: + if (!roce3_need_proc_bond_event(rdev->hwdev)) { + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE don't need proc bond event\n", + __func__); + return -1; + } + if (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) != 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC BOND_UP, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ACTIVE; + + case EVENT_NIC_BOND_DOWN: + if (!roce3_need_proc_bond_event(rdev->hwdev)) { + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE don't need proc bond event\n", + __func__); + return -1; + } + + if (test_and_clear_bit(ROCE3_PORT_EVENT, &rdev->status) == 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC BOND_DOWN, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ERR; + + default: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Nic event unsupported, func_id(%u), event(%hu)\n", + __func__, rdev->glb_func_id, event->type); + return -1; + } +} + +static int roce3_set_nic_event(struct roce3_device *rdev, + const struct hinic3_event_info *event) +{ + switch (event->type) { + case EVENT_NIC_LINK_UP: + if (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) != 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC LINK_UP, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ACTIVE; + + case EVENT_NIC_LINK_DOWN: + if (!roce3_need_proc_link_event(rdev->hwdev)) { + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE don't need proc link event\n", __func__); + return -1; + } + + if (test_and_clear_bit(ROCE3_PORT_EVENT, &rdev->status) == 0) + return -1; + dev_info(rdev->hwdev_hdl, + "[ROCE, WARN] %s: RoCE report NIC LINK_DOWN, func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_EVENT_PORT_ERR; + + case EVENT_NIC_BOND_UP: + case EVENT_NIC_BOND_DOWN: + return roce3_proc_bond_status_change(rdev, event); + + case EVENT_NIC_DCB_STATE_CHANGE: + dev_info(rdev->hwdev_hdl, "[ROCE]%s: DCB state change no longer requires RoCE involvement, func_id(%u)\n", + __func__, rdev->glb_func_id); + return -1; + + default: + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Nic event unsupported, func_id(%u), event(%hu)\n", + __func__, rdev->glb_func_id, event->type); + return -1; + } +} + +static int roce3_set_ib_event(struct roce3_device *rdev, const struct hinic3_event_info *event) +{ + switch (event->service) { + case EVENT_SRV_NIC: + return roce3_set_nic_event(rdev, event); + + case EVENT_SRV_COMM: + return roce3_set_comm_event(rdev, event); + + default: + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: func_id(%u) event from svc(%hu) is not supported, event(%hu)\n", + __func__, rdev->glb_func_id, event->service, event->type); + return -1; + } +} + +void roce3_event(struct hinic3_lld_dev *lld_dev, void *uld_dev, struct hinic3_event_info *event) +{ + struct ib_event ibevent; + struct roce3_device *rdev = NULL; + int type; + int ret = 0; + + roce3_lock_rdev(); + + ret = roce3_get_rdev_by_uld(lld_dev, uld_dev, &rdev, event); + if (ret != 0) { + pr_err("[ROCE] %s: find rdev failed, ret(%d)\n", __func__, ret); + goto err_unlock; + } +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + roce3_handle_bonded_port_state_event(rdev); + goto err_unlock; + } +#endif + type = roce3_set_ib_event(rdev, event); + if (type == -1) + goto err_unlock; + + ibevent.event = (enum ib_event_type)type; + ibevent.device = &rdev->ib_dev; + ibevent.element.port_num = ROCE_DEFAULT_PORT_NUM; + + if (rdev->ib_active) + ib_dispatch_event(&ibevent); + +err_unlock: + roce3_unlock_rdev(); +} + +typedef int (*roce3_adm_func_t)(struct roce3_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +/*lint -e26*/ +static roce3_adm_func_t g_roce3_adm_funcs[COMMON_CMD_VM_COMPAT_TEST] = { + [COMMON_CMD_GET_DRV_VERSION] = roce3_get_drv_version, + +#ifdef __ROCE_DFX__ + [ROCE_CMD_GET_QPC_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_QPC_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_CQC_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_CQC_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_SRQC_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_SRQC_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_MPT_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_MPT_FROM_HOST] = roce3_adm_dfx_query, + [ROCE_CMD_GET_GID_FROM_CACHE] = roce3_adm_dfx_query, + [ROCE_CMD_GET_QPC_CQC_PI_CI] = roce3_adm_dfx_query, + [ROCE_CMD_GET_QP_COUNT] = roce3_adm_dfx_query, + [ROCE_CMD_GET_DEV_ALGO] = roce3_adm_dfx_query, +#ifdef ROCE_PKT_CAP_EN + [ROCE_CMD_START_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_STOP_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_QUERY_CAP_INFO] = roce3_adm_dfx_capture, + [ROCE_CMD_ENABLE_QP_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_DISABLE_QP_CAP_PACKET] = roce3_adm_dfx_capture, + [ROCE_CMD_QUERY_QP_CAP_INFO] = roce3_adm_dfx_capture, +#endif /* ROCE_PKT_CAP_EN */ +#endif /* __ROCE_DFX__ */ + + [ROCE_CMD_ENABLE_BW_CTRL] = roce3_adm_dfx_bw_ctrl, + [ROCE_CMD_DISABLE_BW_CTRL] = roce3_adm_dfx_bw_ctrl, + [ROCE_CMD_CHANGE_BW_CTRL_PARAM] = roce3_adm_dfx_bw_ctrl, + [ROCE_CMD_QUERY_BW_CTRL_PARAM] = roce3_adm_dfx_bw_ctrl, +}; +/*lint +e26*/ + +static int roce3_adm(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct roce3_device *rdev = (struct roce3_device *)uld_dev; + roce3_adm_func_t roce3_adm_func; + + if (uld_dev == NULL || buf_in == NULL || buf_out == NULL || out_size == NULL) { + pr_err("[ROCE] %s: Input params is null\n", __func__); + return -EINVAL; + } + + if (!hinic3_support_roce(rdev->hwdev, NULL)) { + pr_err("[ROCE, ERR] %s: %s Not support RoCE\n", __func__, pci_name(rdev->pdev)); + return -EINVAL; + } + + rdev = (struct roce3_device *)uld_dev; + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (cmd >= COMMON_CMD_VM_COMPAT_TEST) { + dev_err(rdev->hwdev_hdl, "Not support this type(%u)", cmd); + return -EINVAL; + } + + roce3_adm_func = g_roce3_adm_funcs[cmd]; + if (roce3_adm_func == NULL) { + dev_err(rdev->hwdev_hdl, "Not support this type(%u)", cmd); + return -EINVAL; + } + + return roce3_adm_func(rdev, buf_in, in_size, buf_out, out_size); +} + +struct hinic3_uld_info roce3_info = { + .probe = roce3_add, + .remove = roce3_remove, + .suspend = NULL, + .resume = NULL, + .event = roce3_event, + .ioctl = roce3_adm, +}; + +struct hinic3_uld_info *roce3_info_get(void) +{ + return &roce3_info; +} + +/* + **************************************************************************** + Prototype : roce3_service_init + Description : + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** +*/ +static int __init roce3_service_init(void) +{ + int ret; + + INIT_LIST_HEAD(&g_roce_device_list); + init_waitqueue_head(&g_roce_probe_queue); + roce3_service_init_pre(); +#ifdef ROCE_NETLINK_EN + /* init mutex */ + mutex_init(&hiroce_get_adp()->mutex_dev); +#endif + ret = hinic3_register_uld(SERVICE_T_ROCE, roce3_info_get()); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to register uld. ret(%d)\n", __func__, ret); + return ret; + } + + roce3_service_init_ext(); +#ifdef ROCE_BONDING_EN + ret = roce3_bond_init(); +#endif + pr_info("[ROCE] %s: Register roce service done, ret(%d).\n", __func__, ret); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_service_exit + Description : roce service disable + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** +*/ +static void __exit roce3_service_exit(void) +{ +#ifdef ROCE_BONDING_EN + roce3_bond_pre_exit(); +#endif + hinic3_unregister_uld(SERVICE_T_ROCE); + +#ifdef ROCE_BONDING_EN + pr_info("[ROCE] %s: Bond remove all.\n", __func__); + roce3_bond_exit(); +#endif + + pr_info("[ROCE] %s: Unregister roce service successful\n", __func__); +} + +bool roce3_is_roceaa(u8 scence_id) +{ + if ((scence_id == SCENES_ID_STORAGE_ROCEAA_2x100) || + (scence_id == SCENES_ID_STORAGE_ROCEAA_4x25)) + return true; + + return false; +} + +module_init(roce3_service_init); +module_exit(roce3_service_exit); diff --git a/drivers/infiniband/hw/hiroce3/roce_mix.c b/drivers/infiniband/hw/hiroce3/roce_mix.c new file mode 100644 index 000000000..ad907547a --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_mix.c @@ -0,0 +1,1194 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <rdma/ib_mad.h> +#include <rdma/ib_verbs.h> +#include <linux/pci.h> + +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> + +#include "hinic3_crm.h" +#include "hinic3_srv_nic.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_pd.h" +#include "roce_qp.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" + +#include "roce_mix.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +struct net_device *roce3_ib_get_netdev(struct ib_device *ibdev, u8 port_num) +{ + struct roce3_device *rdev = NULL; + struct net_device *netdev = NULL; + + if (ibdev == NULL) { + pr_err("[ROCE] %s: Ibdev is null\n", __func__); + return NULL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + pr_err("[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return NULL; + } + +#ifdef ROCE_BONDING_EN + netdev = roce3_bond_get_netdev(rdev); + if (netdev != NULL) + return netdev; +#endif + + netdev = rdev->ndev; + + dev_hold(netdev); + + return netdev; +} + +static void roce3_parse_fw_version(struct roce3_device *rdev, u64 *fw_ver) +{ + int ret; + int i = 0; + struct hinic3_fw_version fw_version; + char *fw_str = (char *)fw_version.microcode_ver; + char *fw_temp = NULL; + u64 fw_verion[ROCE_FW_VERSION_LEN] = {0}; + + ret = hinic3_get_fw_version(rdev->hwdev, &fw_version, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + pr_warn("[ROCE] %s: get fw version failed\n", __func__); + *fw_ver = ROCE_FW_VER; + return; + } + pr_info("[ROCE] %s: fw ver:%s - %s - %s\n", __func__, fw_version.boot_ver, + fw_version.mgmt_ver, fw_version.microcode_ver); + + while (((fw_temp = strsep(&fw_str, ".")) != NULL) && (i < ROCE_FW_VERSION_LEN)) { + ret = kstrtou64(fw_temp, 10, &fw_verion[i]); + if (ret != 0) { + pr_warn("[ROCE] %s: parse fw version failed\n", __func__); + *fw_ver = ROCE_FW_VER; + return; + } + + i++; + } +/* + * 0 is fw_version array idx, 32 is offset + * 1 is fw_version array idx, 16 is offset + * 2 is fw_version array idx, 8 is offset + * 3 is fw_version array idx + */ + *fw_ver = (((fw_verion[0] & 0xffffffff) << 32) | + ((fw_verion[1] & 0xffff) << 16) | + ((fw_verion[2] & 0xff) << 8) | + (fw_verion[3] & 0xff)); +} + +static void roce3_set_local_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if (((rdma_cap->flags & RDMA_BMME_FLAG_LOCAL_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_REMOTE_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_FAST_REG_WR) != 0)) { + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_MGT_EXTENSIONS; + } +} + +static void roce3_set_bmme_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if ((rdma_cap->flags & RDMA_BMME_FLAG_TYPE_2_WIN) != 0) { + if ((rdma_cap->flags & RDMA_BMME_FLAG_WIN_TYPE_2B) != 0) + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2B; + else + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2A; + } +} + +static void roce3_query_device_props_set(struct roce3_device *rdev, + struct rdma_service_cap *rdma_cap, struct ib_device_attr *props) +{ + props->vendor_id = rdev->pdev->vendor; + props->vendor_part_id = rdev->pdev->device; + roce3_parse_fw_version(rdev, &props->fw_ver); + props->hw_ver = ROCE_HW_VER; + + /* sys_image_guid equal GID */ + props->sys_image_guid = rdev->ib_dev.node_guid; + + props->max_mr_size = ~0ULL; + props->page_size_cap = rdma_cap->page_size_cap; + props->max_qp = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_qps - rdma_cap->reserved_qps); + props->max_qp_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes; + /* + * 4.19 ofed will return the smaller of sq/rq sge num to user space. + * 4.17 We use max_sge to only represent max sq sge num, max_rq_sge is a fixed macro of 16. + */ + props->max_send_sge = rdma_cap->max_sq_sg; + props->max_recv_sge = rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg; + props->max_cq = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs - rdma_cap->reserved_cqs); + props->max_cqe = (int)rdma_cap->max_cqes; + + if ((rdev->board_info.port_num == ROCE_PORT_NUM_2) && + (rdev->board_info.port_speed == ROCE_25G_PORT_SPEED)) { + // 2 smf for 64B cache + props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - + rdma_cap->reserved_mrws) / MEND_CAP_DEVIDE; + props->max_srq = + (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - + rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs) / + MEND_CAP_DEVIDE; + } else { + props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - + rdma_cap->reserved_mrws); + props->max_srq = + (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - + rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs); + } + + props->max_mw = props->max_mr; + props->max_pd = (int)(rdma_cap->num_pds - rdma_cap->reserved_pds); + props->max_qp_rd_atom = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma; + props->max_qp_init_rd_atom = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma; + props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; + + props->max_srq_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_wqes; + props->max_srq_sge = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_sge; + props->max_fast_reg_page_list_len = rdma_cap->max_frpl_len; + props->local_ca_ack_delay = (u8)rdma_cap->local_ca_ack_delay; + props->atomic_cap = ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_ATOMIC) != 0) ? + IB_ATOMIC_HCA : IB_ATOMIC_NONE; + props->masked_atomic_cap = props->atomic_cap; + props->max_pkeys = (u16)rdma_cap->max_pkeys; + props->max_ah = INT_MAX; +} +/* + **************************************************************************** + Prototype : roce3_query_device + Description : query device attribute + Input : struct ib_device *ibdev + struct ib_device_attr *props + struct ib_udata *uhw + Output : struct ib_device_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw) +{ + struct roce3_device *rdev = NULL; + struct rdma_service_cap *rdma_cap = NULL; + + if ((ibdev == NULL) || (props == NULL)) { + pr_err("[ROCE] %s: Ibdev or props is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + memset(props, 0, sizeof(*props)); + + rdma_cap = &rdev->rdma_cap; + + props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN; + + /* APM */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_APM) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_AUTO_PATH_MIG; + + /* rsvd_lKey */ + if ((rdma_cap->flags & RDMA_BMME_FLAG_RESERVED_LKEY) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_LOCAL_DMA_LKEY; + + roce3_set_local_cap_flag(rdma_cap, props); + +#ifndef ROCE_COMPUTE + /* support XRC */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_XRC) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_XRC; +#endif + + /* support MW */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_MEM_WINDOW) != 0) + props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_WINDOW; + + roce3_set_bmme_cap_flag(rdma_cap, props); + + roce3_query_device_props_set(rdev, rdma_cap, props); + return 0; +} + +static void eth_link_get_speed(struct ib_port_attr *props, enum mag_cmd_port_speed speed) +{ + switch (speed) { + /* 10G <==> 1X x 10G */ + case PORT_SPEED_10GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 25G <==> 1X x 25G */ + case PORT_SPEED_25GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 40G <==> 4X x 10G */ + case PORT_SPEED_40GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 100G <==> 4X x 25G */ + case PORT_SPEED_100GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_EDR; + break; + + default: + props->active_width = 0; + props->active_speed = 0; + break; + } +} + +static void roce3_set_ib_port_attr(struct ib_port_attr *props, struct roce3_device *rdev) +{ + props->port_cap_flags = IB_PORT_CM_SUP; + props->gid_tbl_len = (int)rdev->rdma_cap.max_gid_per_port; + props->max_msg_sz = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz; + props->pkey_tbl_len = (u16)rdev->rdma_cap.max_pkeys; + props->max_mtu = IB_MTU_4096; + props->state = IB_PORT_DOWN; + props->phys_state = ROCE_PORT_PHYS_STATE_DISABLED; + props->active_mtu = IB_MTU_256; +} +#ifdef OFED_MLNX_5_8 +static void eth_link_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) +#else +static void eth_link_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) +#endif +{ + struct roce3_device *rdev = NULL; + struct net_device *netdev = NULL; +#ifdef ROCE_BONDING_EN + struct net_device *upper = NULL; +#endif + enum ib_mtu mtu; + enum mag_cmd_port_speed speed = PORT_SPEED_10GB; + int ret = 0; + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u), dev_name(%s).\n", + __func__, rdev->glb_func_id, ibdev->name); + return; + } + + roce3_set_ib_port_attr(props, rdev); + + ret = hinic3_get_speed(rdev->hwdev, &speed, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get speed, func_id(%d)\n", + __func__, rdev->glb_func_id); + props->active_width = 0; + props->active_speed = 0; + return; + } + + eth_link_get_speed(props, speed); + + netdev = roce3_ib_get_netdev(ibdev, ROCE_DEFAULT_PORT_NUM); + if (netdev == NULL) + return; + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + rcu_read_lock(); + upper = netdev_master_upper_dev_get_rcu(netdev); + if (upper != NULL) { + dev_put(netdev); + netdev = upper; + dev_hold(netdev); + } + rcu_read_unlock(); + } +#endif + + if (netif_running(netdev) && netif_carrier_ok(netdev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = ROCE_PORT_PHYS_STATE_LINKUP; + } + + mtu = (enum ib_mtu)iboe_get_mtu((int)netdev->mtu); + + dev_put(netdev); + + props->active_mtu = ROCE_MIN(props->max_mtu, mtu); +} + +/* + **************************************************************************** + Prototype : roce3_query_port + Description : query port attribute + Input : struct ib_device *ibdev + u8 port + struct ib_port_attr *props + Output : struct ib_port_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) +{ + if ((ibdev == NULL) || (props == NULL)) { + pr_err("[ROCE] %s: Ibdev or props is null\n", __func__); + return -EINVAL; + } + + memset(props, 0, sizeof(*props)); + + eth_link_query_port(ibdev, port, props); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_query_gid + Description : query gid + Input : struct ib_device *ibdev + u8 port + int index + union ib_gid *gid + Output : union ib_gid *gid + + 1.Date : 2015/5/8 + Modification : Created function + + 2.Date : 2015/6/8 + Modification : Modify function + +**************************************************************************** +*/ +int roce3_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + struct rdma_gid_entry gid_entry; + + if ((ibdev == NULL) || (gid == NULL)) { + pr_err("[ROCE] %s: Ibdev or gid is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_rdma_get_gid(rdev->hwdev, (u32)port, (u32)index, &gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get gid, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + memcpy((void *)gid->raw, (void *)gid_entry.raw, sizeof(*gid)); + + // 按照OFED的gid生成方式转换GID, 仅IPv4场景需要转换 + if (gid_entry.dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + // 未add的gid直接返回，不需要转换 + if ((gid->global.subnet_prefix == 0) && (gid->global.interface_id == 0)) + return 0; + ipv6_addr_set_v4mapped(*((u32 *)(void *)gid + ROCE_GID_IP_IDX), + (struct in6_addr *)gid); + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_query_pkey + Description : query pkey + Input : struct ib_device *ibdev + u8 port + u16 index + u16 *pkey + Output : u16 *pkey + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +{ + struct roce3_device *rdev = NULL; + + if ((ibdev == NULL) || (pkey == NULL)) { + pr_err("[ROCE] %s: Ibdev or pkey is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (pkey == NULL) { + pr_err("[ROCE] %s: Pkey is null\n", __func__); + return -EINVAL; + } + + *pkey = 0xffff; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_modify_device + Description : modify device attribute + Input : struct ib_device *ibdev + int mask + struct ib_device_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props) +{ + unsigned long flags = 0; + struct roce3_device *rdev = NULL; + + if ((ibdev == NULL) || (props == NULL)) { + pr_err("[ROCE] %s: Ibdev or props is null\n", __func__); + return -EINVAL; + } + + if (((unsigned int)mask & ~IB_DEVICE_MODIFY_NODE_DESC) != 0) { + pr_err("[ROCE] %s: Not supported to modify node description\n", __func__); + return -EOPNOTSUPP; + } + + if ((((u32)mask) & IB_DEVICE_MODIFY_NODE_DESC) == 0) { + pr_info("[ROCE] %s: No need to modify node description\n", __func__); + return 0; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + spin_lock_irqsave(&rdev->node_desc_lock, flags); + memcpy((void *)ibdev->node_desc, (void *)props->node_desc, IB_DEVICE_NODE_DESC_MAX); + spin_unlock_irqrestore(&rdev->node_desc_lock, flags); + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_modify_port + Description : modify port attribute + Input : struct ib_device *ibdev + u8 port + int mask + struct ib_port_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props) +{ + int ret = 0; + struct ib_port_attr attr; + struct roce3_device *rdev = NULL; + + if (ibdev == NULL) { + pr_err("[ROCE] %s: Ibdev is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + memset(&attr, 0, sizeof(struct ib_port_attr)); + + mutex_lock(&rdev->cap_mask_mutex); + + ret = roce3_query_port(ibdev, port, &attr); + if (ret != 0) + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to query port, func_id(%d)\n", + __func__, rdev->glb_func_id); + + mutex_unlock(&rdev->cap_mask_mutex); + + return ret; +} + +static void roce3_alloc_ucontext_set(struct roce3_device *rdev, + struct roce3_alloc_ucontext_resp *resp) +{ + struct rdma_service_cap *rdma_cap = NULL; + + rdma_cap = &rdev->rdma_cap; + + resp->num_qps = rdma_cap->dev_rdma_cap.roce_own_cap.max_qps; + resp->num_xsrqs = rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs; + resp->cqe_size = rdma_cap->cqe_size; + resp->wqebb_size = rdma_cap->wqebb_size; + resp->dwqe_size = rdma_cap->direct_wqe_size; + resp->max_msg_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_msg_sz; + resp->max_comp_vector = rdma_cap->num_comp_vectors; + resp->max_inline_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + + resp->storage_aa_en = roce3_is_roceaa(rdev->cfg_info.scence_id); + resp->phy_port = rdev->hw_info.phy_port; + resp->srq_container_en = rdev->cfg_info.srq_container_en; + resp->srq_container_mode = rdev->cfg_info.srq_container_mode; + resp->xrc_srq_container_mode = rdev->cfg_info.xrc_srq_container_mode; + resp->warn_th = rdev->cfg_info.warn_th; + + roce3_resp_set_ext(rdev, resp); +} + +static int roce3_alloc_ucontext_pre_check(struct ib_device *ibdev, const struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + + if ((ibdev == NULL) || (udata == NULL)) { + pr_err("[ROCE] %s: Ibdev or udata is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if (!rdev->ib_active) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Device is abnormal, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EAGAIN; + } + + return 0; +} + +static int roce3_alloc_ucontext_return(struct roce3_device *rdev, struct ib_udata *udata, + struct roce3_ucontext *context, struct roce3_alloc_ucontext_resp *resp) +{ + int ret; + + resp->db_offset = context->db_dma_addr & ((1 << PAGE_SHIFT) - 1); + resp->dwqe_offset = context->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1); + + if (context->dwqe_dma_addr == 0) + resp->dwqe_size = 0; + + roce3_ucontext_set_ext(rdev, context); + + INIT_LIST_HEAD(&context->db_page_list); + mutex_init(&context->db_page_mutex); + + /* Copy data to user space */ + ret = ib_copy_to_udata_ext(udata, resp); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce3_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) +{ + int ret; + struct roce3_ucontext *context = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + struct roce3_device *rdev = to_roce3_dev(ibucontext->device); + struct roce3_alloc_ucontext_resp *resp = NULL; + + ret = roce3_alloc_ucontext_pre_check(ibucontext->device, udata); + if (ret != 0) + return ret; + + resp = roce3_resp_alloc_ext(); + if (resp == NULL) { + ret = (-ENOMEM); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc ucontext, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err; + } + + roce3_alloc_ucontext_set(rdev, resp); + /* Alloc user space context Doorbell and DWQE */ + ret = hinic3_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr, &context->dwqe_dma_addr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc DB pa, ret(%d), func_id(%u)\n", + __func__, ret, rdev->glb_func_id); + goto err_db; + } + + /* Copy data to user space */ + ret = roce3_alloc_ucontext_return(rdev, udata, context, resp); + if (ret != 0) + goto err_return; + + kfree(resp); + + return 0; +err_return: + hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr); +err_db: + kfree(resp); +err: + return ret; +} + +void roce3_dealloc_ucontext(struct ib_ucontext *ibcontext) +{ + struct roce3_ucontext *context = NULL; + struct roce3_device *rdev = NULL; + + if (ibcontext == NULL) { + pr_err("[ROCE] %s: Ibcontext is null\n", __func__); + return; + } + + context = to_roce3_ucontext(ibcontext); + rdev = to_roce3_dev(ibcontext->device); + + hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr); +} + +/* + **************************************************************************** + Prototype : roce3_mmap + Description : memory map + Input : struct ib_ucontext *ibcontext + struct vm_area_struct *vma + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** +*/ +int roce3_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) +{ + struct roce3_device *rdev = NULL; + struct roce3_ucontext *ucontext = NULL; + unsigned long db_pfn = 0; + unsigned long dwqe_pfn = 0; + int res = 0; + + if ((ibcontext == NULL) || (vma == NULL)) { + pr_err("[ROCE] %s: Ibcontext or vma is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibcontext->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ucontext = to_roce3_ucontext(ibcontext); + db_pfn = ucontext->db_dma_addr >> PAGE_SHIFT; + dwqe_pfn = ucontext->dwqe_dma_addr >> PAGE_SHIFT; + + if ((vma->vm_end - vma->vm_start) != PAGE_SIZE) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: (Vm_end - vm_start) is not equal to PAGE_SIZE, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EINVAL; + } + + /* map hw DB to physical page from user */ + if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* construct vm_start~vm_start+PAGE_SIZE page table + * db_pfn is page number + * vm_page_prot means attr + */ + if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to do db io remap, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } + // DWQE mmap + if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && (rdev->rdma_cap.direct_wqe_size != 0)) { +#ifdef __aarch64__ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + + if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to do dwqe io remap, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } + + res = roce3_mmap_ext(rdev, ucontext, vma); + return res; +} + +enum rdma_link_layer roce3_port_link_layer(struct ib_device *ibdev, u8 port_num) +{ + struct roce3_device *rdev = NULL; + + if (ibdev == NULL) + return IB_LINK_LAYER_UNSPECIFIED; + + rdev = to_roce3_dev(ibdev); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return IB_LINK_LAYER_UNSPECIFIED; + } + + if (port_num != ROCE_DEFAULT_PORT_NUM) + return IB_LINK_LAYER_UNSPECIFIED; + + return IB_LINK_LAYER_ETHERNET; +} + +static void roce3_resolve_cb(int status, struct sockaddr *src_addr, + struct rdma_dev_addr *addr, void *context) +{ + ((struct roce3_resolve_cb_context *)context)->status = status; + complete(&((struct roce3_resolve_cb_context *)context)->comp); +} + +static int roce3_rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid, const union ib_gid *dgid, + u8 *dmac, const struct net_device *ndev, int *hoplimit, struct roce3_device *rdev) +{ + struct rdma_dev_addr dev_addr; + struct roce3_resolve_cb_context ctx; + union { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } sgid_addr, dgid_addr; + int ret; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid); + rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid); + + memset(&dev_addr, 0, sizeof(dev_addr)); + + if (ndev) { + dev_addr.bound_dev_if = ndev->ifindex; + dev_addr.net = dev_net(ndev); + } else { + dev_addr.net = &init_net; + } + + init_completion(&ctx.comp); + ret = rdma_resolve_ip(&sgid_addr._sockaddr, &dgid_addr._sockaddr, &dev_addr, + RESOLVE_IP_TIME_OUT, roce3_resolve_cb, false, &ctx); + if (ret != 0) { + pr_err("[ROCE] %s: rdma_resolve_ip failed. Igonore the err.\n", __func__); + roce3_resolve_cb(0, &sgid_addr._sockaddr, &dev_addr, &ctx); + } + + wait_for_completion(&ctx.comp); + + memcpy(dmac, dev_addr.dst_dev_addr, ETH_ALEN); + if (hoplimit) + *hoplimit = dev_addr.hoplimit; + + return 0; +} + +static int roce3_ah_valid_check(struct ib_global_route *grh, u16 *vlan_id, u8 *dmac) +{ + u8 unicast_gid0[ROCE_GID_LEN] = { 0 }; + u8 unicast_gid1[ROCE_GID_LEN] = { 0 }; + + /* check gid(unicast gid can not be 0 or 1) */ + unicast_gid0[ROCE_GID_HIGHEST_BYTE] = 0; + unicast_gid1[ROCE_GID_HIGHEST_BYTE] = 1; + + if ((ROCE_MEMCMP(grh->dgid.raw, unicast_gid0, sizeof(union ib_gid)) == 0) || + (ROCE_MEMCMP(grh->dgid.raw, unicast_gid1, sizeof(union ib_gid)) == 0)) { + pr_err("[ROCE] %s: Invalid unicast dgid\n", __func__); + return (-EINVAL); + } + + if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) != 0) { + rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, dmac); + *vlan_id = ROCE_DEFAULT_VLAN_ID; + } + + return 0; +} + +static int roce3_fill_gid_attr(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + union ib_gid *sgid, const struct ib_gid_attr **sgid_attr) +{ + int ret = 0; + + ret = rdma_query_gid(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index, sgid); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to query gid func_id(%u),port_num(%d),gid_index(%d),ret(%d)\n", + rdev->glb_func_id, ah_attr->port_num, ah_attr->grh.sgid_index, ret); + return ret; + } + + *sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index); + if (IS_ERR_OR_NULL(*sgid_attr)) { + ret = (int)PTR_ERR(*sgid_attr); + dev_err(rdev->hwdev_hdl, + "[ROCE] : Failed to get sgid_attr, func_id(%u), ret(%d).\n", + rdev->glb_func_id, ret); + return ret; + } + + return ret; +} + +static void roce3_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr) +{ + rdma_put_gid_attr(sgid_attr); +} + +static struct net_device *roce3_fill_netdev(struct roce3_device *rdev, union ib_gid *sgid) +{ + struct net_device *netdev = NULL; + union { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } socket_addr; + + rdma_gid2ip((struct sockaddr *)&socket_addr, sgid); + + /* find netdev,rdev->ndevis not valid in vlan scenario */ + netdev = ip_dev_find(&init_net, + ((const struct sockaddr_in *)&socket_addr._sockaddr)->sin_addr.s_addr); + if (netdev) + dev_put(netdev); + + return netdev; +} + +int roce3_resolve_grh(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata) +{ + int ret = 0; + u8 zero_mac[ETH_ALEN] = { 0 }; + u8 *dmac = NULL; + union ib_gid sgid; + const struct ib_gid_attr *sgid_attr = NULL; + struct net_device *netdev = NULL; + + if ((rdev == NULL) || (ah_attr == NULL) || (vlan_id == NULL)) { + pr_err("[ROCE, ERR] %s: Input pointer is NULL, rdev(%p), ah_attr(%p), vlan_id(%p).\n", + __func__, rdev, ah_attr, vlan_id); + return (-EINVAL); + } + + dmac = ah_attr->roce.dmac; + ret = roce3_ah_valid_check(&ah_attr->grh, vlan_id, dmac); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to check grh input, func_id(%u), ret(%d).\n", + rdev->glb_func_id, ret); + return ret; + } + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) != 0) + return 0; + + ret = roce3_fill_gid_attr(rdev, ah_attr, &sgid, &sgid_attr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to fill gid attr, func_id(%u), ret(%d)\n", + rdev->glb_func_id, ret); + return ret; + } + + netdev = roce3_fill_netdev(rdev, &sgid); + + /* reparse dmac avoiding invalid damc from OFED */ + ret = roce3_rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid, dmac, + netdev, NULL, rdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to find dmac by grh, func_id(%u)\n", + rdev->glb_func_id); + goto resolve_grh_end; + } + + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to find valid dmac, func_id(%u)\n", + rdev->glb_func_id); + ret = (-EINVAL); + goto resolve_grh_end; + } + + *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev); + +resolve_grh_end: + roce3_release_gid_ref_cnt(sgid_attr); + + return ret; +} + +static int ah_get_vlan_id(struct roce3_device *rdev, struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, u32 *vlan_id) +{ + struct net_device *ndev; + + rcu_read_lock(); + ndev = rcu_dereference(ah_attr->grh.sgid_attr->ndev); + if (ndev == NULL) { + rcu_read_unlock(); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Net device is NULL, func_id(%u)\n", rdev->glb_func_id); + return -EINVAL; + } + *vlan_id = rdma_vlan_dev_vlan_id(ndev); + rcu_read_unlock(); + + return 0; +} + +static int create_ib_ah(struct roce3_device *rdev, struct ib_pd *pd, struct roce3_ah *rah, + struct rdma_ah_attr *ah_attr) +{ + int ret; + u8 *dmac = ah_attr->roce.dmac; + u32 vlan_id = 0; + + ret = ah_get_vlan_id(rdev, pd, ah_attr, &vlan_id); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get vlan_id (ret:%d)\n", __func__, ret); + return -EFAULT; + } + + if (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) != 0) { + memcpy((void *)rah->priv_ah.dgid, (void *)ah_attr->grh.dgid.raw, + sizeof(rah->priv_ah.dgid)); + rah->priv_ah.dw2.bs.flow_label = ah_attr->grh.flow_label & 0xfffff; + rah->priv_ah.dw1.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f; + rah->priv_ah.dw1.bs.hoplimit = ah_attr->grh.hop_limit; + rah->priv_ah.dw1.bs.tclass = (u8)(ah_attr->grh.traffic_class | 0x2); + } + rah->priv_ah.dw0.bs.pd = to_roce3_pd(pd)->pdn & 0x3ffff; + rah->priv_ah.dw0.bs.wqe_cos = roce3_get_db_cos_from_vlan_pri(rdev, ah_attr->sl); + rah->priv_ah.dw0.value = cpu_to_be32(rah->priv_ah.dw0.value); + + rah->priv_ah.dw1.bs.port = ah_attr->port_num & 0xf; + rah->priv_ah.dw2.bs.smac_index = rdev->glb_func_id; /* set global Function ID */ + rah->priv_ah.dw2.value = cpu_to_be32(rah->priv_ah.dw2.value); + + rah->priv_ah.dw1.bs.resv = 0; + rah->priv_ah.dw7.bs.vlan_id = vlan_id & 0xfff; + rah->priv_ah.dw7.bs.vlan_pri = ah_attr->sl & 0x7; + + rah->priv_ah.dw1.value = cpu_to_be32(rah->priv_ah.dw1.value); + + rah->priv_ah.dw7.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | dmac[1]; + rah->priv_ah.dw7.value = cpu_to_be32(rah->priv_ah.dw7.value); + + memcpy((void *)&rah->priv_ah.dmac_l32, + (void *)&dmac[ROCE_RAH_DMAC_L32_START], sizeof(rah->priv_ah.dmac_l32)); + + return 0; +} + +int roce3_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata) +{ + struct roce3_ah *rah = to_roce3_ah(ibah); + struct roce3_device *rdev = to_roce3_dev(ibah->device); + struct rdma_ah_attr *ah_attr = init_attr->ah_attr; + enum rdma_ah_attr_type ah_type = ah_attr->type; + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && (((u32)rdma_ah_get_ah_flags(ah_attr) + & IB_AH_GRH) == 0)) + return -EINVAL; + + return create_ib_ah(rdev, ibah->pd, rah, ah_attr); +} + +int roce3_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ + struct roce3_ah *ah = NULL; + struct roce3_priv_ah priv_ah; + + if ((ibah == NULL) || (ah_attr == NULL)) { + pr_err("[ROCE] %s: Ibah or ah_attr is null\n", __func__); + return -EINVAL; + } + + ah = to_roce3_ah(ibah); + memset(ah_attr, 0, sizeof(*ah_attr)); + + priv_ah.dw1.value = be32_to_cpu(ah->priv_ah.dw1.value); + priv_ah.dw2.value = be32_to_cpu(ah->priv_ah.dw2.value); + priv_ah.dw7.value = be32_to_cpu(ah->priv_ah.dw7.value); + + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->sl = priv_ah.dw7.bs.vlan_pri; + ah_attr->port_num = priv_ah.dw1.bs.port; + ah_attr->grh.traffic_class = priv_ah.dw1.bs.tclass; + ah_attr->grh.hop_limit = priv_ah.dw1.bs.hoplimit; + ah_attr->grh.sgid_index = priv_ah.dw1.bs.sgid_index; + ah_attr->grh.flow_label = priv_ah.dw2.bs.flow_label; + + memcpy((void *)ah_attr->grh.dgid.raw, (void *)ah->priv_ah.dgid, + sizeof(ah->priv_ah.dgid)); + return 0; +} + +int roce3_destroy_ah(struct ib_ah *ibah, u32 flags) +{ + return 0; +} + +int roce3_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + struct roce3_device *rdev = to_roce3_dev(ibdev); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; // only rocev2 + + err = ib_query_port(ibdev, port_num, &attr); + if (err != 0) { + pr_err("[ROCE] %s: query ib port failed\n", __func__); + return err; + } + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + + return 0; +} + +int roce3_get_dcb_cfg_cos(struct roce3_device *rdev, struct roce3_get_cos_inbuf *inbuf, u8 *cos) +{ + int ret; + u8 pri; + struct rdma_gid_entry gid; + struct hinic3_dcb_state dcb = { 0 }; + + ret = roce3_rdma_get_gid(rdev->hwdev, inbuf->port_num, inbuf->sgid_index, &gid); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to init gid info\n", __func__); + return (-EINVAL); + } + + ret = hinic3_get_dcb_state(rdev->hwdev, &dcb); + if (ret != 0) { + pr_err("[ROCE] %s: hinic3_get_dcb_state failed.ret: %d.\n", __func__, ret); + return (-EINVAL); + } + + *cos = dcb.default_cos; + gid.dw6_h.value = cpu_to_le16(gid.dw6_h.value); + if ((dcb.trust == ROCE3_DCB_PCP) && (gid.dw6_h.bs.tag == ROCE_GID_VLAN_INVALID)) { + // pcp cfg & no vlan should use default cos + return 0; + } + + pri = (dcb.trust == ROCE3_DCB_PCP) ? inbuf->sl : (inbuf->traffic_class >> ROCE3_DSCP_IDX); + ret = hinic3_get_cos_by_pri(rdev->hwdev, pri, cos); + if (ret != 0) { + pr_err("[ROCE] %s: get_cos_by_pri failed.ret: %d, pri:%u, dcb_on:%u, trust:%u.\n", + __func__, ret, pri, dcb.dcb_on, dcb.trust); + return (-EINVAL); + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_mix.h b/drivers/infiniband/hw/hiroce3/roce_mix.h new file mode 100644 index 000000000..21a495773 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_mix.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_MIX_H +#define ROCE_MIX_H + +#include <net/arp.h> +#include <net/neighbour.h> +#include <net/route.h> +#include <net/addrconf.h> +#include <net/ip6_route.h> +#include <linux/mutex.h> +#include <linux/inetdevice.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <linux/module.h> + +#include <rdma/ib_addr.h> +#include <rdma/ib_verbs.h> + +#include "roce.h" + +#define IB_DEVICE_LOCAL_DMA_LKEY (1 << 15) + +#define ROCE_HW_VER 0ULL +#define ROCE_FW_VER_0 1ULL +#define ROCE_FW_VER_1 4ULL +#define ROCE_FW_VER_2 0ULL +#define ROCE_FW_VER_3 0ULL +#define ROCE_FW_VER ((ROCE_FW_VER_0 << 32) | (ROCE_FW_VER_1 << 16) | \ + (ROCE_FW_VER_2 << 8) | ROCE_FW_VER_3) + +#define RESOLVE_IP_TIME_OUT 1000 +#define ROCE_GID_HIGHEST_BYTE 15 +#define ROCE_FW_VERSION_LEN 4 +#define ROCE_DEFAULT_VLAN_ID 0xFFFF +#define ROCE_RAH_TC_SHIFT 2 +#define ROCE_RAH_DMAC_H16_SHIFT 8 +#define ROCE_RAH_DMAC_L32_START 2 + +#define ROCE3_DCB_PCP 0 +#define ROCE3_DCB_DSCP 1 + +#define ROCE3_DSCP_IDX 2 // dscp begin from bit 2 in traffic_class + +#define IB_DEVICE_NODE_DESC_MAX 64 + +#define MEND_CAP_DEVIDE 2 // to avoid chip cache problem +#define ROCE_PORT_NUM_2 2 +#define ROCE_25G_PORT_SPEED 25 + +#define ROCE_GID_IP_IDX 3 + +enum { + USR_MMAP_DB_OFFSET = 0, + USR_MMAP_DWQE_OFFSET, + USR_MMAP_DFX_OFFSET, + USR_MMAP_DBAR3_OFFSET +}; + +enum roce_port_state_e { + ROCE_PORT_PHYS_STATE_NO_CHANGE = 0, + ROCE_PORT_PHYS_STATE_SLEEP, + ROCE_PORT_PHYS_STATE_POLLING, + ROCE_PORT_PHYS_STATE_DISABLED = 3, + ROCE_PORT_PHYS_STATE_PORTCONFTRAIN, + ROCE_PORT_PHYS_STATE_LINKUP = 5, + ROCE_PORT_PHYS_STATE_LINKERRRECOVER, + ROCE_PORT_PHYS_STATE_PHYTEST +}; + +enum ROCE_MBOX_CMD { + ROCE_MBOX_CMD_SEND_MAIL_BOX, + ROCE_MBOX_CMD_MAX +}; + +struct roce3_priv_ah { + /* DW0 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; + u32 wqe_cos : 3; + u32 fl : 1; /* forcelock flag, it is used in multicast communication */ +#else + u32 f1 : 1; + u32 wqe_cos : 3; + u32 stat_rate : 4; + u32 rsvd0 : 6; + u32 pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 hoplimit : 8; + u32 sgid_index : 7; + u32 resv : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tclass : 8; +#else + u32 tclass : 8; + u32 rsvd1 : 4; + u32 port : 4; + u32 resv : 1; + u32 sgid_index : 7; + u32 hoplimit : 8; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 flow_label : 20; + u32 smac_index : 10; + u32 rsvd : 2; +#else + u32 rsvd : 2; + u32 smac_index : 10; + u32 flow_label : 20; +#endif + } bs; + u32 value; + } dw2; + + /* DW3~6 */ + u8 dgid[16]; + + /* DW7 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 rsvd : 1; + u32 vlan_pri : 3; +#else + u32 vlan_pri : 3; + u32 rsvd : 1; + u32 vlan_id : 12; + u32 dmac_h16 : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + u32 dmac_l32; +}; + +struct roce3_ah { + struct ib_ah ibah; + struct roce3_priv_ah priv_ah; +}; + +union gid_addr { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; +}; + +struct mailbox_header { + u32 dest_host_id; + u32 dest_global_function_id; +}; + +struct roce3_mail_box_buffer { + struct mailbox_header header; + u32 data[0]; //lint !e1501 +}; + +struct roce3_resolve_cb_context { + struct completion comp; + int status; +}; + +struct roce3_get_cos_inbuf { + u8 sl; + u8 sgid_index; + u8 port_num; + u8 traffic_class; +}; + +/* Find struct roce3_ah through ibah */ +static inline struct roce3_ah *to_roce3_ah(const struct ib_ah *ibah) +{ + return container_of(ibah, struct roce3_ah, ibah); +} + +int roce3_resolve_grh(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata); +int roce3_get_dcb_cfg_cos(struct roce3_device *rdev, struct roce3_get_cos_inbuf *inbuf, u8 *cos); + + +#endif /* ROCE_MIX_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_netdev.c b/drivers/infiniband/hw/hiroce3/roce_netdev.c new file mode 100644 index 000000000..f9be5faca --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netdev.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include "roce_netdev.h" +#include "roce_netdev_extension.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" +#include "hinic3_rdma.h" +#include "roce.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +/* + **************************************************************************** + Prototype : roce3_fill_gid_type + Description : fill gid type + Input : struct rdma_gid_entry *gid_entry + Output : None + + 1.Date : 2016/5/27 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_fill_gid_type(struct rdma_gid_entry *gid_entry, enum roce3_gid_type new_gid_type) +{ + static u32 index_roce3_gid_mapping[32][4] = { + {0x120E1436, 3452, 224529784, 0x180E252A}, + {0x120E004A, 3472, 224660876, 0x6C0E403E}, + {0x120E0042, 3464, 224660836, 0x840E4036}, + {0x120E0042, 3464, 224398688, 0xC00E0036}, + {0x1212143A, 3456, 224791932, 0x1812252E}, + {0x1212004E, 3476, 224923024, 0x6C124042}, + {0x12120046, 3468, 224922984, 0x8412403A}, + {0x12120046, 3468, 224660836, 0xC012003A}, + {0x1216143E, 3460, 225054080, 0x18162532}, + {0x12160052, 3480, 225185172, 0x6C164046}, + {0x1216004A, 3472, 225185132, 0x8416403E}, + {0x1216004A, 3472, 224922984, 0xC016003E}, + {0x10301458, 3484, 226626968, 0x1830254C}, + {0x1030006C, 3504, 226758060, 0x6C304060}, + {0x10300064, 3496, 226758020, 0x84304058}, + {0x10300064, 3496, 226495872, 0xC0300058}, + {0x10401468, 838864300, 227675560, 0x1840255C}, + {0x1040007C, 1174408640, 227806652, 0x6C404070}, + {0x10400074, 1040190904, 227806612, 0x84404068}, + {0x10400074, 1040190904, 227544464, 0xC0400068}, + {0x1044146C, 905973168, 227937708, 0x18442560}, + {0x10440080, 1241517508, 228068800, 0x6C444074}, + {0x10440078, 1107299772, 228068760, 0x8444406C}, + {0x10440078, 1107299772, 227806612, 0xC044006C}, + {0x10481470, 973082036, 228199856, 0x18482564}, + {0x10480084, 1308626376, 228330948, 0x6C484078}, + {0x1048007C, 1174408640, 228330908, 0x84484070}, + {0x1048007C, 1174408640, 228068760, 0xC0480070}, + {0x1262148A, 1040190928, 230034892, 0x1862257E}, + {0x1262009E, 1375735268, 230165984, 0x6C624092}, + {0x12620096, 1241517532, 230165944, 0x8462408A}, + {0x12620096, 1241517532, 229903796, 0xC062008A}, + }; + u8 i = 0; + + gid_entry->dw6_h.bs.gid_type = new_gid_type; + gid_entry->dw6_h.bs.gid_update = (new_gid_type == ROCE_IPv4_ROCEv2_GID); + + i = ROCE_GID_MAP_TBL_IDX_GET(gid_entry->dw6_h.bs.tunnel, gid_entry->dw6_h.bs.tag, + (u16)new_gid_type); + gid_entry->hdr_len_value = index_roce3_gid_mapping[i][0]; + if (new_gid_type == ROCE_IPv4_ROCEv2_GID) { + *((u32 *)(void *)gid_entry + 1) = cpu_to_be32(index_roce3_gid_mapping[i][1]); + *((u32 *)(void *)gid_entry + ROCE_GID_MAP_TBL_IDX2) = + cpu_to_be32(index_roce3_gid_mapping[i][ROCE_GID_MAP_TBL_IDX2]); + } +} + +static void roce3_fill_gid_smac(struct net_device *net_dev, struct rdma_gid_entry *gid_entry) +{ + memcpy((void *)gid_entry->smac, (void *)net_dev->dev_addr, sizeof(gid_entry->smac)); +} + +static void roce3_fill_gid_vlan(struct roce3_device *rdev, struct net_device *net_dev, + struct rdma_gid_entry *gid_entry) +{ + gid_entry->dw6_h.bs.tunnel = ROCE_GID_TUNNEL_INVALID; + if (rdma_vlan_dev_vlan_id(net_dev) != 0xffff) { + gid_entry->dw4.bs.cvlan = rdma_vlan_dev_vlan_id(net_dev) & 0xfff; + gid_entry->dw6_h.bs.tag = ROCE_GID_VLAN_VALID; + } +} + +/* + **************************************************************************** + Prototype : roce3_dispatch_event + Description : roce3_dispatch_event + Input : struct roce3_device *rdev + int port_num + enum ib_event_type type + Output : None + + 1.Date : 2016/3/29 + Modification : Created function + +**************************************************************************** +*/ +static void roce3_dispatch_event(struct roce3_device *rdev, int port_num, enum ib_event_type type) +{ + struct ib_event event; + + if (rdev == NULL) { + pr_err("[ROCE, ERR] %s: Rdev is null\n", __func__); + return; + } + + memset(&event, 0, sizeof(event)); + + event.device = &rdev->ib_dev; + event.element.port_num = (u8)port_num; + event.event = type; + + ib_dispatch_event(&event); +} + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) +struct sin6_list { + struct list_head list; + struct sockaddr_in6 sin6; +}; + +static int roce3_gid_fill(struct roce3_device *rdev, struct rdma_gid_entry *gid_entry, + struct net_device *event_netdev, struct sin6_list *sin6_iter) +{ + int ret = 0; + + if (cpu_to_be64(gid_entry->global.subnet_prefix) == ROCE_DEFAULT_GID_SUBNET_PREFIX) + return ret; + + roce3_fill_gid_vlan(rdev, event_netdev, gid_entry); + roce3_fill_gid_smac(event_netdev, gid_entry); + + /* RoCE V2 */ + if (rdma_protocol_roce_udp_encap(&rdev->ib_dev, ROCE_DEFAULT_PORT_NUM)) { + roce3_fill_gid_type(gid_entry, ROCE_IPv6_ROCEv2_GID); + ret = roce3_rdma_update_gid_mac(rdev->hwdev, 0, gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update V2 gid(IPv6), func_id(%d)\n", + __func__, rdev->glb_func_id); + list_del(&sin6_iter->list); + kfree(sin6_iter); + return ret; + } + } + + roce3_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, IB_EVENT_GID_CHANGE); + + return ret; +} + +static int roce3_netdev_event_ipv6_dev_scan(struct roce3_device *rdev, + struct net_device *event_netdev, struct rdma_gid_entry *gid_entry) +{ + struct sin6_list *sin6_temp = NULL; + struct sin6_list *sin6_iter = NULL; + struct inet6_dev *in6_dev = NULL; + struct inet6_ifaddr *ifp = NULL; + struct sin6_list *entry = NULL; + int ret; + + LIST_HEAD(sin6_init_list); + + in6_dev = in6_dev_get(event_netdev); + if (in6_dev == NULL) + return 0; + + read_lock_bh(&in6_dev->lock); + list_for_each_entry(ifp, &in6_dev->addr_list, if_list) { + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (entry == NULL) + continue; + + entry->sin6.sin6_family = AF_INET6; + entry->sin6.sin6_addr = ifp->addr; + list_add_tail(&entry->list, &sin6_init_list); + } + + read_unlock_bh(&in6_dev->lock); + in6_dev_put(in6_dev); + + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + memcpy((void *)gid_entry->raw, (void *)&sin6_iter->sin6.sin6_addr, + sizeof(union ib_gid)); + + ret = roce3_gid_fill(rdev, gid_entry, event_netdev, sin6_iter); + if (ret != 0) + goto err_free; + + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + + return 0; + +err_free: + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + return ret; +} +#endif + +static int roce3_netdev_event_ip_dev_scan(struct roce3_device *rdev, + struct net_device *event_netdev) +{ + struct rdma_gid_entry gid_entry; + int ret; + + memset(&gid_entry, 0, sizeof(gid_entry)); + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) + ret = roce3_netdev_event_ipv6_dev_scan(rdev, event_netdev, &gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to scan IPv6, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } +#endif + + return 0; +} + +static struct roce3_vlan_dev_list *roce3_find_vlan_device_list(const struct list_head *mac_list, + const struct net_device *netdev) +{ + struct roce3_vlan_dev_list *vlan_dev_list = NULL; + + list_for_each_entry(vlan_dev_list, mac_list, list) { + if (vlan_dev_list->net_dev == netdev) + return vlan_dev_list; + } + + return NULL; +} + +static int roce3_add_vlan_device(struct roce3_device *rdev, struct net_device *netdev) +{ + struct roce3_vlan_dev_list *new_list; + + new_list = kzalloc(sizeof(*new_list), GFP_KERNEL); + if (new_list == NULL) + return -ENOMEM; + + new_list->ref_cnt = 1; + new_list->net_dev = netdev; + new_list->vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID(((u32)rdma_vlan_dev_vlan_id(netdev))); + memcpy(new_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN); + INIT_LIST_HEAD(&new_list->list); + list_add_tail(&new_list->list, &rdev->mac_vlan_list_head); + + return 0; +} + +static void roce3_del_vlan_device(struct roce3_device *rdev, struct roce3_vlan_dev_list *old_list) +{ + list_del(&old_list->list); + kfree(old_list); +} + +static int roce3_update_vlan_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + struct roce3_vlan_dev_list *old_list = NULL; + int ret; + + mutex_lock(&rdev->mac_vlan_mutex); + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, netdev); + if (old_list == NULL) { + mutex_unlock(&rdev->mac_vlan_mutex); + return 0; + } + + if (ROCE_MEMCMP(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN) != 0) { + roce3_del_vlan_device_mac(rdev, old_list); + + ret = roce3_add_vlan_device_mac(rdev, netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add mac_vlan, func_id(%d)\n", + __func__, rdev->glb_func_id); + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + + memcpy(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN); + } + + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +static int roce3_update_real_device_mac(struct roce3_device *rdev, struct net_device *netdev) +{ + int ret; + + roce3_del_real_device_mac(rdev); + + ret = roce3_add_real_device_mac(rdev, netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add real device ipsu mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return 0; +} + +void roce3_clean_vlan_device_mac(struct roce3_device *rdev) +{ + struct roce3_vlan_dev_list *pos = NULL; + struct roce3_vlan_dev_list *tmp = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + list_for_each_entry_safe(pos, tmp, &rdev->mac_vlan_list_head, list) { +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + (void)roce3_del_bond_vlan_slave_mac(rdev, pos->mac, (u16)pos->vlan_id); +#endif + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, pos->mac, pos->vlan_id, + rdev->glb_func_id, hinic3_er_id(rdev->hwdev)); + + (void)roce3_del_mac_tbl_mac_entry(rdev->hwdev, pos->mac, pos->vlan_id, + rdev->glb_func_id, rdev->glb_func_id); + + list_del(&pos->list); + kfree(pos); + } + mutex_unlock(&rdev->mac_vlan_mutex); +} + +void roce3_clean_real_device_mac(struct roce3_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) + (void)roce3_del_bond_real_slave_mac(rdev); +#endif + + roce3_del_ipsu_tbl_mac_entry(rdev->hwdev, rdev->mac, 0, rdev->glb_func_id, + hinic3_er_id(rdev->hwdev)); +} + +static bool roce3_rdma_is_upper_dev_rcu(struct net_device *dev, struct net_device *upper) +{ + struct net_device *rdev_upper = NULL; + struct net_device *master = NULL; + bool ret = false; + + if ((upper == NULL) || (dev == NULL)) { + ret = false; + } else { + rdev_upper = rdma_vlan_dev_real_dev(upper); + master = netdev_master_upper_dev_get_rcu(dev); + ret = ((upper == master) || (rdev_upper && (rdev_upper == master)) || + (rdev_upper == dev)); + } + + return ret; +} + +/* check bonding device */ +int roce3_is_eth_port_of_netdev(struct net_device *rdma_ndev, struct net_device *cookie) +{ + struct net_device *real_dev = NULL; + int res; + + if (rdma_ndev == NULL) + return 0; + + rcu_read_lock(); + real_dev = rdma_vlan_dev_real_dev(cookie); + if (real_dev == NULL) + real_dev = cookie; + + res = (roce3_rdma_is_upper_dev_rcu(rdma_ndev, cookie) || (real_dev == rdma_ndev)); + rcu_read_unlock(); + return res; +} + +int roce3_ifconfig_up_down_event_report(struct roce3_device *rdev, u8 net_event) +{ + struct ib_event event = { 0 }; + + if ((net_event == IB_EVENT_PORT_ACTIVE) && + (test_and_set_bit(ROCE3_PORT_EVENT, &rdev->status) != 0)) + return -1; + + if ((net_event == IB_EVENT_PORT_ERR) && + (test_and_clear_bit(ROCE3_PORT_EVENT, &rdev->status) == 0)) + return -1; + + event.device = &rdev->ib_dev; + event.event = net_event; + event.element.port_num = ROCE_DEFAULT_PORT_NUM; + + ib_dispatch_event(&event); + return 0; +} + +static int roce3_netdev_event_raw_dev(unsigned long event, struct roce3_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_REGISTER) { + ret = roce3_add_real_device_mac(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to add real device mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + if (event == NETDEV_CHANGEADDR) { + ret = roce3_update_real_device_mac(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update readl device mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to scan ip_dev, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + if ((event == NETDEV_DOWN) || (event == NETDEV_UP)) + roce3_handle_bonded_port_state_event(rdev); + } else { + if (event == NETDEV_DOWN) + roce3_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ERR); + if (event == NETDEV_UP) + roce3_event_up_extend(rdev); + } +#else + if (event == NETDEV_DOWN) + roce3_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ERR); + if (event == NETDEV_UP) + roce3_event_up_extend(rdev); +#endif + return 0; +} + +static int roce3_netdev_event_vlan_dev(unsigned long event, struct roce3_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_CHANGEADDR) { + ret = roce3_update_vlan_device_mac(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update vlan device mac, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to scan vlan device ip list, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static bool roce3_is_port_of_net_dev(struct roce3_device *rdev, struct net_device *event_netdev) +{ +#ifdef ROCE_BONDING_EN + if (roce3_bond_is_active(rdev)) { + if (roce3_bond_is_eth_port_of_netdev(rdev, event_netdev) == 0) + return false; + } else +#endif + { + if (roce3_is_eth_port_of_netdev(rdev->ndev, event_netdev) == 0) + return false; + } + return true; +} + +static int roce3_netdev_event(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct net_device *event_netdev; + struct roce3_device *rdev = NULL; + struct roce3_notifier *notifier = NULL; + struct net_device *real_dev = NULL; + int ret; + + if (nb == NULL || ptr == NULL) + goto err_out; + + event_netdev = netdev_notifier_info_to_dev(ptr); + if (event_netdev->type != ARPHRD_ETHER) + goto err_out; + + notifier = container_of(nb, struct roce3_notifier, nb); + rdev = container_of(notifier, struct roce3_device, notifier); + + if (roce3_hca_is_present(rdev) == 0) + goto err_out; + + if (!roce3_is_port_of_net_dev(rdev, (void *)event_netdev)) + goto err_out; + + /* get raw netdev from event_netdev */ + real_dev = (rdma_vlan_dev_real_dev(event_netdev) != 0) ? + rdma_vlan_dev_real_dev(event_netdev) : event_netdev; + if (real_dev == event_netdev) { + ret = roce3_netdev_event_raw_dev(event, rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to deal with raw device, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + } else { + ret = roce3_netdev_event_vlan_dev(event, rdev, event_netdev); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to deal with vlan device, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_out; + } + } + +err_out: + return NOTIFY_DONE; +} + +/* + **************************************************************************** + Prototype : roce3_unregister_netdev_event + Description : unregister netdev event related function + Input : struct roce3_device *rdev + Output : None + + 1.Date : 2015/6/18 + Modification : Created function + +**************************************************************************** +*/ +void roce3_unregister_netdev_event(struct roce3_device *rdev) +{ + int ret = 0; + struct roce3_notifier *notifier = &rdev->notifier; + + if (notifier->nb.notifier_call) { + ret = unregister_netdevice_notifier(¬ifier->nb); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to unregister netdev notifier, func_id(%d)\n", + __func__, rdev->glb_func_id); + } + + notifier->nb.notifier_call = NULL; + } +} + +/* + **************************************************************************** + Prototype : roce3_register_netdev_event + Description : register netdev event related function + Input : struct roce3_device *rdev + Output : None + + 1.Date : 2015/6/18 + Modification : Created function + +**************************************************************************** +*/ +int roce3_register_netdev_event(struct roce3_device *rdev) +{ + struct roce3_notifier *notifier = &rdev->notifier; + int ret; + + notifier->nb.notifier_call = roce3_netdev_event; + ret = register_netdevice_notifier(¬ifier->nb); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to register netdev notifier, func_id(%d)\n", + __func__, rdev->glb_func_id); + notifier->nb.notifier_call = NULL; + return ret; + } + + return 0; +} + +static int roce3_set_gid_entry(struct roce3_device *rdev, struct rdma_gid_entry *gid_entry, + const struct ib_gid_attr *attr, const union ib_gid *gid) +{ + static enum roce3_gid_type gid_type_map[ROCE_NETWORK_GID_TYPE_MAX] = {0}; + enum rdma_network_type network_type; + enum roce3_gid_type roce_gid_type; + + gid_type_map[RDMA_NETWORK_IPV4] = ROCE_IPv4_ROCEv2_GID; + gid_type_map[RDMA_NETWORK_IPV6] = ROCE_IPv6_ROCEv2_GID; + + memset(gid_entry, 0, sizeof(*gid_entry)); + network_type = rdma_gid_attr_network_type(attr); + memcpy((void *)gid_entry->raw, (void *)(&attr->gid), sizeof(union ib_gid)); + if ((network_type == RDMA_NETWORK_IB) || (network_type == RDMA_NETWORK_ROCE_V1)) { + pr_err("[ROCE, ERR] %s: IB or RoCE v1 is no longer supported, network_type(%d)\n", + __func__, network_type); + return (-EINVAL); + } + + roce_gid_type = gid_type_map[network_type]; + roce3_fill_gid_vlan(rdev, attr->ndev, gid_entry); + roce3_fill_gid_type(gid_entry, roce_gid_type); + roce3_fill_gid_smac(attr->ndev, gid_entry); + + return 0; +} + +static int roce3_check_and_set_gid_entry(const struct ib_gid_attr *attr, const union ib_gid *gid, + struct rdma_gid_entry *gid_entry, struct roce3_device *rdev, unsigned int index) +{ + if ((cpu_to_be64(gid->global.subnet_prefix) == ROCE_DEFAULT_GID_SUBNET_PREFIX) && + (index > 1)) + return -EINVAL; + + memset(gid_entry, 0, sizeof(struct rdma_gid_entry)); + if (roce3_set_gid_entry(rdev, gid_entry, attr, gid) != 0) { + pr_err("[ROCE, ERR] %s: Failed to set gid entry.\n", __func__); + return (-EINVAL); + } + + return 0; +} + +/* add vlan_mac list or ref_cnt + 1 + * add mac_vlan when new list add + */ +static int roce3_notify_vlan(struct roce3_device *rdev, const struct ib_gid_attr *attr) +{ + int ret; + struct roce3_vlan_dev_list *old_list = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + pr_info("[ROCE] ADD vlan: Func_id:%d, Netdev:%s\n", + rdev->glb_func_id, attr->ndev->name); + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, attr->ndev); + if (old_list != NULL) { + old_list->ref_cnt++; + mutex_unlock(&rdev->mac_vlan_mutex); + return 0; + } + + ret = roce3_add_vlan_device(rdev, attr->ndev); + if (ret != 0) { + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + + ret = roce3_add_vlan_device_mac(rdev, attr->ndev); + if (ret != 0) { + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, attr->ndev); + if (old_list) + roce3_del_vlan_device(rdev, old_list); + + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +int roce3_ib_add_gid(const struct ib_gid_attr *attr, __always_unused void **context) +{ + int ret; + struct rdma_gid_entry gid_entry; + struct roce3_device *rdev = to_roce3_dev(attr->device); /*lint !e78 !e530*/ + unsigned int index = attr->index; /*lint !e530*/ + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + /*lint !e40*/ + ret = roce3_check_and_set_gid_entry(attr, &(attr->gid), &gid_entry, rdev, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: set gid err, func_id(%u)\n", __func__, + rdev->glb_func_id); + return ret; + } + + // 添加第一个gid时，设置ipsu mac + if (index == 0) { + ret = roce3_update_real_device_mac(rdev, attr->ndev); + if (ret != 0) + return ret; + } + + ret = roce3_rdma_update_gid(rdev->hwdev, 0, index, &gid_entry); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to update gid. ret(%d),func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + return ret; + } + + rdev->gid_dev[index] = attr->ndev; + + if (rdma_vlan_dev_real_dev(attr->ndev)) { + ret = roce3_notify_vlan(rdev, attr); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to nofify vlan, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return ret; + } + } + + return 0; +} + +int roce3_ib_del_gid(const struct ib_gid_attr *attr, __always_unused void **context) +{ + struct roce3_vlan_dev_list *old_list = NULL; + struct roce3_device *rdev = NULL; + u32 index = 0; + + if ((attr == NULL) || (attr->device == NULL)) { /*lint !e55 !e58 !e78*/ + pr_err("[ROCE] %s: Attr or attr->device is null\n", __func__); + return (-EINVAL); + } + + rdev = to_roce3_dev(attr->device); /*lint !e78*/ + index = attr->index; + if (index >= rdev->rdma_cap.max_gid_per_port) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: Invalid gid index(%u), func_id(%d)\n", + __func__, index, rdev->glb_func_id); + return (-EINVAL); + } + + /* del vlan_mac list or ref_cnt - 1 */ + /* del mac_vlan when ref_cnt = 0 */ + if (rdev->ndev != rdev->gid_dev[index]) { + mutex_lock(&rdev->mac_vlan_mutex); + old_list = roce3_find_vlan_device_list(&rdev->mac_vlan_list_head, + rdev->gid_dev[index]); + if (old_list) { + old_list->ref_cnt--; + if (old_list->ref_cnt == 0) { + roce3_del_vlan_device_mac(rdev, old_list); + + roce3_del_vlan_device(rdev, old_list); + } + } + mutex_unlock(&rdev->mac_vlan_mutex); + } + /* + * delete gid no longer send cmd to ucode which write 0s to target entry, + * preventing PPE from building malformed packets. + */ + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_netdev.h b/drivers/infiniband/hw/hiroce3/roce_netdev.h new file mode 100644 index 000000000..f0e9b2235 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netdev.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_NETDEV_H +#define ROCE_NETDEV_H + +#include <net/ipv6.h> +#include <net/bonding.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_mix.h" +#include "roce_cmd.h" + +#ifdef ROCE_BONDING_EN +#include "hinic3_srv_nic.h" +#endif +/* the 31 bit is vlan eable bit */ +#define ROCE_GID_SET_VLAN_32BIT_VLAID(vlan_id) ((vlan_id) | (((u32)1) << 31)) +#define ROCE_MAC_ADDR_LEN 6 + +/* ipv4 header len. unit:4B */ +#define IPV4_HDR_LEN 5 + +#define ROCE_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define ROCE_GID_MAP_TBL_IDX_GET(tunnel, tag, new_gid_type) \ + ((u8)(((tunnel) << 4) | ((tag) << 2) | (new_gid_type))) + +#define ROCE_GID_MAP_TBL_IDX2 2 + +#define ROCE_NETWORK_GID_TYPE_MAX 4 + +enum roce3_gid_tunnel_type { + ROCE_GID_TUNNEL_INVALID = 0, + ROCE_GID_TUNNEL_VALID +}; + +enum roce3_gid_vlan_type { + ROCE_GID_VLAN_INVALID = 0, + ROCE_GID_VLAN_VALID +}; + +struct roce3_vlan_dev_list { + u8 mac[6]; + u32 vlan_id; + u32 ref_cnt; + struct net_device *net_dev; + struct list_head list; +}; + +#endif /* ROCE_NETDEV_H */ diff --git a/drivers/infiniband/hw/hiroce3/roce_netlink.c b/drivers/infiniband/hw/hiroce3/roce_netlink.c new file mode 100644 index 000000000..509536bef --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netlink.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <net/net_namespace.h> +#include <linux/module.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/netlink.h> + +#include "roce_mix.h" +#include "roce_cmd.h" +#include "roce_netlink.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifdef ROCE_EXTEND +#include "roce_qp_extension.h" +#endif + +#ifdef ROCE_NETLINK_EN + +static struct hiroce_netlink_dev g_roce_adapter = {0, 0}; + +struct hiroce_netlink_dev *hiroce_get_adp(void) +{ + return &g_roce_adapter; +} + +int roce3_drv_cmd_execute(struct sk_buff *skb, struct genl_info *nl_info); + +static void roce3_nlahdr_push(struct sk_buff *skb, u16 type) +{ + u16 attrlen = (u16)skb->len; + struct nlattr *nlahdr; + u32 padlen = nla_padlen(attrlen); + + nlahdr = (struct nlattr *)skb_push(skb, NLA_HDRLEN); + nlahdr->nla_len = (u16)nla_attr_size(attrlen); + nlahdr->nla_type = type; + + if ((skb->end - skb->tail) >= padlen) + (void)skb_put(skb, padlen); +} + +static struct nlmsghdr *nlmsg_push(struct sk_buff *skb, u32 portid, u32 seq, + int type, int len, int flags) +{ + struct nlmsghdr *nlh; + u32 size = (u32)nlmsg_msg_size(len); + + nlh = (struct nlmsghdr *)skb_push(skb, NLMSG_ALIGN(size)); + if (nlh == NULL) + return NULL; + + nlh->nlmsg_type = (u16)type; + nlh->nlmsg_len = skb->len; + nlh->nlmsg_flags = (u16)flags; + nlh->nlmsg_pid = portid; + nlh->nlmsg_seq = seq; + + if (!__builtin_constant_p(size) || NLMSG_ALIGN(size) - size != 0) + memset((char *)nlmsg_data(nlh) + len, 0, NLMSG_ALIGN(size) - size); + + return nlh; +} + +static inline struct nlmsghdr *roce3_nlmsg_push(struct sk_buff *skb, u32 portid, + u32 seq, int type, int payload, int flags) +{ + if (unlikely(skb_headroom(skb) < (unsigned int)nlmsg_total_size(payload))) + return NULL; + + return nlmsg_push(skb, portid, seq, type, payload, flags); +} + +static void *roce3_genlmsg_push(struct sk_buff *skb, u32 portid, u32 seq, + const struct genl_family *family, int flags, u8 cmd) +{ + struct nlmsghdr *nlh; + struct genlmsghdr *hdr = NULL; + + nlh = roce3_nlmsg_push(skb, portid, seq, (int)family->id, + (int)(GENL_HDRLEN + family->hdrsize), flags); + if (nlh == NULL) + return NULL; + + hdr = nlmsg_data(nlh); + hdr->cmd = cmd; + hdr->version = (u8)family->version; + hdr->reserved = 0; + + return (void *)nlh; +} + +static long roce3_netlink_query_dcb(struct roce3_device *rdev, union roce3_query_dcb_buf *buf_in, + union roce3_query_dcb_buf *dcb_buf) +{ + int ret; + int get_group_id; + struct roce_group_id group_id = {0}; + struct roce3_get_cos_inbuf in_buf; + u8 cos; + +#ifdef ROCE_BONDING_EN + if (roce3_bond_get_dcb_info(rdev) != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb info\n", __func__); + return (-EINVAL); + } +#endif + in_buf.port_num = buf_in->cmd.port; + in_buf.sl = buf_in->cmd.sl; + in_buf.sgid_index = buf_in->cmd.sgid_idx; + in_buf.traffic_class = buf_in->cmd.traffic_class; + ret = roce3_get_dcb_cfg_cos(rdev, &in_buf, &cos); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get cos from dcb info, ret:%d\n", __func__, ret); + return ret; + } + + dcb_buf->resp.cos = cos; + + if (rdev->is_vroce) { + get_group_id = roce3_get_group_id(rdev->glb_func_id, rdev->hwdev, &group_id); + if (get_group_id != 0) { + pr_warn("Failed to get group id, ret(%d)", get_group_id); + } else { + rdev->group_rc_cos = group_id.group_rc_cos; + rdev->group_ud_cos = group_id.group_ud_cos; + rdev->group_xrc_cos = group_id.group_xrc_cos; + } + if (buf_in->cmd.dscp_type == (u8)IB_QPT_RC) + dcb_buf->resp.cos = rdev->group_rc_cos & MAX_COS_NUM; + else if (buf_in->cmd.dscp_type == (u8)IB_QPT_UD) + dcb_buf->resp.cos = rdev->group_ud_cos & MAX_COS_NUM; + else + dcb_buf->resp.cos = rdev->group_xrc_cos & MAX_COS_NUM; + } + + return 0; +} + +int roce3_drv_dcb_info_get(void *buf_in, void *buf_out, u32 *out_size) +{ + struct hiroce_dev_dcbinfo_get *cmd; + struct hiroce_dev_dcbinfo_rsp *rsp; + struct roce3_device *rdev = NULL; + int offset = 0; + int ret; + + cmd = (struct hiroce_dev_dcbinfo_get *)buf_in; + rsp = (struct hiroce_dev_dcbinfo_rsp *)buf_out; + offset = get_instance_of_func_id(cmd->func_id); + rdev = hiroce_get_adp()->netlink->rdev[offset]; + if (rdev == NULL) { + pr_err("[ROCE, ERR]: offset is invild, please check\n"); + return -EINVAL; + } + ret = roce3_netlink_query_dcb(rdev, &cmd->dcb_info, &rsp->dcb_info); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to get dcb info, ret:%d\n", __func__, ret); + return ret; + } + rsp->hdr.cmd = HIROCE_DRV_CMD_DCB_GET; + rsp->hdr.msg_len = (u32)sizeof(*rsp); + rsp->hdr.err_code = 0; + *out_size = (u32)sizeof(*rsp); + + return 0; +} + +static long roce3_netlink_create_ah(struct roce3_device *rdev, union roce3_create_ah_buf *buf, + union roce3_create_ah_buf *rsp) +{ + struct rdma_ah_attr attr; + struct ib_udata udata; + + attr.sl = buf->cmd.attr.sl; + attr.static_rate = buf->cmd.attr.static_rate; + attr.ah_flags = (buf->cmd.attr.is_global != 0) ? IB_AH_GRH : 0; + attr.port_num = buf->cmd.attr.port_num; + attr.grh.flow_label = buf->cmd.attr.grh.flow_label; + attr.grh.sgid_index = buf->cmd.attr.grh.sgid_index; + attr.grh.hop_limit = buf->cmd.attr.grh.hop_limit; + attr.grh.traffic_class = buf->cmd.attr.grh.traffic_class; + memset(attr.roce.dmac, 0, sizeof(attr.roce.dmac)); + memcpy(attr.grh.dgid.raw, buf->cmd.attr.grh.dgid, ROCE_GID_LEN); + + memset(&udata, 0, sizeof(struct ib_udata)); + if (roce3_resolve_grh(rdev, &attr, &buf->resp.vlan_id, &udata) != 0) { + pr_err("[ROCE, ERR] %s: Failed to resolve grh\n", __func__); + return -EINVAL; + } + + memcpy(buf->resp.dmac, attr.roce.dmac, ETH_ALEN); + memcpy(rsp, buf, sizeof(*buf)); + + return 0; +} + +int roce3_drv_create_ah(void *buf_in, void *buf_out, u32 *out_size) +{ + struct hiroce_dev_ah_info *cmd; + struct hiroce_dev_ah_info *rsp; + struct roce3_device *rdev = NULL; + int offset; + int ret; + + cmd = (struct hiroce_dev_ah_info *)buf_in; + rsp = (struct hiroce_dev_ah_info *)buf_out; + offset = get_instance_of_func_id(cmd->func_id); + rdev = hiroce_get_adp()->netlink->rdev[offset]; + if (rdev == NULL) { + pr_err("[ROCE, ERR]: offset is invild, please check\n"); + return -EINVAL; + } + ret = roce3_netlink_create_ah(rdev, &cmd->ah_info, &rsp->ah_info); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to create ah, ret:%d\n", __func__, ret); + return ret; + } + rsp->hdr.cmd = HIROCE_DRV_CMD_CREATE_AH; + rsp->hdr.msg_len = (u32)sizeof(*rsp); + rsp->hdr.err_code = 0; + *out_size = (u32)sizeof(*rsp); + + return 0; +} + +static int roce3_drv_cmd_process(usrnl_drv_msg_hdr_s *hdr, struct sk_buff *reply, u32 *out_size) +{ + int ret = 0; + void *buf_out = (void *)reply->data; + void *buf_in = (void *)hdr; + + switch (hdr->cmd) { + case HIROCE_DRV_CMD_DCB_GET: + ret = roce3_drv_dcb_info_get(buf_in, buf_out, out_size); + break; + case HIROCE_DRV_CMD_CREATE_AH: + ret = roce3_drv_create_ah(buf_in, buf_out, out_size); + break; + default: + pr_err("invalid drv cmd:0x%x", hdr->cmd); + ret = -EINVAL; + break; + } + return ret; +} + +static const struct nla_policy roce3_policy[USRNL_GENL_ATTR_MAX + 1] = { + [USRNL_GENL_ATTR] = { .type = NLA_UNSPEC }, //lint !e26 +}; + +static const struct genl_ops roce3_genl_ops[] = { + { .cmd = USRNL_DEV_DRV_CMD_EXECUTE, + .flags = CAP_NET_ADMIN, /* Requires CAP_NET_ADMIN privilege. */ + .policy = roce3_policy, + .doit = roce3_drv_cmd_execute, + .validate = GENL_DONT_VALIDATE_STRICT + }, +}; + +/* + * Netlink Ops + */ +#define ROCE_GENL_OPS_ARRAY_SIZE 1 + +static struct genl_family roce3_genl_family = { + .hdrsize = 0, + .name = ROCE3_FAMILY, + .version = ROCE3_VERSION, + .maxattr = USRNL_GENL_ATTR_MAX, + .netnsok = true, + .parallel_ops = true, + .ops = roce3_genl_ops, + .n_ops = ROCE_GENL_OPS_ARRAY_SIZE, +}; + +struct genl_family *roce3_genl_families[] = { + &roce3_genl_family, +}; + +int roce3_drv_cmd_execute(struct sk_buff *skb, struct genl_info *nl_info) +{ + u8 cmd; + int rc; + u32 out_size = 0; + struct sk_buff *reply = NULL; + struct nlattr *cmd_attr; + usrnl_drv_msg_hdr_s *hdr = NULL; + struct nlattr **ppstAttr = nl_info->attrs; + + cmd_attr = ppstAttr[USRNL_GENL_ATTR]; + if (cmd_attr == NULL) { + pr_err("[ROCE, ERR] %s: Do drv cmd failed for NULL attr!", __func__); + return -ENOMEM; + } + + hdr = (usrnl_drv_msg_hdr_s *)nla_data(cmd_attr); + + reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (reply == NULL) { + pr_err("Alloc reply buf for drv cmd:%d failed!", hdr->cmd); + return -ENOMEM; + } + skb_reserve(reply, NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN); + + cmd = hdr->cmd; + rc = roce3_drv_cmd_process(hdr, reply, &out_size); + if (rc != 0) { + pr_err("Do drv cmd:%u failed, errcode %d", cmd, rc); + nlmsg_free(reply); + return -EIO; + } + + hdr = (usrnl_drv_msg_hdr_s *)reply->data; + hdr->msg_len = out_size; + hdr->cmd = cmd; + (void)skb_put(reply, out_size); + + roce3_nlahdr_push(reply, USRNL_GENL_ATTR); + roce3_genlmsg_push(reply, nl_info->snd_portid, nl_info->snd_seq, + &roce3_genl_family, 0, USRNL_DEV_DRV_CMD_EXECUTE); + genlmsg_reply(reply, nl_info); + + return rc; +} + +/* pf use 32, vf use 96 */ +int get_instance_of_func_id(int glb_func_id) +{ + return glb_func_id > PF_MAX_SIZE ? glb_func_id % VF_USE_SIZE + PF_MAX_SIZE : + glb_func_id; +} + +int roce3_netlink_init(void) +{ + int err; + + err = genl_register_family(&roce3_genl_family); + + return err; +} + +void roce3_netlink_unit(void) +{ + genl_unregister_family(&roce3_genl_family); +} + +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_netlink.h b/drivers/infiniband/hw/hiroce3/roce_netlink.h new file mode 100644 index 000000000..684fc32eb --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_netlink.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef _ROCE_NETLINK_H_ +#define _ROCE_NETLINK_H_ +#ifdef ROCE_NETLINK_EN +#include <net/netlink.h> +#include <net/net_namespace.h> +#include <linux/netlink.h> + +#include "udk_usrnl.h" +#include "roce_k_ioctl.h" +#include "roce.h" + +#define USRNL_GENL_ATTR_MAX (__USRNL_GENL_ATTR_MAX - 1) + +/* netlink server info in roce ko */ +#define ROCE3_FAMILY "ROCE" +#define ROCE3_VERSION 0x1 +#define ROCE_MAX_FUNCTION 128 +#define PF_MAX_SIZE 32 +#define VF_USE_SIZE 96 + +#define P2PCOS_MAX_VALUE 2 +#define MAX_COS_NUM 0x7 + +#ifndef __NET_GENERIC_NETLINK_H +/* 解决genentlink 头文件冲突问题 */ +struct genlmsghdr { + u8 cmd; + u8 version; + u16 reserved; +}; +#define USER_PTR_SIZE 2 + +/* length of family name */ + +#define GENL_NAMSIZ 16 +#define GENL_HDRLEN NLMSG_ALIGN(sizeof(struct genlmsghdr)) + +struct genl_info { + u32 snd_seq; + u32 snd_portid; + struct nlmsghdr *nlhdr; + struct genlmsghdr *genlhdr; + void *userhdr; + struct nlattr **attrs; + possible_net_t _net; + void *user_ptr[USER_PTR_SIZE]; + struct netlink_ext_ack *extack; +}; + +struct genl_ops { + int (*doit)(struct sk_buff *skb, struct genl_info *info); + int (*start)(struct netlink_callback *cb); + int (*dumpit)(struct sk_buff *skb, struct netlink_callback *cb); + int (*done)(struct netlink_callback *cb); + const struct nla_policy *policy; + unsigned int maxattr; + u8 cmd; + u8 internal_flags; + u8 flags; + u8 validate; +}; + +struct genl_small_ops { + int (*doit)(struct sk_buff *skb, struct genl_info *info); + int (*dumpit)(struct sk_buff *skb, struct netlink_callback *cb); + u8 cmd; + u8 internal_flags; + u8 flags; + u8 validate; +}; + +struct genl_multicast_group { + char name[GENL_NAMSIZ]; +}; + +struct genl_family { +/* private */ + int id; + unsigned int hdrsize; + char name[GENL_NAMSIZ]; + unsigned int version; + unsigned int maxattr; + unsigned int mcgrp_offset; + u8 netnsok : 1; + u8 parallel_ops : 1; + u8 n_ops; + u8 n_small_ops; + u8 n_mcgrps; + const struct nla_policy *policy; + int (*pre_doit)(const struct genl_ops *ops, + struct sk_buff *skb, + struct genl_info *info); + void (*post_doit)(const struct genl_ops *ops, + struct sk_buff *skb, + struct genl_info *info); + const struct genl_ops *ops; + const struct genl_small_ops *small_ops; + const struct genl_multicast_group *mcgrps; + struct module *module; +}; + +static inline int genlmsg_unicast(struct net *net, struct sk_buff *skb, u32 portid) +{ + return nlmsg_unicast(net->genl_sock, skb, portid); +} + +static inline struct net *genl_info_net(struct genl_info *info) +{ + return read_pnet(&info->_net); +} + +static inline int genlmsg_reply(struct sk_buff *skb, struct genl_info *info) +{ + return genlmsg_unicast(genl_info_net(info), skb, info->snd_portid); +} +extern int genl_register_family(struct genl_family *family); +extern int genl_unregister_family(const struct genl_family *family); + +#endif +enum hiroce_drv_cmd_type { + HIROCE_DRV_CMD_DCB_GET = 100, + HIROCE_DRV_CMD_CREATE_AH, + HIROCE_DRV_CMD_MAX +}; + +struct hiroce_dev_dcbinfo_get { + usrnl_drv_msg_hdr_s hdr; + union roce3_query_dcb_buf dcb_info; + uint16_t func_id; + uint16_t rsvd; +}; + +struct hiroce_dev_dcbinfo_rsp { + usrnl_drv_msg_hdr_s hdr; + union roce3_query_dcb_buf dcb_info; +}; + +struct hiroce_dev_ah_info { + usrnl_drv_msg_hdr_s hdr; + union roce3_create_ah_buf ah_info; + uint16_t func_id; + uint16_t rsvd; +}; + +struct netlink_dev { + struct roce3_device *rdev[ROCE_MAX_FUNCTION]; +}; + +struct hiroce_netlink_dev { + int used_dev_num; + struct netlink_dev *netlink; + struct mutex mutex_dev; +}; + +int get_instance_of_func_id(int glb_func_id); +struct hiroce_netlink_dev *hiroce_get_adp(void); +int roce3_netlink_init(void); +void roce3_netlink_unit(void); + +#endif // ROCE_NETLINK_EN +#endif diff --git a/drivers/infiniband/hw/hiroce3/roce_pd.c b/drivers/infiniband/hw/hiroce3/roce_pd.c new file mode 100644 index 000000000..494184106 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_pd.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_pd.h" + +int roce3_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + int ret; + struct roce3_pd *pd = to_roce3_pd(ibpd); + struct roce3_device *rdev = to_roce3_dev(ibpd->device); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_rdma_pd_alloc(rdev->hwdev, &pd->pdn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc pdn, ret(%d), func_id(%u)\n", + __func__, ret, rdev->glb_func_id); + goto err_out; + } + + pd->func_id = rdev->glb_func_id; + if (udata) { + if (ib_copy_to_udata(udata, &pd->pdn, PD_RESP_SIZE) != 0) { + ret = -EFAULT; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%u)\n", + __func__, rdev->glb_func_id); + goto err_copy_to_user; + } + } + + return 0; + +err_copy_to_user: + roce3_rdma_pd_free(rdev->hwdev, pd->pdn); +err_out: + return ret; +} + +int roce3_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + struct roce3_pd *pd = NULL; + struct roce3_device *rdev = NULL; + + if (ibpd == NULL) { + pr_err("[ROCE, ERR] %s: Ibpd is null\n", __func__); + return -EINVAL; + } + + pd = to_roce3_pd(ibpd); + rdev = to_roce3_dev(ibpd->device); + + roce3_rdma_pd_free(rdev->hwdev, pd->pdn); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_pd.h b/drivers/infiniband/hw/hiroce3/roce_pd.h new file mode 100644 index 000000000..020c8354b --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_pd.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_PD_H +#define ROCE_PD_H + +#include <rdma/ib_verbs.h> + +#include "roce.h" + +#define PD_RESP_SIZE (2 * sizeof(u32)) +struct roce3_pd { + struct ib_pd ibpd; + u32 pdn; + u16 func_id; + u16 rsvd; +}; + +static inline struct roce3_pd *to_roce3_pd(const struct ib_pd *ibpd) +{ + return container_of(ibpd, struct roce3_pd, ibpd); +} + +#endif // ROCE_PD_H diff --git a/drivers/infiniband/hw/hiroce3/roce_sysfs.c b/drivers/infiniband/hw/hiroce3/roce_sysfs.c new file mode 100644 index 000000000..1503def0f --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_sysfs.c @@ -0,0 +1,1800 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/types.h> + +#include "hinic3_hw.h" +#include "rdma_context_format.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_cmd.h" +#include "roce_qp.h" + +#include "roce_sysfs.h" + +#define ROCE_DEFAULT_IP_EABLE_EN 1 +#define ROCE_DEFAULT_IP_EABLE_DIS 0 +#define ROCE_DEFAULT_MIN_CNP_PERIOD 16 +#define ROCE_DEFAULT_ALPHA_DEC_PERIOD 160 +#define ROCE_DEFAULT_RATE_DEC_PERIOD 32 +#define ROCE_DEFAULT_RATE_INC_PERIOD 480 +#define ROCE_DEFAULT_ALPHA_THRESHOLD 4 // 32, uint 8 +#define ROCE_DEFAULT_CNP_CNT_THRESHOLD 6 +#define ROCE_DEFAULT_PORT_MODE_25G 0 +#define ROCE_DEFAULT_FACTOR_GITA 7 +#define ROCE_DEFAULT_INITIAL_ALPHA 1023 +#define ROCE_DEFAULT_RATE_INC_AI 2 +#define ROCE_DEFAULT_RATE_INC_HAI 8 +#define ROCE_DEFAULT_RATE_FIRST_SET 1024 +#define ROCE_DEFAULT_RATE_TARGET_CLAMP 1 +#define ROCE_DEFAULT_QUICK_AJ_ENABLE 0 +#define ROCE_DEFAULT_CNP_PRIO_ENABLE 0 +#define ROCE_DEFAULT_CNP_PRIO 0 +#define ROCE_DEFAULT_TOKEN_PERIOD 16 +#define ROCE_DEFAULT_MIN_RATE 1 +#define ROCE_DEFAULT_WND_RESET_RATIO 0 +#define ROCE_DEFAULT_WND_RESET_TIMEOUT 0 +#define ROCE_PORT_MODE_MASK 0xfffffffe +#define ROCE_QUICK_AJ_EN_MASK 0xfffffffe +#define ROCE_CNP_PRIO_EN_MASK 0xfffffffe +#define ROCE_RT_CLAMP_MASK 0xfffffffe +#define ROCE_PRIO_EN_MASK 0xfffffffe +#define ROCE_MIN_RATE_INC_HAI 1 +#define ROCE_MAX_RATE_INC_HAI 255 +#define ROCE_MIN_RATE_INC_AI 1 +#define ROCE_MAX_RATE_INC_AI 63 +#define ROCE_MIN_INITIAL_ALPHA 127 +#define ROCE_MAX_INITIAL_ALPHA 1023 +#define ROCE_MIN_RATE_INC_PERIOD 1 +#define ROCE_MAX_RATE_INC_PERIOD 1024 +#define ROCE_MIN_ALPHA_DEC_PERIOD 1 +#define ROCE_MAX_ALPHA_DEC_PERIOD 1024 +#define ROCE_MIN_TOKEN_PERIOD 4 +#define ROCE_MAX_TOKEN_PERIOD 255 +#define ROCE_MIN_FLOW_MIN_RATE 1 +#define ROCE_MAX_FLOW_MIN_RATE 64 +#define ROCE_MIN_WND_RESET_RATIO 0 +#define ROCE_MAX_WND_RESET_RATIO 0xf +#define ROCE_MIN_WND_RESET_TIMEOUT 0 +#define ROCE_MAX_WND_RESET_TIMEOUT 0xf +#define ROCE_MIN_ALPHA_THRESHOLD 8 +#define ROCE_MAX_ALPHA_THRESHOLD 248 +#define ROCE_MIN_CNP_CNT_THRESHOLD 1 +#define ROCE_MAX_CNP_CNT_THRESHOLD 15 +#define ROCE_MIN_RATE_DEC_PERIOD 1 +#define ROCE_MAX_RATE_DEC_PERIOD 255 +#define ROCE_MIN_MIN_CNP_PERIOD 1 +#define ROCE_MAX_MIN_CNP_PERIOD 255 +#define ROCE_MIN_FACTOR_GITA 1 +#define ROCE_MAX_FACTOR_GITA 15 +#define ROCE_MIN_RATE_FIRST_SET 128 +#define ROCE_MAX_RATE_FIRST_SET 8191 + +#define MAX_STRING_FORMAT_LEN 64 + +/* ECN VER */ +#define ECN_VER_DCQCN 0 +#define ECN_VER_PATHQCN 1 + +enum { + ROCE_DCQCN_NP = 0, + ROCE_DCQCN_RP = 1 +}; + +#define to_roce3_ecn_ctx(_kobj) container_of(_kobj, struct roce3_ecn_ctx, ecn_root) +#define to_roce3_ecn_np_ctx(_kobj) container_of(_kobj, struct roce3_ecn_np_ctx, ecn_np_root) +#define to_roce3_ecn_rp_ctx(_kobj) container_of(_kobj, struct roce3_ecn_rp_ctx, ecn_rp_root) + +#define INIT_ROCE_KOBJ_ATTR(_name, _mode, _show, _store) \ + { \ + .attr = { .name = __stringify(_name), .mode = _mode }, \ + .show = (_show), \ + .store = (_store), \ + } + +#define ROCE_ATTR_RW(_name, _show, _store) \ + static struct kobj_attribute roce_attr_##_name = \ + INIT_ROCE_KOBJ_ATTR(_name, 0640, _show, _store) + +#define ROCE_ATTR_PTR(_name) (&roce_attr_##_name.attr) + +static int roce3_update_cfg_ccf_param(struct roce3_device *rdev, + const struct roce3_ecn_ctx *ecn_ctx) +{ + const struct roce3_prio_enable_ctx *prio_enable_ctx = NULL; + const struct roce3_ip_prio_enable_ctx *ip_prio_enable_ctx = NULL; + struct roce_ccf_param ccf_param; + int ret, i; + + memset(&ccf_param, 0, sizeof(ccf_param)); + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + prio_enable_ctx = &ecn_ctx->np_ctx.enable_ctx.prio_enable[i]; + ccf_param.dw0.bs.np_enable |= (prio_enable_ctx->prio_en << prio_enable_ctx->prio); + prio_enable_ctx = &ecn_ctx->rp_ctx.enable_ctx.prio_enable[i]; + ccf_param.dw0.bs.rp_enable |= (prio_enable_ctx->prio_en << prio_enable_ctx->prio); + ip_prio_enable_ctx = &ecn_ctx->ip_enable_ctx.ip_prio_enable[i]; + ccf_param.dw0.bs.ip_enable |= + (ip_prio_enable_ctx->prio_en << ip_prio_enable_ctx->prio); + } + + ccf_param.dw0.bs.cnp_prio_enable = ecn_ctx->np_ctx.cnp_prio_enable; + ccf_param.dw0.bs.port_mode = ecn_ctx->np_ctx.port_mode; + ccf_param.dw1.bs.cnp_prio = ecn_ctx->np_ctx.cnp_prio; + ccf_param.dw1.bs.cnp_cos = roce3_get_db_cos_from_vlan_pri(rdev, ccf_param.dw1.bs.cnp_prio); + ccf_param.dw1.bs.ccf_appid = ecn_ctx->cc_algo; + + ret = roce3_set_cfg_ccf_param(rdev->hwdev, rdev->glb_func_id, (u32 *)&ccf_param); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to set ccf param, func_id(%d) ret(%d)\n", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce3_update_dcqcn_param(struct roce3_device *rdev, const struct roce3_ecn_ctx *ecn_ctx) +{ + struct roce_dcqcn_param dcqcn_param; + int ret; + + memset(&dcqcn_param, 0, sizeof(dcqcn_param)); + + dcqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + dcqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate; + dcqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + dcqcn_param.dw1.bs.alpha_threshold = ecn_ctx->rp_ctx.alpha_threshold; + dcqcn_param.dw1.bs.cnp_cnt_threshold = ecn_ctx->rp_ctx.cnp_cnt_threshold; + dcqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + dcqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + dcqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + dcqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + dcqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + dcqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita; + dcqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp; + dcqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + // adjust_en + dcqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce3_set_cfg_dcqcn_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&dcqcn_param); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to set dcqcn param, func_id(%d), ret(%d)\n", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce3_update_ipqcn_param(struct roce3_device *rdev, const struct roce3_ecn_ctx *ecn_ctx) +{ + struct roce_ipqcn_param ipqcn_param; + int ret; + + memset(&ipqcn_param, 0, sizeof(ipqcn_param)); + + ipqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + ipqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate; + ipqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + ipqcn_param.dw1.bs.alpha_threshold = ecn_ctx->rp_ctx.alpha_threshold; + ipqcn_param.dw1.bs.cnp_cnt_threshold = ecn_ctx->rp_ctx.cnp_cnt_threshold; + ipqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + ipqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + ipqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + ipqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + ipqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + ipqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita; + ipqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp; + ipqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + ipqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce3_set_cfg_ipqcn_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ipqcn_param); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] : Failed to set ipqcn param, func_id(%d) ret(%d)\n", + rdev->glb_func_id, ret); + } + + return ret; +} + +typedef int (*roce3_update_param_t)(struct roce3_device *rdev, const struct roce3_ecn_ctx *ecn_ctx); + +static roce3_update_param_t roce3_update_param_funcs[] = { + roce3_update_cfg_ccf_param, + roce3_update_dcqcn_param, + roce3_update_ipqcn_param, +}; + +int roce3_update_ecn_param(const struct roce3_ecn_ctx *ecn_ctx) +{ + unsigned int i; + int ret = 0; + struct roce3_device *rdev = container_of(ecn_ctx, struct roce3_device, ecn_ctx); + + for (i = 0; i < (sizeof(roce3_update_param_funcs) / sizeof(roce3_update_param_t)); i++) { + ret = roce3_update_param_funcs[i](rdev, ecn_ctx); + if (ret != 0) + ret = -EINVAL; + } + + return ret; +} + +static ssize_t roce3_show_cc_algo(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + pr_err("[ROCE] %s: attr or buf is null\n", __func__); + return -EINVAL; + } + + switch ((int)ecn_ctx->cc_algo) { + case ROCE_CC_DISABLE: + return sprintf(buf, "%s\n", "disable"); + case ROCE_CC_DCQCN_ALGO: + return sprintf(buf, "%s\n", "dcqcn"); + case ROCE_CC_LDCP_ALGO: + return sprintf(buf, "%s\n", "ldcp"); + case ROCE_CC_IPQCN_ALGO: + return sprintf(buf, "%s\n", "hc3_1/ipqcn"); + default: + return sprintf(buf, "%s\n", "error_type"); + } +} + +static ssize_t roce3_store_cc_algo(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + u32 old_cc_algo = 0; + int cc_algo = 0; + ssize_t ret; + + if ((attr == NULL) || (buf == NULL)) { + pr_err("[ROCE] %s: attr or buf is null\n", __func__); + return -EINVAL; + } + + /* dcqcn -> 0x0, ldcp -> 0x2, hc3_1 -> 0x80 */ + if (strncmp(buf, "disable", strlen("disable")) == 0) { + cc_algo = ROCE_CC_DISABLE; + } else if (strncmp(buf, "dcqcn", strlen("dcqcn")) == 0) { + cc_algo = ROCE_CC_DCQCN_ALGO; + } else if ((strncmp(buf, "ipqcn", strlen("ipqcn")) == 0) || + (strncmp(buf, "hc3_1", strlen("hc3_1")) == 0)) { + cc_algo = ROCE_CC_IPQCN_ALGO; + } else if (strncmp(buf, "ldcp", strlen("ldcp")) == 0) { + cc_algo = ROCE_CC_LDCP_ALGO; + } else { + pr_err("[ROCE] %s: Invalid cc_algo(%d),buf(%s)\n", __func__, cc_algo, buf); + return -EIO; + } + + if (ecn_ctx->cc_algo != (u32)cc_algo) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cc_algo = ecn_ctx->cc_algo; + ecn_ctx->cc_algo = (u32)cc_algo; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to update cc_algo param, ret(%d)", + __func__, (int)ret); + ecn_ctx->cc_algo = old_cc_algo; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce3_prio_enable_ctx, enable); + + return sprintf(buf, "%d\n", (int)prio_enable_ctx->prio_en); +} + +static ssize_t roce3_store_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce3_ecn_rp_ctx *rp_ctx = NULL; + struct roce3_ecn_np_ctx *np_ctx = NULL; + struct roce3_ecn_ctx *ecn_ctx = NULL; + struct roce3_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce3_prio_enable_ctx, enable); + struct roce3_ecn_enable_ctx *ecn_enable_ctx = + (struct roce3_ecn_enable_ctx *)prio_enable_ctx->ecn_enable_ctx; + + if (ecn_enable_ctx->np_rp == ROCE_DCQCN_NP) { + np_ctx = container_of(ecn_enable_ctx, struct roce3_ecn_np_ctx, enable_ctx); + ecn_ctx = container_of(np_ctx, struct roce3_ecn_ctx, np_ctx); + } else { + rp_ctx = container_of(ecn_enable_ctx, struct roce3_ecn_rp_ctx, enable_ctx); + ecn_ctx = container_of(rp_ctx, struct roce3_ecn_ctx, rp_ctx); + } + + ret = kstrtoint(buf, 10, &prio_en); + if (ret != 0) + return -EIO; + + if (((u32)prio_en & ROCE_PRIO_EN_MASK) != 0) + return -EIO; + + if (prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = prio_enable_ctx->prio_en; + prio_enable_ctx->prio_en = (u32)prio_en; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce3_ip_prio_enable_ctx, ip_enable); + + return sprintf(buf, "%d\n", (int)ip_prio_enable_ctx->prio_en); +} + +static ssize_t roce3_store_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce3_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce3_ip_prio_enable_ctx, ip_enable); + struct roce3_ecn_ip_enable_ctx *ip_enable_ctx = + (struct roce3_ecn_ip_enable_ctx *)ip_prio_enable_ctx->ecn_ip_enable_ctx; + struct roce3_ecn_ctx *ecn_ctx = container_of(ip_enable_ctx, + struct roce3_ecn_ctx, ip_enable_ctx); + + ret = kstrtoint(buf, 10, &prio_en); + if (ret != 0) + return -EIO; + + if (((u32)prio_en & ROCE_PRIO_EN_MASK) != 0) + return -EIO; + + if (ip_prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = ip_prio_enable_ctx->prio_en; + ip_prio_enable_ctx->prio_en = (u32)prio_en; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ip_prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_inc_hai(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_inc_hai); +} + +static ssize_t roce3_store_rate_inc_hai(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_inc_hai = 0; + u32 old_rate_inc_hai = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_inc_hai); + if (ret != 0) + return -EIO; + + if ((rate_inc_hai < ROCE_MIN_RATE_INC_HAI) || (rate_inc_hai > ROCE_MAX_RATE_INC_HAI)) + return -EIO; + + if (ecn_rp_ctx->rate_inc_hai != (u32)rate_inc_hai) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_inc_hai = ecn_rp_ctx->rate_inc_hai; + ecn_rp_ctx->rate_inc_hai = (u32)rate_inc_hai; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_inc_hai = old_rate_inc_hai; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_inc_ai(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_inc_ai); +} + +static ssize_t roce3_store_rate_inc_ai(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_inc_ai = 0; + u32 old_rate_inc_ai = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_inc_ai); + if (ret != 0) + return -EIO; + + if ((rate_inc_ai < ROCE_MIN_RATE_INC_AI) || (rate_inc_ai > ROCE_MAX_RATE_INC_AI)) + return -EIO; + + if (ecn_rp_ctx->rate_inc_ai != (u32)rate_inc_ai) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_inc_ai = ecn_rp_ctx->rate_inc_ai; + ecn_rp_ctx->rate_inc_ai = (u32)rate_inc_ai; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_inc_ai = old_rate_inc_ai; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_initial_alpha(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->initial_alpha); +} + +static ssize_t roce3_store_initial_alpha(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int initial_alpha = 0; + u32 old_initial_alpha = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &initial_alpha); + if (ret != 0) + return -EIO; + + if ((initial_alpha < ROCE_MIN_INITIAL_ALPHA) || (initial_alpha > ROCE_MAX_INITIAL_ALPHA)) + return -EIO; + + if (ecn_rp_ctx->initial_alpha != (u32)initial_alpha) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_initial_alpha = ecn_rp_ctx->initial_alpha; + ecn_rp_ctx->initial_alpha = (u32)initial_alpha; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->initial_alpha = old_initial_alpha; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_factor_gita(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->factor_gita); +} + +static ssize_t roce3_store_factor_gita(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int factor_gita = 0; + u32 old_factor_gita = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &factor_gita); + if (ret != 0) + return -EIO; + + if ((factor_gita < ROCE_MIN_FACTOR_GITA) || (factor_gita > ROCE_MAX_FACTOR_GITA)) + return -EIO; + + if (ecn_rp_ctx->factor_gita != (u32)factor_gita) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_factor_gita = ecn_rp_ctx->factor_gita; + ecn_rp_ctx->factor_gita = (u32)factor_gita; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->factor_gita = old_factor_gita; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_inc_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_inc_period); +} + +static ssize_t roce3_store_rate_inc_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_inc_period = 0; + u32 old_rate_inc_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_inc_period); + if (ret != 0) + return -EIO; + + if ((rate_inc_period < ROCE_MIN_RATE_INC_PERIOD) || + (rate_inc_period > ROCE_MAX_RATE_INC_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->rate_inc_period != (u32)rate_inc_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_inc_period = ecn_rp_ctx->rate_inc_period; + ecn_rp_ctx->rate_inc_period = (u32)rate_inc_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_inc_period = old_rate_inc_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_cnp_cnt_threshold(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->cnp_cnt_threshold); +} + +static ssize_t roce3_store_cnp_cnt_threshold(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int cnp_cnt_threshold = 0; + u32 old_cnp_cnt_threshold = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &cnp_cnt_threshold); + if (ret != 0) + return -EIO; + + if ((cnp_cnt_threshold < ROCE_MIN_CNP_CNT_THRESHOLD) || + (cnp_cnt_threshold > ROCE_MAX_CNP_CNT_THRESHOLD)) + return -EIO; + + if (ecn_rp_ctx->cnp_cnt_threshold != (u32)cnp_cnt_threshold) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_cnt_threshold = ecn_rp_ctx->cnp_cnt_threshold; + ecn_rp_ctx->cnp_cnt_threshold = (u32)cnp_cnt_threshold; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->cnp_cnt_threshold = old_cnp_cnt_threshold; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_alpha_threshold(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", + (int)(ecn_rp_ctx->alpha_threshold << ALPHA_THREADHOLD_UNIT_SHIFT)); +} + +static ssize_t roce3_store_alpha_threshold(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + u32 alpha_threshold = 0; + u32 old_alpha_threshold = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &alpha_threshold); + if (ret != 0) + return -EIO; + + if ((alpha_threshold < ROCE_MIN_ALPHA_THRESHOLD) || + (alpha_threshold > ROCE_MAX_ALPHA_THRESHOLD)) + return -EIO; + alpha_threshold = ((u32)alpha_threshold >> ALPHA_THREADHOLD_UNIT_SHIFT); + + if (ecn_rp_ctx->alpha_threshold != (u32)alpha_threshold) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_alpha_threshold = ecn_rp_ctx->alpha_threshold; + ecn_rp_ctx->alpha_threshold = (u32)alpha_threshold; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->alpha_threshold = old_alpha_threshold; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_dec_period); +} + +static ssize_t roce3_store_rate_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_dec_period = 0; + u32 old_rate_dec_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_dec_period); + if (ret != 0) + return -EIO; + + if ((rate_dec_period < ROCE_MIN_RATE_DEC_PERIOD) || + (rate_dec_period > ROCE_MAX_RATE_DEC_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->rate_dec_period != (u32)rate_dec_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_dec_period = ecn_rp_ctx->rate_dec_period; + ecn_rp_ctx->rate_dec_period = (u32)rate_dec_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_dec_period = old_rate_dec_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_token_period(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->token_period); +} + +static ssize_t roce3_store_token_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int token_period = 0; + u32 old_token_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &token_period); + if (ret != 0) + return -EIO; + + if ((token_period < ROCE_MIN_TOKEN_PERIOD) || + (token_period > ROCE_MAX_TOKEN_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->token_period != (u32)token_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_token_period = ecn_rp_ctx->token_period; + ecn_rp_ctx->token_period = (u32)token_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->token_period = old_token_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_min_rate(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->min_rate); +} + +static ssize_t roce3_store_min_rate(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int min_rate = 0; + u32 old_min_rate = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &min_rate); + if (ret != 0) + return -EIO; + + if ((min_rate < ROCE_MIN_FLOW_MIN_RATE) || (min_rate > ROCE_MAX_FLOW_MIN_RATE)) + return -EIO; + + if (ecn_rp_ctx->token_period != (u32)min_rate) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_min_rate = ecn_rp_ctx->min_rate; + ecn_rp_ctx->min_rate = (u32)min_rate; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->min_rate = old_min_rate; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_alpha_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->alpha_dec_period); +} + +static ssize_t roce3_store_alpha_dec_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int alpha_dec_period = 0; + u32 old_alpha_dec_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &alpha_dec_period); + if (ret != 0) + return -EIO; + + if ((alpha_dec_period < ROCE_MIN_ALPHA_DEC_PERIOD) || + (alpha_dec_period > ROCE_MAX_ALPHA_DEC_PERIOD)) + return -EIO; + + if (ecn_rp_ctx->alpha_dec_period != (u32)alpha_dec_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_alpha_dec_period = ecn_rp_ctx->alpha_dec_period; + ecn_rp_ctx->alpha_dec_period = (u32)alpha_dec_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->alpha_dec_period = old_alpha_dec_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_first_set(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_first_set); +} + +static ssize_t roce3_store_rate_first_set(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_first_set = 0; + u32 old_rate_first_set = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_first_set); + if (ret != 0) + return -EIO; + + if ((rate_first_set < ROCE_MIN_RATE_FIRST_SET) || + (rate_first_set > ROCE_MAX_RATE_FIRST_SET)) + return -EIO; + + if (ecn_rp_ctx->rate_first_set != (u32)rate_first_set) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_first_set = ecn_rp_ctx->rate_first_set; + ecn_rp_ctx->rate_first_set = (u32)rate_first_set; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_first_set = old_rate_first_set; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_rp_ctx->rate_target_clamp); +} + +static ssize_t roce3_store_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int rate_target_clamp = 0; + u32 old_rate_target_clamp = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_rp_ctx *ecn_rp_ctx = to_roce3_ecn_rp_ctx(kobj); + + ret = kstrtoint(buf, 10, &rate_target_clamp); + if (ret != 0) + return -EIO; + + if (((u32)rate_target_clamp & ROCE_RT_CLAMP_MASK) != 0) + return -EIO; + + if (ecn_rp_ctx->rate_target_clamp != (u32)rate_target_clamp) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_target_clamp = ecn_rp_ctx->rate_target_clamp; + ecn_rp_ctx->rate_target_clamp = (u32)rate_target_clamp; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_rp_ctx->rate_target_clamp = old_rate_target_clamp; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_min_cnp_period(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->min_cnp_period); +} + +static ssize_t roce3_store_min_cnp_period(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int min_cnp_period = 0; + u32 old_min_cnp_period = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &min_cnp_period); + if (ret != 0) + return -EIO; + + if ((min_cnp_period < ROCE_MIN_MIN_CNP_PERIOD) || + (min_cnp_period > ROCE_MAX_MIN_CNP_PERIOD)) + return -EIO; + + if (ecn_np_ctx->min_cnp_period != (u32)min_cnp_period) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_min_cnp_period = ecn_np_ctx->min_cnp_period; + ecn_np_ctx->min_cnp_period = (u32)min_cnp_period; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->min_cnp_period = old_min_cnp_period; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_port_mode(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->port_mode); +} + +static ssize_t roce3_store_port_mode(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int port_mode = 0; + u32 old_port_mode = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &port_mode); + if (ret != 0) + return -EIO; + + if (((u32)port_mode & ROCE_PORT_MODE_MASK) != 0) + return -EIO; + + if (ecn_np_ctx->port_mode != (u32)port_mode) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_port_mode = ecn_np_ctx->port_mode; + ecn_np_ctx->port_mode = (u32)port_mode; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->port_mode = old_port_mode; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_quick_adjust_en(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->quick_adjust_en); +} + +static ssize_t roce3_store_quick_adjust_en(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int quick_adjust_en = 0; + u32 old_quick_adjust_en = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &quick_adjust_en); + if (ret != 0) + return -EIO; + + if (((u32)quick_adjust_en & ROCE_QUICK_AJ_EN_MASK) != 0) + return -EIO; + + if (ecn_np_ctx->quick_adjust_en != (u32)quick_adjust_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_quick_adjust_en = ecn_np_ctx->quick_adjust_en; + ecn_np_ctx->quick_adjust_en = (u32)quick_adjust_en; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->quick_adjust_en = old_quick_adjust_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->cnp_prio_enable); +} + +static ssize_t roce3_store_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + ssize_t ret = 0; + int cnp_prio_enable = 0; + u32 old_cnp_prio_enable = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &cnp_prio_enable); + if (ret != 0) + return -EIO; + + if (((u32)cnp_prio_enable & ROCE_CNP_PRIO_EN_MASK) != 0) + return -EIO; + + if (ecn_np_ctx->cnp_prio_enable != (u32)cnp_prio_enable) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio_enable = ecn_np_ctx->cnp_prio_enable; + ecn_np_ctx->cnp_prio_enable = (u32)cnp_prio_enable; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->cnp_prio_enable = old_cnp_prio_enable; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce3_show_cnp_prio(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_np_ctx->cnp_prio); +} + +static ssize_t roce3_store_cnp_prio(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = 0; + int cnp_prio = 0; + u32 old_cnp_prio = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj->parent); + struct roce3_ecn_np_ctx *ecn_np_ctx = to_roce3_ecn_np_ctx(kobj); + + ret = kstrtoint(buf, 10, &cnp_prio); + if (ret != 0) + return -EIO; + + if (((u32)cnp_prio) > (PRI_ARRAY_LEN - 1)) + return -EIO; + + if (ecn_np_ctx->cnp_prio != (u32)cnp_prio) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio = ecn_np_ctx->cnp_prio; + ecn_np_ctx->cnp_prio = (u32)cnp_prio; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->cnp_prio = old_cnp_prio; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static int roce3_init_ecn_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_enable_ctx *ecn_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = {"0", "1", "2", "3", "4", "5", "6", "7"}; + struct roce3_prio_enable_ctx *prio_enable = NULL; + + ecn_enable_ctx->enable_root = kobject_create_and_add("enable", kobj); + if (ecn_enable_ctx->enable_root == NULL) { + pr_err("[ROCE, ERR] %s: Failed to create kobject for ecn enable.(ret:%d)\n", + __func__, ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + prio_enable = &ecn_enable_ctx->prio_enable[i]; + prio_enable->ecn_enable_ctx = (void *)ecn_enable_ctx; + prio_enable->prio = (u32)i; + prio_enable->prio_en = 1; + sysfs_attr_init(&prio_enable->enable.attr); + prio_enable->enable.attr.name = prio_arr[i]; + prio_enable->enable.attr.mode = 0640; + prio_enable->enable.show = roce3_show_ecn_prio_enable; + prio_enable->enable.store = roce3_store_ecn_prio_enable; + ret = sysfs_create_file(ecn_enable_ctx->enable_root, &prio_enable->enable.attr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to create file for ecn enable.(ret:%d)\n", + __func__, ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); + + return ret; +} + +static void roce3_remove_ecn_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_enable_ctx *ecn_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); +} + +static int roce3_init_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = {"0", "1", "2", "3", "4", "5", "6", "7"}; + struct roce3_ip_prio_enable_ctx *ip_prio_enable = NULL; + + ip_enable_ctx->ip_enable_root = kobject_create_and_add("ip_enable", kobj); + if (ip_enable_ctx->ip_enable_root == NULL) { + pr_err("[ROCE, ERR] %s: Failed to create kobject for ecn ip enable.(ret:%d)\n", + __func__, ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + ip_prio_enable = &ip_enable_ctx->ip_prio_enable[i]; + ip_prio_enable->ecn_ip_enable_ctx = (void *)ip_enable_ctx; + ip_prio_enable->prio = (u32)i; + ip_prio_enable->prio_en = ROCE_DEFAULT_IP_EABLE_EN; + sysfs_attr_init(&ip_prio_enable->ip_enable.attr); + ip_prio_enable->ip_enable.attr.name = prio_arr[i]; + ip_prio_enable->ip_enable.attr.mode = 0640; + ip_prio_enable->ip_enable.show = roce3_show_ecn_ip_prio_enable; + ip_prio_enable->ip_enable.store = roce3_store_ecn_ip_prio_enable; + ret = sysfs_create_file(ip_enable_ctx->ip_enable_root, + &ip_prio_enable->ip_enable.attr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to create file for ecn ip enable.(ret:%d)\n", + __func__, ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file(ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); + + return ret; +} + +static void roce3_remove_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce3_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file(ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); +} + +ROCE_ATTR_RW(alpha_dec_period, roce3_show_alpha_dec_period, roce3_store_alpha_dec_period); +ROCE_ATTR_RW(rate_dec_period, roce3_show_rate_dec_period, roce3_store_rate_dec_period); +ROCE_ATTR_RW(rate_inc_period, roce3_show_rate_inc_period, roce3_store_rate_inc_period); +ROCE_ATTR_RW(cnp_cnt_threshold, roce3_show_cnp_cnt_threshold, roce3_store_cnp_cnt_threshold); +ROCE_ATTR_RW(alpha_threshold, roce3_show_alpha_threshold, roce3_store_alpha_threshold); +ROCE_ATTR_RW(factor_gita, roce3_show_factor_gita, roce3_store_factor_gita); +ROCE_ATTR_RW(initial_alpha, roce3_show_initial_alpha, roce3_store_initial_alpha); +ROCE_ATTR_RW(rate_inc_ai, roce3_show_rate_inc_ai, roce3_store_rate_inc_ai); +ROCE_ATTR_RW(rate_inc_hai, roce3_show_rate_inc_hai, roce3_store_rate_inc_hai); +ROCE_ATTR_RW(rate_first_set, roce3_show_rate_first_set, roce3_store_rate_first_set); +ROCE_ATTR_RW(rate_target_clamp, roce3_show_rate_target_clamp, roce3_store_rate_target_clamp); +ROCE_ATTR_RW(token_period, roce3_show_token_period, roce3_store_token_period); +ROCE_ATTR_RW(min_rate, roce3_show_min_rate, roce3_store_min_rate); + +static struct attribute *ecn_rp_ctx_attrs[] = { + ROCE_ATTR_PTR(alpha_dec_period), + ROCE_ATTR_PTR(rate_dec_period), + ROCE_ATTR_PTR(rate_inc_period), + ROCE_ATTR_PTR(cnp_cnt_threshold), + ROCE_ATTR_PTR(alpha_threshold), + ROCE_ATTR_PTR(factor_gita), + ROCE_ATTR_PTR(initial_alpha), + ROCE_ATTR_PTR(rate_inc_ai), + ROCE_ATTR_PTR(rate_inc_hai), + ROCE_ATTR_PTR(rate_first_set), + ROCE_ATTR_PTR(rate_target_clamp), + ROCE_ATTR_PTR(token_period), + ROCE_ATTR_PTR(min_rate), + NULL, +}; +ATTRIBUTE_GROUPS(ecn_rp_ctx); + +static void roce_ecn_rp_sysfs_release(struct kobject *kobj) {} + +static struct kobj_type roce_ecn_rp_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_rp_sysfs_release, + .default_groups = ecn_rp_ctx_groups, +}; + +static int roce3_init_ecn_rp_sysfs(struct kobject *kobj, struct roce3_ecn_rp_ctx *ecn_rp_ctx) +{ + int ret = 0; + + ecn_rp_ctx->alpha_dec_period = ROCE_DEFAULT_ALPHA_DEC_PERIOD; + ecn_rp_ctx->rate_dec_period = ROCE_DEFAULT_RATE_DEC_PERIOD; + ecn_rp_ctx->rate_inc_period = ROCE_DEFAULT_RATE_INC_PERIOD; + ecn_rp_ctx->alpha_threshold = ROCE_DEFAULT_ALPHA_THRESHOLD; + ecn_rp_ctx->cnp_cnt_threshold = ROCE_DEFAULT_CNP_CNT_THRESHOLD; + ecn_rp_ctx->factor_gita = ROCE_DEFAULT_FACTOR_GITA; + ecn_rp_ctx->initial_alpha = ROCE_DEFAULT_INITIAL_ALPHA; + ecn_rp_ctx->rate_inc_ai = ROCE_DEFAULT_RATE_INC_AI; + ecn_rp_ctx->rate_inc_hai = ROCE_DEFAULT_RATE_INC_HAI; + ecn_rp_ctx->rate_first_set = ROCE_DEFAULT_RATE_FIRST_SET; + ecn_rp_ctx->rate_target_clamp = ROCE_DEFAULT_RATE_TARGET_CLAMP; + ecn_rp_ctx->token_period = ROCE_DEFAULT_TOKEN_PERIOD; + ecn_rp_ctx->min_rate = ROCE_DEFAULT_MIN_RATE; + ret = kobject_init_and_add(&ecn_rp_ctx->ecn_rp_root, &roce_ecn_rp_ktype, kobj, "roce3_rp"); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add kobject roce3_rp.(ret:%d)\n", __func__, ret); + kobject_put(&ecn_rp_ctx->ecn_rp_root); + return ret; + } + + ecn_rp_ctx->enable_ctx.np_rp = ROCE_DCQCN_RP; + ret = roce3_init_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, &ecn_rp_ctx->enable_ctx); + if (ret != 0) + goto err_init_prio_enable_sysfs; + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_rp_ctx->ecn_rp_root); + + return ret; +} + +static void roce3_remove_ecn_rp_sysfs(struct kobject *kobj, struct roce3_ecn_rp_ctx *ecn_rp_ctx) +{ + roce3_remove_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, &ecn_rp_ctx->enable_ctx); + + kobject_put(&ecn_rp_ctx->ecn_rp_root); +} + +ROCE_ATTR_RW(min_cnp_period, roce3_show_min_cnp_period, roce3_store_min_cnp_period); +ROCE_ATTR_RW(quick_adjust_en, roce3_show_quick_adjust_en, roce3_store_quick_adjust_en); +ROCE_ATTR_RW(port_mode, roce3_show_port_mode, roce3_store_port_mode); +ROCE_ATTR_RW(cnp_prio_enable, roce3_show_cnp_prio_enable, roce3_store_cnp_prio_enable); +ROCE_ATTR_RW(cnp_prio, roce3_show_cnp_prio, roce3_store_cnp_prio); + +static struct attribute *ecn_np_ctx_attrs[] = { + ROCE_ATTR_PTR(min_cnp_period), + ROCE_ATTR_PTR(quick_adjust_en), + ROCE_ATTR_PTR(port_mode), + ROCE_ATTR_PTR(cnp_prio_enable), + ROCE_ATTR_PTR(cnp_prio), + NULL, +}; +ATTRIBUTE_GROUPS(ecn_np_ctx); + +static void roce_ecn_np_sysfs_release(struct kobject *kobj) {} + +static struct kobj_type roce_ecn_np_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_np_sysfs_release, + .default_groups = ecn_np_ctx_groups, +}; + +static int roce3_init_ecn_np_sysfs(struct kobject *kobj, struct roce3_ecn_np_ctx *ecn_np_ctx) +{ + int ret = 0; + + ecn_np_ctx->min_cnp_period = ROCE_DEFAULT_MIN_CNP_PERIOD; + ecn_np_ctx->quick_adjust_en = ROCE_DEFAULT_QUICK_AJ_ENABLE; + ecn_np_ctx->port_mode = ROCE_DEFAULT_PORT_MODE_25G; + ecn_np_ctx->cnp_prio_enable = ROCE_DEFAULT_CNP_PRIO_ENABLE; + ecn_np_ctx->cnp_prio = ROCE_DEFAULT_CNP_PRIO; + ret = kobject_init_and_add(&ecn_np_ctx->ecn_np_root, &roce_ecn_np_ktype, kobj, "roce3_np"); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add kobject roce3_np.(ret:%d)\n", __func__, ret); + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; + } + + ecn_np_ctx->enable_ctx.np_rp = ROCE_DCQCN_NP; + ret = roce3_init_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, &ecn_np_ctx->enable_ctx); + if (ret != 0) + goto err_init_prio_enable_sysfs; + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; +} + +static void roce3_remove_ecn_np_sysfs(struct kobject *kobj, struct roce3_ecn_np_ctx *ecn_np_ctx) +{ + roce3_remove_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, &ecn_np_ctx->enable_ctx); + + kobject_put(&ecn_np_ctx->ecn_np_root); +} + +static ssize_t roce3_show_ecn_ver(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + + return sprintf(buf, "%d\n", (int)ecn_ctx->ecn_ver); +} + +static ssize_t roce3_store_ecn_ver(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = 0; + int ecn_ver = 0; + u32 old_ecn_ver = 0; + struct roce3_ecn_ctx *ecn_ctx = to_roce3_ecn_ctx(kobj); + + ret = kstrtoint(buf, 10, &ecn_ver); + if (ret != 0) + return -EIO; + + if ((ecn_ver < ECN_VER_DCQCN) || (ecn_ver > ECN_VER_PATHQCN)) + return -EIO; + + if (ecn_ctx->ecn_ver != (u32)ecn_ver) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ecn_ver = ecn_ctx->ecn_ver; + ecn_ctx->ecn_ver = (u32)ecn_ver; + ret = (ssize_t)roce3_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_ctx->ecn_ver = old_ecn_ver; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_ATTR_RW(ecn_ver, roce3_show_ecn_ver, roce3_store_ecn_ver); +ROCE_ATTR_RW(cc_algo, roce3_show_cc_algo, roce3_store_cc_algo); + +static struct attribute *ecn_ctx_attrs[] = { + ROCE_ATTR_PTR(ecn_ver), + ROCE_ATTR_PTR(cc_algo), + NULL, +}; +ATTRIBUTE_GROUPS(ecn_ctx); + +static void roce_ecn_sysfs_release(struct kobject *kobj) {} + +static struct kobj_type roce_ecn_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_sysfs_release, + .default_groups = ecn_ctx_groups, +}; + +static int roce3_init_ecn_sysfs(struct net_device *ndev, struct roce3_ecn_ctx *ecn_ctx) +{ + int ret = 0; + + memset(ecn_ctx, 0, sizeof(*ecn_ctx)); + + mutex_init(&ecn_ctx->ecn_mutex); + + ecn_ctx->ecn_ver = ECN_VER_DCQCN; +#ifdef ROCE_COMPUTE +#ifdef EULER_2_10_OFED_4_18 + ecn_ctx->cc_algo = ROCE_CC_LDCP_ALGO; +#else + ecn_ctx->cc_algo = ROCE_CC_DCQCN_ALGO; +#endif +#elif defined(ROCE_VBS_EN) || defined(ROCE_STANDARD) + ecn_ctx->cc_algo = ROCE_CC_LDCP_ALGO; +#else + ecn_ctx->cc_algo = ROCE_CC_DISABLE; +#endif + + ret = kobject_init_and_add(&ecn_ctx->ecn_root, &roce_ecn_ktype, &ndev->dev.kobj, "ecn"); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to add kobject ecn.(ret:%d)\n", __func__, ret); + kobject_put(&ecn_ctx->ecn_root); + return ret; + } + + ret = roce3_init_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + if (ret != 0) + goto err_init_np_sysfs; + + ret = roce3_init_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + if (ret != 0) + goto err_init_rp_sysfs; + + ret = roce3_init_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ip_enable_ctx); + if (ret != 0) + goto err_init_ip_enable_sysfs; + + ret = roce3_update_ecn_param(ecn_ctx); + if (ret != 0) + goto err_update; + + return 0; + +err_update: + roce3_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ip_enable_ctx); +err_init_ip_enable_sysfs: + roce3_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); +err_init_rp_sysfs: + roce3_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); +err_init_np_sysfs: + kobject_put(&ecn_ctx->ecn_root); + return ret; +} + +static void roce3_prase_qpc(struct roce_qp_context *qpc) +{ + qpc->chip_seg.sqc.dw2.value = be32_to_cpu(qpc->chip_seg.sqc.dw2.value); + qpc->chip_seg.sqc.dw3.value = be32_to_cpu(qpc->chip_seg.sqc.dw3.value); + qpc->chip_seg.sqac.dw3.value = be32_to_cpu(qpc->chip_seg.sqac.dw3.value); + qpc->chip_seg.sqac.dw7.value = be32_to_cpu(qpc->chip_seg.sqac.dw7.value); + qpc->chip_seg.rqc.dw3.value = be32_to_cpu(qpc->chip_seg.rqc.dw3.value); + qpc->chip_seg.rqc.dw7.value = be32_to_cpu(qpc->chip_seg.rqc.dw7.value); + qpc->sw_seg.ucode_seg.common.dw3.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.common.dw3.value); + qpc->sw_seg.ucode_seg.sq_ctx.dw9.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.sq_ctx.dw9.value); + qpc->sw_seg.ucode_seg.sq_ctx.ack_ctx.dw15.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.sq_ctx.ack_ctx.dw15.value); + qpc->sw_seg.ucode_seg.rq_ctx.dw22.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.rq_ctx.dw22.value); + qpc->sw_seg.ucode_seg.rq_ctx.ack_ctx.dw27.value = + be32_to_cpu(qpc->sw_seg.ucode_seg.rq_ctx.ack_ctx.dw27.value); +} + +static ssize_t roce3_store_dfx_qpc(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 qpn; + struct roce3_dfx_qpc_ctx *qpc_ctx = NULL; + struct roce3_dfx_ctx *dfx_ctx = NULL; + struct roce3_device *rdev = NULL; + struct roce_qp_context qpc; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + pr_err("[ROCE] %s: Invalid input para\n", __func__); + return -EINVAL; + } + + qpc_ctx = container_of(attr, struct roce3_dfx_qpc_ctx, kattr); + dfx_ctx = container_of(qpc_ctx, struct roce3_dfx_ctx, qpc_ctx); + rdev = container_of(dfx_ctx, struct roce3_device, dfx_ctx); + + memset(&qpc, 0, sizeof(struct roce_qp_context)); + + ret = (int)kstrtou32(buf, 10, &qpn); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to do sscanf_s, ret(%d)\n", __func__, ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if ((qpn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_qps) || (qpn < ROCE_MIN_QPN)) { + pr_err("[ROCE] %s: Invalid qpn(%u), max_qps(%u)\n", + __func__, qpn, rdma_cap->dev_rdma_cap.roce_own_cap.max_qps); + return -EIO; + } + + ret = roce3_dfx_cmd_query_qp(rdev, qpn, &qpc); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to query qpc, ret(%d)\n", __func__, ret); + return (ssize_t)ret; + } + + roce3_prase_qpc(&qpc); + + roce3_dfx_print("qpn(0x%x): sqc pi(0x%x), sqc ci(0x%x), ccf_appid(0x%x)\n", + qpn, qpc.chip_seg.sqc.dw2.bs.sq_pi, qpc.chip_seg.sqc.dw3.bs.sq_ci, + qpc.sw_seg.ucode_seg.common.dw0.bs.ccf_appid); + roce3_dfx_print("sqac ci(0x%x), sqac wqe_prefetch_ci(0x%x), rqc pi(0x%x), rqc ci(0x%x)\n", + qpc.chip_seg.sqac.dw3.bs.sqa_ci, qpc.chip_seg.sqac.dw7.bs.sqa_wqe_prefetch_ci, + qpc.chip_seg.rqc.dw7.bs.rq_pi, qpc.chip_seg.rqc.dw3.bs.rq_ci); + roce3_dfx_print("sq_ssn(0x%x), sq_rcv_msn(0x%x), rq_last_msn(0x%x), rqa_msn(0x%x)\n", + qpc.sw_seg.ucode_seg.sq_ctx.dw9.bs.ssn, + qpc.sw_seg.ucode_seg.sq_ctx.ack_ctx.dw15.bs.sq_rcv_msn, + qpc.sw_seg.ucode_seg.rq_ctx.dw22.bs.last_msn, + qpc.sw_seg.ucode_seg.rq_ctx.ack_ctx.dw27.bs.msn); + + return (ssize_t)count; +} + +static ssize_t roce3_store_dfx_cqc(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 cqn; + struct roce3_dfx_cqc_ctx *cqc_ctx = NULL; + struct roce3_dfx_ctx *dfx_ctx = NULL; + struct roce3_device *rdev = NULL; + struct roce_cq_context cqc; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + pr_err("[ROCE] %s: Invalid input para\n", __func__); + return -EINVAL; + } + + cqc_ctx = container_of(attr, struct roce3_dfx_cqc_ctx, kattr); + dfx_ctx = container_of(cqc_ctx, struct roce3_dfx_ctx, cqc_ctx); + rdev = container_of(dfx_ctx, struct roce3_device, dfx_ctx); + + memset(&cqc, 0, sizeof(struct roce_cq_context)); + + ret = (int)kstrtou32(buf, 10, &cqn); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to do sscanf_s, ret(%d)\n", __func__, ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (cqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs) { + pr_err("[ROCE] %s: Invalid cqn(%u), max_cqs(%u)\n", __func__, cqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs); + return -EIO; + } + + ret = roce3_dfx_cmd_query_cq(rdev, cqn, &cqc); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to query cqc, ret(%d)\n", __func__, ret); + return (ssize_t)ret; + } + + cqc.dw2.value = be32_to_cpu(cqc.dw2.value); + cqc.dw1.value = be32_to_cpu(cqc.dw1.value); + + roce3_dfx_print("cqn(0x%x): pi(0x%x), ci(0x%x)\n", cqn, cqc.dw2.bs.pi, cqc.dw1.bs.ci); + + return (ssize_t)count; +} + +static ssize_t roce3_store_dfx_srqc(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 srqn; + struct roce3_dfx_srqc_ctx *srqc_ctx = NULL; + struct roce3_dfx_ctx *dfx_ctx = NULL; + struct roce3_device *rdev = NULL; + struct roce_srq_context srqc; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + pr_err("[ROCE] %s: Invalid input para\n", __func__); + return -EINVAL; + } + + srqc_ctx = container_of(attr, struct roce3_dfx_srqc_ctx, kattr); + dfx_ctx = container_of(srqc_ctx, struct roce3_dfx_ctx, srqc_ctx); + rdev = container_of(dfx_ctx, struct roce3_device, dfx_ctx); + + memset(&srqc, 0, sizeof(struct roce_srq_context)); + + ret = (int)kstrtou32(buf, 10, &srqn); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to do sscanf_s, ret(%d)\n", __func__, ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (srqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs) { + pr_err("[ROCE] %s: Invalid srqn(%u), max_srqs(%u)\n", __func__, srqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs); + return -EIO; + } + + ret = roce3_dfx_cmd_query_srq(rdev, srqn, &srqc); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to query srqc, ret(%d)\n", __func__, ret); + return (ssize_t)ret; + } + + srqc.dw4.value = be32_to_cpu(srqc.dw4.value); + + roce3_dfx_print("srqn(0x%x): pcnt(0x%x), ccnt(0x%x)\n", + srqn, srqc.dw4.bs.pcnt, srqc.dw4.bs.ccnt); + + return (ssize_t)count; +} + +static int roce3_init_dfx_sub_ctx_sysfs(struct kobject *kobj, struct roce3_dfx_ctx *dfx_ctx) +{ + int ret; + struct kobj_attribute *qpc_kattr = NULL; + struct kobj_attribute *cqc_kattr = NULL; + struct kobj_attribute *srqc_kattr = NULL; + + if (kobj == NULL || dfx_ctx == NULL) { + pr_err("[ROCE] %s: Invalid kobj or dfx_ctx\n", __func__); + return -EINVAL; + } + + qpc_kattr = &dfx_ctx->qpc_ctx.kattr; + cqc_kattr = &dfx_ctx->cqc_ctx.kattr; + srqc_kattr = &dfx_ctx->srqc_ctx.kattr; + + sysfs_attr_init(&qpc_kattr->attr); + qpc_kattr->attr.name = "qpc"; + qpc_kattr->attr.mode = 0640; + qpc_kattr->store = roce3_store_dfx_qpc; + ret = sysfs_create_file(kobj, &qpc_kattr->attr); + if (ret != 0) { + pr_err("[ROCE] %s: QPC failed to do sysfs_create_file, ret(%d)\n", __func__, ret); + return ret; + } + + sysfs_attr_init(&cqc_kattr->attr); + cqc_kattr->attr.name = "cqc"; + cqc_kattr->attr.mode = 0640; + cqc_kattr->store = roce3_store_dfx_cqc; + ret = sysfs_create_file(kobj, &cqc_kattr->attr); + if (ret != 0) { + pr_err("[ROCE] %s: CQC failed to do sysfs_create_file, ret(%d)\n", __func__, ret); + goto err_create_cqc_sysfs; + } + + sysfs_attr_init(&srqc_kattr->attr); + srqc_kattr->attr.name = "srqc"; + srqc_kattr->attr.mode = 0640; + srqc_kattr->store = roce3_store_dfx_srqc; + ret = sysfs_create_file(kobj, &srqc_kattr->attr); + if (ret != 0) { + pr_err("[ROCE] %s: SRQC failed to do sysfs_create_file, ret(%d)\n", __func__, ret); + goto err_create_srqc_sysfs; + } + + return 0; + +err_create_srqc_sysfs: + sysfs_remove_file(kobj, &cqc_kattr->attr); + +err_create_cqc_sysfs: + sysfs_remove_file(kobj, &qpc_kattr->attr); + + return ret; +} + +static int roce3_init_dfx_sysfs(struct net_device *ndev, struct roce3_dfx_ctx *dfx_ctx) +{ + int ret; + + memset(dfx_ctx, 0, sizeof(*dfx_ctx)); + + dfx_ctx->dfx_root = kobject_create_and_add("roce3_dfx", &ndev->dev.kobj); + if (dfx_ctx->dfx_root == NULL) { + pr_err("[ROCE] %s: Failed to do kobject_create_and_add\n", __func__); + return -ENOMEM; + } + + ret = roce3_init_dfx_sub_ctx_sysfs(dfx_ctx->dfx_root, dfx_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to init dfx sub ctx sysfs, ret(%d)\n", __func__, ret); + kobject_put(dfx_ctx->dfx_root); + return ret; + } + + return 0; +} + +static void roce3_remove_ecn_sysfs(struct roce3_ecn_ctx *ecn_ctx) +{ + roce3_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ip_enable_ctx); + + roce3_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + + roce3_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + + kobject_put(&ecn_ctx->ecn_root); +} + +int roce3_init_sysfs(struct roce3_device *rdev) +{ + int ret; + + ret = roce3_init_ecn_sysfs(rdev->ndev, &rdev->ecn_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to init ecn sysfs, ret(%d)\n", __func__, ret); + return ret; + } + + ret = roce3_init_dfx_sysfs(rdev->ndev, &rdev->dfx_ctx); + if (ret != 0) { + pr_err("[ROCE] %s: Failed to init dfx sysfs, ret(%d)\n", __func__, ret); + return ret; + } + + return 0; +} + +static void roce3_remove_dfx_sub_sysfs(struct roce3_dfx_ctx *dfx_ctx) +{ + if (dfx_ctx->dfx_root != NULL) { + sysfs_remove_file(dfx_ctx->dfx_root, &dfx_ctx->srqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, &dfx_ctx->cqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, &dfx_ctx->qpc_ctx.kattr.attr); + + kobject_put(dfx_ctx->dfx_root); + + dfx_ctx->dfx_root = NULL; + } +} + + +void roce3_remove_sysfs(struct roce3_device *rdev) +{ + roce3_remove_dfx_sub_sysfs(&rdev->dfx_ctx); + roce3_remove_ecn_sysfs(&rdev->ecn_ctx); +} diff --git a/drivers/infiniband/hw/hiroce3/roce_sysfs.h b/drivers/infiniband/hw/hiroce3/roce_sysfs.h new file mode 100644 index 000000000..956e823fe --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_sysfs.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_SYSFS_H +#define ROCE_SYSFS_H + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/kobject.h> + +#define ROCE_MIN_QPN 2 +#define BYTE_TO_BIT 8 + +#define PRI_ARRAY_LEN 8 + +#define ROCE_MAX_PORT_CNT 8 + +#define HRN0_K_MAGIC_NUM_3_SYSFS 3 +#define HRN0_K_MAGIC_NUM_7_SYSFS 7 +#define HRN0_K_MAGIC_NUM_8_SYSFS 8 +#define HRN0_K_MAGIC_NUM_M_SYSFS 1000000 + +struct roce3_prio_enable_ctx { + struct kobj_attribute enable; + void *ecn_enable_ctx; + u32 prio_en; + u32 prio; +}; +struct roce3_ip_prio_enable_ctx { + struct kobj_attribute ip_enable; + void *ecn_ip_enable_ctx; + u32 prio_en; + u32 prio; +}; + +struct roce3_ecn_ip_enable_ctx { + struct kobject *ip_enable_root; + struct roce3_ip_prio_enable_ctx ip_prio_enable[PRI_ARRAY_LEN]; +}; + +struct roce3_ecn_enable_ctx { + struct kobject *enable_root; + struct roce3_prio_enable_ctx prio_enable[PRI_ARRAY_LEN]; + u32 np_rp; +}; + +struct roce3_dfx_qpc_ctx { + struct kobj_attribute kattr; +}; + +struct roce3_dfx_cqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce3_dfx_srqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce3_dfx_ctx { + struct kobject *dfx_root; + struct roce3_dfx_qpc_ctx qpc_ctx; + struct roce3_dfx_cqc_ctx cqc_ctx; + struct roce3_dfx_srqc_ctx srqc_ctx; +}; + +struct roce3_ecn_rp_ctx { + struct kobject ecn_rp_root; + u32 alpha_dec_period; + u32 rate_dec_period; + u32 rate_inc_period; + u32 alpha_threshold; + u32 cnp_cnt_threshold; + u32 factor_gita; + u32 initial_alpha; + u32 rate_inc_ai; + u32 rate_inc_hai; + u32 rate_first_set; + u32 rate_target_clamp; + u32 token_period; + u32 min_rate; + struct roce3_ecn_enable_ctx enable_ctx; +}; + +struct roce3_ecn_np_ctx { + struct kobject ecn_np_root; + u32 min_cnp_period; + u32 quick_adjust_en; + u32 port_mode; + u32 cnp_prio_enable; + u32 cnp_prio; + struct roce3_ecn_enable_ctx enable_ctx; +}; + +struct roce3_ecn_ctx { + struct kobject ecn_root; + struct mutex ecn_mutex; + u32 ecn_ver; + u32 cc_algo; + struct roce3_ecn_np_ctx np_ctx; + struct roce3_ecn_rp_ctx rp_ctx; + struct roce3_ecn_ip_enable_ctx ip_enable_ctx; +}; + +int roce3_update_ecn_param(const struct roce3_ecn_ctx *ecn_ctx); + +#endif // ROCE_SYSFS_H diff --git a/drivers/infiniband/hw/hiroce3/roce_user.h b/drivers/infiniband/hw/hiroce3/roce_user.h new file mode 100644 index 000000000..fd931d734 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_user.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_USER_H +#define ROCE_USER_H + +#include <linux/types.h> + +struct roce3_alloc_ucontext_resp { + u32 num_qps; + u32 num_xsrqs; + u32 cqe_size; + u32 wqebb_size; /* 64B or 128B */ + u32 dwqe_size; /* 256B */ + u32 max_msg_size; /* (2G -1)B */ + u32 max_comp_vector; + u32 max_inline_size; + u32 dev_caps; + u8 phy_port; + u8 storage_aa_en; + u16 rsvd; + u16 db_offset; + u16 dwqe_offset; + u8 srq_container_en; + u8 srq_container_mode; + u8 xrc_srq_container_mode; + u8 warn_th; +}; + +struct roce3_create_cq_cmd { + u64 buf_addr; + u64 db_addr; +}; + +struct roce3_resize_cq_cmd { + u64 buf_addr; /* resize cq's 'buf vaַ */ + u64 stage; /* describe the resize stage,0 or 1; */ +}; + +struct roce3_create_srq_cmd { + u64 buf_addr; + u64 db_addr; + u32 rsvd; +}; + +struct create_qp_cmd { + u64 buf_addr; /* describe the qp buf address that used to store cqe; */ + u64 db_addr; /* describe the qp sw db address */ + u32 comp_mask; /* kernel can judge whether handle receive inline through comp_mask */ + u8 log_sq_bb_count; /* wqebb number = 2 << log_sq_bb_count */ + u8 log_sq_stride; /* wqebb size = 2 << log_sq_stride */ + u16 reserved; /* reserved is used to aligned cmd */ +}; + +struct roce3_modify_srq_cmd { + u8 container_flag; + u8 container_warn_th; + u16 rsvd; +}; + +struct roce3_reg_frmr_cmd { + u64 pbl_vaddr; +}; + +#endif // ROCE_USER_H diff --git a/drivers/infiniband/hw/hiroce3/roce_xrc.c b/drivers/infiniband/hw/hiroce3/roce_xrc.c new file mode 100644 index 000000000..4d535d1ce --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_xrc.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_xrc.h" + +static int roce3_rdma_xrcd_alloc(void *hwdev, u32 *xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (xrcdn == NULL)) { + pr_err("%s: Hwdev or xrcdn is null\n", __func__); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return -EINVAL; + } + + *xrcdn = rdma_bitmap_alloc(&comp_priv->xrcd_bitmap); + if (*xrcdn == RDMA_INVALID_INDEX) { + pr_err("%s: Can't get valid xrcdn, err(%d)\n", __func__, -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +static void roce3_rdma_xrcd_free(void *hwdev, u32 xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + pr_err("%s: Hwdev is null\n", __func__); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + pr_err("%s: Comp_priv is null\n", __func__); + return; + } + + rdma_bitmap_free(&comp_priv->xrcd_bitmap, xrcdn); +} + +static int roce3_init_xrcd(struct ib_device *ibdev, struct roce3_device *rdev, + struct roce3_xrcd *xrcd) +{ + int ret = 0; + struct ib_cq_init_attr cq_attr = { 0 }; + + ret = roce3_rdma_xrcd_alloc(rdev->hwdev, &xrcd->xrcdn); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma xrcd, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_xrcdn; + } + + xrcd->pd = ib_alloc_pd(ibdev, 0); /*lint !e119*/ + + if (IS_ERR(xrcd->pd)) { + ret = (int)PTR_ERR(xrcd->pd); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc pd, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_alloc_pd; + } + + cq_attr.cqe = 1; + xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr); + if (IS_ERR(xrcd->cq)) { + ret = (int)PTR_ERR(xrcd->cq); + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create cq, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_create_cq; + } + return 0; + +err_create_cq: + ib_dealloc_pd(xrcd->pd); + +err_alloc_pd: + roce3_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + +err_alloc_xrcdn: + return ret; +} + +int roce3_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce3_device *rdev = to_roce3_dev(ibxrcd->device); + struct roce3_xrcd *xrcd = to_roce3_xrcd(ibxrcd); + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + return roce3_init_xrcd(ibxrcd->device, rdev, xrcd); +} + +int roce3_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce3_device *rdev = NULL; + struct roce3_xrcd *xrcd = NULL; + + if (ibxrcd == NULL) { + pr_err("[ROCE, ERR] %s: Ibxrcd is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibxrcd->device); + xrcd = to_roce3_xrcd(ibxrcd); + ib_destroy_cq(xrcd->cq); + + ib_dealloc_pd(xrcd->pd); + roce3_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + return 0; +} diff --git a/drivers/infiniband/hw/hiroce3/roce_xrc.h b/drivers/infiniband/hw/hiroce3/roce_xrc.h new file mode 100644 index 000000000..8e040828c --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/roce_xrc.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_XRC_H +#define ROCE_XRC_H + +#include <rdma/ib_verbs.h> + +#include "roce.h" + +struct roce3_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; + struct ib_pd *pd; + struct ib_cq *cq; +}; + +static inline struct roce3_xrcd *to_roce3_xrcd(const struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct roce3_xrcd, ibxrcd); +} + +#endif // ROCE_XRC_H diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq.h b/drivers/infiniband/hw/hiroce3/srq/roce_srq.h new file mode 100644 index 000000000..a5e4c723a --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2024 Huawei Technologies Co., Ltd */ + +#ifndef ROCE_SRQ_H +#define ROCE_SRQ_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic3_rdma.h" +#include "hinic3_cqm.h" + +#include "roce.h" +#include "roce_pd.h" +#include "roce_db.h" + +#include "rdma_context_format.h" + +#define ROCE_SRQ_MAX_SGE 15 +#define ROCE_SRQ_MID_SGE 7 +#define ROCE_SRQ_MIN_SGE 3 + +#define ROCE_SRQN_INVLD 0XFFFFFFFF +#define ROCE_SRQ_CONTAINER_LWM_MASK 0xFFFF +#define ROCE_SRQ_CONTAINER_WARTH_MASK 0xF + +#define MAX_SUPPORT_CONTAINER_MODE 3 +#define DWORD_LEN 32 +#define XRC_CQN_FIRST_LEN 10 +#define XRC_CQN_SECOND_LEN 3 +#define SRQ_GPA_SIG_LEN 3 +/* (2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 */ +#define SRQ_WQEBB_SIZE_CAL_SECTTOR 4 +#define RDMA_PREFETCH_WQE_MAX 7 +#define RDMA_PREFETCH_MTT_LEN_MAX 3 +#define ROCE_WQE_BB_SIZE_MIN 64 + +/* + * Lbit:0 - The next SGE is present in the list + * Lbit:1 - The last SGE, no SGE is present in the list + */ +#define LAST_SGE_NO_PRESENT 0x80000000UL + +/* + * Ebit:b0 - Normal format, without extension. + * Ebit:b1 - The pointer inside of SGE points to the next SGL. + * "Length" and "Key" fields are + */ +#define NORMAL_FMT_AND_NEXT_SGE_PRESENT 0x3FFFFFFFUL +#define NORMAL_FMT_AND_LAST_SGE_NO_PRESENT 0xBFFFFFFFUL + +/** + * container_mode: + * mode: 0 -> container_size: 16 + * mode: 1 -> container_size: 8 + * mode: 2 -> container_size: 4 + * mode: 3 -> container_size: 2 + */ +enum roce3_srq_mode { + ROCE_SRQ_MODE_0 = 0, + ROCE_SRQ_MODE_1, + ROCE_SRQ_MODE_2, + ROCE_SRQ_MODE_3 +}; + +/** + * CHIP container_mode: + * chip mode: 0 -> Not container + * chip mode: 1 -> container_size: 2 + * chip mode: 2 -> container_size: 4 + * chip mode: 3 -> container_size: 8 + * chip mode: 4 -> container_size: 16 + */ +enum roce3_chip_srq_mode { + ROCE_CHIP_SRQ_MODE_N = 0, + ROCE_CHIP_SRQ_MODE_1, + ROCE_CHIP_SRQ_MODE_2, + ROCE_CHIP_SRQ_MODE_3, + ROCE_CHIP_SRQ_MODE_4 +}; + +enum roce3_srq_cont_num_mode { + ROCE_SRQ_CONT_NUM_MODE3 = 2, + ROCE_SRQ_CONT_NUM_MODE2 = 4, + ROCE_SRQ_CONT_NUM_MODE1 = 8, + ROCE_SRQ_CONT_NUM_MODE0 = 16 +}; + +enum srq_state { + ROCE_SRQ_STATE_INVALID = 0x0, + ROCE_SRQ_STATE_ERR = 0x1, + ROCE_SRQ_STATE_VALID = 0xf, + ROCE_SRQ_STATE_MEM_INIT = 0xa +}; + +#define ROCE_SRQ_STATE_CHECK_VALUE 0x0 + +struct roce3_srq_query_outbuf { + struct roce_srq_context srqc; + u32 srq_ctr_vld; + u32 srq_empty_ctr; + u32 reserved[6]; +}; + +struct roce3_wqe_srq_next_seg { + u16 reserved1; + __be16 pcnt; /* indicate the pi */ + u8 signature; + u8 reserved2; + __be16 next_wqe_index; + u32 reserved3[2]; +}; + +struct roce3_wqe_container_srq_next_seg { + u32 next_gpa_h; + + struct { + u32 rsvd : 11; + u32 next_gpa_vd : 1; + u32 next_gpa_h : 20; /* indicate the pi */ + } dw1; + + struct { + u32 next_idx : 16; + u32 rsvd : 16; + } dw2; + + struct { + u32 rsvd2 : 30; + u32 link_flag : 1; + u32 rsvd : 1; + } dw3; + + struct { + u32 osd_next_idx : 16; + u32 osd_cur_idx : 16; + } dw4; +}; + +#define ROCE_SRQ_SGE_LAST 1 +#define ROCE_SRQ_SGE_NLAST 0 + +#define ROCE_SRQ_SGE_LKEY_NOEXT 0 +#define ROCE_SRQ_SGE_LKEY_EXT 1 + +struct roce3_srq { + struct ib_srq ibsrq; /* ibsrq */ + struct tag_cqm_queue *cqm_srq; + + u32 srqn; + int max_depth; + int max_gs; + int wqe_shift; + + struct tag_cqm_buf *buf; + struct roce3_db db; + u64 *wrid; + spinlock_t lock; + int head; + int tail; + u16 wqe_ctr; + u8 xrc_en; + u8 rsvd; + struct ib_umem *umem; + struct rdma_mtt mtt; + struct mutex mutex; + u32 rqe_cnt_th; + u8 container_flag; + u8 container_size; + u8 container_mode; + u8 container_warn_th; + + int buf_sz; +}; + +static inline struct roce3_srq *to_roce3_srq(const struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct roce3_srq, ibsrq); +} + +static inline struct roce3_srq *cqmobj_to_roce3_srq(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_srq; + + cqm_srq = container_of(object, struct tag_cqm_queue, object); + return (struct roce3_srq *)cqm_srq->priv; +} + +void roce3_free_srq_wqe(struct roce3_srq *srq, int wqe_index); +void roce3_srq_async_event(struct roce3_device *rdev, struct roce3_srq *srq, int type); +void *roce3_srq_get_wqe(struct roce3_srq *srq, int n); +u8 roce3_get_container_sz(u32 container_mode); +u8 roce3_calculate_cont_th(u32 srq_limit); +u8 roce3_srq_mode_chip_adapt(u8 cfg_mode); +u32 roce3_srq_max_avail_wr_set(struct roce3_srq *rsrq); + +int roce3_create_srq_common(struct roce3_device *rdev, struct roce3_srq *rsrq, struct roce3_pd *pd, + struct ib_srq_init_attr *init_attr, struct ib_udata *udata, u32 index); + +#endif // ROCE_SRQ_H diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c b/drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c new file mode 100644 index 000000000..49df482a2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq_comm.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_pub_cmd.h" + +u8 roce3_srq_mode_chip_adapt(u8 cfg_mode) +{ + switch (cfg_mode) { + case ROCE_CHIP_SRQ_MODE_1: + return ROCE_SRQ_MODE_3; + case ROCE_CHIP_SRQ_MODE_2: + return ROCE_SRQ_MODE_2; + case ROCE_CHIP_SRQ_MODE_3: + return ROCE_SRQ_MODE_1; + case ROCE_CHIP_SRQ_MODE_4: + return ROCE_SRQ_MODE_0; + default: + return ROCE_SRQ_MODE_3; + } +} + +static void *roce3_srq_buf_offset(struct tag_cqm_buf *buf, int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce3_srq_get_wqe(struct roce3_srq *srq, int n) +{ + return roce3_srq_buf_offset(srq->buf, (int)((u32)n << (unsigned int)srq->wqe_shift)); +} + +void roce3_srq_async_event(struct roce3_device *rdev, struct roce3_srq *srq, int type) +{ + struct ib_srq *ibsrq = &srq->ibsrq; + struct ib_event event; + + memset(&event, 0, sizeof(event)); + if (ibsrq->event_handler) { + event.device = ibsrq->device; + event.element.srq = ibsrq; + switch (type) { + case ROCE_EVENT_TYPE_SRQ_LIMIT: + event.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + event.event = IB_EVENT_SRQ_ERR; + break; + + default: + dev_warn_ratelimited(rdev->hwdev_hdl, + "[ROCE] %s: unexpected event type(%d) on SRQ(%06x), func_id(%d)\n", + __func__, type, srq->srqn, rdev->glb_func_id); + return; + } + + ibsrq->event_handler(&event, ibsrq->srq_context); + } +} + +/* + **************************************************************************** + Prototype : roce3_free_srq_wqe + Description : roce3_free_srq_wqe + Input : struct roce3_srq *srq + int wqe_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +void roce3_free_srq_wqe(struct roce3_srq *srq, int wqe_index) +{ + struct roce3_wqe_srq_next_seg *next = NULL; + + spin_lock(&srq->lock); + + next = (struct roce3_wqe_srq_next_seg *)roce3_srq_get_wqe(srq, srq->tail); + next->next_wqe_index = (u16)wqe_index; + srq->tail = wqe_index; + + spin_unlock(&srq->lock); +} diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c b/drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c new file mode 100644 index 000000000..3f9e5d76e --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq_create.c @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> +#ifndef __PC_LINT__ +#include <linux/kernel.h> +#endif + +#include "roce.h" +#include "roce_compat.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_verbs_attr.h" +#include "roce_verbs_format.h" + +#include "roce_srq_extension.h" +#include "roce_main_extension.h" +#include "hinic3_hmm.h" +#include "roce_pub_cmd.h" + +/* + **************************************************************************** + Prototype : roce3_srq_sw2hw + Description : roce3_srq_sw2hw + Input : struct roce3_device *rdev + struct roce3_srq *rsrq + u32 pdn + u32 cqn + u16 xrcdn + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + + 1.Date : 2015/7/27 + Modification : modify function +**************************************************************************** +*/ +static void roce3_srq_sw2hw_set(struct tag_roce_verbs_srq_attr *srq_attr, struct roce3_srq *rsrq, + u32 pdn, u32 cqn, u16 xrcdn, int page_shift) +{ + srq_attr->dw0.bs.size = (u32)(ROCE_ILOG2((unsigned int)rsrq->max_depth)); + srq_attr->dw0.bs.page_size = (u32)page_shift; + srq_attr->dw0.bs.wqebb_size = (u32)rsrq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->dw0.bs.mtt_page_size = (rsrq->mtt.mtt_page_shift - PAGE_SHIFT_4K); + srq_attr->dw0.bs.xrcd = xrcdn; + + srq_attr->dw1.bs.dma_attr_idx = 0; + srq_attr->dw1.bs.so_ro = 1; + srq_attr->dw1.bs.state = ROCE_SRQ_STATE_VALID; + + srq_attr->dw3.bs.container = rsrq->container_flag; + srq_attr->dw3.bs.lth_pre_en = (rsrq->container_flag == 0); // rdma engine prefetch + srq_attr->dw3.bs.pcnt_on_chip = 0; + srq_attr->dw3.bs.rkey_en = 1; + srq_attr->dw3.bs.pd = pdn; + + srq_attr->srqn = cpu_to_be32(rsrq->srqn); + srq_attr->xrc_cqn = cpu_to_be32(cqn); + + if (rsrq->container_flag == 0) { // normal srq + srq_attr->dw1.bs.ep = 0; + srq_attr->dw1.bs.cos = 0; + + srq_attr->dw2.value = 0; + // rdma engine wqe prefetch max num + srq_attr->dw2.bs.wqe_prefetch_max_num = RDMA_PREFETCH_WQE_MAX; + // srq_attr->dw2.bs.wqe_prefetch_min_num = 0; + // srq_attr->dw2.bs.wqe_cache_thd_sel = 0; + srq_attr->dw2.bs.wqecnt_lth = 0xe; + // srq_attr->dw2.bs.wqecnt_ctrl_en = 0; + // srq_attr->dw2.bs.wqecnt_rctl = 0; + srq_attr->dw2.bs.mtt_prefetch_maxlen = 0; // rdma engine mtt prefetch max len + // srq_attr->dw2.bs.next_wqe_idx = 0; + + srq_attr->dw3.bs.lth_pre_en = 1; // rdma engine prefetch + srq_attr->dw3.bs.lth_gap = 0; + } else { + srq_attr->cont.dw2.bs.head_idx = 0; + srq_attr->cont.dw2.bs.warn_th = 0; // Do NOT init warn_th until user arm srq + srq_attr->cont.dw2.bs.cont_size = + (MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode) & 0x3; + srq_attr->cont.dw2.value = cpu_to_be32(srq_attr->cont.dw2.value); + srq_attr->record_gpa_at_hop_num = cpu_to_be64((rsrq->mtt.mtt_layers & 0x3)); + } + + srq_attr->dw0.value = cpu_to_be32(srq_attr->dw0.value); + srq_attr->dw1.value = cpu_to_be32(srq_attr->dw1.value); + srq_attr->dw2.value = cpu_to_be32(srq_attr->dw2.value); + srq_attr->dw3.value = cpu_to_be32(srq_attr->dw3.value); + + srq_attr->l0mtt_gpa = rsrq->mtt.mtt_paddr; + + srq_attr->l0mtt_gpa = cpu_to_be64(srq_attr->l0mtt_gpa); + + srq_attr->record_gpa_at_hop_num = cpu_to_be64(rsrq->db.dma | (rsrq->mtt.mtt_layers & 0x3)); +} + +static int roce3_srq_sw2hw(struct roce3_device *rdev, struct roce3_srq *rsrq, + u32 pdn, u32 cqn, u16 xrcdn, int page_shift) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_verbs_srq_attr *srq_attr = NULL; + struct tag_roce_uni_cmd_creat_srq *srq_sw2hw_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_uni_cmd_creat_srq), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_sw2hw_inbuf = (struct tag_roce_uni_cmd_creat_srq *)cqm_cmd_inbuf->buf; + srq_sw2hw_inbuf->com.index = cpu_to_be32((u32)rsrq->srqn); + srq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + srq_attr = &srq_sw2hw_inbuf->srq_attr; + + roce3_srq_sw2hw_set(srq_attr, rsrq, pdn, cqn, xrcdn, page_shift); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_SW2HW_SRQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to send SW2HW_SRQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(SW2HW_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, rsrq->srqn, rdev->glb_func_id); + + /* update dev status after cmdq timeout or exception, notify PCIe reset */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static int roce3_create_srq_check(const struct ib_pd *ibpd, + const struct ib_srq_init_attr *init_attr) +{ + struct roce3_device *rdev = to_roce3_dev(ibpd->device); + + if ((init_attr->attr.max_wr > (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes) || + (init_attr->attr.max_sge > + (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge)) { + pr_err("[ROCE, ERR] %s: Invalid input parms\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int roce3_create_srq_write_mtt_and_db(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_create_srq_cmd *ucmd, struct roce3_pd *pd) +{ + int ret = 0; + + ret = roce3_umem_write_mtt(rdev, &rsrq->mtt, rsrq->umem); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + ret = roce3_db_map_user(to_roce3_ucontext(pd->ibpd.uobject->context), + ucmd->db_addr, &rsrq->db); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to map db to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + return ret; +} + +static int roce3_create_srq_get_umem(struct roce3_create_srq_cmd *ucmd, + struct ib_udata *udata, struct roce3_srq *rsrq, + struct roce3_device *rdev, struct roce3_pd *pd) +{ + int ret = 0; + + if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0) { + ret = -EFAULT; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy from user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + rsrq->umem = ib_umem_get(&rdev->ib_dev, ucmd->buf_addr, (size_t)rsrq->buf_sz, 0); + if (IS_ERR(rsrq->umem)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to get ib_umem, func_id(%d) rsrq->buf_sz:%u\n", + __func__, rdev->glb_func_id, rsrq->buf_sz); + ret = (int)PTR_ERR(rsrq->umem); + return ret; + } + + return ret; +} + +static int roce3_create_user_srq_update(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_pd *pd, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + struct roce_srq_context *srqc = NULL; + int ret = 0; + +#if defined(OFED_MLNX_5_8) || defined(OFED_VER_4_19) + cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce3_cq(init_attr->ext.cq)->cqn : 0); /*lint !e40*/ +#endif + xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce3_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; + roce3_create_user_srq_update_ext(&cqn, rsrq->srqn); + + srqc = (struct roce_srq_context *)((void *)rsrq->cqm_srq->q_ctx_vaddr); + srqc->dw2.bs.state = ROCE_SRQ_STATE_MEM_INIT; + srqc->dw2.value = cpu_to_be32(srqc->dw2.value); + + /* set SRQC */ + ret = roce3_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq_sw2hw, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_unmap; + } + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + + if (ib_copy_to_udata(udata, &rsrq->srqn, sizeof(u32)) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", + __func__, rdev->glb_func_id); + ret = -EFAULT; + goto err_unmap; + } + + return 0; + +err_unmap: + roce3_db_unmap_user(to_roce3_ucontext(pd->ibpd.uobject->context), &rsrq->db); + return ret; +} + +u32 roce3_srq_max_avail_wr_set(struct roce3_srq *rsrq) +{ + u32 link_wqe_count = 0; + + if (rsrq->container_flag != 0) { + link_wqe_count = (u32)rsrq->max_depth / rsrq->container_size; + return ((u32)rsrq->max_depth - link_wqe_count) - (rsrq->container_size - 1); + } else { + return (u32)rsrq->max_depth - 1; + } +} + +/* + **************************************************************************** + Prototype : roce3_create_user_srq + Description : roce3_create_user_srq + Input : struct roce3_device *rdev + struct roce3_srq *rsrq + struct roce3_pd *pd + struct ib_srq_init_attr *init_attr + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_user_srq(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_pd *pd, struct ib_srq_init_attr *init_attr, struct ib_udata *udata, u32 index) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + struct roce3_create_srq_cmd ucmd = { 0 }; + + /* dispatch SRQN and SRQC */ + rsrq->cqm_srq = + cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SRQ, 0, rsrq, false, index); + if (rsrq->cqm_srq == NULL) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to create rdma queue by cqm_object, func_id(%d) index(%d)\n", + __func__, rdev->glb_func_id, index); + return (-ENOMEM); + } + + rsrq->srqn = rsrq->cqm_srq->index; + + ret = roce3_create_srq_get_umem(&ucmd, udata, rsrq, rdev, pd); + if (ret != 0) + goto err_free_cqm_srq; + + npages = (u32)ib_umem_num_pages(rsrq->umem); + page_shift = PAGE_SHIFT; + + + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &rsrq->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_release_umem; + } + + ret = roce3_create_srq_write_mtt_and_db(rdev, rsrq, &ucmd, pd); + if (ret != 0) + goto err_free_mtt; + + ret = roce3_create_user_srq_update(rdev, rsrq, pd, init_attr, udata, page_shift); + if (ret != 0) + goto err_free_mtt; + + init_attr->attr.max_wr = roce3_srq_max_avail_wr_set(rsrq); + + return 0; + +err_free_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + +err_release_umem: + ib_umem_release(rsrq->umem); + +err_free_cqm_srq: + hiudk_cqm_object_delete(rdev->hwdev, &rsrq->cqm_srq->object); + + return ret; +} + +static void roce3_get_cqn_xrcdn(u32 *cqn, u16 *xrcdn, + struct ib_srq_init_attr *init_attr, struct roce3_device *rdev) +{ +#if defined(OFED_MLNX_5_8) || defined(OFED_VER_4_19) + *cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce3_cq(init_attr->ext.cq)->cqn : 0); /*lint !e40*/ +#endif + *xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce3_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; +} + +static int roce3_create_kernel_srq_update(struct roce3_device *rdev, + struct roce3_srq *rsrq, struct roce3_pd *pd, + struct ib_srq_init_attr *init_attr, int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + struct roce_srq_context *srqc = NULL; + int ret; + + ret = roce3_buf_write_mtt(rdev, &rsrq->mtt, rsrq->buf); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to write mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_mtt; + } + + rsrq->wrid = kmalloc((size_t)((u32)rsrq->max_depth * sizeof(u64)), + GFP_KERNEL); + if (rsrq->wrid == NULL) { + rsrq->wrid = vzalloc((size_t)((u32)rsrq->max_depth * sizeof(u64))); + if (rsrq->wrid == NULL) { + ret = -ENOMEM; + goto err_free_mtt; + } + } + + roce3_get_cqn_xrcdn(&cqn, &xrcdn, init_attr, rdev); + + srqc = (struct roce_srq_context *)((void *)rsrq->cqm_srq->q_ctx_vaddr); + srqc->dw2.bs.state = ROCE_SRQ_STATE_MEM_INIT; + srqc->dw2.value = cpu_to_be32(srqc->dw2.value); + + ret = roce3_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq sw2hw, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_wrid; + } + + return 0; + +err_free_wrid: + kvfree(rsrq->wrid); + rsrq->wrid = NULL; + +err_free_mtt: + hmm_rdma_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_create_kernel_srq + Description : roce3_create_kernel_srq + Input : struct roce3_device *rdev + struct roce3_srq *rsrq + struct roce3_pd *pd + struct ib_srq_init_attr *init_attr + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_create_kernel_srq(struct roce3_device *rdev, struct roce3_srq *rsrq, + struct roce3_pd *pd, struct ib_srq_init_attr *init_attr, u32 index) +{ + int i = 0; + int ret = 0; + int page_shift = 0; + struct roce3_wqe_srq_next_seg *next = NULL; + struct roce3_wqe_srq_data_seg *scatter = NULL; + + /* alloc queue Buf/ Soft DB/SRQN/SRQC */ + rsrq->cqm_srq = cqm_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SRQ, (u32)rsrq->buf_sz, rsrq, true, index); + if (rsrq->cqm_srq == NULL) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create rdma queue by cqm_object, func_id(%d)\n", + __func__, rdev->glb_func_id); + return -ENOMEM; + } + + rsrq->srqn = rsrq->cqm_srq->index; + rsrq->buf = &rsrq->cqm_srq->q_room_buf_1; + + /* set Soft DB */ + rsrq->db.db_record = (__be32 *)(void *)(&rsrq->cqm_srq->q_header_vaddr->doorbell_record); + rsrq->db.dma = rsrq->cqm_srq->q_header_paddr; + *rsrq->db.db_record = 0; + + rsrq->head = 0; + rsrq->tail = rsrq->max_depth - 1; + rsrq->wqe_ctr = 0; + + for (i = 0; i < rsrq->max_depth; ++i) { + next = (struct roce3_wqe_srq_next_seg *)roce3_srq_get_wqe(rsrq, i); + next->next_wqe_index = cpu_to_be16((u16)((u32)(i + 1) & + (u32)(rsrq->max_depth - 1))); + + scatter = (struct roce3_wqe_srq_data_seg *)(next + 1); + scatter->dw2.length = 0; + + /* first SGE = last SGE */ + scatter->dw3.lkey = LAST_SGE_NO_PRESENT; + scatter->dw3.lkey = cpu_to_be32(scatter->dw3.lkey); + } + + page_shift = ROCE_ILOG2(rsrq->buf->buf_size); + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_rdma_mtt_alloc(rdev->hwdev, rsrq->buf->buf_number, (u32)page_shift, + &rsrq->mtt, SERVICE_T_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc rdma rdma_mtt, func_id(%d)\n", + __func__, rdev->glb_func_id); + goto err_free_cqm_srq; + } + + ret = roce3_create_kernel_srq_update(rdev, rsrq, pd, init_attr, page_shift); + if (ret != 0) + goto err_free_cqm_srq; + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + init_attr->attr.max_wr = (u32)rsrq->max_depth - 1; + return 0; + +err_free_cqm_srq: + hiudk_cqm_object_delete(rdev->hwdev, &rsrq->cqm_srq->object); + + return ret; +} + +u8 roce3_get_container_sz(u32 container_mode) +{ + switch (container_mode) { + case ROCE_SRQ_MODE_0: + return ROCE_SRQ_CONT_NUM_MODE0; + case ROCE_SRQ_MODE_1: + return ROCE_SRQ_CONT_NUM_MODE1; + case ROCE_SRQ_MODE_2: + return ROCE_SRQ_CONT_NUM_MODE2; + case ROCE_SRQ_MODE_3: + return ROCE_SRQ_CONT_NUM_MODE3; + default: + return ROCE_SRQ_CONT_NUM_MODE3; + } +} + +static void roce3_set_srq_depth(struct ib_srq_init_attr *init_attr, struct roce3_srq *rsrq) +{ + u32 link_wqe_count = 0; + u32 remain_wqe_count = 0; + + /* + * +---+---+---+---+---+---+---+---+ + * |wqe|wqe|wqe|wqe|wqe|wqe|wqe|nil| + * +---+---+---+---+---+---+---+---+ + * ^ + * | + * full condition:head==tail==nil + */ + if (rsrq->container_flag != 0) { + link_wqe_count = init_attr->attr.max_wr / (rsrq->container_size - 1u); + remain_wqe_count = init_attr->attr.max_wr % (rsrq->container_size - 1u); + link_wqe_count += ((remain_wqe_count == 0) ? + 0 : (rsrq->container_size - remain_wqe_count)); + rsrq->max_depth = (int)(init_attr->attr.max_wr + link_wqe_count + + rsrq->container_size); + } else { + rsrq->max_depth = (int)(init_attr->attr.max_wr + 1); + } + rsrq->max_depth = (int)((ROCE_ROUNDUP_POW_OF_TWO( + (u32)rsrq->max_depth)) & 0xffffffff); //lint !e587 +} + +static void roce3_rsrq_init(struct ib_srq_init_attr *init_attr, + struct roce3_srq *rsrq, struct roce3_device *rdev) +{ + int buf_size = 0; + int desc_size = 0; + + roce3_srq_container_init(init_attr, rsrq, rdev); + + mutex_init(&rsrq->mutex); + /*lint -e708*/ + spin_lock_init(&rsrq->lock); + /*lint +e708*/ + + roce3_set_srq_depth(init_attr, rsrq); + + rsrq->max_gs = (init_attr->attr.max_sge <= ROCE_SRQ_MIN_SGE) ? + ROCE_SRQ_MIN_SGE : + ((init_attr->attr.max_sge <= ROCE_SRQ_MID_SGE) ? + ROCE_SRQ_MID_SGE : ROCE_SRQ_MAX_SGE); + init_attr->attr.max_sge = (u32)rsrq->max_gs; + + /* 0 <= max_gs <= 3, desc_size = 64 + * 4 <= max_gs <= 7, desc_size = 128 + */ + desc_size = (int)sizeof(struct roce3_wqe_srq_next_seg) + + rsrq->max_gs * (int)sizeof(struct roce3_wqe_srq_data_seg); + desc_size = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)desc_size)); //lint !e587 + desc_size = ROCE_MAX(ROCE_WQE_BB_SIZE_MIN, desc_size); // align with min bb_size + rsrq->wqe_shift = ROCE_ILOG2((unsigned int)desc_size); + buf_size = rsrq->max_depth * desc_size; + + /* WQEBB align 1 PAGE */ + if (buf_size < (int)PAGE_SIZE) + rsrq->max_depth = (int)((u64)PAGE_SIZE >> (unsigned int)rsrq->wqe_shift); + + /* align with PAGE SIZE */ + buf_size = (int)ROCE_ALIGN((u32)buf_size, PAGE_SIZE); + rsrq->buf_sz = buf_size; +} + +int roce3_create_srq_common(struct roce3_device *rdev, struct roce3_srq *rsrq, struct roce3_pd *pd, + struct ib_srq_init_attr *init_attr, struct ib_udata *udata, u32 index) +{ + int ret; + + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + ret = roce3_create_srq_check(&pd->ibpd, init_attr); + if (ret != 0) { + pr_err("[ROCE, ERR] %s: Failed to check srq info\n", __func__); + return ret; + } + + roce3_rsrq_init(init_attr, rsrq, rdev); + if (rsrq->ibsrq.uobject) { + ret = roce3_create_user_srq(rdev, rsrq, pd, init_attr, udata, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create user srq, func_id(%u), index(%u)\n", + __func__, rdev->glb_func_id, index); + } + } else { + ret = roce3_create_kernel_srq(rdev, rsrq, pd, init_attr, index); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to create kernel srq, func_id(%u)\n", + __func__, rdev->glb_func_id); + } + } + + return ret; +} + +int roce3_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct roce3_pd *pd = to_roce3_pd(ibsrq->pd); + struct roce3_device *rdev = to_roce3_dev(ibsrq->device); + struct roce3_srq *rsrq = to_roce3_srq(ibsrq); + + return roce3_create_srq_common(rdev, rsrq, pd, init_attr, udata, ROCE_SRQN_INVLD); +} diff --git a/drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c b/drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c new file mode 100644 index 000000000..6f97d48a7 --- /dev/null +++ b/drivers/infiniband/hw/hiroce3/srq/roce_srq_ctrl.c @@ -0,0 +1,570 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2024 Huawei Technologies Co., Ltd + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "hinic3_hmm.h" +#include "roce_main_extension.h" +#include "roce_pub_cmd.h" + +u8 roce3_calculate_cont_th(u32 srq_limit) +{ + u8 cont_th = 0; + u32 srq_limit_tmp = srq_limit; + + srq_limit_tmp >>= 1; + while (srq_limit_tmp != 0) { + srq_limit_tmp >>= 1; + ++cont_th; + } + + return cont_th; +} + +static u8 roce3_cal_srq_container_num(u32 avail_wr, struct roce3_srq *srq) +{ + u32 container_num = 0; + u32 srqe_num = 0; + + /* cal container_num by avail_wr from user */ + srqe_num = srq->container_size - 1; + + /* explain the formula: round up by srqe_num */ + container_num = (avail_wr + srqe_num - 1) / srqe_num; + return roce3_calculate_cont_th(container_num); +} + +/* + **************************************************************************** + Prototype : roce3_srq_arm + Description : roce3_srq_arm + Input : struct roce3_device *rdev + struct roce3_srq *srq + u32 srq_limit + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_srq_arm(struct roce3_device *rdev, struct roce3_srq *srq, u32 srq_limit) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_srq_arm *srq_arm_inbuf = NULL; + u8 warth = 0; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_arm), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_arm_inbuf = (struct tag_roce_cmd_srq_arm *)cqm_cmd_inbuf->buf; + srq_arm_inbuf->com.index = cpu_to_be32((u32)(srq->srqn)); + srq_arm_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + if (srq->container_flag != 0) { + srq_arm_inbuf->bs.cont_en = srq->container_flag; + srq_arm_inbuf->bs.th_up_en = 1; // Valid for setting up container warn_threshold + warth = roce3_cal_srq_container_num(srq_limit, srq); + srq_arm_inbuf->bs.warth = warth & ROCE_SRQ_CONTAINER_WARTH_MASK; + srq_arm_inbuf->limitwater = cpu_to_be32(srq_arm_inbuf->limitwater); + } else { + srq_arm_inbuf->bs.lwm = srq_limit; + srq_arm_inbuf->limitwater = cpu_to_be32(srq_arm_inbuf->limitwater); + } + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_ARM_SRQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s:send ARM_SRQ command Failed, ret(%d), func_id(%d), warth(%d)\n", + __func__, ret, rdev->glb_func_id, warth); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(ARM_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, srq->srqn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + ret = -1; + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_modify_srq + Description : roce3_modify_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *attr + enum ib_srq_attr_mask attr_mask + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + int ret = 0; + struct roce3_device *rdev = NULL; + struct roce3_srq *srq = NULL; + struct roce3_modify_srq_cmd ucmd = { 0 }; + u32 max_avail_wr = 0; + + if ((ibsrq == NULL) || (attr == NULL)) { + pr_err("[ROCE, ERR] %s: Ibsrq or attr is null\n", __func__); + return (-EINVAL); + } + + if (((unsigned int)attr_mask & IB_SRQ_MAX_WR) != 0) { + pr_err("[ROCE, ERR] %s: Not support resizing SRQs\n", __func__); + return (-EINVAL); + } + + rdev = to_roce3_dev(ibsrq->device); + srq = to_roce3_srq(ibsrq); + + if ((udata != NULL) && (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0)) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: SRQN:(%d), Failed to copy from user space, func_id(%d)\n", + __func__, srq->srqn, rdev->glb_func_id); + return (-EFAULT); + } + + if (((unsigned int)attr_mask & IB_SRQ_LIMIT) != 0) { + /* calculate max_wr */ + max_avail_wr = roce3_srq_max_avail_wr_set(srq); + if (attr->srq_limit > max_avail_wr) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: srq_limit > max_wr, func_id(%d), max_avail_wr(%d)\n", + __func__, rdev->glb_func_id, max_avail_wr); + return (-EINVAL); + } + + mutex_lock(&srq->mutex); + ret = roce3_srq_arm(rdev, srq, attr->srq_limit); + mutex_unlock(&srq->mutex); + + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq arm, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_srq_query + Description : roce3_srq_query + Input : struct roce3_device *rdev + struct roce3_srq *srq + u32 *srq_limit + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_srq_query(struct roce3_device *rdev, struct roce3_srq *srq, u32 *srq_limit) +{ + int ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_roce_cmd_srq_query *srq_query_inbuf = NULL; + struct roce3_srq_query_outbuf *srq_query_outbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_query), &cqm_cmd_outbuf, + (u16)sizeof(struct roce3_srq_query_outbuf)); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_query_inbuf = (struct tag_roce_cmd_srq_query *)cqm_cmd_inbuf->buf; + srq_query_outbuf = (struct roce3_srq_query_outbuf *)cqm_cmd_outbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_QUERY_SRQ, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send QUERY_SRQ command, ret(%d), func_id(%d)\n", + __func__, ret, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(QUERY_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, srq->srqn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto err_cmd; + } + + srq_query_outbuf->srqc.dw3.value = be32_to_cpu(srq_query_outbuf->srqc.dw3.value); + *srq_limit = srq_query_outbuf->srqc.dw3.bs.lth; + + if (srq->container_flag != 0) { + /* parse srqc to get warn_th */ + u32 warn_th = srq_query_outbuf->srqc.dw2.bs_c.warn_th; + *srq_limit = (warn_th == 0) ? 0 : + ((u32)((srq->container_size - 1) * (1U << warn_th))); + } + +err_cmd: + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + + return ret; +} + +/* + **************************************************************************** + Prototype : roce3_query_srq + Description : roce3_query_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *srq_attr + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +int roce3_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + int ret = 0; + u32 limit_water = 0; + struct roce3_device *rdev = NULL; + struct roce3_srq *rsrq = NULL; + + if ((ibsrq == NULL) || (srq_attr == NULL)) { + pr_err("[ROCE, ERR] %s: Ibsrq or srq_attr is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibsrq->device); + if (roce3_hca_is_present(rdev) == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s): HCA not present(return fail), func_id(%u)\n", + __func__, rdev->glb_func_id); + return -EPERM; + } + + rsrq = to_roce3_srq(ibsrq); + + ret = roce3_srq_query(rdev, rsrq, &limit_water); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to query srq, func_id(%d)\n", + __func__, rdev->glb_func_id); + return ret; + } + + srq_attr->srq_limit = limit_water; + srq_attr->max_wr = roce3_srq_max_avail_wr_set(rsrq); + srq_attr->max_sge = (u32)rsrq->max_gs; + + return 0; +} + +/* + **************************************************************************** + Prototype : roce3_srq_hw2sw + Description : roce3_srq_hw2sw + Input : struct roce3_device *rdev + struct roce3_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** +*/ +static int roce3_srq_hw2sw(struct roce3_device *rdev, struct roce3_srq *srq) +{ + int ret = 0; + struct rdma_service_cap *rdma_cap = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_roce_cmd_srq_hw2sw *srq_hw2sw_inbuf = NULL; + + ret = roce3_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct tag_roce_cmd_srq_hw2sw), NULL, 0); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)\n", + __func__, rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_hw2sw_inbuf = (struct tag_roce_cmd_srq_hw2sw *)cqm_cmd_inbuf->buf; + rdma_cap = &rdev->rdma_cap; + srq_hw2sw_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); //lint !e778 + srq_hw2sw_inbuf->srq_buf_len = cpu_to_be32((u32)srq->buf_sz); + + srq_hw2sw_inbuf->mtt_info.mtt_flags = 0; + srq_hw2sw_inbuf->mtt_info.mtt_num = 0; + srq_hw2sw_inbuf->mtt_info.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + srq_hw2sw_inbuf->mtt_info.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + srq_hw2sw_inbuf->mtt_info.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + + srq_hw2sw_inbuf->wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + srq_hw2sw_inbuf->wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + srq_hw2sw_inbuf->wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); + + ret = cqm_send_cmd_box(rdev->hwdev, HINIC3_MOD_ROCE, ROCE_CMD_HW2SW_SRQ, + cqm_cmd_inbuf, NULL, NULL, ROCE_CMD_TIME_CLASS_A, HINIC3_CHANNEL_ROCE); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to send HW2SW_SRQ command, func_id(%d)\n", + __func__, rdev->glb_func_id); + + if (roce3_hca_is_present(rdev) == 0) { + ret = 0; + } else { + dev_err(rdev->hwdev_hdl, + "[ROCE] %s: HCA is present(HW2SW_SRQ), srqn(0x%x), func_id(%u)\n", + __func__, srq->srqn, rdev->glb_func_id); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + + ret = -1; + } + } + + roce3_cqm_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void roce3_free_srq(struct ib_srq *ibsrq, struct roce3_device *rdev, + struct roce3_srq *srq, struct ib_udata *udata) +{ + hmm_rdma_mtt_free(rdev->hwdev, &srq->mtt, SERVICE_T_ROCE); + + if (ibsrq->uobject) { + struct roce3_ucontext *ucontext = rdma_udata_to_drv_context( + udata, struct roce3_ucontext, ibucontext); + roce3_db_unmap_user(ucontext, &srq->db); + ib_umem_release(srq->umem); + } else { + kfree(srq->wrid); + } + + hiudk_cqm_object_delete(rdev->hwdev, &srq->cqm_srq->object); +} + +int roce3_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + int ret = 0; + int times = 0; + int read_back_flag = 0; + struct roce3_device *rdev = NULL; + struct roce3_srq *srq = NULL; + struct roce_srq_context *srqc = NULL; + struct roce_srq_context check_srqc; + + if (ibsrq == NULL) { + pr_err("[ROCE, ERR] %s: Ibsrq is null\n", __func__); + return -EINVAL; + } + + rdev = to_roce3_dev(ibsrq->device); + srq = to_roce3_srq(ibsrq); + times = rdev->try_times; + + ret = roce3_srq_hw2sw(rdev, srq); + if (ret != 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to handle srq hw2sw, func_id(%u), ret:%d\n", + __func__, rdev->glb_func_id, ret); + return ret; + } + + srqc = (struct roce_srq_context *)((void *)srq->cqm_srq->q_ctx_vaddr); + while ((times--) != 0) { + if (roce3_hca_is_present(rdev) == 0) + goto err_roce_srq_free; + check_srqc.dw2.value = be32_to_cpu(srqc->dw2.value); + if (check_srqc.dw2.bs.state == ROCE_SRQ_STATE_CHECK_VALUE) { + read_back_flag = 1; + break; + } + + ROCE_UDELAY(US_PERF_DELAY); + } + + if (read_back_flag == 0) { + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Failed to read srq state back after try %d times, func_id(%u), Srqn(0x%x), state_dw(0x%x)\n", + __func__, rdev->try_times, rdev->glb_func_id, srq->srqn, srqc->dw2.value); + return -1; + } + +err_roce_srq_free: + roce3_free_srq(ibsrq, rdev, srq, udata); + + return 0; +} + +static int roce3_check_sge_length(const struct ib_recv_wr *wr, struct roce3_device *rdev, + const struct ib_recv_wr **bad_wr, struct roce3_wqe_srq_data_seg *scat) +{ + int i = 0; + int ret = 0; + u32 data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + if (ROCE_UNLIKELY(wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - 1)) { + ret = -EINVAL; + *bad_wr = wr; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Sge data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + return ret; + } + + data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(data_len > rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + ret = -EINVAL; + *bad_wr = wr; + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Data length is over range, data_len(0x%x), max_msg_sz(0x%x), func_id(%d)\n", + __func__, data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return ret; + } + + scat[i].dw2.length = cpu_to_be32(wr->sg_list[i].length); + scat[i].dw3.lkey = cpu_to_be32((u32)(wr->sg_list[i].lkey & + NORMAL_FMT_AND_NEXT_SGE_PRESENT)); + scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); + } + + if (ROCE_UNLIKELY(i == 0)) { + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + scat[0].dw2.length = 0; + } else { + scat[i - 1].dw3.lkey = + cpu_to_be32((u32)((wr->sg_list[i - 1].lkey | LAST_SGE_NO_PRESENT) & + NORMAL_FMT_AND_LAST_SGE_NO_PRESENT)); + } + + return ret; +} + +int roce3_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + int ret = 0; + int nreq = 0; + unsigned long flags = 0; + struct roce3_srq *srq = to_roce3_srq(ibsrq); + struct roce3_device *rdev = NULL; + struct roce3_wqe_srq_next_seg *next = NULL; + struct roce3_wqe_srq_data_seg *scat = NULL; + const struct ib_recv_wr *wr_tmp = wr; + + rdev = to_roce3_dev(ibsrq->device); + if (roce3_hca_is_present(rdev) == 0) + return -EPERM; + + spin_lock_irqsave(&srq->lock, flags); + + for (nreq = 0; wr_tmp; ++nreq, wr_tmp = wr_tmp->next) { + if (ROCE_UNLIKELY((wr_tmp->num_sge > srq->max_gs) || (srq->head == srq->tail))) { + ret = (wr_tmp->num_sge > srq->max_gs) ? (-EINVAL) : (-ENOMEM); + *bad_wr = wr_tmp; + dev_err(rdev->hwdev_hdl, + "[ROCE, ERR] %s: Num_sge(%d) > max_gs(%d) or srq is empty(head(%d) == tail(%d)), func_id(%d)\n", + __func__, wr_tmp->num_sge, srq->max_gs, + rdev->glb_func_id, srq->head, srq->tail); + break; + } + + next = (struct roce3_wqe_srq_next_seg *)roce3_srq_get_wqe(srq, srq->head); + scat = (struct roce3_wqe_srq_data_seg *)(next + 1); + + ret = roce3_check_sge_length(wr_tmp, rdev, bad_wr, scat); + if (ret != 0) + goto sge_len_err; + + next->pcnt = (u16)cpu_to_be16(srq->wqe_ctr + (u16)nreq); + srq->wrid[srq->head] = wr_tmp->wr_id; + srq->head = be16_to_cpu(next->next_wqe_index); + } + +sge_len_err: + if (ROCE_UNLIKELY(nreq != 0)) { + srq->wqe_ctr += (u16)nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *srq->db.db_record = cpu_to_be32(srq->wqe_ctr); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return ret; +}