[PATCH OLK-6.6] infiniband/hroce5: Add hiroce5 driver
From: maogangpin <maogangpin@huawei.com>
driver inclusion
category: feature
bugzilla:
https://atomgit.com/openeuler/kernel/issues/9708
CVE: NA
---------------------------------
Add hiroce5 driver to support hi1825 chip.
Signed-off-by: maogangpin <maogangpin@huawei.com>
Reviewed-by: Qiujun <roland.qiu@huawei.com>
Reviewed-by: chiqijun <chiqijun@huawei.com>
---
.../hw/hiroce5/drv_intf/roce_drv_msg_intf.h | 14 +
.../hw/hiroce5/kernel/CMakeLists.txt | 62 +
drivers/infiniband/hw/hiroce5/kernel/Makefile | 543 ++++
.../hw/hiroce5/kernel/base/cq/roce_cq.h | 230 ++
.../hiroce5/kernel/base/cq/roce_cq_common.c | 299 ++
.../hw/hiroce5/kernel/base/cq/roce_cq_cqe.c | 763 ++++++
.../hiroce5/kernel/base/cq/roce_cq_create.c | 636 +++++
.../hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c | 964 +++++++
.../hiroce5/kernel/base/cq/roce_cq_destroy.c | 304 +++
.../hiroce5/kernel/base/cq/roce_cq_format.h | 29 +
.../hiroce5/kernel/base/cq/roce_cq_intern.c | 98 +
.../hiroce5/kernel/base/cq/roce_cq_intern.h | 24 +
.../hw/hiroce5/kernel/base/event/roce_event.c | 437 +++
.../hw/hiroce5/kernel/base/event/roce_event.h | 25 +
.../base/extension/roce_event_extension.c | 56 +
.../base/extension/roce_main_extension.c | 170 ++
.../kernel/base/extension/roce_mr_extension.c | 40 +
.../kernel/base/extension/roce_qp_extension.c | 222 ++
.../base/extension/roce_srq_extension.c | 55 +
.../hw/hiroce5/kernel/base/mr/roce_mr.c | 1281 +++++++++
.../hw/hiroce5/kernel/base/mr/roce_mr.h | 113 +
.../hw/hiroce5/kernel/base/qp/roce_post.h | 171 ++
.../hw/hiroce5/kernel/base/qp/roce_qp.h | 317 +++
.../hiroce5/kernel/base/qp/roce_qp_create.c | 1506 +++++++++++
.../kernel/base/qp/roce_qp_ctx_format.h | 278 ++
.../hiroce5/kernel/base/qp/roce_qp_destroy.c | 342 +++
.../hw/hiroce5/kernel/base/qp/roce_qp_exp.h | 80 +
.../hw/hiroce5/kernel/base/qp/roce_qp_ext.c | 39 +
.../hw/hiroce5/kernel/base/qp/roce_qp_ext.h | 19 +
.../hw/hiroce5/kernel/base/qp/roce_qp_inner.h | 17 +
.../hiroce5/kernel/base/qp/roce_qp_modify.c | 2398 +++++++++++++++++
.../kernel/base/qp/roce_qp_post_recv.c | 238 ++
.../kernel/base/qp/roce_qp_post_send.c | 1205 +++++++++
.../hw/hiroce5/kernel/base/qp/roce_qp_query.c | 543 ++++
.../hw/hiroce5/kernel/base/rdma/rdma_comp.c | 22 +
.../hw/hiroce5/kernel/base/rdma/rdma_comp.h | 166 ++
.../hiroce5/kernel/base/rdma/rdma_comp_gid.c | 489 ++++
.../hiroce5/kernel/base/rdma/rdma_comp_init.c | 443 +++
.../kernel/base/rdma/rdma_comp_mw_mr.c | 120 +
.../kernel/base/rdma/rdma_comp_mw_mr.h | 12 +
.../hiroce5/kernel/base/rdma/rdma_comp_pd.c | 51 +
.../hiroce5/kernel/base/rdma/rdma_comp_res.c | 140 +
.../hw/hiroce5/kernel/base/rdma/roce_com.h | 78 +
.../hw/hiroce5/kernel/base/rdma/roce_mix.c | 1594 +++++++++++
.../hw/hiroce5/kernel/base/rdma/roce_mix.h | 269 ++
.../hw/hiroce5/kernel/base/rdma/roce_pd.c | 138 +
.../hw/hiroce5/kernel/base/rdma/roce_pd.h | 15 +
.../hw/hiroce5/kernel/base/srq/roce_srq.h | 191 ++
.../hiroce5/kernel/base/srq/roce_srq_comm.c | 116 +
.../hiroce5/kernel/base/srq/roce_srq_create.c | 712 +++++
.../hiroce5/kernel/base/srq/roce_srq_ctrl.c | 635 +++++
.../hiroce5/kernel/base/srq/roce_srq_format.h | 36 +
.../hw/hiroce5/kernel/base/uld/roce_uld_api.c | 729 +++++
.../hiroce5/kernel/base/uld/roce_uld_inner.h | 77 +
.../hw/hiroce5/kernel/base/xrc/roce_xrc.c | 169 ++
.../hw/hiroce5/kernel/base/xrc/roce_xrc.h | 17 +
.../hw/hiroce5/kernel/dev/card/roce_card.c | 162 ++
.../hw/hiroce5/kernel/dev/card/roce_card.h | 27 +
.../hw/hiroce5/kernel/dev/cmd/roce_cmd.c | 1270 +++++++++
.../hw/hiroce5/kernel/dev/cmd/roce_cmd.h | 82 +
.../hw/hiroce5/kernel/dev/db/roce_db.c | 86 +
.../hw/hiroce5/kernel/dev/db/roce_db.h | 30 +
.../hw/hiroce5/kernel/dev/init/roce_main.c | 1701 ++++++++++++
.../hiroce5/kernel/dev/netdev/roce_netdev.c | 1082 ++++++++
.../hiroce5/kernel/dev/netdev/roce_netdev.h | 70 +
.../hiroce5/kernel/dev/netlink/roce_netlink.c | 413 +++
.../hiroce5/kernel/dev/netlink/roce_netlink.h | 53 +
.../kernel/dev/netlink/roce_netlink_process.c | 832 ++++++
.../kernel/dev/netlink/roce_netlink_process.h | 133 +
.../hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c | 74 +
.../hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h | 21 +
.../kernel/dev/sysfs/roce_multipath_sysfs.c | 240 ++
.../kernel/dev/sysfs/roce_multipath_sysfs.h | 30 +
.../hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c | 2157 +++++++++++++++
.../hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h | 241 ++
.../hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c | 472 ++++
.../hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h | 73 +
.../extern/chip_ext/roce_main_chip_ext.c | 13 +
.../extern/chip_ext/roce_main_chip_ext.h | 17 +
.../kernel/extern/chip_ext/roce_mr_chip_ext.c | 348 +++
.../kernel/extern/chip_ext/roce_mr_chip_ext.h | 24 +
.../kernel/extern/chip_ext/roce_qp_chip_ext.c | 44 +
.../extern/chip_ext/roce_srq_chip_ext.c | 26 +
.../extern/hyper_roce/hyper_roce_common.h | 13 +
.../extern/hyper_roce/hyper_roce_netlink.c | 511 ++++
.../extern/hyper_roce/hyper_roce_netlink.h | 82 +
.../kernel/extern/hyper_roce/hyper_roce_qp.c | 29 +
.../kernel/extern/hyper_roce/hyper_roce_qp.h | 35 +
.../extern/hyper_roce/tool/hyper_roce_tool.c | 60 +
.../extern/hyper_roce/tool/hyper_roce_tool.h | 21 +
.../hyper_roce/tool/hyper_roce_tool_pktdrop.c | 142 +
.../hyper_roce/tool/hyper_roce_tool_pktdrop.h | 23 +
.../hw/hiroce5/kernel/include/roce.h | 665 +++++
.../hw/hiroce5/kernel/include/roce_compat.h | 87 +
.../kernel/include/roce_event_extension.h | 17 +
.../hw/hiroce5/kernel/include/roce_gid.h | 48 +
.../hw/hiroce5/kernel/include/roce_infra.h | 77 +
.../kernel/include/roce_kernel_compat_gen.h | 6 +
.../kernel/include/roce_main_extension.h | 77 +
.../kernel/include/roce_mr_extension.h | 14 +
.../hiroce5/kernel/include/roce_os_compat.h | 237 ++
.../kernel/include/roce_qp_extension.h | 61 +
.../hw/hiroce5/kernel/include/roce_restore.h | 88 +
.../kernel/include/roce_srq_extension.h | 38 +
.../hw/hiroce5/kernel/include/roce_user.h | 100 +
.../kernel/include/roce_verbs_compat.h | 24 +
.../hw/hiroce5/kernel/util/bond/roce_bond.h | 79 +
.../hiroce5/kernel/util/bond/roce_bond_cfm.c | 490 ++++
.../kernel/util/bond/roce_bond_device.c | 490 ++++
.../kernel/util/bond/roce_bond_internal.h | 97 +
.../hiroce5/kernel/util/bond/roce_bond_mac.c | 260 ++
.../kernel/util/bond/roce_bond_slave.c | 459 ++++
.../hiroce5/kernel/util/bond/roce_bond_tx.c | 94 +
.../hw/hiroce5/kernel/util/debug/roce_debug.h | 25 +
.../hw/hiroce5/kernel/util/dfx/roce_dfx.c | 375 +++
.../hw/hiroce5/kernel/util/dfx/roce_dfx.h | 378 +++
.../hiroce5/kernel/util/dfx/roce_dfx_attack.c | 828 ++++++
.../hiroce5/kernel/util/dfx/roce_dfx_debug.c | 35 +
.../hiroce5/kernel/util/dfx/roce_dfx_query.c | 851 ++++++
.../kernel/util/obj_ext/roce_obj_ext.c | 83 +
.../kernel/util/obj_ext/roce_obj_ext.h | 40 +
.../hiroce5/kernel/util/option/roce_option.c | 125 +
.../hiroce5/kernel/util/option/roce_option.h | 30 +
123 files changed, 36652 insertions(+)
create mode 100644 drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/Makefile
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c
create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h
diff --git a/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h b/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h
new file mode 100644
index 000000000..4875a1f1a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h
@@ -0,0 +1,14 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+#ifndef ROCE_DRV_MSG_INTF_H
+#define ROCE_DRV_MSG_INTF_H
+
+#include "udk_usrnl.h"
+
+typedef struct {
+ usrnl_drv_msg_hdr_s common_hdr;
+ unsigned long long card_id;
+} roce5_drv_msg_hdr_s;
+
+#endif /* ROCE_DRV_MSG_INTF_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt b/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt
new file mode 100644
index 000000000..e67756615
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt
@@ -0,0 +1,62 @@
+option(ENABLE_DEBUG "Enable debug in RoCE kernel driver" OFF)
+option(ENABLE_HYPER_ROCE "Enable hyper roce" OFF)
+
+if("${BUILD_VERSION}" MATCHES "_debug$")
+ set(ENABLE_DEBUG ON)
+endif()
+
+# ubx kernel hyper_roce使能宏
+if (DEFINED ENV{build_roce_mode})
+ if ($ENV{build_roce_mode} STREQUAL "hyperroce")
+ set(ENABLE_HYPER_ROCE ON)
+ endif()
+endif()
+
+if("${BUILD_VERSION}" MATCHES "ub_ascend")
+ set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core-v1)
+elseif("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl")
+ set(UBUS_DIR ${TOP_DIR}/drivers/ubus)
+ set(KDIR ${KERNEL_WORK_DIR}/../linux-4.19)
+ set(UMMU_CORE_BUILD_DIR ${UBUS_DIR}/kernel/ummu-core-v1)
+else()
+ set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core)
+endif()
+
+set(UBUS_UBC_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ubus)
+set(HISDK5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/sdk/knldk/lld)
+set(HINIC5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/nic/linux)
+set(HIROCE5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/roce/linux/kernel)
+
+if("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl")
+ set(hiroce5_depends hisdk hinic)
+ add_device_ko(LOCAL_MODULE hiroce5
+ KO_SRC_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}
+ MAKE_ARGS "IS_ASCEND=true"
+ TARGETE_DPENDS "${hiroce5_depends}")
+else()
+ add_custom_target(hiroce5_ko
+ COMMENT echo "build ${CMAKE_CURRENT_SOURCE_DIR} start."
+ COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/Makefile ${CMAKE_CURRENT_BINARY_DIR}
+ COMMAND ${MAKE} -j64
+ -C ${KDIR}
+ M=${CMAKE_CURRENT_BINARY_DIR}
+ src=${CMAKE_CURRENT_SOURCE_DIR}
+ UBUS_UBC_BUILD_DIR=${UBUS_UBC_BUILD_DIR}
+ HI1823_TRUNK_DIR=${TOP_DIR}/ChipSolution
+ HI1823_BUILD_DIR=${TOP_DIR}/ChipSolution/build
+ CONFIG_UBUS_DEVICE=y
+ HI1823_OS_TYPE=openEuler24.03
+ VERBOSE=1
+ HISDK5_SYMVERS=${HISDK5_BUILD_DIR}/Module.symvers
+ HINIC5_SYMVERS=${HINIC5_BUILD_DIR}/Module.symvers
+ CHIP_VERSION=hi1825v100
+ VERSION=ROCE_STANDARD
+ ENABLE_DEBUG=${ENABLE_DEBUG}
+ ENABLE_HYPER_ROCE=${ENABLE_HYPER_ROCE}
+ COMMAND cp -f *.ko ${CMAKE_INSTALL_PREFIX}/ko
+ DEPENDS kernel
+ )
+ add_dependencies(hiroce5_ko hisdk5_ko)
+ add_dependencies(hiroce5_ko hinic5_ko)
+ add_dependencies(hiroce5_ko ubus_ko)
+endif()
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/Makefile b/drivers/infiniband/hw/hiroce5/kernel/Makefile
new file mode 100644
index 000000000..0527b1b8a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/Makefile
@@ -0,0 +1,543 @@
+ifeq ($(KERNELRELEASE), )
+
+.PHONY: all kernel install_kernel install clean clean_kernel
+
+CWD=$(shell pwd)
+TOP_DIR=$(CWD)/../../../../../../..
+INSTALL_MOD_PATH=$(TOP_DIR)/ci/bin
+
+KERN_VERSION=$(shell uname -r)
+KSRC=/lib/modules/$(shell uname -r)/build
+ifeq ($(CONFIG_UB_UNIFIED_UBUS), y)
+ KERN_VERSION ?= 6.6.0-132.0.0.111.oe2403sp3.aarch64
+ KSRC=${UBUS_BUILD_KERNEL_DIR}
+endif
+
+ENABLE_DEBUG ?= OFF
+ENABLE_HYPER_ROCE ?= ON
+
+PANGEA_EULER_2107=4.19.90-vhulk2107.1.0.h699.eulerosv2r10.aarch64
+PANGEA_EULER_2108=4.19.90-vhulk2108.6.0.h832.eulerosv2r10.aarch64
+PANGEA_EULER_2202=4.19.90-vhulk2202.2.0.h1064.eulerosv2r10.aarch64
+EULER_V2R12_H1032=5.10.0-136.12.0.86.h1032.eulerosv2r12.x86_64
+EULER_V2R12_ARM_H1032=5.10.0-136.12.0.86.h1032.eulerosv2r12.aarch64
+EULER_V2R12_ARM_H1258=5.10.0-136.12.0.86.h1258.eulerosv2r12.aarch64
+EULER_V2R11_X86=5.10.0-60.18.0.50.h509.eulerosv2r11.x86_64
+EULER_V2R11_ARM=5.10.0-60.18.0.50.h665.eulerosv2r11.aarch64
+HCE_2_ARM_R865_35=5.10.0-60.18.0.50.r865_35.hce2.aarch64
+HCE_2_ARM_R1526_92=5.10.0-136.12.0.86.r1526_92.hce2.aarch64
+HCE_3_ARM_R1040_72=6.6.0-72.0.0.76.r1040_72.hce3.aarch64
+HCE_3_ARM_R1245_84=6.6.0-72.0.0.76.r1245_84.hce3.aarch64
+OPEN_EULER_153_2203_ARM=5.10.0-153.12.0.92.oe2203sp2.aarch64
+OPEN_EULER_136_2203_ARM=5.10.0-136.12.0.86.oe2203sp1.aarch64
+OPEN_EULER_2403_X86=6.6.0-28.0.0.34.oe2403.x86_64
+OPEN_EULER_2403_ARM=6.6.0-28.0.0.34.oe2403.aarch64
+OPEN_EULER_2403_SP1_ARM=6.6.0-72.0.0.76.oe2403sp1.aarch64
+OPEN_EULER_2403_SP2_ARM=6.6.0-98.0.0.103.oe2403sp2.aarch64
+OPEN_EULER_2403_SP3_ARM=6.6.0-132.0.0.111.oe2403sp3.aarch64
+EULER_V2R9_ARM=4.19.90-vhulk2009.2.0.h310.eulerosv2r9.aarch64
+
+OS_NAME := $(shell grep -E '^NAME=' /etc/os-release | cut -d= -f2 | tr -d '"')
+OS_VERSION_ID := $(shell grep -E '^VERSION_ID=' /etc/os-release | cut -d= -f2 | tr -d '"')
+OS_VERSION := $(shell grep -E '^VERSION=' /etc/os-release | cut -d= -f2 | tr -d '"')
+
+KBUILD_EXTRA_SYMBOLS+=$(TOP_DIR)/ci/lib/SDK_Module.symvers
+KBUILD_EXTRA_SYMBOLS+=$(TOP_DIR)/ci/lib/NIC_Module.symvers
+
+# Set CFLAGS based on public macro definations
+CFLAGS += -DHW_CONVERT_ENDIAN
+CFLAGS += -DROCE_SERVICE
+CFLAGS += -DROCE_DRIVER_COMPILE
+CFLAGS += -D__ROCE_DFX__
+CFLAGS += -DROCE_CC_EN
+CFLAGS += -DRDMA_ROCE_MODULE
+CFLAGS += $(HI1823_FLAGS)
+CFLAGS += -Werror -Wall
+CFLAGS += -DROCE_DWQE_EN
+
+# Set CFLAGS, KBUILD_EXTRA_SYMBOLS and other THIRDPARTY* variables
+# from OS and kernel version information
+ifeq ($(OFED_VERSION), MLNX_OFED)
+ THIRDPARTY = /opt/buildtools/mlnx-ofa_kernel-5.8
+ KBUILD_EXTRA_SYMBOLS+=/opt/buildtools/mlnx-ofa_kernel-5.8/Module.symvers
+else ifeq ($(KERN_VERSION),$(filter $(KERN_VERSION), $(PANGEA_EULER_2107) $(PANGEA_EULER_2108) $(PANGEA_EULER_2202)))
+ KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/pangea/output/aarch64/euleros/ofed_knl/include/Module.symvers
+ THIRDPARTY_INC = $(TOP_DIR)/pangea/output/aarch64/euleros/ofed_knl/
+ CFLAGS += -DCONFIG_INFINIBAND_USER_MEM
+ CFLAGS += -DHAVE_GET_USER_PAGES_GUP_FLAGS
+ CFLAGS += -DCONFIG_MODULE_UPGRADE
+ CFLAGS += -DCONFIG_NVMF_AA
+else ifeq ($(KERN_VERSION),$(filter $(KERN_VERSION), $(EULER_V2R12_H1032) $(EULER_V2R12_ARM_H1032) $(EULER_V2R12_ARM_H1258) \
+ $(EULER_V2R11_ARM) $(OPEN_EULER_153_2203_ARM) $(HCE_2_ARM_R865_35) $(HCE_2_ARM_R1526_92) $(OPEN_EULER_136_2203_ARM) \
+ $(EULER_V2R9_ARM) $(OPEN_EULER_2403_X86) $(OPEN_EULER_2403_ARM) $(OPEN_EULER_2403_SP1_ARM) $(OPEN_EULER_2403_SP2_ARM) $(OPEN_EULER_2403_SP3_ARM) \
+ $(HCE_3_ARM_R1040_72) $(HCE_3_ARM_R1245_84)))
+ THIRDPARTY_INC = /usr/src/kernels/$(KERN_VERSION)/include
+else ifeq ($(OS_NAME),openEuler)
+ ifeq ($(OS_VERSION_ID),24.03)
+ ifneq (,$(findstring SP4,$(OS_VERSION)))
+ THIRDPARTY_INC = /usr/src/kernels/$(KERN_VERSION)/include
+ endif
+ endif
+else ifeq ($(KERN_VERSION), $(filter $(KERN_VERSION), $(EULER_V2R11_X86)))
+ THIRDPARTY =/usr/src/compat-rdma-4.17
+ KBUILD_EXTRA_SYMBOLS+=/usr/src/compat-rdma/Module.symvers
+ THIRDPARTY_INC = /usr/src/compat-rdma
+else
+ ifeq ($(CROSS_COMPILE),)
+ THIRDPARTY =/usr/src/compat-rdma-4.17
+ KBUILD_EXTRA_SYMBOLS+=/usr/src/compat-rdma/Module.symvers
+ THIRDPARTY_INC = /usr/src/compat-rdma
+# centos compat 417 need 6 para
+ CFLAGS += -DHAVE_NLA_PARSE_6_PARAMS
+ else
+ THIRDPARTY =/arm/cross_compile/install/sysroot/usr/src/compat-rdma-4.8
+ KBUILD_EXTRA_SYMBOLS+=/arm/cross_compile/install/sysroot/usr/src/compat-rdma/Module.symvers
+ THIRDPARTY_INC = /arm/cross_compile/install/sysroot/usr/src/compat-rdma
+ endif
+endif
+
+# Set CFLAGS from compiled driver version
+ifeq ($(VERSION),ROCE_STANDARD)
+ CFLAGS += -DROCE_STANDARD
+ CFLAGS += -DROCE_BONDING_EN
+ CFLAGS += -DSIGN_MTT_EN
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond
+ CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond
+ # CFLAGS += -DROCE_LATENCY_PROF_EN #
时延统计工具，需手动开启
+else ifeq ($(VERSION),ROCE_VBS)
+ CFLAGS += -DROCE_STANDARD
+ CFLAGS += -DROCE_BONDING_EN
+ CFLAGS += -DSIGN_MTT_EN
+ CFLAGS += -DROCE_VBS_EN
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond
+ CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond
+ CFLAGS += -I$(TOP_DIR)/test/at_test/host/service/vbs/extend/roce_kernel/
+else ifeq ($(VERSION),CHIP_TEST)
+ CFLAGS += -DROCE_CHIP_TEST
+ CFLAGS += -DSIGN_MTT_EN
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/chip_ext
+else ifeq ($(VERSION),ROCE_COMPUTE)
+ CFLAGS += -DROCE_STANDARD
+ CFLAGS += -DROCE_COMPUTE
+else ifeq ($(VERSION),ROCE_VROCE)
+ CFLAGS += -DROCE_VROCE_EN
+ CFLAGS += -I$(TOP_DIR)/src/dpu_extended_library/host/service/roce/linux/kernel/extern/vroce/include
+endif
+
+ifeq ($(ENABLE_DEBUG),ON)
+ CFLAGS += -DENABLE_DEBUG
+endif
+
+ifeq ($(ENABLE_HYPER_ROCE),ON)
+ CFLAGS += -DENABLE_HYPER_ROCE
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce/tool
+ CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/cqm
+endif
+
+# Set CFLAGS from default file directories
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce/hi182x
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/include
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/drv_intf
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/cq
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/event
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/mr
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/qp
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/rdma
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/srq
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/xrc
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/uld
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/card
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/cmd
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/db
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netdev
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netlink
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sdk_api
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sysfs
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/include
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/dfx
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/option
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/debug
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/obj_ext
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_cfm_intf/hmm
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/urts
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/crm
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/include
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/rdma
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ccp
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include
+CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg
+CFLAGS += -I$(TOP_DIR)/platform/huawei_secure_c/include
+CFLAGS += -I$(TOP_DIR)/platform/huawei_secure_c/src
+CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/roce
+CFLAGS += -DSECUREC_EXPORT_KERNEL_SYMBOL=0
+
+##Safe compilation options
+ifneq ($(HI1823_OS_TYPE), BCLINUX)
+ CFLAGS += -fstack-protector-all -Wl,-z,noexecstack
+endif
+
+# Set CFLAGS from Driver version info,
+# Currently not in use.
+CFLAGS += -D HIROCE5_DRV_VER=\"$(HIROCE5_DRV_VERSION)\"
+
+export KERN_VERSION
+export KBUILD_EXTRA_SYMBOLS
+
+DEPMOD = /sbin/depmod
+INSTALL_MOD_DIR ?= $(shell test -f /etc/redhat-release && echo extra/ofa_kernel || echo updates)
+
+##Define compilation target
+all: kernel
+
+install: install_kernel
+install_kernel: install_modules
+
+##Test whether there are autoconf.h, kconfig.h files in the kernel directory, and if so, refer to the header file later
+autoconf_h=$(shell /bin/ls -1 $(KSRC)/include/*/autoconf.h 2> /dev/null | head -1)
+kconfig_h=$(shell /bin/ls -1 $(KSRC)/include/*/kconfig.h 2> /dev/null | head -1)
+
+ifneq ($(kconfig_h),)
+ KCONFIG_H = -include $(kconfig_h)
+endif
+
+##Test whether V variable is defined. If so, "make" will print some information during subsequent compilation
+V ?= 0
+
+##KBUILD_NOCMDDEP=1, when "make" again after "make", it will not recompile
+##LINUXINCLUDE, reconfigure kernel module compilation options
+##__OFED_BUILD__, use macro definition configured by OFED instead of that configured by kernel
+
+linux_include := -include $(autoconf_h) \
+ $(KCONFIG_H) \
+ $(BACKPORT_INCLUDES) \
+ -I$(THIRDPARTY)/include \
+ -I$(THIRDPARTY)/include/uapi \
+ $$(if $$(CONFIG_XEN),-I$$(srctree)/arch/x86/include/mach-xen) \
+ $$(if $$(CONFIG_XEN),-D__XEN_INTERFACE_VERSION__=$$(CONFIG_XEN_INTERFACE_VERSION)) \
+ -I$$(srctree)/arch/$$(SRCARCH)/include \
+ -I$$(srctree)/arch/$$(SRCARCH)/include/uapi \
+ -Iarch/$$(SRCARCH)/include/generated \
+ -Iarch/$$(SRCARCH)/include/generated/uapi \
+ -Iinclude \
+ $$(if $$(KBUILD_SRC),-Iinclude2 -I$$(srctree)/include) \
+ -I$$(srctree)/include/uapi \
+ -I$$(srctree)/arch/$$(SRCARCH)/include \
+ -Iinclude/generated/uapi \
+ -Iarch/$$(SRCARCH)/include/generated
+
+#########################
+# make kernel #
+#########################
+#NB: The LINUXINCLUDE value comes from main kernel Makefile
+# with local directories prepended. This eventually affects
+# CPPFLAGS in the kernel Makefile
+kernel: $(COMPAT_CONFIG) $(COMPAT_AUTOCONF)
+ @echo '============== INFO =============='
+ @echo 'Building kernel modules'
+ @echo 'Driver Version: $(VERSION)'
+ @echo 'Kernel Version: $(KERN_VERSION)'
+ @echo 'Modules Directory: $(INSTALL_MOD_PATH)/$(MODULES_DIR)'
+ @echo 'Kernel Sources: $(KSRC)'
+ @echo 'Thirdparty Source: $(THIRDPARTY)'
+ @echo 'Thirdparty Symbols: $(KBUILD_EXTRA_SYMBOLS)'
+ @echo 'TOP_DIR: $(TOP_DIR)'
+ @echo 'HI1823_TRUNK_DIR: $(HI1823_TRUNK_DIR)'
+ @echo '=================================='
+ $(MAKE) -C $(KSRC) M="$(CWD)" V=$(V) KBUILD_NOCMDDEP=1 $(WITH_MAKE_PARAMS) EXTRA_CFLAGS="$(CFLAGS)" \
+ LINUXINCLUDE='$(linux_include)' ENABLE_DEBUG=$(ENABLE_DEBUG) ENABLE_HYPER_ROCE=$(ENABLE_HYPER_ROCE) modules
+
+#########################
+# Install kernel #
+#########################
+install_modules:
+ @echo "Installing kernel modules"
+
+ $(MAKE) -C $(KSRC) M="$(CWD)" \
+ INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) \
+ INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \
+ $(WITH_MAKE_PARAMS) modules_install;
+ if [ ! -n "$(INSTALL_MOD_PATH)" ]; then $(DEPMOD) $(KERN_VERSION);fi;
+
+clean: clean_kernel
+
+clean_kernel:
+ @echo "clean kernel modules"
+ $(MAKE) -C $(KSRC) M="$(CWD)" $(WITH_MAKE_PARAMS) clean
+ @/bin/rm -f $(clean-files)
+
+clean-files := Module.symvers modules.order Module.markers compat/modules.order
+clean-files += $(COMPAT_CONFIG) $(COMPAT_AUTOCONF)
+
+help:
+ @echo
+ @echo "kernel: build kernel modules"
+ @echo "all: build kernel modules"
+ @echo
+ @echo "install_kernel: install kernel modules under $(INSTALL_MOD_PATH)/$(MODULES_DIR)"
+ @echo "install: run install_kernel"
+ @echo
+ @echo "clean: delete kernel modules binaries"
+ @echo "clean_kernel: delete kernel modules binaries"
+ @echo
+
+else
+
+obj-m += hiroce5.o
+
+ENABLE_DEBUG ?= OFF
+$(info kernel ENABLE_DEBUG: $(ENABLE_DEBUG))
+
+ifeq ($(CONFIG_UBUS_DEVICE), y)
+TOP_DIR=$(HI1823_TRUNK_DIR)
+
+GLOBAL_VERSION=$(shell cat $(HI1823_TRUNK_DIR)/src/GLOBAL_VERSION_NEW | grep driver | awk -F ':' '{print $$2}')
+ccflags-y += -DGLOBAL_VERSION_STR=\"$(GLOBAL_VERSION)\"
+ccflags-y += -DHW_CONVERT_ENDIAN
+ccflags-y += -DROCE_SERVICE
+ccflags-y += -DROCE_DRIVER_COMPILE
+ccflags-y += -D__ROCE_DFX__
+ccflags-y += -DROCE_CC_EN
+ccflags-y += -DRDMA_ROCE_MODULE
+ccflags-y += $(HI1823_FLAGS)
+# ccflags-y += -Werror -Wall
+ccflags-y += -Wall
+ccflags-y += -DROCE_DWQE_EN
+
+KBUILD_EXTRA_SYMBOLS += $(HISDK5_SYMVERS)
+KBUILD_EXTRA_SYMBOLS += $(HINIC5_SYMVERS)
+# ccflags-y = /usr/src/kernels/$(KERN_VERSION)/include
+
+ccflags-y += -DROCE_STANDARD
+ccflags-y += -DROCE_BONDING_EN
+ccflags-y += -DSIGN_MTT_EN
+
+ifeq ($(ENABLE_DEBUG),ON)
+ ccflags-y += -DENABLE_DEBUG
+endif
+ifeq ($(ENABLE_HYPER_ROCE),ON)
+ ccflags-y += -DENABLE_HYPER_ROCE
+ ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce
+ ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce/tool
+ ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/cqm
+endif
+
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce/hi182x
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/include
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/drv_intf
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/cq
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/event
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/mr
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/qp
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/rdma
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/srq
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/xrc
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/uld
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/card
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/cmd
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/db
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netdev
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netlink
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sdk_api
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sysfs
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/include
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/dfx
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/option
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/debug
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/obj_ext
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_cfm_intf/hmm
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/urts
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/hwsdk
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/rdma
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/crm
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/include
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ccp
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/roce
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg
+ccflags-y += -I$(TOP_DIR)/platform/huawei_secure_c/include
+ccflags-y += -I$(TOP_DIR)/platform/huawei_secure_c/src
+ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/knldk/rdma
+ccflags-y += -I$(TOP_DIR)/..//opensource/openeuler/kernel/olk-6.6/include/linux/
+ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface
+ccflags-y += -I$(TOP_DIR)/../opensource/openeuler/kernel/olk-6.6/include
+ccflags-y += -nostdinc
+endif
+
+# List files of release here.
+ccflags-y += -DSECUREC_EXPORT_KERNEL_SYMBOL=0
+hiroce5-y := ../../../../../../../platform/huawei_secure_c/src/securecutil.o \
+ ../../../../../../../platform/huawei_secure_c/src/secureinput_a.o \
+ ../../../../../../../platform/huawei_secure_c/src/secureprintoutput_a.o \
+ ../../../../../../../platform/huawei_secure_c/src/memset_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/memcpy_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/strncpy_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/strncat_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/sprintf_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/vsprintf_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/vsnprintf_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/snprintf_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/vsscanf_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/sscanf_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/memmove_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/strcpy_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/strcat_s.o \
+ ../../../../../../../platform/huawei_secure_c/src/strtok_s.o \
+ ../../../../cfm/hmm/hmm_buddy.o \
+ ../../../../cfm/hmm/hmm_common.o \
+ ../../../../cfm/hmm/hmm_em.o \
+ ../../../../cfm/hmm/hmm_init.o \
+ ../../../../cfm/hmm/hmm_mpt.o \
+ ../../../../cfm/hmm/hmm_mr.o \
+ ../../../../cfm/hmm/hmm_mtt.o \
+ ../../../../cfm/hmm/hmm_umem.o \
+ ../../../../cfm/rdma/rdma_bitmap.o \
+ base/cq/roce_cq_common.o \
+ base/cq/roce_cq_cqe.o \
+ base/cq/roce_cq_create.o \
+ base/cq/roce_cq_ctrl.o \
+ base/cq/roce_cq_destroy.o \
+ base/cq/roce_cq_intern.o \
+ base/extension/roce_event_extension.o \
+ base/extension/roce_main_extension.o \
+ base/extension/roce_mr_extension.o \
+ base/extension/roce_qp_extension.o \
+ base/extension/roce_srq_extension.o \
+ base/mr/roce_mr.o \
+ base/qp/roce_qp_create.o \
+ base/qp/roce_qp_destroy.o \
+ base/qp/roce_qp_ext.o \
+ base/qp/roce_qp_modify.o \
+ base/qp/roce_qp_post_recv.o \
+ base/qp/roce_qp_post_send.o \
+ base/qp/roce_qp_query.o \
+ base/rdma/rdma_comp.o \
+ base/rdma/rdma_comp_res.o \
+ base/rdma/rdma_comp_gid.o \
+ base/rdma/rdma_comp_init.o \
+ base/rdma/rdma_comp_pd.o \
+ base/rdma/rdma_comp_mw_mr.o \
+ base/rdma/roce_pd.o \
+ base/rdma/roce_mix.o \
+ base/srq/roce_srq_comm.o \
+ base/srq/roce_srq_create.o \
+ base/srq/roce_srq_ctrl.o \
+ base/xrc/roce_xrc.o \
+ base/event/roce_event.o \
+ base/uld/roce_uld_api.o \
+ dev/card/roce_card.o \
+ dev/db/roce_db.o \
+ dev/cmd/roce_cmd.o \
+ dev/init/roce_main.o \
+ dev/netdev/roce_netdev.o \
+ dev/netlink/roce_netlink_process.o \
+ dev/netlink/roce_netlink.o \
+ dev/sdk_api/roce_cqm_cmd.o \
+ dev/sysfs/roce_sysfs.o \
+ dev/sysfs/roce_ucc_sysfs.o \
+ dev/sysfs/roce_multipath_sysfs.o \
+ util/dfx/roce_dfx.o \
+ util/dfx/roce_dfx_query.o \
+ util/option/roce_option.o \
+ util/obj_ext/roce_obj_ext.o
+
+ifeq ($(ENABLE_DEBUG),ON)
+ # List debug files here.
+ hiroce5-y += util/dfx/roce_dfx_debug.o
+ hiroce5-y += util/dfx/roce_dfx_attack.o
+endif
+
+ifeq ($(ENABLE_HYPER_ROCE),ON)
+ # List hyper roce files here.
+ hiroce5-y += extern/hyper_roce/hyper_roce_netlink.o
+ hiroce5-y += extern/hyper_roce/hyper_roce_qp.o
+ hiroce5-y += extern/hyper_roce/tool/hyper_roce_tool.o
+ hiroce5-y += extern/hyper_roce/tool/hyper_roce_tool_pktdrop.o
+endif
+
+# Add hiroce5 objects from compiled driver version
+# from dpu_extended_library directories
+ifeq ($(VERSION),ROCE_STANDARD)
+ hiroce5-y +=util/bond/roce_bond_mac.o
+ hiroce5-y +=util/bond/roce_bond_slave.o
+ hiroce5-y +=util/bond/roce_bond_cfm.o
+ hiroce5-y +=util/bond/roce_bond_device.o
+ hiroce5-y +=util/bond/roce_bond_tx.o
+else ifeq ($(VERSION),ROCE_COMPUTE)
+ # hiroce5-y +=util/bond/roce_bond_common.o
+else ifeq ($(VERSION),CHIP_TEST)
+# CHIP_TEST or ROCE_VBS, as VBS driver currently runs on chip test
+ hiroce5-y +=extern/chip_ext/roce_main_chip_ext.o
+ hiroce5-y +=extern/chip_ext/roce_mr_chip_ext.o
+ hiroce5-y +=extern/chip_ext/roce_qp_chip_ext.o
+ hiroce5-y +=extern/chip_ext/roce_srq_chip_ext.o
+else ifeq ($(VERSION),ROCE_VROCE)
+ hiroce5-y +=util/bond/roce_bond_mac.o
+ hiroce5-y +=util/bond/roce_bond_slave.o
+ hiroce5-y +=util/bond/roce_bond_cfm.o
+ hiroce5-y +=util/bond/roce_bond_device.o
+ hiroce5-y +=util/bond/roce_bond_tx.o
+ hiroce5-y +=../../../../../../../src/dpu_extended_library/host/roce/vroce/roce_cmd_ext.o
+else ifeq ($(VERSION),ROCE_VBS)
+ hiroce5-y +=util/bond/roce_bond_mac.o
+ hiroce5-y +=util/bond/roce_bond_slave.o
+ hiroce5-y +=util/bond/roce_bond_cfm.o
+ hiroce5-y +=util/bond/roce_bond_device.o
+ hiroce5-y +=util/bond/roce_bond_tx.o
+ hiroce5-y +=../../../../../../../test/at_test/host/service/vbs/extend/roce_kernel/roce_vbs_qp.o
+endif
+
+# Driver Extension
+# DPU_HIROCE5_DRV_EXTEND_MK, DPU_DRV_MK_DIR, DPU_DRV_HIROCE5_EXTEND_OBJS对外提供用户使用，修改需要评审
+ifneq ($(DPU_HIROCE5_DRV_EXTEND_MK),)
+ $(info Using roce extension $(DPU_HIROCE5_DRV_EXTEND_MK))
+ export DPU_DRV_MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+ include $(DPU_HIROCE5_DRV_EXTEND_MK)
+ hiroce5-y += $(DPU_DRV_HIROCE5_EXTEND_OBJS)
+endif
+
+build_info:
+ mkdir -p build
+ @echo "CURDIR=$(CURDIR)" > build/build_src.txt
+ @for obj in $(hiroce5-y); do \
+ echo $$(realpath $$obj) >> build/build_src.txt; \
+ done
+
+endif
+
+include $(HI1823_BUILD_DIR)/host/linux/Makefile.ko
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h
new file mode 100644
index 000000000..37562da9c
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_CQ_H
+#define ROCE_CQ_H
+
+#include <rdma/ib_verbs.h>
+#include <linux/slab.h>
+
+#include "hinic5_hw.h"
+#include "hinic5_rdma.h"
+
+#include "roce.h"
+#include "roce_srq.h"
+
+#include "roce_option.h"
+#include "hinic5_hmm.h"
+
+#include "securec.h"
+
+/* DB type of ARM_CQ */
+enum { ROCE_CQ_DB_REQ_NOT_SOL = 1, ROCE_CQ_DB_REQ_NOT = 2 };
+
+#define CQ_DW_TO_BYTE 4
+#define ROCE_CQE_UP_THRESHOLD 1024
+
+/*
+1.Same type of operation as SQ WQE
+8'h00-Send
+8'h01-Send with Invalidate
+8'h02-Send with Immediate Data
+8'h03-reserved
+
+8'h04-RDMA Write
+8'h05-RDMA Write with Immediate Data
+8'h06-reserved
+8'h07-reserved
+
+8'h08-RDMA Read
+8'h09-reserved
+8'h0a-reserved
+8'h0b-reserved
+
+8'h0c-Atomic Compare & Swap
+8'h0d-Atomic Fetch & Add
+8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation)
+8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation)
+
+8'h10-Fast Register PMR
+8'h11-Local Invalidate
+8'h12-Bind Memory Window Type1/2
+8'h13-Local opreation(Extended for further local opreation)
+other-Reserved
+
+2.Receive
+00000 - RDMA Write with Immediate
+00001 - Send
+00010 - Send with Immediate
+00011 - Send & Invalidate
+
+3.The following are general
+11110 Error coding
+10110 Resize coding
+*/
+enum roce5_cqe_send_opcode {
+ ROCE_OPCODE_SEND = 0x0,
+ ROCE_OPCODE_SEND_WITH_INV = 0x1,
+ ROCE_OPCODE_SEND_WITH_IMM = 0x2,
+ /* 0x3 reserved */
+
+ ROCE_OPCODE_RDMA_WRITE = 0x4,
+ ROCE_OPCODE_RDMA_WRITE_WITH_IMM = 0x5,
+ /* 0x6 and 0x7 reserved */
+
+ ROCE_OPCODE_RDMA_READ = 0x8,
+ /* 0x9~0xb reserved */
+
+ ROCE_OPCODE_ATOMIC_COMP_AND_SWP = 0xc,
+ ROCE_OPCODE_ATOMIC_FETCH_AND_ADD = 0xd,
+ ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP = 0xe,
+ ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD = 0xf,
+
+ ROCE_OPCODE_FAST_REG_PMR = 0x10,
+ ROCE_OPCODE_LOCAL_INV = 0x11,
+ ROCE_OPCODE_BIND_TYPE2_MW = 0x12,
+ ROCE_OPCODE_REG_SIG_MR = 0x13,
+
+ ROCE_OPCODE_RESIZE_CQE = 0x16,
+ ROCE_OPCODE_ERR = 0x1e,
+ ROCE_OPCODE_CQE_UNUSED = 0x1f /* Be used in new CQ buf when reszie cq */
+};
+
+enum roce5_cqe_recv_opcode {
+ ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM = 0x0,
+ ROCE_RECV_OPCODE_SEND = 0x1,
+ ROCE_RECV_OPCODE_SEND_WITH_IMM = 0x2,
+ ROCE_RECV_OPCODE_SEND_WITH_INV = 0x3
+};
+
+/* Define the state type of the CQ */
+enum cq_state {
+ ROCE_CQ_STATE_INVALID = 0x0,
+ ROCE_CQ_STATE_ERR = 0x1,
+ ROCE_CQ_STATE_OVERFLOW = 0x2,
+ ROCE_CQ_STATE_VALID = 0xf,
+ ROCE_CQ_STATE_MEM_INIT = 0xa /* Initial value of Host Memory */
+};
+
+#define ROCE_CQ_TIME_OUT_CHECK_VALUE 0xe
+#define ROCE_CQ_STATE_CHECK_VALUE 0x0
+
+#define ROCE_CQE_RQ_INLINE 1
+#define ROCE_CQE_RQ_NORMAL 0
+#define ROCE_CQE_SEND_COMP 1
+#define ROCE_CQE_RECV_COMP 0
+
+#define ATOMIC_DATA_LEN 8 /* Specified as 8B by protocol */
+
+#define ROCE_CQE_INVALID_VALUE 0xff
+#define ROCE_CQ_RESIZE_POLL_TIMES 100
+
+/* roce Commands, bufs and data structures related to cq */
+struct roce5_cq_buf {
+ struct tag_cqm_buf *buf; /* pointer to describe the buf structure b cqm */
+ struct mtt
+ mtt; /* the mtt struct used by kernel mode and user mode to discribe buf */
+ int entry_size; /* the size of cqe */
+ int buf_size; /* the size of cq_buf */
+};
+
+struct roce5_cq_resize_buf {
+ struct roce5_cq_buf buf; /* the size of buf that was resized */
+ int cqe; /* the number of resized cqe */
+};
+
+#define ROCE_RCQ_ULD_DEF_LEN 64
+#define ROCE_RCQ_MAGIC 0x5a5ae0e0
+struct roce5_rcq_uld_def {
+ u8 uld_def[ROCE_RCQ_ULD_DEF_LEN];
+};
+
+struct roce5_cq {
+ struct ib_cq ibcq;
+ cqm_queue_s *cqm_cq; /* Save the handle obtained from cqm */
+ struct roce5_db
+ db; /* The address information of the software DB that stores the CI user mode/kernel mode */
+ __be32 *set_ci_db; /* Kernel-mode software DB */
+ u32 cons_index; /* consumer pointer */
+
+ u32 arm_sn; /* the serial number of arm */
+ int arm_flag; /* Used to determine whether the arm command has been sent, to avoid repeated sending */
+ u32 cqn;
+ unsigned int vector; /* associated to the eq used */
+ struct roce5_cq_buf buf; /* pointer describing the buf structure of cqm */
+ struct roce5_cq_resize_buf *resize_buf; /* resize buf struction */
+
+ spinlock_t lock; /* Need to lock when operating cq */
+ struct mutex resize_mutex; /* resize the mutex of cq */
+ struct ib_umem *umem; /* record the infomation mapped by User-mode buf */
+ struct ib_umem *
+ resize_umem; /* record the infomation mapped by buf which the User-mode resized */
+
+ struct list_head send_qp_list; /* send queue qp */
+ struct list_head recv_qp_list; /* receive queue qp */
+
+ int reset_notify_added;
+ struct list_head reset_notify;
+ u32 cqe_mask; /* mask of the cq size */
+
+ /*
+ * 'left_num' and 'wqebb_index' are only valid in the merged CQE within the RQ.
+ * 'left_num' records the remaining count of the last merged CQE.
+ * 'wqebb_index' records the wqebb index of the current CQE.
+ */
+ u32 left_num;
+ u32 wqebb_index;
+ struct roce5_option *opt;
+ u8 shadow;
+ u32 shadow_vfid;
+
+ void (*reset_flow_comp)(struct roce5_cq *);
+
+ struct roce5_rcq_uld_def
+ uld_def; /* must placed at cq_magic before, preventstop uld access memory stomping, support up to 64B space */
+ u32 rcq_magic;
+};
+
+/* Used when destroy QP */
+void roce5_cq_clean(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq);
+void roce5_cq_async_event(struct roce5_device *rdev, struct roce5_cq *cq,
+ int type);
+void roce5_cq_clean_process(struct roce5_cq *cq, u32 qpn,
+ struct roce5_srq *srq);
+void roce5_cq_put_umem(struct roce5_device *rdev, struct roce5_cq_buf *buf,
+ struct ib_umem **umem, u8 shadow_cq);
+int roce5_create_cq_common(struct ib_device *ibdev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata, struct roce5_cq *rcq,
+ u32 index);
+int roce5_cq_get_umem(struct roce5_device *rdev, struct ib_ucontext *context,
+ struct roce5_cq *rcq, u64 buf_addr, int cqe,
+ bool is_resize);
+void *roce5_get_sw_cqe(struct roce5_cq *cq, unsigned int n);
+void *roce5_get_cqe_from_buf(struct roce5_cq_buf *buf, unsigned int n);
+void roce5_cq_set_ci(struct roce5_cq *cq);
+void roce5_cq_buf_init(struct roce5_cq_buf *buf);
+void *roce5_get_cqe(struct roce5_cq *cq, unsigned int n);
+void roce5_reset_flow_comp(struct roce5_cq *rcq);
+
+void roce5_lock_cqs(struct roce5_cq *roce5_send_cq,
+ struct roce5_cq *roce5_recv_cq)
+ __acquires(&roce5_send_cq->lock) __acquires(&roce5_recv_cq->lock);
+void roce5_unlock_cqs(struct roce5_cq *roce5_send_cq,
+ struct roce5_cq *roce5_recv_cq)
+ __releases(&roce5_send_cq->lock) __releases(&roce5_recv_cq->lock);
+
+struct roce5_cq *roce5_get_rcq_by_cqn(struct roce5_device *rdev, u32 cqn);
+void roce5_cq_completion(void *svc_hd, u32 cqn, void *cq_handler);
+
+static inline void roce5_rcq_put(struct roce5_cq *rcq)
+{
+ if (rcq == NULL) {
+ return;
+ }
+ roce5_option_put_value(rcq->opt);
+ cqm5_object_put(&rcq->cqm_cq->object);
+}
+
+#endif // ROCE_CQ_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c
new file mode 100644
index 000000000..f212acecc
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c
@@ -0,0 +1,299 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+
+#include "hinic5_hw.h"
+
+#include "roce.h"
+#include "roce_srq.h"
+#include "roce_qp.h"
+#include "roce_mix.h"
+#include "roce_xrc.h"
+#include "roce_user.h"
+#include "roce_cq.h"
+#include "roce_xqe_format.h"
+#include "roce_option.h"
+#include "hinic5_hmm.h"
+#include "securec.h"
+#include "roce_os_compat.h"
+#include "roce_aeq_format.h"
+
+/* ****************************************************************************
+ Prototype : roce5_cq_async_event
+ Description : roce5_cq_async_event
+ Input : struct roce5_device *rdev
+ struct roce5_cq *cq
+ int type
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_cq_async_event(struct roce5_device *rdev, struct roce5_cq *cq,
+ int type)
+{
+ struct ib_cq *ibcq = &cq->ibcq;
+ struct ib_event event;
+ int ret;
+
+ ret = memset_s(&event, sizeof(event), 0, sizeof(event));
+ if (ret != 0) {
+ ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset event.(ret:%d)",
+ ret);
+ }
+ if (type != ROCE_EVENT_TYPE_CQ_ERROR) {
+ ROCE_DEV_WARN_LIMIT(
+ rdev,
+ "Unexpected event type(0x%x) on CQ(%06x), func_id(%d)",
+ type, cq->cqn, rdev->glb_func_id);
+ return;
+ }
+
+ if (ibcq->event_handler) {
+ event.device = ibcq->device;
+ event.event = IB_EVENT_CQ_ERR;
+ event.element.cq = ibcq;
+ ibcq->event_handler(&event, ibcq->cq_context);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_buf_init
+ Description : roce5_cq_buf_init
+ Input : struct roce5_cq_buf *buf
+ Output : None
+
+ 1.Date : 2015/11/9
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_cq_buf_init(struct roce5_cq_buf *buf)
+{
+ /* optype was initialized into ROCE_OPCODE_CQE_UNUSED(0x1f) */
+ int ret;
+ ret = memset_s(buf->buf->direct.va,
+ (unsigned long)((unsigned)buf->buf_size),
+ ROCE_CQE_INVALID_VALUE,
+ (unsigned long)((unsigned)buf->buf_size));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset va.(ret:%d)", ret);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_put_umem
+ Description : Release umem and corresponding mtt, corresponding to put
+ Input : struct roce5_device *rdev
+ struct roce5_cq_buf *buf
+ struct ib_umem **umem
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_cq_put_umem(struct roce5_device *rdev, struct roce5_cq_buf *buf,
+ struct ib_umem **umem, u8 shadow_cq)
+{
+ if (shadow_cq != 0) {
+ return;
+ }
+ /* free MTT of Buffer */
+ hmm_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE);
+
+ /* release umem */
+ ib_umem_release(*umem);
+}
+
+static void *roce5_cq_buf_offset(struct tag_cqm_buf *buf, unsigned int offset)
+{
+ return (void *)((char *)buf->direct.va + offset);
+}
+
+void *roce5_get_cqe_from_buf(struct roce5_cq_buf *buf, unsigned int n)
+{
+ return roce5_cq_buf_offset(buf->buf, (n * buf->entry_size));
+}
+
+void *roce5_get_cqe(struct roce5_cq *cq, unsigned int n)
+{
+ return roce5_get_cqe_from_buf(&cq->buf, n);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_get_sw_cqe
+ Description : roce5_get_sw_cqe
+ Input : struct roce5_cq *cq
+ unsigned int n
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+void *roce5_get_sw_cqe(struct roce5_cq *cq, unsigned int n)
+{
+ struct roce_cqe *cqe = NULL;
+ struct roce_cqe tmp_cqe;
+ unsigned int tmp_val;
+
+ cqe = (struct roce_cqe *)roce5_get_cqe(
+ cq, (n & ((unsigned int)cq->cqe_mask)));
+ if (cqe == NULL) {
+ ROCE_ERR("Get sw cqe failed.");
+ return NULL;
+ }
+ tmp_cqe.dw0.value = roce5_convert_cpu32(cqe->dw0.value);
+ tmp_cqe.dw1.value = roce5_convert_cpu32(cqe->dw1.value);
+
+ /* Add judgment condition: the optype of CQE cannot be UNUSED, UNUSED means that it has been initialized in resize
+ cq */
+ tmp_val = ((n & ((unsigned int)cq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) ==
+ 0) ?
+ 1 :
+ 0;
+ if ((ROCE_LIKELY(tmp_cqe.dw1.bs.op_type != ROCE_OPCODE_CQE_UNUSED)) &&
+ ((tmp_cqe.dw0.bs.owner ^ tmp_val) != 0)) {
+ return cqe;
+ }
+
+ return NULL;
+}
+
+int roce5_cq_get_umem(struct roce5_device *rdev, struct ib_ucontext *context,
+ struct roce5_cq *rcq, u64 buf_addr, int cqe,
+ bool is_resize)
+{
+ int ret = 0;
+ u32 npages = 0;
+ int page_shift = 0;
+ int cqe_size = (int)rdev->rdma_cap.cqe_size;
+ struct roce5_cq_buf *buf = NULL;
+ struct ib_umem **umem = NULL;
+
+ if (rcq->shadow != 0) {
+ return 0;
+ }
+ if (is_resize) {
+ buf = &rcq->resize_buf->buf;
+ umem = &rcq->resize_umem;
+ } else {
+ buf = &rcq->buf;
+ umem = &rcq->umem;
+ }
+
+ *umem = roce5_umem_get(&rdev->ib_dev, context, buf_addr,
+ (unsigned long)(cqe * cqe_size),
+ IB_ACCESS_LOCAL_WRITE, 1);
+ if (IS_ERR(*umem)) {
+ ROCE_DEV_ERR(rdev, "Failed to get ib_umem, func_id(%d)",
+ rdev->glb_func_id);
+ return (int)PTR_ERR(*umem);
+ }
+
+ page_shift = (int)roce5_umem_get_buf_page_shift(rdev, *umem, buf_addr);
+ npages = roce5_umem_get_mtt_npages(rdev, *umem, page_shift);
+
+ buf->mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &buf->mtt,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma_mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_buf;
+ }
+
+ ret = roce5_umem_write_mtt(rdev, &buf->mtt, *umem, true);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt for umem, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_mtt;
+ }
+
+ return 0;
+
+err_mtt:
+ hmm_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE);
+
+err_buf:
+ ib_umem_release(*umem);
+
+ return ret;
+}
+
+void roce5_reset_flow_comp(struct roce5_cq *rcq)
+{
+ struct ib_cq *ibcq = &rcq->ibcq;
+
+ ibcq->comp_handler(ibcq, ibcq->cq_context);
+}
+
+/**
+ * @brief Get roce cq by cqn. When querying succeeded, you have to call roce5_rcq_put() to free
+ the cqm_obj and opt. But there is no need to call roce5_rcq_put() if querying failed.
+ * @return Success: return the roce cq. Failure: return NULL.
+ */
+struct roce5_cq *roce5_get_rcq_by_cqn(struct roce5_device *rdev, u32 cqn)
+{
+ cqm_object_s *cqm_obj = NULL;
+ cqm_queue_s *cqm_queue = NULL;
+ struct roce5_option *opt = NULL;
+ struct roce5_cq *rcq = NULL;
+
+ cqm_obj = cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false);
+ if (cqm_obj == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Can't find rcq according to cqn(0x%x), func_id(%d)",
+ cqn, rdev->glb_func_id);
+ goto err_out;
+ }
+
+ cqm_queue = container_of(cqm_obj, cqm_queue_s, object);
+ opt = (struct roce5_option *)cqm_queue->priv;
+ if (opt == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to get option, priv ptr is NULL.");
+ goto err_out_cqm_put;
+ }
+
+ rcq = (struct roce5_cq *)roce5_option_get_value(opt);
+ if (rcq == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to get rcq from option.");
+ goto err_out_cqm_put;
+ }
+
+ return rcq;
+
+err_out_cqm_put:
+ cqm5_object_put(cqm_obj);
+err_out:
+ return NULL;
+}
+
+void roce5_cq_completion(void *svc_hd, u32 cqn, void *cq_handler)
+{
+ struct roce5_option *opt = NULL;
+ struct roce5_cq *rcq = NULL;
+
+ if (cq_handler == NULL) {
+ ROCE_ERR("Cq_handler is null");
+ return;
+ }
+
+ opt = (struct roce5_option *)cq_handler;
+ rcq = (struct roce5_cq *)roce5_option_get_value(opt);
+ if (rcq == NULL) {
+ ROCE_WARN("Failed to get rcq from option.");
+ return;
+ }
+
+ ++rcq->arm_sn;
+ rcq->arm_flag = 0;
+
+ rcq->ibcq.comp_handler(&rcq->ibcq, rcq->ibcq.cq_context);
+
+ roce5_option_put_value(opt);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c
new file mode 100644
index 000000000..b79aad2dc
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c
@@ -0,0 +1,763 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+
+#include "hinic5_hw.h"
+#include "securec.h"
+
+#include "roce.h"
+#include "roce_srq.h"
+#include "roce_qp.h"
+#include "roce_mix.h"
+#include "roce_xrc.h"
+#include "roce_cq.h"
+#include "roce_restore.h"
+#include "roce_aeq_format.h"
+#include "roce_xqe_format.h"
+
+#include "roce_main_extension.h"
+
+/* ****************************************************************************
+ Prototype : roce5_next_cqe_sw
+ Description : roce5_next_cqe_sw
+ Input : struct roce5_cq *cq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static struct roce_cqe *roce5_next_cqe_sw(struct roce5_cq *cq)
+{
+ return (struct roce_cqe *)roce5_get_sw_cqe(cq, cq->cons_index);
+}
+
+static int g_error_cqe_to_wc[ROCE_COMPLETE_EVENT_MAX] = {
+ [ROCE_COMPLETE_EVENT_LOC_LEN_ERR] = IB_WC_LOC_LEN_ERR,
+ [ROCE_COMPLETE_EVENT_LOC_QP_OPERATION_ERR] = IB_WC_LOC_QP_OP_ERR,
+ [ROCE_COMPLETE_EVENT_LOC_PROTECTION_ERR] = IB_WC_LOC_PROT_ERR,
+ [ROCE_COMPLETE_EVENT_WR_FLUSH_ERR] = IB_WC_WR_FLUSH_ERR,
+
+ [ROCE_COMPLETE_EVENT_MW_BIND_ERR] = IB_WC_MW_BIND_ERR,
+ [ROCE_COMPLETE_EVENT_BAD_RESP_ERR] = IB_WC_BAD_RESP_ERR,
+
+ [ROCE_COMPLETE_EVENT_LOC_ACCESS_ERR] = IB_WC_LOC_ACCESS_ERR,
+ [ROCE_COMPLETE_EVENT_REM_INV_REQ_ERR] = IB_WC_REM_INV_REQ_ERR,
+ [ROCE_COMPLETE_EVENT_REM_ACCESS_ERR] = IB_WC_REM_ACCESS_ERR,
+ [ROCE_COMPLETE_EVENT_REM_OPERATION_ERR] = IB_WC_REM_OP_ERR,
+ [ROCE_COMPLETE_EVENT_RETRY_CTR_EXCEED_ERR] = IB_WC_RETRY_EXC_ERR,
+
+ [ROCE_COMPLETE_EVENT_RNR_RETRY_CTR_EXCEED_ERR] =
+ IB_WC_RNR_RETRY_EXC_ERR,
+
+ [ROCE_COMPLETE_EVENT_REM_ABORTED_ERR] = IB_WC_REM_ABORT_ERR,
+ [ROCE_COMPLETE_EVENT_XRC_VIOLATION_ERR] = IB_WC_REM_INV_RD_REQ_ERR,
+};
+
+/* ****************************************************************************
+ Prototype : roce5_handle_error_cqe
+ Description : roce5_handle_error_cqe
+ Input : struct roce_err_cqe *cqe
+ struct ib_wc *wc
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_handle_error_cqe(struct roce_err_cqe *cqe, struct ib_wc *wc)
+{
+ if (cqe->dw7.bs.syndrome >= ROCE_COMPLETE_EVENT_MAX) {
+ wc->status = IB_WC_GENERAL_ERR;
+ } else if (cqe->dw7.bs.syndrome > 0) {
+ wc->status = g_error_cqe_to_wc[cqe->dw7.bs.syndrome];
+ if (wc->status == 0) {
+ wc->status = IB_WC_GENERAL_ERR;
+ }
+ }
+
+ wc->vendor_err = cqe->dw7.bs.vendor_err;
+}
+
+static void roce5_get_local_opcode_from_cqe(const struct roce_cqe *cqe,
+ struct ib_wc *wc)
+{
+ switch (cqe->dw1.bs.op_type) {
+ case ROCE_OPCODE_LOCAL_INV:
+ wc->opcode = IB_WC_LOCAL_INV;
+ break;
+ case ROCE_OPCODE_REG_SIG_MR:
+ wc->opcode = IB_WC_REG_MR;
+ break;
+
+ default:
+ ROCE_WARN("Unknown cqe optype");
+ break;
+ }
+}
+
+static void roce5_get_opcode_type_part_1(struct roce_cqe *cqe, struct ib_wc *wc)
+{
+ switch (cqe->dw1.bs.op_type) {
+ case ROCE_OPCODE_RDMA_WRITE_WITH_IMM:
+ wc->opcode = IB_WC_RDMA_WRITE;
+ wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM);
+ break;
+
+ case ROCE_OPCODE_RDMA_WRITE:
+ wc->opcode = IB_WC_RDMA_WRITE;
+ break;
+
+ case ROCE_OPCODE_SEND_WITH_IMM:
+ wc->opcode = IB_WC_SEND;
+ wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM);
+ break;
+
+ case ROCE_OPCODE_SEND:
+ wc->opcode = IB_WC_SEND;
+ break;
+
+ case ROCE_OPCODE_SEND_WITH_INV:
+ wc->opcode = IB_WC_SEND;
+ break;
+ default:
+ roce5_get_local_opcode_from_cqe(cqe, wc);
+ break;
+ }
+}
+
+static void roce5_get_opcode_type_part_2(struct roce_cqe *cqe, struct ib_wc *wc)
+{
+ switch (cqe->dw1.bs.op_type) {
+ case ROCE_OPCODE_RDMA_READ:
+ wc->opcode = IB_WC_RDMA_READ;
+ wc->byte_len = cqe->byte_cnt;
+ break;
+
+ case ROCE_OPCODE_ATOMIC_COMP_AND_SWP:
+ wc->opcode = IB_WC_COMP_SWAP;
+ wc->byte_len = ATOMIC_DATA_LEN;
+ break;
+
+ case ROCE_OPCODE_ATOMIC_FETCH_AND_ADD:
+ wc->opcode = IB_WC_FETCH_ADD;
+ wc->byte_len = ATOMIC_DATA_LEN;
+ break;
+
+ case ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP:
+ wc->opcode = IB_WC_MASKED_COMP_SWAP;
+ wc->byte_len = ATOMIC_DATA_LEN;
+ break;
+
+ case ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD:
+ wc->opcode = IB_WC_MASKED_FETCH_ADD;
+ wc->byte_len = ATOMIC_DATA_LEN;
+ break;
+ case ROCE_OPCODE_FAST_REG_PMR:
+ wc->opcode = IB_WC_REG_MR;
+ break;
+
+ default:
+ roce5_get_local_opcode_from_cqe(cqe, wc);
+ break;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_get_opcode_from_scqe
+ Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe
+ Input : struct roce_cqe *cqe
+ struct ib_wc *wc
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_get_opcode_from_scqe(struct roce_cqe *cqe, struct ib_wc *wc)
+{
+ wc->wc_flags = 0;
+
+ if (cqe->dw1.bs.op_type < ROCE_OPCODE_RDMA_READ) {
+ roce5_get_opcode_type_part_1(cqe, wc);
+ } else {
+ roce5_get_opcode_type_part_2(cqe, wc);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_get_opcode_from_rcqe
+ Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe
+ Input : struct roce_cqe *cqe
+ struct ib_wc *wc
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_get_opcode_from_rcqe(struct roce_cqe *cqe, struct ib_wc *wc)
+{
+ wc->byte_len = cqe->byte_cnt;
+
+ switch (cqe->dw1.bs.op_type) {
+ case ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM:
+ wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+ wc->wc_flags = IB_WC_WITH_IMM;
+ /* The driver does not perform big or small endian conversion for immediate data, but the incoming CQE
+ has been converted to the CPU endian, so it needs to be converted back */
+ wc->ex.imm_data = roce5_convert_be32(cqe->imm_invalid_rkey);
+ break;
+
+ case ROCE_RECV_OPCODE_SEND_WITH_INV:
+ wc->opcode = IB_WC_RECV;
+ wc->wc_flags = IB_WC_WITH_INVALIDATE;
+ wc->ex.invalidate_rkey =
+ roce5_convert_be32(cqe->imm_invalid_rkey);
+ break;
+
+ case ROCE_RECV_OPCODE_SEND:
+ wc->opcode = IB_WC_RECV;
+ wc->wc_flags = 0;
+ break;
+
+ case ROCE_RECV_OPCODE_SEND_WITH_IMM:
+ wc->opcode = IB_WC_RECV;
+ wc->wc_flags = IB_WC_WITH_IMM;
+ /* The driver does not perform big or small endian conversion for immediate data, but the incoming CQE
+ has been converted to the CPU endian, so it needs to be converted back */
+ wc->ex.imm_data = roce5_convert_be32(cqe->imm_invalid_rkey);
+ break;
+
+ default:
+ ROCE_WARN("Not supported");
+ break;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_poll_and_resize
+ Description : roce5_cq_poll_and_resize
+ Input : struct roce5_cq *cq
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_cq_poll_and_resize(struct roce5_device *rdev,
+ struct roce5_cq *cq)
+{
+ if (ROCE_LIKELY(cq->resize_buf != NULL)) {
+ /* Release the original Buffer of CQ */
+ cqm5_object_resize_free_old(&cq->cqm_cq->object);
+ cq->buf = cq->resize_buf->buf;
+ cq->ibcq.cqe = cq->resize_buf->cqe;
+
+ kfree(cq->resize_buf);
+ cq->resize_buf = NULL;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_get_cur_qp
+ Description : roce5_cq_get_cur_qp
+ Input : struct roce5_cq *cq
+ struct roce5_qp **cur_qp
+ struct roce_cqe *cqe
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_get_cur_qp(struct roce5_cq *cq, struct roce5_qp **cur_qp,
+ struct roce_cqe *cqe)
+{
+ struct roce5_device *rdev = to_roce5_dev(cq->ibcq.device);
+
+ if ((*cur_qp == NULL) || (cqe->dw0.bs.qpn != (u32)(*cur_qp)->qpn)) {
+ if (*cur_qp != NULL) {
+ roce5_rqp_put(*cur_qp);
+ }
+
+ *cur_qp = roce5_get_rqp_by_qpn(rdev, cqe->dw0.bs.qpn);
+ if (*cur_qp == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "CQ(%06x) with entry for unknown QPN(%06x), func_id(%d)",
+ cq->cqn, cqe->dw0.bs.qpn, rdev->glb_func_id);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_get_xrc_srq
+ Description : roce5_cq_get_xrc_srq
+ Input : struct roce5_cq *cq
+ struct roce5_srq **srq
+ struct roce_cqe *cqe
+ u32 qp_type
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_get_xrc_srq(struct roce5_cq *cq, struct roce5_srq **srq,
+ struct roce_cqe *cqe, u32 qp_type)
+{
+ u32 srqn = cqe->dw6.bs.srqn_rqpn;
+ struct roce5_device *rdev = to_roce5_dev(cq->ibcq.device);
+
+ if (qp_type == IB_QPT_XRC_TGT) {
+ if (*srq != NULL) {
+ roce5_rsrq_put(*srq);
+ }
+
+ *srq = roce5_get_rsrq_by_srqn(rdev, cqe->dw6.bs.srqn_rqpn);
+ if (*srq == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "CQ(%06x) with entry for unknown SRQN(%06x), func_id(%d)",
+ cq->cqn, srqn, rdev->glb_func_id);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int roce5_poll_recv_cqe(struct roce5_cq *rcq, struct roce5_qp *cur_qp,
+ struct roce5_srq *srq, struct ib_wc *wc)
+{
+ u32 wqe_ctr = rcq->wqebb_index;
+ unsigned int tail = 0;
+ struct roce5_srq *srq_tmp = NULL;
+
+ if (cur_qp->ibqp.srq) {
+ srq_tmp = to_roce5_srq(cur_qp->ibqp.srq);
+ /* Determine whether to overflow */
+ if (wqe_ctr > (int)srq_tmp->max_depth) {
+ ROCE_ERR("Get wqe index(0x%x) from cqe", wqe_ctr);
+ return -EINVAL;
+ }
+
+ wc->wr_id = srq_tmp->wrid[wqe_ctr];
+ roce5_free_srq_wqe(srq_tmp, wqe_ctr);
+ } else if (srq != NULL) {
+ /* Determine whether to overflow */
+ if (wqe_ctr > srq->max_depth) {
+ ROCE_ERR("Get wqe index(0x%x) from cqe", wqe_ctr);
+ return -EINVAL;
+ }
+
+ wc->wr_id = srq->wrid[wqe_ctr];
+ roce5_free_srq_wqe(srq, wqe_ctr);
+ } else {
+ struct roce5_wq *wq = &(cur_qp->rq);
+ tail = (u32)(wq->tail & ((unsigned int)wq->wqebb_cnt - 1));
+ wc->wr_id = wq->wrid[tail];
+ ++wq->tail;
+ }
+
+ return 0;
+}
+
+static void roce5_poll_move_tail(const struct roce5_qp *cur_qp,
+ struct roce5_wq *wq,
+ struct roce_cqe *trans_cqe)
+{
+ u16 wqe_ctr = 0;
+ if (cur_qp->sq_signal_bits == 0) {
+ wqe_ctr = trans_cqe->dw7.bs.wqe_cnt;
+ wq->tail += (u16)(wqe_ctr - (u16)wq->tail);
+ }
+}
+
+static void roce5_poll_one_get_av_info(const struct roce5_qp *cur_qp,
+ struct ib_wc *wc,
+ struct roce_cqe trans_cqe,
+ struct roce_cqe *cqe)
+{
+ int ret;
+
+ if ((cur_qp->qp_type == IB_QPT_UD) || (cur_qp->qp_type == IB_QPT_GSI)) {
+ wc->sl = trans_cqe.dw4.bs.vlan_pri;
+ wc->src_qp = trans_cqe.dw6.bs.srqn_rqpn;
+ wc->network_hdr_type = trans_cqe.dw6.bs.stp;
+ wc->smac[0] =
+ (u8)(trans_cqe.dw4.bs.smac_h >>
+ 8); /* Take the first Byte data, shift 8bit */
+ wc->smac[1] = (u8)(trans_cqe.dw4.bs.smac_h & 0xff);
+ ret = memcpy_s(&wc->smac[2], sizeof(cqe->smac_l), &cqe->smac_l,
+ sizeof(cqe->smac_l)); //2 : smac array idx
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy smac.(ret:%d)", ret);
+ }
+ wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_SMAC);
+ if (trans_cqe.dw6.bs.vlan_pre != 0) {
+ wc->vlan_id = trans_cqe.dw4.bs.vlan_id;
+ wc->wc_flags =
+ (int)((u32)wc->wc_flags | IB_WC_WITH_VLAN);
+ }
+ }
+}
+
+static int roce5_poll_one_get_qp_and_srq(struct roce5_cq *cq,
+ struct roce5_qp **cur_qp,
+ struct ib_wc *wc,
+ struct roce_cqe *trans_cqe,
+ struct roce5_srq **srq)
+{
+ int ret = 0;
+ ret = roce5_cq_get_cur_qp(cq, cur_qp, trans_cqe);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get current qp");
+ return ret;
+ }
+
+ wc->qp = &((*cur_qp)->ibqp);
+
+ ret = roce5_cq_get_xrc_srq(cq, srq, trans_cqe, wc->qp->qp_type);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get xrc srq");
+ return ret;
+ }
+
+ return ret;
+}
+
+static void roce5_bytes_trans(u32 *addr, int dw_num)
+{
+ u32 *dw = addr;
+ int i = 0;
+
+ for (i = 0; i < dw_num; i++) {
+ *dw = roce5_convert_cpu32(*dw);
+ dw++;
+ }
+}
+
+static int roce5_get_and_trans_cqe(struct roce5_device *rdev,
+ struct roce5_cq *cq,
+ struct roce_cqe *trans_cqe, int *is_send,
+ struct roce_cqe **cqe)
+{
+ int ret;
+
+ do {
+ *cqe = roce5_next_cqe_sw(cq);
+ if (*cqe == NULL) {
+ return -EAGAIN;
+ }
+
+ /* Later, it is necessary to perform big and small end conversion on CQE.
+ In order to avoid turning back after processing, copy a copy */
+ ret = memcpy_s((void *)trans_cqe, sizeof(struct roce_cqe),
+ (void *)*cqe, sizeof(struct roce_cqe));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy trans_cqe.(ret:%d).", ret);
+ return -ENOMEM;
+ }
+
+ /*
+ * Make sure we read CQ entry contents after we've checked the
+ * ownership bit.
+ */
+ rmb();
+
+ /* Convert the CQE into the CPU endian corresponding to the running environment */
+ roce5_bytes_trans(
+ (u32 *)(void *)trans_cqe,
+ (int)(sizeof(struct roce_cqe) / CQ_DW_TO_BYTE));
+
+ *is_send = trans_cqe->dw1.bs.s_r;
+ /* Resize CQ operation in progress */
+ if (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type ==
+ ROCE_OPCODE_RESIZE_CQE)) {
+ roce5_cq_poll_and_resize(rdev, cq);
+ }
+ } while (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type ==
+ ROCE_OPCODE_RESIZE_CQE));
+
+ return 0;
+}
+
+static void roce5_poll_send_cqe(struct roce5_qp *cur_qp,
+ struct roce_cqe *trans_cqe, struct ib_wc *wc)
+{
+ struct roce5_wq *wq = &(cur_qp->sq);
+ /* QP of type IB_SIGNAL_REQ_WR, tail needs to skip several WRs that do not generate CQE */
+ roce5_poll_move_tail(cur_qp, wq, trans_cqe);
+
+ wc->wr_id = wq->wrid[wq->tail & ((unsigned int)wq->wqebb_cnt - 1)];
+ ++wq->tail;
+
+ if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) {
+ roce5_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe,
+ wc);
+ return;
+ }
+
+ roce5_get_opcode_from_scqe(trans_cqe, wc);
+}
+
+static void roce5_init_wc(struct roce_cqe *trans_cqe, struct ib_wc *wc)
+{
+ roce5_get_opcode_from_rcqe(trans_cqe, wc);
+ wc->wc_flags = (int)((unsigned int)wc->wc_flags | IB_WC_GRH);
+ wc->pkey_index = 0;
+ wc->vlan_id = 0xffff;
+ wc->sl = 0;
+ /* avoid cm_req_handler()->ib_lid_be16() trigger call trace */
+ wc->slid = 0;
+ return;
+}
+
+static int roce5_poll_one_cqe(struct roce5_cq *rcq, struct roce5_qp *cur_qp,
+ int is_send, struct roce_cqe *trans_cqe,
+ struct roce_cqe *cqe, struct roce5_srq *srq,
+ struct ib_wc *wc)
+{
+ int ret = 0;
+
+ wc->status = IB_WC_SUCCESS;
+ if (is_send != 0) {
+ roce5_poll_send_cqe(cur_qp, trans_cqe, wc);
+ return 0;
+ }
+
+ ret = roce5_poll_recv_cqe(rcq, cur_qp, srq, wc);
+ if (ret != 0) {
+ return ret;
+ }
+
+ if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) {
+ roce5_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe,
+ wc);
+ return 0;
+ }
+
+ roce5_init_wc(trans_cqe, wc);
+
+ roce5_poll_one_get_av_info((cur_qp), wc, *trans_cqe, cqe);
+
+ /* Kernel mode does not support receiving inline */
+ if (trans_cqe->dw1.bs.inline_r == ROCE_CQE_RQ_INLINE) {
+ ROCE_ERR("Receive inline not supported in kernel space");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_update_cons_index(struct roce5_device *rdev,
+ struct roce5_cq *rcq,
+ struct roce_cqe *trans_cqe)
+{
+ if (trans_cqe->dw1.bs.flush_op != 0) {
+ if (trans_cqe->dw1.bs.s_r == ROCE_CQE_SEND_COMP) {
+ ROCE_DEV_ERR(rdev, "Unexpected flush operation in SQ.");
+ ++rcq->cons_index;
+ return -EBADE;
+ }
+
+ if (rcq->left_num == 0) {
+ rcq->left_num = trans_cqe->wqe_num;
+ rcq->wqebb_index = trans_cqe->dw1.bs.wqebb_cnt;
+ }
+ } else {
+ rcq->left_num = 1;
+ rcq->wqebb_index = trans_cqe->dw1.bs.wqebb_cnt;
+ }
+
+ if (rcq->left_num == 1) {
+ ++rcq->cons_index;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_poll_one
+ Description : roce5_poll_one
+ Input : struct roce5_cq *cq
+ struct roce5_qp **cur_qp
+ struct ib_wc *wc
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_poll_one(struct roce5_device *rdev, struct roce5_cq *cq,
+ struct roce5_qp **cur_qp, struct ib_wc *wc)
+{
+ int ret = 0;
+ struct roce_cqe *cqe = NULL;
+ struct roce_cqe trans_cqe;
+ struct roce5_srq *srq = NULL;
+ int is_send = 0;
+ bool need_poll = true;
+
+ while (need_poll) {
+ ret = roce5_get_and_trans_cqe(rdev, cq, &trans_cqe, &is_send,
+ &cqe);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ ret = roce5_update_cons_index(rdev, cq, &trans_cqe);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ ret = roce5_poll_one_get_qp_and_srq(cq, cur_qp, wc, &trans_cqe,
+ &srq);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ /* Poll cq exception handling, generally cq judges user state information, xrc judges wrid */
+ if ((*cur_qp) == NULL) {
+ ROCE_ERR("cur_qp is NULL when poll cq");
+ ret = -EACCES;
+ goto err_out;
+ }
+ if ((*cur_qp)->umem ||
+ ((*cur_qp)->ibqp.xrcd && srq != NULL && !(srq->wrid))) {
+ ROCE_ERR(
+ "qp(%u) create in user space , but poll cq in kernel. NOT PERMIT!",
+ (*cur_qp)->qpn);
+ ret = -EACCES;
+ goto err_out;
+ }
+
+ if (trans_cqe.dw1.bs.fake != 0) {
+ ROCE_INFO("Fake cqe go repoll.");
+ continue;
+ }
+
+ ret = roce5_poll_one_cqe(cq, *cur_qp, is_send, &trans_cqe, cqe,
+ srq, wc);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ --cq->left_num;
+ need_poll = false;
+ }
+
+err_out:
+ roce5_rsrq_put(srq);
+ return ret;
+}
+
+/* the format of cq_ci DB:the bits of ci less than 24bit */
+void roce5_cq_set_ci(struct roce5_cq *cq)
+{
+ *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff);
+}
+
+int roce5_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+ struct roce5_device *rdev = NULL;
+ struct roce5_cq *rcq = to_roce5_cq(ibcq);
+ struct roce5_qp *cur_qp = NULL;
+ unsigned long flags = 0; /* type required by kernel API */
+ int npolled = 0;
+ int ret = 0;
+
+ spin_lock_irqsave(&rcq->lock, flags);
+
+ rdev = to_roce5_dev(ibcq->device);
+
+ for (npolled = 0; npolled < num_entries; ++npolled) {
+ ret = roce5_poll_one(rdev, rcq, &cur_qp, wc + npolled);
+ if (ret != 0) {
+ break;
+ }
+ }
+
+ roce5_cq_set_ci(rcq);
+
+ spin_unlock_irqrestore(&rcq->lock, flags);
+
+ if ((ret == 0) || (ret == -EAGAIN)) {
+ ret = npolled;
+ goto err_out;
+ }
+
+err_out:
+ roce5_rqp_put(cur_qp);
+ return ret;
+}
+
+static void roce5_cq_arm(struct roce5_cq *cq, u32 cmd, void __iomem *uar_page)
+{
+ struct roce_db_cq_arm db_value;
+ int ret;
+
+ ret = memset_s(&db_value, sizeof(db_value), 0, sizeof(db_value));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset db_value.(ret:%d)", ret);
+ }
+
+ db_value.dw0.bs.type = 3;
+ db_value.dw0.bs.cos = 0; /* arm_cq don't need cos */
+ db_value.dw0.bs.cp = 1; /* 1 for cp */
+ db_value.dw0.bs.non_filter = 1;
+ db_value.dw0.bs.cqc_type = 0;
+ db_value.dw0.bs.cqn = cq->cqn;
+ db_value.dw0.value = roce5_convert_be32(db_value.dw0.value);
+
+ db_value.dw1.bs.cmd_sn = cq->arm_sn;
+ db_value.dw1.bs.cmd = cmd;
+ db_value.dw1.bs.ci = cq->cons_index;
+ db_value.dw1.value = roce5_convert_be32(db_value.dw1.value);
+
+ /*
+ * Make sure that the doorbell record in host memory is
+ * written before ringing the doorbell via PCI MMIO.
+ */
+ wmb();
+
+ roce5_write64((u32 *)(void *)&db_value, uar_page);
+}
+
+int roce5_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
+{
+ unsigned long lock_flags = 0; /* type required by kernel API */
+ struct roce5_cq *rcq = NULL;
+
+ if (CHECK_2LAYER_IS_NULL(ibcq, device)) {
+ return -EINVAL;
+ }
+
+ rcq = to_roce5_cq(ibcq);
+
+ spin_lock_irqsave(&rcq->lock, lock_flags);
+ if (rcq->arm_flag != 0) {
+ spin_unlock_irqrestore(&rcq->lock, lock_flags);
+ return 0;
+ }
+
+ rcq->arm_flag = 1;
+ spin_unlock_irqrestore(&rcq->lock, lock_flags);
+
+ /* Only 64-bit is supported, 64-bit writes are atomic, no need to lock */
+ roce5_cq_arm(rcq,
+ (((unsigned int)flags & IB_CQ_SOLICITED_MASK) ==
+ IB_CQ_SOLICITED) ?
+ ROCE_CQ_DB_REQ_NOT_SOL :
+ ROCE_CQ_DB_REQ_NOT,
+ to_roce5_dev(ibcq->device)->kernel_db_map);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c
new file mode 100644
index 000000000..0df5d34e4
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c
@@ -0,0 +1,636 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "comm_defs.h"
+#include "roce_cq.h"
+#include "roce_npu_cmd.h"
+#include "roce_cqm_cmd.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "roce_os_compat.h"
+#include "roce_user.h"
+#include "roce_option.h"
+#include "roce_cq_intern.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_uld_inner.h"
+
+#define ROCE_CQE_BASE_SIZE_BYTES 16
+
+/* ****************************************************************************
+ Prototype :roce5_cq_attr_assign
+ Description : Assign value to CQC
+ Input : struct roce_cq_attr *cq_attr
+ struct roce5_cq *rcq
+ int eqn
+ int page_shift
+ Output : None
+
+ 1.Date : 2019/10/13
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_cq_attr_assign(roce_verbs_cq_attr_s *cq_attr,
+ struct roce5_cq *rcq, int eqn, int page_shift)
+{
+ /* Assign value to CQ */
+ /* dw0 */
+ cq_attr->dw0.bs.size = (u32)ROCE_ILOG2(
+ (unsigned)(rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD));
+ cq_attr->dw0.bs.page_size = (u32)page_shift;
+ cq_attr->dw0.bs.cqe_size = (u32)ROCE_ILOG2(
+ (unsigned)(rcq->buf.entry_size / ROCE_CQE_BASE_SIZE_BYTES));
+ cq_attr->dw0.bs.mtt_page_size =
+ rcq->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K;
+ cq_attr->dw0.bs.tss_timer_num =
+ 7; /* 7 : The maximum number of timers supported by cq */
+ cq_attr->dw0.bs.arm_timer_en = 0;
+ cq_attr->dw0.bs.timer_mode = 0;
+ cq_attr->dw0.bs.cnt_adjust_en = 1;
+ cq_attr->dw0.bs.cnt_clear_en = 1;
+ cq_attr->dw0.bs.ci_on_chip = 0;
+ cq_attr->dw0.value = cpu_to_be32(cq_attr->dw0.value);
+
+ /* dw1 */
+ cq_attr->dw1.bs.dma_attr_idx = 0;
+ cq_attr->dw1.bs.so_ro = 0;
+ cq_attr->dw1.bs.state = ROCE_CQ_STATE_VALID;
+ cq_attr->dw1.value = cpu_to_be32(cq_attr->dw1.value);
+
+ /* dw2 */
+ cq_attr->dw2.bs.idle_max_count = 0;
+ cq_attr->dw2.bs.cqecnt_lth = 10; /* update ci threshhold: 2^10=1024 */
+ cq_attr->dw2.bs.cqecnt_rctl_en = 0;
+ cq_attr->dw2.bs.ceqe_en = 1;
+ cq_attr->dw2.bs.arm_ceqe_en = 1;
+ cq_attr->dw2.bs.ceqn = (u8)eqn;
+ cq_attr->dw2.value = cpu_to_be32(cq_attr->dw2.value);
+
+ /* dw3 */
+ cq_attr->dw3.bs.timeout =
+ 0; /* The timeout mechanism is disabled by default when CQ is created */
+ cq_attr->dw3.bs.max_cnt =
+ 0; /* The overtime mechanism is disabled by default when CQ is created */
+ cq_attr->dw3.value = cpu_to_be32(cq_attr->dw3.value);
+
+ /* dw4 - dw5 */
+ cq_attr->cqc_l0mtt_gpa = rcq->buf.mtt.mtt_paddr;
+ cq_attr->cqc_l0mtt_gpa = cpu_to_be64(cq_attr->cqc_l0mtt_gpa);
+
+ /* dw6 - dw7 */
+ cq_attr->ci_record_gpa_at_hop_num = cpu_to_be64(
+ (rcq->db.dma & (~0x3uLL)) | rcq->buf.mtt.mtt_layers);
+}
+
+static int roce5_cq_fill_create_inbuf(struct roce5_device *rdev,
+ struct roce5_cq *rcq, int vector,
+ int page_shift,
+ cqm_cmd_buf_s *cqm_cmd_inbuf)
+{
+ int eqn = 0;
+ roce_verbs_cq_attr_s *cq_attr = NULL;
+ roce_uni_cmd_create_cq_s *cq_sw2hw_inbuf = NULL;
+
+ cq_sw2hw_inbuf = (roce_uni_cmd_create_cq_s *)cqm_cmd_inbuf->buf;
+ cq_sw2hw_inbuf->com.index = cpu_to_be32(rcq->cqn);
+ cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK));
+ if (rcq->shadow != 0) {
+ cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ cq_sw2hw_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(rcq->shadow_vfid);
+ }
+ cq_attr = &cq_sw2hw_inbuf->cq_attr;
+
+ /* Get the EQN of the CEQ based on the Vector index */
+ eqn = hinic5_vector_to_eqn(rdev->hwdev, SERVICE_T_ROCE, vector);
+ if (eqn < 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to get eqn from hinic vector, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ roce5_cq_attr_assign(cq_attr, rcq, eqn, page_shift);
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_sw2hw
+ Description : Send the cqc configuration command
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int vector
+ int page_shift
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_sw2hw(struct roce5_device *rdev, struct roce5_cq *rcq,
+ int vector, int page_shift)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_create_cq_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ ret = roce5_cq_fill_create_inbuf(rdev, rcq, vector, page_shift,
+ cqm_cmd_inbuf);
+ if (ret != 0) {
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_SW2HW_CQ,
+ cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send SW2HW_CQ command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+
+ /* CMDq times out or CMDq does not work, update the device status, notify the PCIe module to reset
+ the device through OFED */
+ if ((ret == -ETIMEDOUT) || (ret == -EPERM)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+
+ ret = -1;
+ goto err_send_cmd;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ rcq->cons_index = 0;
+ rcq->arm_sn = 1;
+ rcq->arm_flag = 0;
+ rcq->vector = (u32)vector;
+
+ return 0;
+
+err_send_cmd:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_cq_check
+ Description : roce5_create_cq_check
+ Input : struct ib_device *ibdev
+ int entries
+ int vector
+ struct ib_ucontext *ibcontext
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_cq_check(struct ib_device *ibdev, int entries,
+ int vector)
+{
+ struct roce5_device *rdev = NULL;
+
+ if (ibdev == NULL) {
+ ROCE_ERR("Ibdev is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+ if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Cqe number invalid, entries(%d), max_cqes(%d), func_id(%d)",
+ entries, (int)rdev->rdma_cap.max_cqes,
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if (vector > (int)rdev->rdma_cap.num_comp_vectors) {
+ ROCE_DEV_ERR(rdev, "Vector over range, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_check_cq_create_flags(u32 flags)
+{
+ /*
+ * It returns non-zero value for unsupported CQ
+ * create flags, otherwise it returns zero.
+ */
+ return (int)(flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN |
+ IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION));
+}
+
+static int roce5_cq_cqc_cfg(struct roce5_device *rdev, struct roce5_cq *rcq,
+ int vector, struct ib_udata *udata, u64 cq_buf_addr)
+{
+ int ret = 0;
+ int page_shift = (int)roce5_umem_get_buf_page_shift(rdev, rcq->umem,
+ cq_buf_addr) -
+ PAGE_SHIFT_4K;
+
+ /* configurate CQC */
+ ret = roce5_cq_sw2hw(rdev, rcq, vector, page_shift);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to handle cq sw2hw, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ /* User mode requires outgoing CQN */
+ if (ib_copy_to_udata(udata, &rcq->cqn, sizeof(u32)) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy data to user space, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -EFAULT;
+ return ret;
+ }
+
+ return 0;
+}
+
+static int roce5_create_user_cq(struct roce5_device *rdev, struct roce5_cq *rcq,
+ int cqe_num, const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ int vector = attr->comp_vector;
+ struct roce5_create_cq_cmd ucmd = { 0 };
+ roce_user_handle_t hdl = { 0 };
+
+ if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy from user space, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -EFAULT;
+ goto err_out;
+ }
+
+ ROCE_CONSTUCT_ULD_HDL_RCQ(&hdl, rdev, rcq, udata, &rcq->buf.mtt,
+ &rcq->db.dma, cqe_num);
+ ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rcq->shadow, &rcq->shadow_vfid,
+ NULL);
+ ret = roce5_uld_cq_create((roce_uld_handle)&hdl);
+ if (ret != 0) {
+ ROCE_ERR("Failed to execute uld cq create, ret(%d)", ret);
+ goto err_out;
+ }
+ /* Entries has been decreased by one when it was input by User Mode, and it is the expected value exactly
+ after increased by one by Kernel Mode */
+ ret = roce5_cq_get_umem(rdev, ibcontext, rcq, ucmd.buf_addr, cqe_num,
+ false);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get umem, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_destroy_uld;
+ }
+
+ ret = roce5_db_map_user(to_roce5_ucontext(ibcontext),
+ (unsigned long)ucmd.db_addr, &rcq->db,
+ rcq->shadow);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to map kernel_mem to user_mem, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_mtt;
+ }
+
+ ret = roce5_cq_cqc_cfg(rdev, rcq, vector, udata, ucmd.buf_addr);
+ if (ret != 0) {
+ goto err_unmap_db;
+ }
+
+ return 0;
+
+err_unmap_db:
+ roce5_db_unmap_user(to_roce5_ucontext(ibcontext), &rcq->db,
+ rcq->shadow);
+
+err_free_mtt:
+ roce5_cq_put_umem(rdev, &rcq->buf, &rcq->umem, rcq->shadow);
+ rcq->umem = NULL;
+
+err_destroy_uld:
+ roce5_uld_cq_destroy((roce_uld_handle)&hdl);
+
+err_out:
+
+ return ret;
+}
+
+static void roce5_fill_rcq_info(struct roce5_cq *rcq)
+{
+ rcq->cqn = rcq->cqm_cq->index;
+ rcq->buf.buf = &rcq->cqm_cq->q_room_buf_1;
+
+ /* Initialize buf to unused */
+ roce5_cq_buf_init(&rcq->buf);
+
+ /* Software DB assignment */
+ rcq->set_ci_db = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr
+ ->doorbell_record);
+ *rcq->set_ci_db = 0;
+ rcq->db.db_record = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr
+ ->doorbell_record);
+ rcq->db.dma = rcq->cqm_cq->q_header_paddr;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_kernel_cq
+ Description : roce5_create_kernel_cq
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ int vector
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_kernel_cq(struct roce5_device *rdev,
+ struct roce5_cq *rcq, int entries, int vector,
+ u32 index)
+{
+ int ret = 0;
+ int page_shift = 0;
+
+ rcq->buf.buf_size = entries * (int)rdev->rdma_cap.cqe_size;
+ rcq->cqm_cq = cqm5_object_rdma_queue_create(
+ rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_SCQ,
+ (u32)rcq->buf.buf_size, rcq->opt, true, index, 0, 0);
+ if (rcq->cqm_cq == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create rdma_queue from cqm_object, func_id(%d), index(%d)",
+ rdev->glb_func_id, index);
+ return (-ENOMEM);
+ }
+
+ /* Buffer is obtained from room1 when the queue is just created */
+ if (rcq->cqm_cq->current_q_room != CQM_RDMA_Q_ROOM_1) {
+ ROCE_DEV_ERR(rdev, "Not use CQM_RDMA_Q_ROOM_1, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -EINVAL;
+ goto err_free_cqm_cq;
+ }
+
+ roce5_fill_rcq_info(rcq);
+
+ page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size);
+
+ /* allocate MTT */
+ rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, rcq->buf.buf->buf_number,
+ (u32)page_shift, &rcq->buf.mtt, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_cqm_cq;
+ }
+
+ /* Write the PA of the CQ Buffer to the MTT */
+ ret = roce5_buf_write_mtt(rdev, &rcq->buf.mtt, rcq->buf.buf);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_mtt;
+ }
+
+ page_shift = (int)ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size) -
+ PAGE_SHIFT_4K;
+
+ /* configurate CQC */
+ ret = roce5_cq_sw2hw(rdev, rcq, vector, page_shift);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to configure CQC, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_mtt;
+ }
+
+ return 0;
+
+err_free_mtt:
+ hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE);
+
+err_free_cqm_cq:
+ cqm5_object_delete(&rcq->cqm_cq->object);
+
+ return ret;
+}
+
+static int roce5_cq_depth_init(struct roce5_device *rdev, struct roce5_cq *rcq,
+ const struct ib_cq_init_attr *attr, int *cqe_num)
+{
+ u32 entries = (u32)attr->cqe;
+
+ /* The CQE queue should reserve four special CQE for the HOST CI can be updated to the CQC CI */
+ entries += ROCE_CQE_UP_THRESHOLD;
+ entries = ROCE_ROUNDUP_POW_OF_TWO(entries) & 0xffffffff;
+ /* Chip Constraints: Minimum queue depth needs to be page-aligned */
+ if ((entries * rdev->rdma_cap.cqe_size) < PAGE_SIZE) {
+ entries = (PAGE_SIZE >> ROCE_ILOG2(rdev->rdma_cap.cqe_size));
+ }
+ if (entries == 0) {
+ ROCE_DEV_ERR(rdev, "Invalid entries after align, func_id(%u)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+ rcq->cqe_mask = entries - 1;
+ /* Check if max spec is exceeded */
+ if (rdev->rdma_cap.max_cqes > UINT_MAX - ROCE_CQE_UP_THRESHOLD) {
+ ROCE_DEV_ERR(
+ rdev,
+ "max_cqes overflow detected, max_cqes(%u), func_id(%u)",
+ rdev->rdma_cap.max_cqes, rdev->glb_func_id);
+ return -EINVAL;
+ }
+ if (entries > (rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Over range after align, entries(%u), max_cqes(%u), func_id(%u)",
+ entries,
+ rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD,
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if ((int)entries > INT_MAX) {
+ ROCE_DEV_ERR(rdev, "Bug detected: Int Overflow!");
+ return -EINVAL;
+ }
+
+ *cqe_num = (int)entries;
+ rcq->ibcq.cqe = (int)entries - ROCE_CQE_UP_THRESHOLD;
+ return 0;
+}
+
+static int roce5_do_create_cq(struct ib_device *ibdev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata, struct roce5_cq *rcq,
+ u32 index)
+{
+ struct roce5_device *rdev = NULL;
+ struct roce5_ucontext *context = NULL;
+ int ret = 0;
+ int vector = attr->comp_vector;
+ int cqe_num = 0;
+
+ rdev = to_roce5_dev(ibdev);
+ ret = roce5_cq_depth_init(rdev, rcq, attr, &cqe_num);
+ if (ret != 0) {
+ return ret;
+ }
+
+ mutex_init(&rcq->resize_mutex);
+ spin_lock_init(&rcq->lock);
+ rcq->resize_buf = NULL;
+ rcq->resize_umem = NULL;
+ rcq->buf.entry_size = (int)rdev->rdma_cap.cqe_size;
+ rcq->rcq_magic = ROCE_RCQ_MAGIC;
+
+ rcq->opt = roce5_some(rcq);
+ if (rcq->opt == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to init some, cqn(%u).", rcq->cqn);
+ return -ENOMEM;
+ }
+
+ INIT_LIST_HEAD(&rcq->send_qp_list);
+ INIT_LIST_HEAD(&rcq->recv_qp_list);
+ if (udata != NULL) {
+ ret = roce5_create_cq_alloc_cqn(rdev, rcq, index);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm cq resource, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+ context = roce5_queue_to_ucontext((void *)(&rcq->ibcq), udata,
+ ROCE_Q_TYPE_CQ);
+ ret = roce5_create_user_cq(rdev, rcq, cqe_num, attr,
+ &context->ibucontext, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to create user_cq, func_id(%d)",
+ rdev->glb_func_id);
+ cqm5_object_delete(&rcq->cqm_cq->object);
+ return ret;
+ }
+ } else {
+ ret = roce5_create_kernel_cq(rdev, rcq, cqe_num, vector, index);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to create kernel_cq, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+ }
+
+ rcq->reset_flow_comp = roce5_reset_flow_comp;
+
+ return 0;
+}
+
+int roce5_create_cq_common(struct ib_device *ibdev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata, struct roce5_cq *rcq,
+ u32 index)
+{
+ int ret;
+ int vector = attr->comp_vector;
+ int entries = (int)attr->cqe;
+
+ if (roce5_check_cq_create_flags(attr->flags) != 0) {
+ ROCE_ERR("Not support the cq_create flag(%x)", attr->flags);
+ return -EOPNOTSUPP;
+ }
+
+ ret = roce5_create_cq_check(ibdev, entries, vector);
+ if (ret != 0) {
+ ROCE_ERR("Failed to check cq information");
+ return ret;
+ }
+
+ return roce5_do_create_cq(ibdev, attr, udata, rcq, index);
+}
+
+int roce5_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
+{
+ struct roce5_cq *rcq = NULL;
+
+ if ((ibcq == NULL) || (attr == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ rcq = to_roce5_cq(ibcq);
+ return roce5_create_cq_common(ibcq->device, attr, udata, rcq,
+ ROCE_CQN_INVLD);
+}
+
+struct ib_cq *roce5_create_cq_ofa(struct ib_device *ibdev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_cq *rcq = NULL;
+ struct ib_uobject uobject;
+
+ if ((ibdev == NULL) || (attr == NULL)) {
+ ROCE_ERR("ibdev or attr is NULL");
+ return ERR_PTR(-EINVAL);
+ }
+
+ uobject.context = ibcontext;
+
+ rcq = (struct roce5_cq *)kzalloc(sizeof(*rcq), GFP_KERNEL);
+ if (rcq == NULL) {
+ ret = -ENOMEM;
+ ROCE_ERR("Failed to alloc rcq");
+ goto out;
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ rcq->ibcq.device = ibdev;
+#if defined(IB_CQ_HAVE_UOBJECT)
+ rcq->ibcq.uobject = ((udata != NULL) ? &uobject : NULL);
+#endif
+ ret = roce5_create_cq(&rcq->ibcq, attr, udata);
+ if (ret != 0) {
+ kfree(rcq);
+ rcq = NULL;
+ goto out;
+ }
+
+out:
+ /* disassociate anyway */
+ if (rcq != NULL) {
+ rcq->ibcq.uobject = NULL;
+ }
+ return ((ret != 0) ? (struct ib_cq *)ERR_PTR((long)ret) : &rcq->ibcq);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c
new file mode 100644
index 000000000..eaf8e22f4
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c
@@ -0,0 +1,964 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "comm_defs.h"
+#include "roce_cq.h"
+#include "roce_cqm_cmd.h"
+#include "roce_npu_cmd.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "roce_infra.h"
+#include "roce_compat.h"
+#include "roce_os_compat.h"
+#include "roce_cqe_format.h"
+#include "roce_user.h"
+#include "roce_com.h"
+
+/* and above next is V100 adaptation data structure */
+typedef struct tag_roce_cmd_resize_cq {
+ roce_verbs_cmd_com_s com;
+
+ u32 rsvd;
+ u32 page_size; /* Size of the resize buf page. */
+ u32 log_cq_size; /* Cq depth after resize */
+ u32 mtt_layer_num; /* Number of mtt levels after resize */
+ /* DW4~5 */
+ union {
+ u64 mtt_base_addr; /* Start address of mr or mw */
+ u32 cqc_l0mtt_gpa[2];
+ };
+ u32 mtt_page_size; /* Size of the mtt page after resize. */
+ roce_xq_mtt_info_s mtt_info;
+} roce_cmd_resize_cq_s;
+
+typedef struct tag_roce_cmd_modify_cq {
+ roce_verbs_cmd_com_s com;
+ u32 max_cnt;
+ u32 timeout;
+} roce_cmd_modify_cq_s;
+
+static int roce5_uni_cmd_cq_modify(struct roce5_device *rdev,
+ struct roce5_cq *cq, u32 cnt, u32 period)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_modify_cq_s *cq_modify_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_modify_cq_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ cq_modify_inbuf = (roce_uni_cmd_modify_cq_s *)cqm_cmd_inbuf->buf;
+ cq_modify_inbuf->com.index = cpu_to_be32(cq->cqn);
+ cq_modify_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK);
+ if (cq->shadow != 0) {
+ cq_modify_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ cq_modify_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(cq->shadow_vfid);
+ }
+ cq_modify_inbuf->cq_modify.max_cnt = cpu_to_be32(cnt);
+ cq_modify_inbuf->cq_modify.timeout = cpu_to_be32(period);
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_MODIFY_CQ, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send MODIFY_CQ, cqn(0x%x), ret(%d), func_id(%hu)",
+ cq->cqn, ret, rdev->glb_func_id);
+ /* CMDq may return a positive number, so its return value cannot be used directly */
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+static int roce5_cq_modify(struct roce5_device *rdev, struct roce5_cq *cq,
+ u32 cnt, u32 period)
+{
+ if (ROCE5_SUPPORT_CQ_MODIFY_MSG_V2(rdev) != 0) {
+ return roce5_uni_cmd_cq_modify(rdev, cq, cnt, period);
+ }
+ return -EINVAL;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_modify_cq
+ Description : OFED_3_12
+ Input : struct ib_cq *ibcq
+ u16 cq_count
+ u16 cq_period
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period)
+{
+ int ret = 0;
+ struct roce5_cq *cq = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if (ibcq == NULL) {
+ ROCE_ERR(" Ibcq is null");
+ return -EINVAL;
+ }
+
+ cq = to_roce5_cq(ibcq);
+ rdev = to_roce5_dev(ibcq->device);
+
+ ret = roce5_cq_modify(rdev, cq, cq_count, cq_period);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to modify cq, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_alloc_resize_buf
+ Description : Kernel mode applies for resize_buf, and its corresponding MTT
+ When there is an error in execution, this function ensures that all intermediate resources are released;
+ when the execution is successful, this function can confirm that all resources are acquired.
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_alloc_resize_buf(struct roce5_device *rdev,
+ struct roce5_cq *rcq, int entries)
+{
+ int ret = 0;
+ int page_shift = 0;
+
+ if (rcq->resize_buf != NULL) {
+ ROCE_DEV_ERR(rdev, "Resize buffer is busy, func_id(%d)",
+ rdev->glb_func_id);
+ return -EBUSY;
+ }
+
+ /* Apply for resize_buf and assign it */
+ rcq->resize_buf = (struct roce5_cq_resize_buf *)kmalloc(
+ sizeof(*rcq->resize_buf), GFP_ATOMIC);
+ if (rcq->resize_buf == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc resize buffer, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ rcq->resize_buf->buf.entry_size = rcq->buf.entry_size;
+ rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size;
+ ret = cqm5_object_resize_alloc_new(&rcq->cqm_cq->object,
+ (u32)rcq->resize_buf->buf.buf_size);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to resize cq buffer, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_resize_buf_alloc;
+ }
+
+ /* If the application is successful, record the buffer information of the new buffer */
+ rcq->resize_buf->cqe = entries - ROCE_CQE_UP_THRESHOLD;
+ rcq->resize_buf->buf.buf = &rcq->cqm_cq->q_room_buf_2;
+
+ /* Initialize new buf to unused state */
+ roce5_cq_buf_init(&rcq->resize_buf->buf);
+
+ page_shift = (int)ROCE_ILOG2(rcq->resize_buf->buf.buf->buf_size);
+
+ /* Apply MTT for resize_buf */
+ rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, rcq->resize_buf->buf.buf->buf_number,
+ (u32)page_shift, &rcq->resize_buf->buf.mtt,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_mtt_alloc;
+ }
+
+ /* configurate MTT, write PA of resize_buf to MTT */
+ ret = roce5_buf_write_mtt(rdev, &rcq->resize_buf->buf.mtt,
+ rcq->resize_buf->buf.buf);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_mtt_write;
+ }
+
+ return 0;
+
+err_mtt_write:
+ hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE);
+
+err_mtt_alloc:
+ /* free resize_buf */
+ cqm5_object_resize_free_new(&rcq->cqm_cq->object);
+
+err_resize_buf_alloc:
+ kfree(rcq->resize_buf);
+ rcq->resize_buf = NULL;
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_free_resize_buf
+ Description : Kernel Mode releases resize_buf and its corresponding MTT
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_cq_free_resize_buf(struct roce5_device *rdev,
+ struct roce5_cq *rcq)
+{
+ /* Release the MTT of resize_buf first */
+ hmm_mtt_free(rdev->hwdev, &(rcq->resize_buf->buf.mtt), SERVICE_T_ROCE);
+
+ /* free resize_buf */
+ cqm5_object_resize_free_new(&(rcq->cqm_cq->object));
+
+ /* free the resize_buf pointer itself */
+ kfree(rcq->resize_buf);
+ rcq->resize_buf = NULL;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_alloc_resize_umem
+ Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_alloc_resize_umem(struct roce5_device *rdev,
+ struct roce5_cq *rcq, int entries,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_resize_cq_cmd ucmd = { 0 };
+ struct roce5_ucontext *context = roce5_queue_to_ucontext(
+ (void *)(&rcq->ibcq), udata, ROCE_Q_TYPE_CQ);
+
+ /* If a resize is being executed, it is not allowed to execute another resize at the same time */
+ if (rcq->resize_umem != NULL) {
+ ROCE_DEV_ERR(rdev, "Resize_umem is busy, func_id(%d)",
+ rdev->glb_func_id);
+ return -EBUSY;
+ }
+
+ if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy from user space, func_id(%d)",
+ rdev->glb_func_id);
+ return -EFAULT;
+ }
+
+ rcq->resize_buf = (struct roce5_cq_resize_buf *)kmalloc(
+ sizeof(*rcq->resize_buf), GFP_ATOMIC);
+ if (rcq->resize_buf == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc cq resize buffer, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ ret = roce5_cq_get_umem(rdev, &context->ibucontext, rcq, ucmd.buf_addr,
+ entries, true);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get umem, func_id(%d)",
+ rdev->glb_func_id);
+
+ kfree(rcq->resize_buf);
+ rcq->resize_buf = NULL;
+
+ return ret;
+ }
+
+ rcq->resize_buf->buf.entry_size = rcq->buf.entry_size;
+ rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size;
+ rcq->resize_buf->cqe = entries - ROCE_CQE_UP_THRESHOLD;
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_free_resize_umem
+ Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_cq_free_resize_umem(struct roce5_device *rdev,
+ struct roce5_cq *rcq)
+{
+ /* Free MTT and umem of resize_buf */
+ roce5_cq_put_umem(rdev, &rcq->resize_buf->buf, &rcq->resize_umem, 0);
+ rcq->resize_umem = NULL;
+
+ /* free resize_buf itself */
+ kfree(rcq->resize_buf);
+ rcq->resize_buf = NULL;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_get_outstanding_cqes
+ Description : roce5_cq_get_outstanding_cqes
+ Input : struct roce5_cq *cq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_get_outstanding_cqes(struct roce5_cq *cq)
+{
+ u32 i = 0;
+
+ i = cq->cons_index;
+ while (roce5_get_sw_cqe(cq, i)) {
+ ++i;
+ }
+
+ return (int)(i - cq->cons_index);
+}
+
+static int roce5_cq_get_next_cqe(struct roce_cqe **cqe, struct roce5_cq *cq,
+ unsigned int *i, u32 *times,
+ const struct roce_cqe *start_cqe)
+{
+ do {
+ *cqe = (struct roce_cqe *)roce5_get_sw_cqe(cq, ++(*i));
+ if (*cqe == NULL) {
+ ROCE_DELAY();
+ --(*times);
+ }
+ } while ((*cqe == NULL) && (*times != 0));
+
+ if ((*cqe == start_cqe) || (*cqe == NULL)) {
+ ROCE_ERR("Failed to get resize CQE, CQN(0x%x)", cq->cqn);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_resize_copy_cqes
+ Description : roce5_cq_resize_copy_cqes
+ Input : struct roce5_cq *cq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_cq_resize_copy_cqes(struct roce5_cq *cq)
+{
+ struct roce_cqe *cqe = NULL;
+ struct roce_cqe *new_cqe = NULL;
+ struct roce_cqe *start_cqe = NULL;
+ u32 times;
+ unsigned int i = 0;
+ int ret = 0;
+
+ i = cq->cons_index;
+
+ times = ROCE_CQ_RESIZE_POLL_TIMES;
+ do {
+ cqe = (struct roce_cqe *)roce5_get_sw_cqe(cq, i);
+ if (cqe == NULL) {
+ ROCE_DELAY();
+ --times;
+ }
+ } while ((cqe == NULL) && (times != 0));
+
+ if (cqe == NULL) {
+ ROCE_ERR("Failed to get resize CQE, CQN(0x%x)", cq->cqn);
+ return -ENOMEM;
+ }
+
+ start_cqe = cqe;
+
+ /* r_cqe:resize_cqe
+ CI
+ +--------+-----+-----+-----+----------+
+ old_buf: | | CQE | CQE |R_CQE| |
+ +--------+-----+-----+-----+----------+
+ | |
+ | |
+ +--------+-----+-----+-----+----------+
+ new_buf: | | CQE | CQE | | |
+ +--------+-----+-----+-----+----------+
+ */
+ /* Convert the fields needed by CQE to little endian */
+ cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value);
+ cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value);
+ while (cqe->dw1.bs.op_type != ROCE_OPCODE_RESIZE_CQE) {
+ /* The resized PI can be inherited, and the index of the original CQE remains unchanged in resize_buf */
+ new_cqe = (struct roce_cqe *)roce5_get_cqe_from_buf(
+ &cq->resize_buf->buf,
+ (i & ((unsigned)cq->resize_buf->cqe)));
+
+ ret = memcpy_s((void *)new_cqe, sizeof(struct roce_cqe),
+ (void *)cqe, sizeof(struct roce_cqe));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy new_cqe.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ /* If the CQE has been wrapped in resize_buf, the corresponding Owner bit needs to be modified
+ to be owned by the software.
+ * The rule is: when the owner bit is owned by CQE software, the O bit is opposite to the high bit of ci;
+ when the owner bit is owned by hardware, the O bit is the same as the high bit of CI.
+ */
+ new_cqe->dw0.bs.owner = ((i & ((unsigned)cq->resize_buf->cqe +
+ ROCE_CQE_UP_THRESHOLD)) != 0) ?
+ 1 :
+ 0;
+
+ /* After processing, turn the DW0/DW1 of CQE back to big endian */
+ cqe->dw0.value = roce5_convert_be32(cqe->dw0.value);
+ cqe->dw1.value = roce5_convert_be32(cqe->dw1.value);
+
+ /* Convert DW0/DW1 of the CQE in the new queue back to big endian */
+ new_cqe->dw0.value = roce5_convert_be32(new_cqe->dw0.value);
+ new_cqe->dw1.value = roce5_convert_be32(new_cqe->dw1.value);
+
+ /* Get the next CQE */
+ ret = roce5_cq_get_next_cqe(&cqe, cq, &i, ×, start_cqe);
+ if (ret != 0) {
+ return ret;
+ }
+
+ /* Convert DW0/DW1 of CQE to little endian */
+ cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value);
+ cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value);
+ }
+
+ return 0;
+}
+
+static int roce5_uni_cmd_cq_resize_inbuf_fill(struct roce5_device *rdev,
+ struct roce5_cq *rcq,
+ int page_shift,
+ cqm_cmd_buf_s **cqm_cmd_inbuf)
+{
+ int ret = 0;
+ struct dev_roce_svc_own_cap *rdma_own_cap = NULL;
+ u32 cq_size = 0;
+ u32 mtt_layer = 0;
+ u64 mtt_paddr = 0;
+ u32 mtt_page_size = 0;
+ roce_uni_cmd_resize_cq_s *cq_resize_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_resize_cq_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ rdma_own_cap = &rdev->rdma_cap.dev_rdma_cap.roce_own_cap;
+ cq_size = (u32)rcq->resize_buf->cqe + ROCE_CQE_UP_THRESHOLD;
+ cq_size = (u32)ROCE_ILOG2(cq_size);
+ mtt_layer = rcq->resize_buf->buf.mtt.mtt_layers;
+ mtt_paddr = rcq->resize_buf->buf.mtt.mtt_paddr;
+ mtt_page_size = rcq->resize_buf->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K;
+
+ cq_resize_inbuf = (roce_uni_cmd_resize_cq_s *)(*cqm_cmd_inbuf)->buf;
+ cq_resize_inbuf->com.index = cpu_to_be32(rcq->cqn);
+ cq_resize_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK));
+ if (rcq->shadow != 0) {
+ cq_resize_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ cq_resize_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(rcq->shadow_vfid);
+ }
+ cq_resize_inbuf->cq_resize.page_size = cpu_to_be32((u32)page_shift);
+ cq_resize_inbuf->cq_resize.log_cq_size = cpu_to_be32(cq_size);
+ cq_resize_inbuf->cq_resize.mtt_layer_num = cpu_to_be32(mtt_layer);
+ cq_resize_inbuf->cq_resize.mtt_base_addr = mtt_paddr;
+ cq_resize_inbuf->cq_resize.mtt_base_addr =
+ cpu_to_be64(cq_resize_inbuf->cq_resize.mtt_base_addr);
+ cq_resize_inbuf->cq_resize.mtt_page_size = cpu_to_be32(mtt_page_size);
+
+ cq_resize_inbuf->cq_resize.cmtt_cache.mtt_flags = 0;
+ cq_resize_inbuf->cq_resize.cmtt_cache.mtt_num = 0;
+ cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_start =
+ cpu_to_be32(rdma_own_cap->cmtt_cl_start);
+ cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_end =
+ cpu_to_be32(rdma_own_cap->cmtt_cl_end);
+ cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_size =
+ cpu_to_be32(rdma_own_cap->cmtt_cl_sz);
+ return 0;
+}
+
+static int roce5_cq_resize(struct roce5_device *rdev, struct roce5_cq *rcq,
+ int page_shift)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+
+ if (ROCE5_SUPPORT_IB_CMD_RESIZE_CQ_MSG_V2(rdev) != 0) {
+ ret = roce5_uni_cmd_cq_resize_inbuf_fill(rdev, rcq, page_shift,
+ &cqm_cmd_inbuf);
+ }
+
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_RESIZE_CQ, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RESIZE_CQ command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+
+ /* When CMDq times out or CMDq cannot work, update the device status and notify the PCIe module to reset
+ the device through OFED */
+ if ((ret == -ETIMEDOUT) || (ret == -EPERM)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_user_cq_resize
+ Description : roce5_user_cq_resize
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_user_cq_resize(struct roce5_device *rdev, struct roce5_cq *rcq,
+ int entries, struct ib_udata *udata)
+{
+ int page_shift = 0;
+ int ret = 0;
+
+ /* Cannot exceed max size after power-of-2 alignment */
+ if (entries > ((int)rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) {
+ ROCE_DEV_ERR(rdev, "Over range after align, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ ret = roce5_cq_alloc_resize_umem(rdev, rcq, entries, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc resize_umem, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+#if defined(UMEM_HAVE_PAGE_SHIFT)
+ page_shift = rcq->resize_umem->page_shift - PAGE_SHIFT_4K;
+#else
+ page_shift = PAGE_SHIFT - PAGE_SHIFT_4K;
+#endif
+
+ /* Send the cq_resize command to configure CQC. After the configuration is successful, the new CQE is written to
+ resize_buf, and the old buffer may still retain the old CQE. */
+ ret = roce5_cq_resize(rdev, rcq, page_shift);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to resize cq, func_id(%d)",
+ rdev->glb_func_id);
+ roce5_cq_free_resize_umem(rdev, rcq);
+ return ret;
+ }
+
+ rcq->ibcq.cqe = rcq->resize_buf->cqe;
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_resize_user_cq
+ Description : roce5_resize_user_cq
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_resize_user_cq(struct roce5_device *rdev, struct roce5_cq *rcq,
+ int entries, struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_resize_cq_cmd ucmd = { 0 };
+ int cqe_entries = entries;
+
+ if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy from user space, func_id(%d)",
+ rdev->glb_func_id);
+ return -EFAULT;
+ }
+
+ mutex_lock(&rcq->resize_mutex);
+
+ if (ucmd.stage == 1) {
+ goto release_flow;
+ }
+
+ /* The CQE queue should reserve four special CQE for the HOST CI can be updated to the CQC CI */
+ cqe_entries += ROCE_CQE_UP_THRESHOLD;
+ cqe_entries =
+ (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff);
+ //Minimum queue depth needs to be aligned by page
+ if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) {
+ cqe_entries =
+ (int)(PAGE_SIZE >> (unsigned int)ROCE_ILOG2(
+ rdev->rdma_cap.cqe_size));
+ }
+ if (cqe_entries == 0) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid cqe_entries after align, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -EINVAL;
+ goto out;
+ }
+ rcq->cqe_mask = cqe_entries - 1;
+
+ /* The new depth of CQ cannot be the same as the old depth. Four special CQE space is reserved when CQ is created,
+ so the value of ibcq->cqe is 4 smaller than the actual value */
+ if (cqe_entries == (rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) {
+ ROCE_DEV_ERR(rdev, "No need to resize cq, func_id(%d)",
+ rdev->glb_func_id);
+ ret = 0;
+ goto out;
+ }
+
+ ret = roce5_user_cq_resize(rdev, rcq, cqe_entries, udata);
+ if (ret != 0) {
+ goto out;
+ }
+
+ mutex_unlock(&rcq->resize_mutex);
+
+ return 0;
+
+release_flow:
+ /* free old MTT */
+ hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE);
+
+ rcq->buf = rcq->resize_buf->buf;
+ ib_umem_release(rcq->umem);
+ rcq->umem = rcq->resize_umem;
+
+ kfree(rcq->resize_buf);
+ rcq->resize_buf = NULL;
+ rcq->resize_umem = NULL;
+
+out:
+ mutex_unlock(&rcq->resize_mutex);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_kernel_cq_resize
+ Description : roce5_kernel_cq_resize
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_kernel_cq_resize(struct roce5_device *rdev,
+ struct roce5_cq *rcq, int entries)
+{
+ int outst_cqe = 0;
+ int page_shift = 0;
+ int ret = 0;
+
+ /* Cannot exceed max size after power-of-2 alignment */
+ if (entries > ((int)rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) {
+ ROCE_DEV_ERR(rdev, "Over range after align, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ /* The number of resized CQEs cannot be smaller than the number of outstanding CQEs */
+ outst_cqe = roce5_cq_get_outstanding_cqes(rcq);
+ if (entries < (outst_cqe + ROCE_CQE_UP_THRESHOLD)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Can't resize, because smaller than the number of outstanding CQES, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ ret = roce5_cq_alloc_resize_buf(rdev, rcq, entries);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc resize buffer, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ page_shift = (int)ROCE_ILOG2(rcq->cqm_cq->q_room_buf_2.buf_size) -
+ PAGE_SHIFT_4K;
+
+ /* Send the cq_resize command to configure CQC. After the configuration is successful, the new CQE is written to
+ resize_buf, and the old buffer may still retain the old CQE. */
+ ret = roce5_cq_resize(rdev, rcq, page_shift);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to resize cq, func_id(%d)",
+ rdev->glb_func_id);
+ roce5_cq_free_resize_buf(rdev, rcq);
+ return ret;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_resize_kernel_cq
+ Description : roce5_resize_kernel_cq
+ Input : struct roce5_device *rdev
+ struct roce5_cq *rcq
+ int entries
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_resize_kernel_cq(struct roce5_device *rdev,
+ struct roce5_cq *rcq, int entries)
+{
+ int ret = 0;
+ int old_cqe = 0;
+ struct mtt old_mtt;
+ int cqe_entries = entries;
+
+ mutex_lock(&rcq->resize_mutex);
+
+ cqe_entries += ROCE_CQE_UP_THRESHOLD;
+ cqe_entries =
+ (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff);
+ /* Minimum queue depth needs to be aligned by page */
+ if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) {
+ cqe_entries = (PAGE_SIZE >> (unsigned int)ROCE_ILOG2(
+ rdev->rdma_cap.cqe_size));
+ }
+
+ /* The new depth of CQ cannot be the same as the old depth. Four special CQE space is reserved when CQ is created,
+ so the value of ibcq->cqe is four smaller than the actual value */
+ if (cqe_entries == (rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) {
+ ROCE_DEV_ERR(rdev, "No need to resize cq, func_id(%d)",
+ rdev->glb_func_id);
+ mutex_unlock(&rcq->resize_mutex);
+ return 0;
+ }
+
+ ret = roce5_kernel_cq_resize(rdev, rcq, cqe_entries);
+ if (ret != 0) {
+ goto out;
+ }
+
+ /* free old MTT */
+ old_mtt = rcq->buf.mtt;
+
+ /* When copying CQE from the old buffer to resize_buf, the user may be polling cqe from the old buf, so
+ it needs to be locked.If CQE of the old buffer has been polled in the polling process, polling process will
+ free old buffer and switch to new buffer.
+ */
+ spin_lock_irq(&rcq->lock);
+
+ /* If the new buf has been switched to, nothing needs to be done */
+ if (rcq->resize_buf != NULL) {
+ ret = roce5_cq_resize_copy_cqes(rcq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Have not polled resize cqe(fuc:%d, cqn:%u, ret:%d)",
+ rdev->glb_func_id, rcq->cqn, ret);
+ roce5_cq_free_resize_buf(rdev, rcq);
+ spin_unlock_irq(&rcq->lock);
+ mutex_unlock(&rcq->resize_mutex);
+ return ret;
+ }
+ old_cqe = rcq->ibcq.cqe;
+ rcq->buf = rcq->resize_buf->buf;
+ rcq->ibcq.cqe = rcq->resize_buf->cqe;
+
+ kfree(rcq->resize_buf);
+ rcq->resize_buf = NULL;
+ }
+
+ spin_unlock_irq(&rcq->lock);
+
+ /* Non-0 means that the above resize_buf non-empty branch has been entered, the old buffer needs to be released */
+ if (old_cqe != 0) {
+ hmm_mtt_free(rdev->hwdev, &old_mtt, SERVICE_T_ROCE);
+ cqm5_object_resize_free_old(&rcq->cqm_cq->object);
+ }
+
+out:
+ mutex_unlock(&rcq->resize_mutex);
+
+ return ret;
+}
+
+static int roce5_resize_cq_check(struct ib_cq *ibcq, int entries,
+ const struct ib_udata *udata)
+{
+ struct roce5_device *rdev = NULL;
+
+ if (ibcq == NULL) {
+ ROCE_ERR("Ibcq is null");
+ return -EINVAL;
+ }
+
+ if ((ibcq->uobject != NULL) && (udata == NULL)) {
+ ROCE_ERR("Udata is null, but uobject is not null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibcq->device);
+ if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) {
+ ROCE_DEV_ERR(rdev,
+ "Resize CQEs invalid. entries(%d), func_id(%d)",
+ entries, rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_resize_cq
+ Description : roce5_resize_cq
+ Input : struct ib_cq *ibcq
+ int entries
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_cq *rcq = NULL;
+ struct roce5_device *rdev = NULL;
+
+ ret = roce5_resize_cq_check(ibcq, entries, udata);
+ if (ret != 0) {
+ ROCE_ERR("Failed to check resize_cq");
+ return ret;
+ }
+
+ rcq = to_roce5_cq(ibcq);
+ rdev = to_roce5_dev(ibcq->device);
+
+ if (ibcq->uobject) {
+ ret = roce5_resize_user_cq(rdev, rcq, entries, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to resize user cq, func_id(%u) ret(%d)",
+ (u32)rdev->glb_func_id, ret);
+ return ret;
+ }
+ } else {
+ ret = roce5_resize_kernel_cq(rdev, rcq, entries);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to resize kernel cq, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+void roce5_lock_cqs(struct roce5_cq *roce5_send_cq,
+ struct roce5_cq *roce5_recv_cq)
+ __acquires(&roce5_send_cq->lock) __acquires(&roce5_recv_cq->lock)
+{
+ if (roce5_send_cq == roce5_recv_cq) {
+ spin_lock_irq(&roce5_send_cq->lock);
+ __acquire(&roce5_recv_cq->lock);
+ } else if (roce5_send_cq->cqn < roce5_recv_cq->cqn) {
+ spin_lock_irq(&roce5_send_cq->lock);
+ spin_lock_nested(&roce5_recv_cq->lock, SINGLE_DEPTH_NESTING);
+ } else {
+ spin_lock_irq(&roce5_recv_cq->lock);
+ spin_lock_nested(&roce5_send_cq->lock, SINGLE_DEPTH_NESTING);
+ }
+}
+
+void roce5_unlock_cqs(struct roce5_cq *roce5_send_cq,
+ struct roce5_cq *roce5_recv_cq)
+ __releases(&roce5_send_cq->lock) __releases(&roce5_recv_cq->lock)
+{
+ if (roce5_send_cq == roce5_recv_cq) {
+ __release(&roce5_recv_cq->lock);
+ spin_unlock_irq(&roce5_send_cq->lock);
+ } else if (roce5_send_cq->cqn < roce5_recv_cq->cqn) {
+ spin_unlock(&roce5_recv_cq->lock);
+ spin_unlock_irq(&roce5_send_cq->lock);
+ } else {
+ spin_unlock(&roce5_send_cq->lock);
+ spin_unlock_irq(&roce5_recv_cq->lock);
+ }
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c
new file mode 100644
index 000000000..c4e442598
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+
+#include "comm_defs.h"
+#include "hinic5_hw.h"
+
+#include "roce.h"
+#include "roce_srq.h"
+#include "roce_qp.h"
+#include "roce_mix.h"
+#include "roce_xrc.h"
+#include "roce_cq.h"
+#include "roce_cqm_cmd.h"
+#include "roce_npu_cmd.h"
+#include "hinic5_hmm.h"
+#include "securec.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "roce_compat.h"
+#include "roce_os_compat.h"
+#include "roce_cq_intern.h"
+#include "roce_com.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_uld_inner.h"
+
+/* and above next is V100 adaptation data structure */
+typedef struct tag_roce_cmd_cq_cache_invalidate {
+ roce_verbs_cmd_com_s com;
+ roce_xq_mtt_info_s mtt_info;
+} roce_cmd_cq_cache_invalidate_s;
+#define CQC_CHECK_RESCHED_LOOP_TIMES 0xffffU
+static int roce5_check_cqc_data_state(struct roce5_device *rdev,
+ const struct roce5_cq *cq,
+ struct roce5_cq_context *cqc_data,
+ u32 check_state)
+{
+ int read_back_flag = 0;
+ u32 times = (u32)rdev->try_times;
+ struct roce5_cq_context check_cqc_data;
+ while ((times--) != 0) {
+ check_cqc_data.dw2.value = be32_to_cpu(cqc_data->dw2.value);
+ if (check_cqc_data.dw2.bs.state == check_state) {
+ read_back_flag = 1;
+ break;
+ }
+ ROCE_DELAY();
+
+ // limit schedule cond_resched count, preventstop thread schedule frequency past high
+ if ((times & CQC_CHECK_RESCHED_LOOP_TIMES) == 0) {
+ cond_resched();
+ }
+ }
+
+ if (read_back_flag == 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to read back after try %d times, Cqn(0x%x), CQ state(0x%x), func_id(%u)",
+ rdev->try_times, cq->cqn, cqc_data->dw2.value,
+ (u32)rdev->glb_func_id);
+ return -1;
+ }
+
+ return 0;
+}
+/* ****************************************************************************
+ Prototype : roce5_cq_hw2sw
+ Description : roce5_cq_hw2sw
+ Input : struct roce5_device *rdev
+ struct roce5_cq *cq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_cq_hw2sw(struct roce5_device *rdev, struct roce5_cq *cq)
+{
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ struct roce5_cq_context *cqc_data = NULL;
+ roce_uni_cmd_cq_hw2sw_s *cq_hw2sw_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_cq_hw2sw_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ cq_hw2sw_inbuf = (roce_uni_cmd_cq_hw2sw_s *)cqm_cmd_inbuf->buf;
+ cq_hw2sw_inbuf->com.index = cpu_to_be32((u32)cq->cqn);
+ cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK);
+ if (cq->shadow != 0) {
+ cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ cq_hw2sw_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(cq->shadow_vfid);
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_HW2SW_CQ,
+ cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send HW2SW_CQ command, func_id(%u), cqn(0x%x), (ret:%d)",
+ (u32)rdev->glb_func_id, cq->cqn, ret);
+
+ ret = (-EINVAL);
+
+ if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) {
+ ROCE_DEV_ERR(rdev,
+ "Card not present, return OK, func_id(%d)",
+ rdev->glb_func_id);
+ ret = 0;
+ }
+
+ /* CMDq may return a positive number, so its return value cannot be used directly */
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ /* HI1825_V100 will retry load cqc in cmdq */
+ if (ROCE5_SUPPORT_RETRY_LOAD_CQC_MSG_FEATURE(rdev) != 0) {
+ cqc_data = (struct roce5_cq_context *)((void *)cq->cqm_cq
+ ->q_ctx_vaddr);
+ return roce5_check_cqc_data_state(rdev, cq, cqc_data,
+ ROCE_CQ_TIME_OUT_CHECK_VALUE);
+ }
+ return 0;
+}
+
+static int roce5_uni_cmd_cq_cache_out(struct roce5_device *rdev,
+ struct roce5_cq *cq,
+ cqm_cmd_buf_s **cqm_cmd_inbuf)
+{
+ int ret = 0;
+ struct dev_roce_svc_own_cap *roce_own_cap = NULL;
+ roce_uni_cmd_cq_hw2sw_s *cq_cache_invld_inbuf = NULL;
+
+ /* Send the cache out command */
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_cq_hw2sw_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ roce_own_cap = &rdev->rdma_cap.dev_rdma_cap.roce_own_cap;
+ cq_cache_invld_inbuf = (roce_uni_cmd_cq_hw2sw_s *)(*cqm_cmd_inbuf)->buf;
+ cq_cache_invld_inbuf->com.index = cpu_to_be32((u32)cq->cqn);
+ cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK));
+ if (cq->shadow != 0) {
+ cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ cq_cache_invld_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(cq->shadow_vfid);
+ }
+ cq_cache_invld_inbuf->cmtt_cache.mtt_flags = 0;
+ cq_cache_invld_inbuf->cmtt_cache.mtt_num = 0;
+ cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_start =
+ cpu_to_be32(roce_own_cap->cmtt_cl_start);
+ cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_end =
+ cpu_to_be32(roce_own_cap->cmtt_cl_end);
+ cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_size =
+ cpu_to_be32(roce_own_cap->cmtt_cl_sz);
+
+ return 0;
+}
+
+static int roce5_cq_cache_out(struct roce5_device *rdev, struct roce5_cq *cq)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+
+ if (ROCE5_SUPPORT_CQ_CACHE_OUT_MSG_V2(rdev) != 0) {
+ ret = roce5_uni_cmd_cq_cache_out(rdev, cq, &cqm_cmd_inbuf);
+ }
+
+ if (ret != 0) {
+ return ret;
+ }
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_MISC_CQ_CACHE_INVLD, cqm_cmd_inbuf,
+ NULL, NULL, ROCE_CMD_TIME_CLASS_A,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send CQ_CACHE_INVLD command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) {
+ ROCE_DEV_ERR(rdev,
+ "Card not present, return OK, func_id(%d)",
+ rdev->glb_func_id);
+ return 0;
+ }
+
+ return -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ return 0;
+}
+
+int roce5_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_device *rdev = NULL;
+ struct roce5_cq *rcq = NULL;
+ struct roce5_ucontext *ucontext = NULL;
+ char waiting_msg[ROCE_WAITING_MSG_MAX_SIZE];
+ roce_user_handle_t hdl = { 0 };
+
+ if (ibcq == NULL) {
+ ROCE_ERR("Ibcq is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibcq->device);
+ rcq = to_roce5_cq(ibcq);
+
+ ret = sprintf_s(waiting_msg, ROCE_WAITING_MSG_MAX_SIZE,
+ "Waiting for roce cq(%u) release...", rcq->cqn);
+ if (ret < 0) {
+ ROCE_DEV_WARN(rdev, "Failed to setup waiting message, ret(%d).",
+ ret);
+ }
+
+ roce5_option_unset(rcq->opt, waiting_msg, sizeof(waiting_msg));
+
+ /* Modify CQC to be owned by the software */
+ ret = roce5_cq_hw2sw(rdev, rcq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify CQC, ret(%d), func_id(%d), cqn(0x%x)",
+ ret, rdev->glb_func_id, rcq->cqn);
+ return ret;
+ }
+
+ /* Send the cache out command */
+ ret = roce5_cq_cache_out(rdev, rcq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send cache out command, ret(%d), func_id(%hu)",
+ ret, rdev->glb_func_id);
+ return ret;
+ }
+
+ ROCE_CONSTUCT_ULD_HDL_RCQ(&hdl, rdev, rcq, NULL, NULL, NULL, 0);
+ roce5_uld_cq_destroy((roce_uld_handle)&hdl);
+
+ /* If it is user mode, you also need to cancel the mapping of the corresponding software DB */
+ if (ibcq->uobject != NULL) {
+ ucontext = roce5_queue_to_ucontext((void *)ibcq, udata,
+ ROCE_Q_TYPE_CQ);
+ roce5_db_unmap_user(ucontext, &rcq->db, rcq->shadow);
+ roce5_cq_put_umem(rdev, &rcq->buf, &rcq->umem, rcq->shadow);
+ } else {
+ hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE);
+ }
+
+ roce5_destroy_cq_dealloc_cqn(rdev, rcq);
+ return 0;
+}
+
+int roce5_destroy_cq_ofa(struct ib_cq *ibcq)
+{
+ int ret = 0;
+ struct roce5_cq *rcq = NULL;
+
+ if (ibcq == NULL) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ ret = roce5_destroy_cq(ibcq, NULL);
+ if (ret == 0) {
+ rcq = to_roce5_cq(ibcq);
+ kfree(rcq);
+ }
+
+ return ret;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h
new file mode 100644
index 000000000..3c1cb6640
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h
@@ -0,0 +1,29 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_CQ_FORMAT_H
+#define ROCE_CQ_FORMAT_H
+
+#include <linux/types.h>
+
+#include "roce_qp_ctx_format.h"
+
+typedef struct roce5_cq_context {
+ u32 rsvd[2];
+ union {
+ struct {
+ u32 rsvd : 28;
+ u32 state : 4;
+ } bs;
+ u32 value;
+ } dw2;
+
+ u32 rsvd1[12];
+
+ struct roce5_qpc_timer_seg timer_seg;
+
+ u32 rsvd2[8];
+} roce5_cq_context_s;
+
+#endif /* ROCE_CQ_FORMAT_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c
new file mode 100644
index 000000000..119fafba9
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c
@@ -0,0 +1,98 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+
+#include "hinic5_hw.h"
+#include "securec.h"
+
+#include "roce.h"
+#include "roce_user.h"
+#include "roce_cq.h"
+#include "roce_os_compat.h"
+#include "roce_cq_intern.h"
+
+/*
+ * high performance cq, based on reserved of cq, first to firstget get.
+ * 特 point: supports 23V200 & 25V100 two types NIC, according to device type different to allocate different of cq.
+ * later get of cq so bind qp performance will has so next decrease.
+*/
+
+#define ROCE_CQN_PERF_BASE 1
+
+int roce5_create_cq_alloc_cqn(struct roce5_device *rdev, struct roce5_cq *rcq,
+ u32 index)
+{
+ u32 cqn = index;
+ u32 offset = ROCE_CQN_PERF_BASE;
+ u32 step = rdev->cqn_ctrl.rsvd_cq_step;
+ u32 rsvd_cqs = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs;
+
+ mutex_lock(&rdev->cqn_ctrl.cq_ctrl_mutex);
+
+ if (rdev->cqn_ctrl.alloc_cq_cnt == rdev->cqn_ctrl.high_perf_cq_cnt) {
+ /* The reserved high performance CQs have been fully allocated. */
+ goto cqn_alloc;
+ }
+
+ while (rdev->cqn_ctrl.alloc_cq_cnt < rdev->cqn_ctrl.high_perf_cq_cnt &&
+ (offset < rsvd_cqs)) {
+ if (test_and_set_bit(offset, &rdev->cqn_ctrl.cq_bitmap) != 0) {
+ offset += step;
+ continue;
+ }
+ cqn = offset;
+ rdev->cqn_ctrl.alloc_cq_cnt++;
+ ROCE_DEV_INFO(
+ rdev,
+ "Alloc High performance CQ, func_id(%d), total_high_perf_cq(%u), cqn(%d)",
+ rdev->glb_func_id, rdev->cqn_ctrl.alloc_cq_cnt, cqn);
+ break;
+ }
+
+cqn_alloc:
+ mutex_unlock(&rdev->cqn_ctrl.cq_ctrl_mutex);
+
+ rcq->cqm_cq = cqm5_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE,
+ CQM_OBJECT_RDMA_SCQ, 0,
+ rcq->opt, false, cqn, 0, 0);
+ if (rcq->cqm_cq == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create rdma queue from cqm object, func_id(%d), index(%d)",
+ rdev->glb_func_id, index);
+ return -ENOMEM;
+ }
+
+ rcq->cqn = rcq->cqm_cq->index;
+
+ return 0;
+}
+
+void roce5_destroy_cq_dealloc_cqn(struct roce5_device *rdev,
+ struct roce5_cq *rcq)
+{
+ u32 cqn = rcq->cqn;
+ u32 rsvd_cqs = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs;
+ /* Call the CQM interface to release CQN and CQC. Since there is no cq_buf and software DB in user mode,
+ it does not need to be released; since kernel mode has both, it needs to be released */
+ cqm5_object_delete(&rcq->cqm_cq->object);
+ kfree(rcq->opt);
+ rcq->opt = NULL;
+
+ mutex_lock(&rdev->cqn_ctrl.cq_ctrl_mutex);
+
+ if ((cqn < rsvd_cqs) &&
+ test_and_clear_bit(cqn, &rdev->cqn_ctrl.cq_bitmap) != 0 &&
+ rdev->cqn_ctrl.alloc_cq_cnt > 0) {
+ rdev->cqn_ctrl.alloc_cq_cnt--;
+ ROCE_DEV_INFO(
+ rdev,
+ "Dealloc High performance CQ, func_id(%d), total_high_perf_cq(%u), cqn(%d)",
+ rdev->glb_func_id, rdev->cqn_ctrl.alloc_cq_cnt, cqn);
+ }
+ mutex_unlock(&rdev->cqn_ctrl.cq_ctrl_mutex);
+
+ return;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h
new file mode 100644
index 000000000..a2cb56943
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h
@@ -0,0 +1,24 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_CQ_INTERN_H
+#define ROCE_CQ_INTERN_H
+
+#include <rdma/ib_verbs.h>
+
+#include "hinic5_hw.h"
+#include "hinic5_rdma.h"
+#include "hinic5_cqm.h"
+
+#include "roce.h"
+#include "roce_cq.h"
+
+#define ROCE_CQN_INVLD CQM_INDEX_INVALID
+
+int roce5_create_cq_alloc_cqn(struct roce5_device *rdev, struct roce5_cq *rcq,
+ u32 index);
+void roce5_destroy_cq_dealloc_cqn(struct roce5_device *rdev,
+ struct roce5_cq *rcq);
+
+#endif //ROCE_CQ_INTERN_H
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c
new file mode 100644
index 000000000..0ce3ffb25
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c
@@ -0,0 +1,437 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "roce_event_extension.h"
+#include "roce_aeq_format.h"
+#include "hinic5_mt.h"
+#include "roce_restore.h"
+#include "roce_infra.h"
+#include "roce_cq.h"
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+#include "roce_event.h"
+
+static void roce5_event_report_common(const struct roce5_device *rdev,
+ int event_str_index)
+{
+ switch (event_str_index) {
+ case OFED_ET_PATH_MIG:
+ ROCE_DEV_INFO_LIMIT(rdev,
+ "OFED event: Path mig. function id: %hu",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_COMM_EST:
+ ROCE_INFO_ONCE(
+ "OFED event: Communication establish. function id: %hu",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_SQ_DRAINED:
+ ROCE_DEV_ERR_LIMIT(rdev,
+ "OFED event: Sq drained. function id: %hu",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_SRQ_QP_LAST_WQE:
+ ROCE_ERR_ONCE("OFED event: Srq/qp last wqe. function id: %hu",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_WQ_CATAS_ERR:
+ ROCE_DEV_ERR_LIMIT(
+ rdev, "OFED event: Wq catas error. function id: %hu",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_PATH_MIG_FAILED:
+ ROCE_DEV_ERR_LIMIT(
+ rdev, "OFED event: Path mig failed. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_WQ_INVAL_REQ_ERR:
+ ROCE_DEV_ERR_LIMIT(
+ rdev, "OFED event: Wq inval req error. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_WQ_ACCESS_ERR:
+ ROCE_DEV_ERR_LIMIT(
+ rdev, "OFED event: Wq access error. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_CQ_ERR:
+ ROCE_DEV_ERR_LIMIT(rdev,
+ "OFED event: Cq error. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_SRQ_LIMIT:
+ ROCE_DEV_ERR_LIMIT(rdev,
+ "OFED event: Srq limit. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case OFED_ET_SRQ_CATAS_ERR:
+ ROCE_DEV_ERR_LIMIT(
+ rdev, "OFED event: Srq catas error. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case NON_OFED_ET_QPC_LOOKUP_ERR:
+ ROCE_DEV_ERR_LIMIT(
+ rdev,
+ "[non ofed event type] Qpc lookup error. function id: %u",
+ rdev->glb_func_id);
+ break;
+ case NON_OFED_ET_OTHER_TYPE_ERR:
+ ROCE_ERR_ONCE(
+ "[non ofed event type] other type error. function id: %u",
+ rdev->glb_func_id);
+ break;
+ default:
+ break;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_event_report
+ Description : roce5_event_report
+ Input : struct roce5_device *rdev
+ int event_str_index
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_event_report(struct roce5_device *rdev, u8 *val,
+ int event_str_index)
+{
+ if (event_str_index <= NON_OFED_ET_OTHER_TYPE_ERR) {
+ roce5_event_report_common(rdev, event_str_index);
+ } else {
+ roce5_event_report_extend(rdev, val, event_str_index);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : to_qp_event_str_index
+ Description : to_qp_event_str_index
+ Input : u8 event_type
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int to_qp_event_str_index(u8 event_type)
+{
+ switch (event_type) {
+ case ROCE_EVENT_TYPE_PATH_MIG:
+ return OFED_ET_PATH_MIG;
+
+ case ROCE_EVENT_TYPE_COMM_EST:
+ return OFED_ET_COMM_EST;
+
+ case ROCE_EVENT_TYPE_SQ_DRAINED:
+ return OFED_ET_SQ_DRAINED;
+
+ case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE:
+ return OFED_ET_SRQ_QP_LAST_WQE;
+
+ case ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+ return OFED_ET_WQ_CATAS_ERR;
+
+ case ROCE_EVENT_TYPE_PATH_MIG_FAILED:
+ return OFED_ET_PATH_MIG_FAILED;
+
+ case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ return OFED_ET_WQ_INVAL_REQ_ERR;
+
+ default:
+ return OFED_ET_WQ_ACCESS_ERR;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_handle_qp_async_event
+ Description : roce5_handle_qp_async_event
+ Input : struct roce5_device *rdev
+ u32 qpn
+ u8 event_type
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_handle_qp_async_event(struct roce5_device *rdev, u32 qpn,
+ u8 event_type)
+{
+ struct roce5_qp *rqp = NULL;
+ cqm_object_s *cqm_obj_queue = NULL;
+
+ cqm_obj_queue = cqm5_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX,
+ qpn, false);
+ if (cqm_obj_queue == NULL) {
+ ROCE_DEV_ERR_LIMIT(
+ rdev,
+ "Failed to get cqm obj queue, func_id(%d), qpn:%u, event_type:%u",
+ rdev->glb_func_id, qpn, event_type);
+ return;
+ }
+
+ rqp = cqmobj_to_roce_qp(cqm_obj_queue);
+ roce5_qp_async_event(rdev, rqp, (int)event_type);
+ cqm5_object_put(cqm_obj_queue);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_handle_cq_async_event
+ Description : roce5_handle_cq_async_event
+ Input : struct roce5_device *rdev
+ u32 cqn
+ u8 event_type
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_handle_cq_async_event(struct roce5_device *rdev, u32 cqn,
+ u8 event_type)
+{
+ struct roce5_cq *rcq = NULL;
+
+ rcq = roce5_get_rcq_by_cqn(rdev, cqn);
+ if (rcq == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to get roce cq by cqn(%u).", cqn);
+ return;
+ }
+
+ roce5_cq_async_event(rdev, rcq, (int)event_type);
+ roce5_rcq_put(rcq);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_handle_srq_async_event
+ Description : roce5_handle_srq_async_event
+ Input : struct roce5_device *rdev
+ u32 srqn
+ u8 event_type
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_handle_srq_async_event(struct roce5_device *rdev, u32 srqn,
+ u8 event_type)
+{
+ struct roce5_srq *rsrq = NULL;
+ cqm_object_s *cqm_obj_queue = NULL;
+
+ cqm_obj_queue =
+ cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false);
+ if (cqm_obj_queue == NULL) {
+ ROCE_DEV_ERR_LIMIT(
+ rdev,
+ "Failed to get cqm obj queue, func_id(%d), srqn:%u, event_type:%u",
+ rdev->glb_func_id, srqn, event_type);
+ return;
+ }
+
+ rsrq = cqmobj_to_roce5_srq(cqm_obj_queue);
+ roce5_srq_async_event(rdev, rsrq, (int)event_type);
+ cqm5_object_put(cqm_obj_queue);
+}
+
+static bool roce5_async_event_need_reset(u8 event_status)
+{
+ if ((event_status == API_FORMAT_ERROR) ||
+ (event_status == UNKNOWN_RESERVED_ERROR1)) {
+ return true;
+ }
+
+ return false;
+}
+
+u8 roce5_async_event_level(void *svc_hd, u8 event_type, u8 *val)
+{
+ u8 event_level = FAULT_LEVEL_MAX;
+ u8 event_status;
+ u64 param = *((u64 *)val);
+
+ if (svc_hd == NULL) {
+ ROCE_ERR_LIMIT("Svc_hd is null");
+ return FAULT_LEVEL_SUGGESTION;
+ }
+
+ event_status = param & 0xff;
+
+ switch (event_type) {
+ case ROCE_EVENT_TYPE_PATH_MIG:
+ case ROCE_EVENT_TYPE_COMM_EST:
+ case ROCE_EVENT_TYPE_SQ_DRAINED:
+ case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE:
+ case ROCE_EVENT_TYPE_SRQ_LIMIT:
+ case ROCE_EVENT_TYPE_DEBUG:
+ event_level = FAULT_LEVEL_SUGGESTION;
+ break;
+
+ case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR:
+ case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: //ROCE_AEQE_EVENT_QP_REQ_ERR
+ if (event_status == API_FORMAT_ERROR) {
+ return FAULT_LEVEL_SERIOUS_RESET;
+ }
+ break;
+
+ case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: //ROCE_AEQE_EVENT_WQE_CATAS
+ if (roce5_async_event_need_reset(event_status)) {
+ return FAULT_LEVEL_SERIOUS_RESET;
+ }
+ break;
+
+ case ROCE_EVENT_TYPE_CQ_ERROR: //CQ ERR
+ if (roce5_async_event_need_reset(event_status)) {
+ return FAULT_LEVEL_SERIOUS_RESET;
+ }
+ break;
+
+ default:
+ event_level = FAULT_LEVEL_GENERAL;
+ break;
+ }
+
+ return event_level;
+}
+
+static int roce5_async_event_handle_common(u8 event_type, u8 *val,
+ struct roce5_device *rdev)
+{
+ u32 xid = 0;
+ int event_str_index = 0;
+
+ switch (event_type) {
+ case ROCE_EVENT_TYPE_PATH_MIG:
+ case ROCE_EVENT_TYPE_COMM_EST:
+ case ROCE_EVENT_TYPE_SQ_DRAINED:
+ case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE:
+ case ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+ case ROCE_EVENT_TYPE_PATH_MIG_FAILED:
+ case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR:
+ xid = *((u32 *)val);
+ roce5_handle_qp_async_event(rdev, xid, event_type);
+ event_str_index = to_qp_event_str_index(event_type);
+ break;
+
+ case ROCE_EVENT_TYPE_CQ_ERROR:
+ xid = *((u32 *)val);
+ roce5_handle_cq_async_event(rdev, xid, event_type);
+ event_str_index = OFED_ET_CQ_ERR;
+ break;
+
+ case ROCE_EVENT_TYPE_SRQ_LIMIT:
+ xid = *((u32 *)val);
+ roce5_handle_srq_async_event(rdev, xid, event_type);
+ event_str_index = OFED_ET_SRQ_LIMIT;
+ break;
+
+ case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
+ xid = *((u32 *)val);
+ roce5_handle_srq_async_event(rdev, xid, event_type);
+ event_str_index = OFED_ET_SRQ_CATAS_ERR;
+ break;
+
+ default:
+ event_str_index = to_unkown_event_str_index(event_type, val);
+ break;
+ }
+
+ return event_str_index;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_async_event
+ Description : roce5_async_event
+ Input : void *svc_hd
+ u8 event_type
+ u64 val
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_async_event(void *svc_hd, u8 event_type, u8 *val)
+{
+ int event_str_index = 0;
+ struct roce5_device *rdev = NULL;
+
+ if (svc_hd == NULL) {
+ ROCE_ERR_LIMIT("Svc_hd is null");
+ return;
+ }
+
+ rdev = (struct roce5_device *)svc_hd;
+ if (event_type <= ROCE_EVENT_TYPE_ODP_PAGE_FAULT) {
+ event_str_index =
+ roce5_async_event_handle_common(event_type, val, rdev);
+ } else {
+ event_str_index =
+ roce5_async_event_handle_extend(event_type, val, rdev);
+ }
+
+ roce5_event_report(rdev, val, event_str_index);
+}
+
+#ifdef ROCE_BONDING_EN
+void roce5_handle_bonded_port_state_event(struct roce5_device *rdev)
+{
+ struct net_device *master = netdev_master_upper_dev_get_rcu(rdev->ndev);
+ enum ib_port_state bonded_port_state = IB_PORT_NOP;
+ struct roce5_bond_device *bond_dev = NULL;
+ enum ib_port_state curr_port_state;
+ enum ib_event_type event;
+ struct roce5_bond_slave *slave = NULL;
+ u32 i;
+
+ bond_dev = rdev->bond_dev;
+ if (bond_dev == NULL) {
+ ROCE_DEV_ERR(rdev, "No bond_dev found");
+ return;
+ }
+
+ if (master == NULL) {
+ bonded_port_state = IB_PORT_DOWN;
+ } else {
+ if (netif_running(master) != 0) {
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ slave = &bond_dev->slaves[i];
+ if (slave->netdev == NULL) {
+ continue;
+ }
+
+ curr_port_state =
+ (netif_running(slave->netdev) &&
+ netif_carrier_ok(slave->netdev)) ?
+ IB_PORT_ACTIVE :
+ IB_PORT_DOWN;
+ bonded_port_state =
+ (bonded_port_state != IB_PORT_ACTIVE) ?
+ curr_port_state :
+ IB_PORT_ACTIVE;
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+ } else {
+ bonded_port_state = IB_PORT_DOWN;
+ }
+ }
+
+ if (rdev->port_state != bonded_port_state) {
+ event = (bonded_port_state == IB_PORT_ACTIVE) ?
+ IB_EVENT_PORT_ACTIVE :
+ IB_EVENT_PORT_ERR;
+ if (rdev->port_state != IB_PORT_NOP) {
+ roce5_ifconfig_up_down_event_report(rdev, event);
+ }
+ rdev->port_state = bonded_port_state;
+ }
+}
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h
new file mode 100644
index 000000000..b5acc9b15
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h
@@ -0,0 +1,25 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_EVENT_H
+#define ROCE_EVENT_H
+
+#include <linux/types.h>
+
+#define ROCE_M_DELAY 100
+#define API_FORMAT_ERROR 0x2
+#define UNKNOWN_RESERVED_ERROR1 0xe
+#define QPC_LOOKUP_ERR_VALUE 0x300d00016
+
+struct roce5_aeqe_event_data {
+ u32 xid;
+
+ //microcode side by ROCE_AEQE_EVENT construct, driver side byte order differs from microcode side
+ u32 err_code : 8;
+ u32 type : 12;
+ u32 subtype : 8;
+ u32 module_type : 4;
+};
+
+#endif /* ROCE_EVENT_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c
new file mode 100644
index 000000000..c33bab11a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c
@@ -0,0 +1,56 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
+
+
+#include <linux/types.h>
+
+#include "roce_event_extension.h"
+#include "roce_aeq_format.h"
+
+int to_unkown_event_str_index(uint8_t event_type, const uint8_t *val)
+{
+ switch (event_type) {
+ case ROCE_EVENT_TYPE_OFED_NO_DEF:
+ if (*((u64 *)val) == QPC_LOOKUP_ERR_VALUE) {
+ return NON_OFED_ET_QPC_LOOKUP_ERR;
+ }
+ return NON_OFED_ET_OTHER_TYPE_ERR;
+
+ case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR:
+ return NON_OFED_ET_OTHER_TYPE_ERR;
+
+ case ROCE_EVENT_TYPE_DEBUG:
+ return NON_OFED_ET_DEBUG_ERR;
+
+ default:
+ return INVAL_ET_ERR;
+ }
+}
+
+void roce5_event_report_extend(const struct roce5_device *rdev, u8 *val,
+ int event_str_index)
+{
+ if (event_str_index == NON_OFED_ET_DEBUG_ERR) {
+ struct roce5_aeqe_event_data *data =
+ (struct roce5_aeqe_event_data *)val;
+ /* debug subtype according to ROCE_AEQE_DEBUG_SUBTYPE_E */
+ ROCE_ERR_ONCE(
+ "[non ofed event type] Debug extend event subtype 0x%x. function id: %u",
+ data->subtype, rdev->glb_func_id);
+ return;
+ }
+
+ ROCE_ERR_ONCE(
+ "[non ofed event type] Invalid extend event type error. function id: %u",
+ rdev->glb_func_id);
+}
+
+int roce5_async_event_handle_extend(u8 event_type, u8 *val,
+ struct roce5_device *rdev)
+{
+ int event_str_index = 0;
+
+ event_str_index = to_unkown_event_str_index(event_type, val);
+
+ return event_str_index;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c
new file mode 100644
index 000000000..759ba598f
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c
@@ -0,0 +1,170 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "roce_main_extension.h"
+#include "roce_infra.h"
+#include "roce_os_compat.h"
+#include "roce_kernel_compat_gen.h"
+
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+
+static struct mutex g_rdev_mutex;
+#endif
+
+void roce5_service_init_pre(void)
+{
+#ifdef ROCE_BONDING_EN
+ mutex_init(&g_rdev_mutex);
+#endif
+
+ return;
+}
+
+void roce5_lock_rdev(void)
+{
+#ifdef ROCE_BONDING_EN
+ mutex_lock(&g_rdev_mutex);
+#endif
+}
+
+void roce5_unlock_rdev(void)
+{
+#ifdef ROCE_BONDING_EN
+ mutex_unlock(&g_rdev_mutex);
+#endif
+}
+
+int roce5_get_rdev_by_uld(struct hinic5_lld_dev *lld_dev, void *uld_dev,
+ struct roce5_device **rdev,
+ struct hinic5_event_info *event)
+{
+#ifdef ROCE_BONDING_EN
+ int ret;
+
+ ret = roce5_bond_event_cfg_rdev(lld_dev, uld_dev, rdev, false);
+ if (ret != 0) {
+ return -1;
+ }
+#else
+ *rdev = (struct roce5_device *)uld_dev;
+#endif
+ return 0;
+}
+
+#ifndef ROCE_CHIP_TEST
+void roce5_init_dev_ext_handlers(struct roce5_device *rdev)
+{
+ return;
+}
+#endif
+
+int roce5_board_cfg_check(struct roce5_device *rdev)
+{
+ int ret = 0;
+ int port_num = 0;
+ int port_speed = 0;
+
+ port_num = rdev->board_info.port_num;
+ port_speed = rdev->board_info.port_speed;
+ if ((port_num == ROCE5_2_PORT_NUM) &&
+ (port_speed == ROCE5_100G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_2_100G_HCA;
+ } else if ((port_num == ROCE5_4_PORT_NUM) &&
+ (port_speed == ROCE5_25G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_4_25G_HCA;
+ } else if ((port_num == ROCE5_2_PORT_NUM) &&
+ (port_speed == ROCE5_25G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_2_25G_HCA;
+ } else if ((port_num == ROCE5_1_PORT_NUM) &&
+ (port_speed == ROCE5_100G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_1_100G_HCA;
+ } else if ((port_num == ROCE5_2_PORT_NUM) &&
+ (port_speed == ROCE5_200G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_2_200G_HCA;
+ } else if ((port_num == ROCE5_2_PORT_NUM) &&
+ (port_speed == ROCE5_400G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_2_400G_HCA;
+ } else if ((port_num == ROCE5_4_PORT_NUM) &&
+ (port_speed == ROCE5_200G_PORT_SPEED)) {
+ rdev->hw_info.hca_type = ROCE5_4_200G_HCA;
+ } else {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid fw cfg, port_num(%d), port_speed(%d), hca_type(%d)",
+ port_num, port_speed, rdev->hw_info.hca_type);
+ ret = (-EINVAL);
+ }
+
+ return ret;
+}
+
+void *roce5_rdev_alloc_ext(void)
+{
+#if defined(HAVE__IB_ALLOC_DEVICE)
+ return (void *)ib_alloc_device(roce5_device, ib_dev);
+#else
+ return (void *)ib_alloc_device(sizeof(struct roce5_device));
+#endif
+}
+
+int roce5_set_comm_event(const struct roce5_device *rdev,
+ const struct hinic5_event_info *event)
+{
+ ROCE_DEV_INFO(rdev, "Comm event unsupported, func_id(%d), event(%d)",
+ rdev->glb_func_id, event->type);
+ return -1;
+}
+
+struct roce5_ucontext *
+roce5_queue_to_ucontext(void *ibq, struct ib_udata *udata, int q_type)
+{
+ struct roce5_ucontext *rctx = NULL;
+
+#if defined(UVERBS_IOCTL_HAVE_RDMA_UDATA_TO_DRV_CONTEXT)
+ rctx = rdma_udata_to_drv_context(udata, struct roce5_ucontext,
+ ibucontext);
+#else
+ struct ib_qp *ibqp = NULL;
+ struct ib_cq *ibcq = NULL;
+ struct ib_srq *ibsrq = NULL;
+ switch (q_type) {
+ case ROCE_Q_TYPE_QP:
+ ibqp = (struct ib_qp *)ibq;
+ rctx = to_roce5_ucontext(ibqp->uobject->context);
+ break;
+ case ROCE_Q_TYPE_CQ:
+ ibcq = (struct ib_cq *)ibq;
+ rctx = to_roce5_ucontext(ibcq->uobject->context);
+ break;
+ case ROCE_Q_TYPE_SRQ:
+ ibsrq = (struct ib_srq *)ibq;
+ rctx = to_roce5_ucontext(ibsrq->uobject->context);
+ break;
+ default:
+ ROCE_ERR("Unknown queue type(%d)", q_type);
+ break;
+ }
+#endif
+ return rctx;
+}
+
+#ifndef ROCE_VROCE_EN
+void roce5_fill_gid_entry_ext(const struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry)
+{
+ return;
+}
+
+void roce5_cfg_info_update_ext(struct roce5_device *rdev,
+ roce_get_cfg_info_resp_s *resp)
+{
+ return;
+}
+
+int roce5_add_check_ext(const struct hinic5_lld_dev *lld_dev)
+{
+ return 0;
+}
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c
new file mode 100644
index 000000000..7f2e14669
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c
@@ -0,0 +1,40 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+#include "roce_mr.h"
+#include "roce_kernel_compat_gen.h"
+#include "roce_main_extension.h"
+#include "roce_mr_extension.h"
+
+#ifndef ROCE_CHIP_TEST
+int roce5_check_alloc_mr_type(enum ib_mr_type mr_type)
+{
+ int ret = 0;
+
+ if (mr_type != IB_MR_TYPE_MEM_REG) {
+ ret = -EINVAL;
+ ROCE_ERR("mr_type is invalid. mr_type:%u", mr_type);
+ }
+
+ return ret;
+}
+
+enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type)
+{
+ switch (ib_mr_type) {
+#if defined(IB_MR_TYPE_HAVE_USER) && defined(IB_MR_TYPE_HAVE_INTEGRITY)
+ case IB_MR_TYPE_MEM_REG:
+ return RDMA_FRMR;
+ case IB_MR_TYPE_USER:
+ return RDMA_USER_MR;
+ case IB_MR_TYPE_INTEGRITY:
+ return RDMA_SIG_MR;
+#else
+ case IB_MR_TYPE_MEM_REG:
+ return RDMA_DMA_MR;
+#endif
+ default:
+ return RDMA_DMA_MR;
+ }
+}
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c
new file mode 100644
index 000000000..0e0170e9d
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c
@@ -0,0 +1,222 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "roce_qp_extension.h"
+#include "roce_ctx_format.h"
+#include "roce_restore.h"
+
+#ifdef ROCE_EXTEND
+#include "roce_pd.h"
+#endif
+
+int to_roce5_qp_type(enum ib_qp_type qp_type)
+{
+ switch (qp_type) {
+ case IB_QPT_RC:
+ return ROCE_QP_ST_RC;
+
+ case IB_QPT_UC:
+ return ROCE_QP_ST_UC;
+
+ case IB_QPT_UD:
+ return ROCE_QP_ST_UD;
+
+ case IB_QPT_XRC_INI:
+ case IB_QPT_XRC_TGT:
+ return ROCE_QP_ST_XRC;
+
+ case IB_QPT_GSI:
+ return ROCE_QP_ST_UD;
+
+ default:
+ return -1;
+ }
+}
+
+bool roce5_check_qp_modify_ok(enum ib_qp_state cur_state,
+ enum ib_qp_state next_state, enum ib_qp_type type,
+ enum ib_qp_attr_mask mask,
+ enum rdma_link_layer ll)
+{
+#if defined(IB_MODIFY_QP_IS_OK_HAVE_RDMA_LINK_LAYER)
+ return (bool)ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll);
+#else
+ return (bool)ib_modify_qp_is_ok(cur_state, next_state, type, mask);
+#endif
+}
+
+int roce5_is_qp_normal(struct roce5_qp *rqp, struct ib_qp_init_attr *init_attr)
+{
+ return 1;
+}
+
+#ifdef ROCE_EXTEND
+static struct roce5_qp *
+roce5_cdev_lookup_and_check_rqp(struct roce5_device *rdev, u32 qpn)
+{
+ struct roce5_qp *rqp = NULL;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Can't find rqp according to qpn(0x%x), func_id(%d)",
+ qpn, rdev->glb_func_id);
+ return NULL;
+ }
+
+ if (rqp->qpn >= QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC) {
+ ROCE_DEV_ERR(rdev,
+ "qpn[%u] more than sqpc num offset(%d) for sqpc.",
+ rqp->qpn, QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC);
+ roce5_rqp_put(rqp);
+ return NULL;
+ }
+
+ return rqp;
+}
+
+static int roce5_alloc_sqpc(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct roce5_vbs_qp *vbs_rqp)
+{
+ cqm_qpc_mpt_s *qpc_info = NULL;
+
+ /* Call the CQM interface to allocate QPNs and QPCs */
+ /* For SQPC, do not need to allocate QPN, QPN is directly spcified in rsvd segment */
+ qpc_info = cqm5_object_qpc_mpt_create(
+ rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, NULL,
+ rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, 0, 0);
+ if (qpc_info == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create qpc by cqm object, func_id(%d), qpn(%u)",
+ rdev->glb_func_id,
+ rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC);
+ return -ENOMEM;
+ }
+
+ if (qpc_info->xid != (rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Create qpc error, func_id(%d), expect qpn(%d), actual qpn(%d)",
+ rdev->glb_func_id,
+ rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC,
+ qpc_info->xid);
+ cqm5_object_delete(&(qpc_info->object));
+ return -EFAULT;
+ }
+
+ vbs_rqp->vbs_sqpc_info = qpc_info;
+ rqp->vbs_qp_ptr = (void *)vbs_rqp;
+
+ return 0;
+}
+
+long roce5_vbs_create_sqpc(struct roce5_device *rdev, void *buf)
+{
+ struct roce5_modify_qp_vbs_cmd cmd;
+ struct roce5_vbs_qp *vbs_rqp = NULL;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_pd *pd = NULL;
+ int ret;
+
+ ret = (int)copy_from_user(&cmd, buf, sizeof(cmd));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to copy data from user");
+ return (long)ret;
+ }
+
+ rqp = roce5_cdev_lookup_and_check_rqp(rdev, cmd.qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp");
+ return -EINVAL;
+ }
+
+ vbs_rqp = (struct roce5_vbs_qp *)kzalloc(sizeof(*vbs_rqp), GFP_KERNEL);
+ if (vbs_rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc vbs_rqp, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -ENOMEM;
+ goto put_rqp;
+ }
+
+ pd = roce5_get_pd(rqp);
+ ret = roce5_db_map_user(to_roce5_ucontext(pd->ibpd.uobject->context),
+ cmd.ci_record_addr, &vbs_rqp->db, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to map db page to user, func_id(%d)",
+ rdev->glb_func_id);
+ goto free_rqp;
+ }
+
+ ret = roce5_alloc_sqpc(rdev, rqp, vbs_rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc sqpc, func_id(%d)",
+ rdev->glb_func_id);
+ goto free_rqp;
+ }
+
+ roce5_rqp_put(rqp);
+ return 0;
+
+free_rqp:
+ kfree(vbs_rqp);
+put_rqp:
+ roce5_rqp_put(rqp);
+ return (long)ret;
+}
+#endif
+
+#ifndef ROCE_CHIP_TEST
+void roce5_set_qp_dif_attr(struct roce5_qp *rqp,
+ const struct ib_qp_init_attr *init_attr,
+ const struct roce5_device *rdev)
+{
+ if (((unsigned int)init_attr->create_flags &
+ IB_QP_CREATE_SIGNATURE_EN) != 0) {
+ rqp->signature_en = true;
+ ROCE_DEV_INFO(rdev,
+ "func(%d) qp(%u) roce5_create_qp signature_en.",
+ rdev->glb_func_id, rqp->qpn);
+ }
+ return;
+}
+#endif
+
+#ifndef ROCE_VBS_EN
+void roce5_set_attr_ext(struct roce5_qp *rqp, const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr, int attr_mask)
+{
+ return;
+}
+
+void roce5_transmit_ci_record_addr_ext(struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+ return;
+}
+#endif
+
+#ifndef ROCE_VBS_EN
+int roce5_modify_qp_pre_ext(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata)
+{
+ return 0;
+}
+
+void roce5_destroy_pcqc_ext(struct roce5_qp *rqp)
+{
+ return;
+}
+
+#endif
+
+#ifndef ROCE_VROCE_EN
+
+void roce5_cos_update_ext(struct roce5_device *rdev, u8 *cos,
+ enum ib_qp_type qp_type)
+{
+ *cos = *cos & 0x7;
+}
+#endif
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c
new file mode 100644
index 000000000..7067e8c80
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c
@@ -0,0 +1,55 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/types.h>
+#include "roce_srq_extension.h"
+
+u8 roce5_get_container_sz(u32 container_mode)
+{
+ switch (container_mode) {
+ case ROCE_SRQ_MODE_0:
+ return ROCE_SRQ_CONT_NUM_MODE0;
+ case ROCE_SRQ_MODE_1:
+ return ROCE_SRQ_CONT_NUM_MODE1;
+ case ROCE_SRQ_MODE_2:
+ return ROCE_SRQ_CONT_NUM_MODE2;
+ case ROCE_SRQ_MODE_3:
+ return ROCE_SRQ_CONT_NUM_MODE3;
+ default:
+ return ROCE_SRQ_CONT_NUM_MODE3;
+ }
+}
+
+u8 roce5_calculate_cont_th(u32 srq_limit)
+{
+ u8 cont_th = 0;
+ u32 srq_limit_tmp = srq_limit;
+
+ srq_limit_tmp >>= 1;
+ while (srq_limit_tmp != 0) {
+ srq_limit_tmp >>= 1;
+ ++cont_th;
+ }
+
+ return cont_th;
+}
+
+#ifndef ROCE_CHIP_TEST
+void roce5_srq_container_init(struct ib_srq_init_attr *init_attr,
+ struct roce5_srq *rsrq, struct roce5_device *rdev)
+{
+ rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC);
+
+ if (rsrq->xrc_en != 0) {
+ rsrq->container_flag = rdev->cfg_info.srq_container_en;
+ }
+ rsrq->container_mode = (rsrq->xrc_en != 0) ?
+ rdev->cfg_info.xrc_srq_container_mode :
+ rdev->cfg_info.srq_container_mode;
+ rsrq->container_warn_th =
+ roce5_calculate_cont_th(init_attr->attr.srq_limit);
+ rsrq->rqe_cnt_th = rdev->cfg_info.warn_th;
+ rsrq->container_size = roce5_get_container_sz(rsrq->container_mode);
+}
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c
new file mode 100644
index 000000000..f7f9117f4
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c
@@ -0,0 +1,1281 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "roce_mr.h"
+#include "roce_mr_extension.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "rdma_comp_mw_mr.h"
+#include "hmm_common.h"
+#include "roce_os_compat.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_uld_inner.h"
+
+/* ****************************************************************************
+ Prototype : get_key_from_index
+ Description : get_key_from_index
+ Input : u32 mpt_index
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static u32 get_key_from_index(u32 mpt_index)
+{
+ /* Algorithm of mr key is obtained by index shift calculation */
+ return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) |
+ (mpt_index << MR_KEY_LEFT_SHIFT_OFS);
+}
+
+/* ****************************************************************************
+ Prototype : convert_ib_access
+ Description : convert ib access to rdma comp access
+ Input : int access_flag
+ Output : None
+
+ 1.Date : 2015/7/16
+ Modification : Created function
+
+**************************************************************************** */
+static u32 convert_ib_access(int access_flag)
+{
+ u32 u_access_flag = (u32)access_flag;
+ u32 access;
+
+ access = (u32)((((u_access_flag & IB_ACCESS_REMOTE_ATOMIC) != 0) ?
+ RDMA_IB_ACCESS_REMOTE_ATOMIC :
+ 0) |
+ (((u_access_flag & IB_ACCESS_REMOTE_WRITE) != 0) ?
+ RDMA_IB_ACCESS_REMOTE_WRITE :
+ 0) |
+ (((u_access_flag & IB_ACCESS_REMOTE_READ) != 0) ?
+ RDMA_IB_ACCESS_REMOTE_READ :
+ 0) |
+ (((u_access_flag & IB_ACCESS_LOCAL_WRITE) != 0) ?
+ RDMA_IB_ACCESS_LOCAL_WRITE :
+ 0) |
+ (((u_access_flag & IB_ACCESS_MW_BIND) != 0) ?
+ RDMA_IB_ACCESS_MW_BIND :
+ 0) |
+ (((u_access_flag & IB_ZERO_BASED) != 0) ?
+ RDMA_IB_ACCESS_ZERO_BASED :
+ 0));
+
+ return access;
+}
+
+/* ****************************************************************************
+ Prototype : check_ib_access
+ Description : check remote access without local write
+ Input : int access_flag
+ Output : None
+
+ 1.Date : 2017/11/18
+ Modification : Created function
+
+**************************************************************************** */
+static int check_ib_access(int access_flag)
+{
+ /*
+ * Local write permission is required if remote write or
+ * remote atomic permission is also requested.
+ */
+
+ if (((((u32)access_flag) &
+ (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE)) != 0) &&
+ ((((u32)access_flag) & IB_ACCESS_LOCAL_WRITE) == 0)) {
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+u32 roce5_get_mpt_xid_mode(struct roce5_device *rdev)
+{
+ u8 mode = rdev->mpt_xid_mode;
+
+ if (mode == ROCE_MPT_XID_SEQUENTIAL)
+ return CQM_INDEX_INVALID;
+
+ return CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, CQM_XID_LOW_BIT_0_0_1,
+ (u32)mode);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_alloc_tpt
+ Description : alloc mpt and mtt
+ Input : struct roce5_device *rdev
+ struct hmm_rdma *mr
+ u32 npages
+ u32 page_shift
+ Output : None
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_alloc_tpt(struct roce5_device *rdev, struct hmm_rdma *mr, u32 npages,
+ u32 page_shift, u32 mpt_xid_mode)
+{
+ int ret = 0;
+
+ ret = hmm_mpt_alloc_templated(rdev->hwdev, &mr->mpt, SERVICE_T_ROCE,
+ mpt_xid_mode);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mpt, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ return ret;
+ }
+
+ mr->enabled = RDMA_MPT_EN_SW;
+
+ /* npages = 0 is a legal case, when npages = 0 or npages = 1, MTT does not need to do address translation */
+ ret = hmm_mtt_alloc(rdev->hwdev, npages, page_shift, &mr->mtt,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mtt, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto err_alloc_mtt;
+ }
+
+ return 0;
+
+err_alloc_mtt:
+ hmm_mpt_free(rdev->hwdev, &mr->mpt);
+ mr->enabled = RDMA_MPT_DISABLED;
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_free_tpt
+ Description : free mpt and mtt
+ Input : struct roce5_device *rdev
+ struct hmm_rdma *mr
+ Output : None
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_free_tpt(struct roce5_device *rdev, struct hmm_rdma *mr)
+{
+ hmm_mtt_free(rdev->hwdev, &mr->mtt, SERVICE_T_ROCE);
+
+ hmm_mpt_free(rdev->hwdev, &mr->mpt);
+
+ mr->enabled = RDMA_MPT_DISABLED;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_rdma_mr
+ Description : set the member of rdma_mr
+ Input : struct hmm_rdma *mr
+ enum rdma_mr_type mr_type
+ u32 pdn
+ u64 iova
+ u64 size
+ u32 access
+ Output : None
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_set_rdma_mr(struct hmm_rdma *mr, enum rdma_mr_type mr_type, u32 pdn,
+ u64 iova, u64 size, u32 access)
+{
+ mr->iova = iova;
+ mr->size = size;
+ mr->pdn = pdn;
+ mr->access = convert_ib_access((
+ int)access); /* Convert ib permissions to rdma component permissions */
+ mr->key = get_key_from_index(
+ mr->mpt.mpt_index); /* Convert from mpt index to key */
+ mr->type = roce5_mr_type2hmm(mr_type);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_rdma_mw
+ Description : set the member of rdma_mw
+ Input : struct rdma_mw *mw
+ u32 pdn
+ enum ib_mw_type type
+ Output : None
+
+ 1.Date : 2016/6/25
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_set_rdma_mw(struct rdma_mw *mw, u32 pdn, enum ib_mw_type type)
+{
+ mw->enabled = RDMA_MPT_EN_SW;
+ mw->pdn = pdn;
+ mw->key = get_key_from_index(mw->mpt.mpt_index);
+ mw->type = (type == IB_MW_TYPE_1 ? RDMA_MW_TYPE_1 : RDMA_MW_TYPE_2);
+}
+#ifdef __ROCE_DFX__
+static void roce5_store_mr_list(struct roce5_device *rdev,
+ struct list_head *node)
+{
+ struct roce5_list *roce_list = NULL;
+
+ if (rdev->device_res == NULL) {
+ ROCE_INFO("Hasn't been created resource, don't need store.");
+ return;
+ }
+
+ roce_list = &rdev->device_res->mr_list;
+ mutex_lock(&roce_list->node_lock);
+ list_add(node, &roce_list->node);
+ mutex_unlock(&roce_list->node_lock);
+}
+
+static void roce5_delete_mr_list(struct roce5_device *rdev,
+ struct list_head *node)
+{
+ struct roce5_list *roce_list = NULL;
+
+ if (rdev->device_res == NULL) {
+ ROCE_INFO("Resource has been release, don't need delete.");
+ return;
+ }
+
+ roce_list = &rdev->device_res->mr_list;
+ mutex_lock(&roce_list->node_lock);
+ list_del(node);
+ mutex_unlock(&roce_list->node_lock);
+}
+#endif //__ROCE_DFX__
+
+/* ****************************************************************************
+ Prototype : roce5_get_dma_mr
+ Description : register DMA_MR
+ Input : struct ib_pd *ibpd
+ int access
+ Output : None
+
+ 1.Date : 2015/4/24
+ Modification : Created function
+
+**************************************************************************** */
+struct ib_mr *roce5_get_dma_mr(struct ib_pd *ibpd, int access)
+{
+ int ret = 0;
+ u32 mpt_xid_mode;
+ struct roce5_mr *mr = NULL;
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if ((ibpd == NULL) || (check_ib_access(access) != 0)) {
+ ret = -EINVAL;
+ ROCE_ERR("Invalid Param.p1:%d", access);
+ goto err_out;
+ }
+
+ pd = to_roce5_pd(ibpd);
+ rdev = to_roce5_dev(ibpd->device);
+
+ mr = (struct roce5_mr *)kzalloc(sizeof *mr, GFP_KERNEL);
+ if (mr == NULL) {
+ ret = -ENOMEM;
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc memory for dma mr, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_out;
+ }
+
+ mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE;
+ mpt_xid_mode = roce5_get_mpt_xid_mode(rdev);
+ ret = roce5_alloc_tpt(rdev, &mr->rdmamr, 0, 0, mpt_xid_mode);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_alloc_tpt;
+ }
+
+ /* Set the content in rdma_mr */
+ roce5_set_rdma_mr(&mr->rdmamr, RDMA_DMA_MR, pd->pdn, 0ULL,
+ ROCE_DMA_MR_SIZE, (u32)access);
+
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(mr->rdmamr),
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to enable mpt of DMA mr, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_enable_mpt;
+ }
+#ifdef __ROCE_DFX__
+ roce5_store_mr_list(rdev, &mr->node);
+#endif
+ mr->ibmr.lkey = mr->rdmamr.key;
+ mr->ibmr.rkey = mr->rdmamr.key;
+ mr->ibmr.device = ibpd->device;
+
+ ROCE_DEV_INFO(rdev, "Roce reg dma mr success, mr mpt index is %u",
+ mr->rdmamr.mpt.mpt_index);
+
+ return &mr->ibmr;
+
+err_enable_mpt:
+ roce5_free_tpt(rdev, &mr->rdmamr);
+
+err_alloc_tpt:
+ kfree(mr);
+
+err_out:
+ return (struct ib_mr *)ERR_PTR((long)ret);
+}
+
+static int roce5_alloc_priv_pages(struct ib_device *ibdev, struct roce5_mr *rmr,
+ u32 max_pages)
+{
+ int ret;
+
+ rmr->page_map_size =
+ roundup(max_pages * sizeof(u64), ROCE5_MR_PAGES_ALIGN);
+ rmr->pages = (__be64 *)(void *)(uintptr_t)get_zeroed_page(GFP_KERNEL);
+ if (rmr->pages == NULL) {
+ ROCE_ERR("Failed to alloc rmr->pages");
+ ret = -ENOMEM;
+ goto err_out;
+ }
+
+ rmr->page_map = dma_map_single(ibdev->dev.parent, rmr->pages,
+ rmr->page_map_size, DMA_TO_DEVICE);
+ if (dma_mapping_error(ibdev->dev.parent, rmr->page_map) != 0) {
+ ROCE_ERR("Failed to do dma mapping");
+ ret = -ENOMEM;
+ goto err_free_pages;
+ }
+
+ return 0;
+
+err_free_pages:
+ free_page((unsigned long)(uintptr_t)(void *)rmr->pages);
+
+err_out:
+ return ret;
+}
+
+void roce5_free_frmr_umem(struct ib_umem **fumem, int fumem_nr)
+{
+ int i;
+
+ ROCE_INFO("%d frmr umems to be free.", fumem_nr);
+ for (i = 0; i < fumem_nr; i++) {
+ ib_umem_release(fumem[i]);
+ }
+}
+
+static void roce5_free_priv_pages(struct roce5_mr *rmr)
+{
+ struct ib_device *ib_dev = rmr->ibmr.device;
+
+ if (rmr->pages != NULL) {
+ ib_dev = rmr->ibmr.device;
+ dma_unmap_single(ib_dev->dev.parent, rmr->page_map,
+ rmr->page_map_size, DMA_TO_DEVICE);
+ free_page((unsigned long)(uintptr_t)(void *)rmr->pages);
+ rmr->pages = NULL;
+ }
+}
+
+static int roce5_alloc_mr_param_validate(const struct ib_pd *ibpd,
+ enum ib_mr_type mr_type,
+ u32 max_num_sg)
+{
+ int ret = 0;
+
+ if (ibpd == NULL) {
+ ret = -EINVAL;
+ ROCE_ERR("Ibpd is null");
+ goto err_out;
+ }
+
+ ret = roce5_check_alloc_mr_type(mr_type);
+ if (ret != 0) {
+ ret = -EINVAL;
+ ROCE_ERR("mr_type is invalid");
+ goto err_out;
+ }
+
+ if (max_num_sg >= ROCE_FRMR_MAX_PAGES) {
+ ROCE_ERR("Invalid max_num_sg(%d)", max_num_sg);
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ return ret;
+}
+
+static int roce5_alloc_mr_param_check(struct roce5_device *rdev,
+ struct roce5_mr **mr, u32 max_num_sg)
+{
+ *mr = (struct roce5_mr *)kzalloc(sizeof(struct roce5_mr), GFP_KERNEL);
+ if ((*mr) == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mr memory, func(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ (*mr)->max_pages = max_num_sg;
+ return 0;
+}
+/* ****************************************************************************
+ Prototype : roce5_alloc_mr
+ Description : register DMA_MR
+ Input : struct ib_pd *ibpd
+ enum ib_mr_type mr_type
+ u32 max_num_sg
+ Output : None
+
+ 1.Date : 2015/4/24
+ Modification : Created function
+
+**************************************************************************** */
+struct ib_mr *roce5_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+ u32 max_num_sg)
+{
+ int ret = 0;
+ u16 sign_val = 0;
+ u32 mpt_xid_mode;
+ struct roce5_mr *mr = NULL;
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+
+ ret = roce5_alloc_mr_param_validate(ibpd, mr_type, max_num_sg);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ pd = to_roce5_pd(ibpd);
+ rdev = to_roce5_dev(ibpd->device);
+
+ ret = roce5_alloc_mr_param_check(rdev, &mr, max_num_sg);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE;
+ mpt_xid_mode = roce5_get_mpt_xid_mode(rdev);
+
+ ret = roce5_alloc_tpt(rdev, &mr->rdmamr, max_num_sg, 0,
+ mpt_xid_mode); //alloc mptc and mtt
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc tpt, func(%d)",
+ rdev->glb_func_id);
+ goto err_alloc_tpt;
+ }
+
+ ret = roce5_alloc_priv_pages(ibpd->device, mr,
+ max_num_sg); //alloc memory to store pa
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc memory, func(%u)",
+ rdev->glb_func_id);
+ goto err_alloc_priv_pages;
+ }
+
+ roce5_set_rdma_mr(&mr->rdmamr, roce5_get_mrtype(mr_type), pd->pdn, 0ULL,
+ max_num_sg * (1 << PAGE_SHIFT),
+ ROCE5_MR_DEFAULT_ACCESS);
+
+#ifdef SIGN_MTT_EN
+ /*
+ MTT table entry of bit[11:2] used for fill in addr_signature, to hardware used for check MTT table entry complete validity. [1:1] is vmr, indicates current page whether be swap out.
+ bit[0:0] is vld bit indicates pointed by of next one level DMTT or Data Buffer of GPA whether valid. use sign_val record [11:0] this 12bit data, exist mr in.
+ send down frmr wqe after chip will do dma use pbl table fill MTT table, here need in advance calculate check value, in roce5_set_page() in write pbl table.
+ */
+ sign_val = (u16)(hmm_gen_mtt_sign(mr->rdmamr.mtt.mtt_paddr,
+ mr->rdmamr.mtt.mtt_type) &
+ 0xffff);
+#endif
+ mr->sign_val = ((sign_val << 1) | MTT_PA_VALID);
+
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(mr->rdmamr),
+ HINIC5_CHANNEL_ROCE); // send to chip by cmdq
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to enable mpt, func(%d)",
+ rdev->glb_func_id);
+ goto err_enale_mpt;
+ }
+
+#ifdef __ROCE_DFX__
+ roce5_store_mr_list(rdev, &mr->node);
+#endif
+
+ mr->ibmr.lkey = mr->rdmamr.key;
+ mr->ibmr.rkey = mr->rdmamr.key;
+
+ return &mr->ibmr;
+
+err_enale_mpt:
+ roce5_free_priv_pages(mr);
+
+err_alloc_priv_pages:
+ roce5_free_tpt(rdev, &mr->rdmamr);
+
+err_alloc_tpt:
+ kfree(mr);
+
+err_out:
+ return (struct ib_mr *)ERR_PTR((long)ret);
+}
+
+static int roce5_set_page(struct ib_mr *ibmr, u64 addr)
+{
+ struct roce5_mr *rmr = NULL;
+
+ if (ibmr == NULL) {
+ ROCE_ERR("Ibmr is null");
+ return -EINVAL;
+ }
+
+ rmr = to_roce5_mr(ibmr);
+ if (ROCE_UNLIKELY(rmr->npages >= rmr->max_pages)) {
+ ROCE_ERR("Invalid npages(0x%x), can't set page", rmr->npages);
+ return -ENOMEM;
+ }
+
+ rmr->pages[rmr->npages] = cpu_to_be64(addr | rmr->sign_val);
+ rmr->npages++;
+
+ return 0;
+}
+
+/* RoCE supports FRMR, for at privileged user, can via post send WR of way for at at free status of MR perform fast register.
+ here need in advance via input parameter of sgl available physical page
整 理 in pbl table 里, later via frmr wqe send down. */
+int roce5_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+ int sg_nents, unsigned int *sg_offset)
+{
+ int ret;
+ struct roce5_mr *rmr = NULL;
+
+ /* sg_offset can be null */
+ if ((ibmr == NULL) || (sg == NULL)) {
+ ROCE_ERR("Ibmr or sg is null");
+ return -EINVAL;
+ }
+
+ rmr = to_roce5_mr(ibmr);
+ rmr->npages = 0;
+
+ if (ROCE_UNLIKELY(((u32)sg_nents) > rmr->max_pages)) {
+ ROCE_ERR("Invalid sg_nents(0x%x), max(0x%x)", sg_nents,
+ rmr->max_pages);
+ return -EINVAL;
+ }
+
+ ib_dma_sync_single_for_cpu(ibmr->device, rmr->page_map,
+ rmr->page_map_size, DMA_TO_DEVICE);
+
+ ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, roce5_set_page);
+
+ ib_dma_sync_single_for_device(ibmr->device, rmr->page_map,
+ rmr->page_map_size, DMA_TO_DEVICE);
+
+ return ret;
+}
+
+static int roce5_umem_write_mtt_check(const struct roce5_device *rdev,
+ const struct mtt *mtt,
+ const struct ib_umem *umem)
+{
+ if ((rdev == NULL) || (mtt == NULL) || (umem == NULL)) {
+ ROCE_ERR("Rdev or mtt or umem is null");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_umem_write_mtt_update(struct roce5_device *rdev,
+ struct mtt *mtt, struct ib_umem *umem,
+ u64 *page_list, bool use_huge)
+{
+ int ret = 0;
+ u32 npages = 0; /* The number of recorded pages */
+ u32 start_index = 0; /* page need to be written to mtt */
+ struct ib_block_iter biter;
+
+ rdma_umem_for_each_dma_block(umem, &biter, BIT(mtt->buf_page_shift)) {
+ page_list[npages++] = rdma_block_iter_dma_address(&biter);
+ /* The size of the memory pointed to by page_list is one page, which can store up to PAGE_SIZE / sizeof(u64)
+ pas. After a page is full, mtt needs to be written */
+ if (npages == BIT(mtt->mtt_page_shift) / PA_SIZE) {
+ ret = hmm_mtt_write(rdev->hwdev, mtt, start_index,
+ npages, page_list, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to write mtt, ret(%d), start_index(%u), func_id(%u).",
+ ret, start_index, rdev->glb_func_id);
+ goto out;
+ }
+ start_index += npages;
+ npages = 0;
+ }
+ }
+
+ if (npages != 0) {
+ ret = hmm_mtt_write(rdev->hwdev, mtt, start_index, npages,
+ page_list, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to write mtt, ret(%d), start_index(%u), func_id(%u).",
+ ret, start_index, rdev->glb_func_id);
+ goto out;
+ }
+ }
+
+out:
+ return ret;
+}
+
+int roce5_umem_write_mtt(struct roce5_device *rdev, struct mtt *mtt,
+ struct ib_umem *umem, bool use_huge)
+{
+ int ret = 0;
+ u64 *page_list = NULL; /* page_list to write to mtt */
+
+ ret = roce5_umem_write_mtt_check(rdev, mtt, umem);
+ if (ret != 0) {
+ return ret;
+ }
+
+ page_list = (u64 *)kzalloc(PAGE_SIZE, GFP_KERNEL);
+ if (page_list == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc memory for page list, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ ret = roce5_umem_write_mtt_update(rdev, mtt, umem, page_list, use_huge);
+ if (ret != 0) {
+ ROCE_ERR("Failed to write mtt.");
+ }
+
+ kfree(page_list);
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_buf_write_mtt
+ Description : roce5_buf_write_mtt
+ Input : struct roce5_device *rdev
+ struct mtt *mtt
+ struct tag_cqm_buf *buf
+ Output : None
+
+ 1.Date : 2015/4/24
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_buf_write_mtt(struct roce5_device *rdev, struct mtt *mtt,
+ struct tag_cqm_buf *buf)
+{
+ u64 *page_list = NULL;
+ int ret = 0;
+ u32 i = 0;
+
+ if ((rdev == NULL) || (mtt == NULL) || (buf == NULL)) {
+ ROCE_ERR("Rdev or mtt or buf is null");
+ return (-EINVAL);
+ }
+
+ page_list =
+ (u64 *)kzalloc(buf->buf_number * sizeof *page_list, GFP_KERNEL);
+ if (page_list == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc memory for page list, func_id(%d)",
+ rdev->glb_func_id);
+ return (-ENOMEM);
+ }
+
+ /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */
+ for (i = 0; i < buf->buf_number; ++i) {
+ page_list[i] = buf->buf_list[i].pa;
+ }
+
+ ret = hmm_mtt_write(rdev->hwdev, mtt, 0, buf->buf_number, page_list,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)",
+ rdev->glb_func_id);
+ }
+
+ kfree(page_list);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_is_umem_dma_contiguous
+ Description : Determine whether the input ib_umem occupies a contiguous
+ region in DMA address space, i.e., all scatter-gather entries
+ are adjacent in DMA addresses and the total length matches
+ umem->length.
+ Input : struct roce5_device *rdev
+ struct ib_umem *umem
+ Output : bool - true if the umem is DMA-contiguous, false otherwise
+ 1.Date : 2026/3/12
+ Modification : Created function
+**************************************************************************** */
+static inline bool roce5_is_umem_dma_contiguous(struct roce5_device *rdev,
+ struct ib_umem *umem)
+{
+ struct scatterlist *sg = NULL;
+ int i;
+ dma_addr_t expected_dma, dma;
+ uint64_t total_len = 0, len;
+
+#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE)
+ sg = umem->sgt_append.sgt.sgl;
+#else
+ sg = umem->sg_head.sgl;
+#endif
+
+ expected_dma = sg_dma_address(sg);
+
+#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE)
+ for_each_sg(umem->sgt_append.sgt.sgl, sg, umem->sgt_append.sgt.nents, i)
+#else
+ for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)
+#endif
+ {
+ dma = sg_dma_address(sg);
+ len = sg_dma_len(sg);
+
+ if (i > 0 && dma != expected_dma) {
+ ROCE_DEV_WARN(
+ rdev,
+ "Enable huge page, but discontinuity at sg[%d]: expected 0x%llx, got 0x%llx\n",
+ i, (u64)expected_dma, (u64)dma);
+ return false;
+ }
+
+ total_len += len;
+ expected_dma = dma + len;
+ }
+
+ /* Check the total length. */
+ if (total_len != umem->length) {
+ ROCE_DEV_WARN(rdev, "Total length %llu != umem length(%zu)\n",
+ total_len, umem->length);
+ return false;
+ }
+
+ return true;
+}
+
+static int roce5_get_page_info(struct roce5_device *rdev, struct ib_umem *umem,
+ u32 *npages, u32 *page_shift, u64 virt_addr)
+{
+ *page_shift = roce5_umem_get_buf_page_shift(rdev, umem, virt_addr);
+ *npages = roce5_umem_get_mtt_npages(rdev, umem, *page_shift);
+
+ if (rdev->huge_page_en && *page_shift > PAGE_SHIFT) {
+ if (*page_shift != PAGE_SHIFT_2MB) {
+ ROCE_DEV_WARN(
+ rdev,
+ "RoCE reg huge MR expect page_size(2MB), but found %lu.",
+ BIT(*page_shift));
+ }
+ ROCE_DEV_INFO(rdev,
+ "RoCE reg huge MR: npages(%u), page_shift(%u).",
+ *npages, *page_shift);
+ return ROCE_USE_HUGEPAGE;
+ }
+
+ return ROCE_NOT_USE_HUGEPAGE;
+}
+
+int roce5_user_mr_reg(struct roce5_device *rdev, struct roce5_mr *mr, u32 pdn,
+ u64 virt_addr, u64 length, int access)
+{
+ int ret = 0;
+ u32 npages = 0;
+ u32 page_shift = PAGE_SHIFT;
+ u32 mpt_xid_mode;
+ int use_huge;
+ struct hinic5_hwdev *hwdev = (struct hinic5_hwdev *)rdev->hwdev;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("hwdev is null");
+ return 0;
+ }
+
+ mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE;
+
+ use_huge = roce5_get_page_info(rdev, mr->umem, &npages, &page_shift,
+ virt_addr);
+ mpt_xid_mode = roce5_get_mpt_xid_mode(rdev);
+
+ ret = roce5_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift,
+ mpt_xid_mode);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, func_id(%d)",
+ hinic5_global_func_id(hwdev));
+ goto err_alloc_tpt;
+ }
+
+ roce5_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length,
+ (u32)access);
+
+ ret = roce5_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem, use_huge);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)",
+ hinic5_global_func_id(hwdev));
+ goto err_write_mtt;
+ }
+
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, hwdev, &mr->rdmamr,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to enable mpt of user mr, func_id(%d)",
+ hinic5_global_func_id(hwdev));
+ goto err_write_mtt;
+ }
+
+ return 0;
+
+err_write_mtt:
+ roce5_free_tpt(rdev, &mr->rdmamr);
+
+err_alloc_tpt:
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_reg_user_mr
+ Description : register MR for user
+ Input : struct ib_pd *ibpd
+ u64 start
+ u64 length
+ u64 virt_addr
+ int access
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2015/4/24
+ Modification : Created function
+
+**************************************************************************** */
+struct ib_mr *roce5_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
+ u64 virt_addr, int access,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_mr *mr = NULL;
+ struct roce5_device *rdev = NULL;
+ struct roce5_pd *rpd = to_roce5_pd(ibpd);
+ roce_user_handle_t *handle = NULL;
+
+ if (ibpd == NULL) {
+ ret = -EINVAL;
+ ROCE_ERR("Ibpd is null");
+ goto err_out;
+ }
+
+ rdev = to_roce5_dev(ibpd->device);
+
+ mr = (struct roce5_mr *)kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (mr == NULL) {
+ ret = -ENOMEM;
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc memory for roce mr, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_out;
+ }
+#if defined(IB_UMEM_GET_PEER)
+ mr->umem = roce5_mr_umem_get_peer(rdev, (size_t)length, access, start,
+ udata, true);
+#else
+ mr->umem = roce5_umem_get(&rdev->ib_dev, ibpd->uobject->context, start,
+ (size_t)length, access, 0);
+#endif
+ if (IS_ERR(mr->umem)) {
+ handle = (roce_user_handle_t *)kzalloc(
+ sizeof(roce_user_handle_t), GFP_KERNEL);
+ if (handle == NULL) {
+ ret = -ENOMEM;
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc memory for handle, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_get_umem;
+ }
+ handle->magic = ROCE_ULD_HANDLE_MAGIC_NUM;
+ handle->rdev = rdev;
+ handle->mr = mr;
+ handle->virt_addr = virt_addr;
+ handle->length = length;
+ handle->pdn = rpd->pdn;
+ handle->access = access;
+
+ mr->david_en = true;
+ mr->private_data = (void *)handle;
+
+ /* call ULD interface */
+ ret = roce5_uld_reg_user_mr((roce_uld_handle)handle);
+ if (ret != 0) {
+ ROCE_DEV_WARN(
+ rdev,
+ "Failed to reg user mr uld, func_id(%d), ret(%d), err(%d)",
+ rdev->glb_func_id, ret, (int)PTR_ERR(mr->umem));
+ ret = (int)PTR_ERR(mr->umem);
+ kfree(handle);
+ mr->private_data = NULL;
+ goto err_get_umem;
+ }
+
+ mr->umem = NULL;
+ } else {
+ ret = roce5_user_mr_reg(rdev, mr, rpd->pdn, virt_addr, length,
+ access);
+ if (ret != 0) {
+ goto err_mr_update;
+ }
+ }
+#ifdef __ROCE_DFX__
+ roce5_store_mr_list(rdev, &mr->node);
+#endif
+ mr->ibmr.lkey = mr->rdmamr.key;
+ mr->ibmr.rkey = mr->rdmamr.key;
+
+ ROCE_DEV_INFO(rdev, "Roce reg user mr success, mr mpt index is %u",
+ mr->rdmamr.mpt.mpt_index);
+
+ return &mr->ibmr;
+
+err_mr_update:
+ ib_umem_release(mr->umem);
+
+err_get_umem:
+ kfree(mr);
+
+err_out:
+ return (struct ib_mr *)ERR_PTR((long)ret);
+}
+
+int roce5_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_mr *mr = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if (ibmr == NULL) {
+ ROCE_ERR("Ibmr is null");
+ return -EINVAL;
+ }
+
+ mr = to_roce5_mr(ibmr);
+ rdev = to_roce5_dev(ibmr->device);
+
+ if (mr->rdmamr.type == HMM_RDMA_FRMR) {
+ roce5_free_priv_pages(mr);
+ }
+
+ if (mr->david_en) {
+ if (mr->private_data == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid parameter, private data is NULL.");
+ return -EINVAL;
+ }
+
+ ret = roce5_uld_dereg_user_mr(
+ (roce_uld_handle)mr->private_data);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "roce dereg user mr uld failed, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ kfree(mr->private_data);
+ mr->private_data = NULL;
+ } else {
+ ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev,
+ &mr->rdmamr, SERVICE_T_ROCE,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Mr may has mw bind on it, ret(%d), mr_key(%#x), func_id(%d)",
+ ret, mr->rdmamr.key, rdev->glb_func_id);
+ if (ret == (-CMDQ_TIMEOUT)) {
+ rdev->dev_status_to_ofed =
+ ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+ return ret;
+ }
+
+ roce5_free_tpt(rdev, &mr->rdmamr);
+
+ ib_umem_release(mr->umem);
+ }
+
+ if (mr->fumem_nr != 0) {
+ roce5_free_frmr_umem(mr->fumem, mr->fumem_nr);
+ mr->fumem_nr = 0;
+ }
+#ifdef __ROCE_DFX__
+ roce5_delete_mr_list(rdev, &mr->node);
+#endif
+
+ kfree(mr);
+
+ return 0;
+}
+
+int roce5_dereg_mr_ofa(struct ib_mr *ibmr)
+{
+ return roce5_dereg_mr(ibmr, NULL);
+}
+
+static void roce5_err_enable_mpt_handler(void *hwdev, struct roce5_mw *mw)
+{
+ hmm_mpt_free(hwdev, &mw->rdmamw.mpt);
+ mw->rdmamw.enabled = RDMA_MPT_DISABLED;
+ return;
+}
+
+static void roce5_mw2hmm_adapter(const struct rdma_mw *rdmamw,
+ struct hmm_rdma *hmmrdma)
+{
+ hmmrdma->mpt = rdmamw->mpt;
+ hmmrdma->key = rdmamw->key;
+ hmmrdma->pdn = rdmamw->pdn;
+ hmmrdma->type = (rdmamw->type == RDMA_MW_TYPE_1) ? HMM_RDMA_MW_TYPE_1 :
+ HMM_RDMA_MW_TYPE_2;
+ hmmrdma->enabled = rdmamw->enabled;
+}
+
+int roce5_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
+{
+ int ret;
+ u32 mpt_xid_mode;
+ struct roce5_mw *mw = NULL;
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+ struct hmm_rdma hmmrdma = { 0 };
+
+ if (ibmw == NULL) {
+ ROCE_ERR("Ibmw is null");
+ return -EINVAL;
+ }
+
+ mw = to_roce5_mw(ibmw);
+ pd = to_roce5_pd(ibmw->pd);
+ rdev = to_roce5_dev(ibmw->device);
+
+ mpt_xid_mode = roce5_get_mpt_xid_mode(rdev);
+ ret = hmm_mpt_alloc_templated(rdev->hwdev, &mw->rdmamw.mpt,
+ SERVICE_T_ROCE, mpt_xid_mode);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mpt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err;
+ }
+
+ roce5_set_rdma_mw(&mw->rdmamw, pd->pdn, ibmw->type);
+ roce5_mw2hmm_adapter(&mw->rdmamw, &hmmrdma);
+
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &hmmrdma,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to enable mpt of mw, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ if (ret == (-CMDQ_TIMEOUT)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+ goto err_enable_mpt;
+ }
+
+ mw->rdmamw.enabled = hmmrdma.enabled;
+ mw->ibmw.rkey = mw->rdmamw.key;
+ return 0;
+err_enable_mpt:
+ roce5_err_enable_mpt_handler(rdev->hwdev, mw);
+err:
+ return ret;
+}
+
+struct ib_mw *roce5_alloc_mw_ofa(struct ib_pd *ibpd, enum ib_mw_type type,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_mw *mw = NULL;
+
+ if (ibpd == NULL) {
+ ret = -EINVAL;
+ ROCE_ERR("Ibpd is null");
+ goto out;
+ }
+
+ mw = (struct roce5_mw *)kzalloc(sizeof *mw, GFP_KERNEL);
+ if (mw == NULL) {
+ ret = -ENOMEM;
+ ROCE_ERR("Failed to alloc memory for roce mw");
+ goto out;
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ mw->ibmw.pd = ibpd;
+ mw->ibmw.device = ibpd->device;
+ mw->ibmw.type = type;
+ ret = roce5_alloc_mw(&mw->ibmw, udata);
+ if (ret != 0) {
+ kfree(mw);
+ }
+
+out:
+ return (ret != 0 ? (struct ib_mw *)ERR_PTR((long)ret) : &mw->ibmw);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_dealloc_mw
+ Description : dealloc mw and release related resource
+ Input : struct ib_mw *ibmw
+ Output : None
+
+ 1.Date : 2015/4/24
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_dealloc_mw(struct ib_mw *ibmw)
+{
+ int ret = 0;
+ struct roce5_mw *mw = NULL;
+ struct roce5_device *rdev = NULL;
+ struct hmm_rdma hmmrdma;
+
+ if (ibmw == NULL) {
+ ROCE_ERR("Ibmw is null");
+ return -EINVAL;
+ }
+
+ mw = to_roce5_mw(ibmw);
+ rdev = to_roce5_dev(ibmw->device);
+ roce5_mw2hmm_adapter(&mw->rdmamw, &hmmrdma);
+
+ ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, &hmmrdma,
+ SERVICE_T_ROCE, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to disable mpt of mw, func_id(%d)",
+ rdev->glb_func_id);
+ if (ret == (-CMDQ_TIMEOUT)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+ return ret;
+ }
+ mw->rdmamw.enabled = hmmrdma.enabled;
+
+ hmm_mpt_free(rdev->hwdev, &mw->rdmamw.mpt);
+ mw->rdmamw.enabled = RDMA_MPT_DISABLED;
+
+ return ret;
+}
+
+int roce5_dealloc_mw_ofa(struct ib_mw *ibmw)
+{
+ int ret = 0;
+ struct roce5_mw *mw = NULL;
+
+ ret = roce5_dealloc_mw(ibmw);
+ if (ret == 0) {
+ mw = to_roce5_mw(ibmw);
+ kfree(mw);
+ }
+
+ return ret;
+}
+
+int roce5_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
+ struct ib_mr_status *mr_status)
+{
+ int ret = 0;
+ struct roce5_mr *mr = NULL;
+
+ if ((ibmr == NULL) || (mr_status == NULL)) {
+ ROCE_ERR("ibmr or mr_status is NULL ptr.");
+ return -EINVAL;
+ }
+
+ mr = to_roce5_mr(ibmr);
+ if ((check_mask & (~IB_MR_CHECK_SIG_STATUS)) != 0) {
+ ROCE_ERR("check_mask(%#x) invalid", check_mask);
+ return -EINVAL;
+ }
+
+ if (mr->signature_en == 0) {
+ ROCE_ERR("mr not configure dif info");
+ ret = -EINVAL;
+ return ret;
+ }
+
+ mr->sig.sig_status_checked = true;
+ mr_status->fail_status = 0;
+
+ /* If err does not exist, return 0 */
+ if (!mr->sig.sig_err_exists) {
+ return 0;
+ }
+
+ ret = memcpy_s((void *)&mr_status->sig_err, sizeof(struct ib_sig_err),
+ (void *)&mr->sig.sig_err_item,
+ sizeof(struct ib_sig_err));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy sig_err.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ mr->sig.sig_err_exists = false;
+ mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
+
+ return 0;
+}
+
+/**
+ * @brief Get mpt by index. When querying succeeded, you have to call roce5_mpt_put() to free
+ the cqm_obj_mpt. But there is no need to call roce5_mpt_put() if querying failed.
+ * @return Success: return the mpt. Failure: return NULL.
+ */
+struct mpt *roce5_get_mpt_by_index(struct roce5_device *rdev, u32 mpt_index)
+{
+ cqm_object_s *cqm_obj_mpt = NULL;
+ cqm_qpc_mpt_s *cqm_mpt = NULL;
+ struct mpt *mpt = NULL;
+
+ cqm_obj_mpt =
+ cqm5_object_get(rdev->hwdev, CQM_OBJECT_MPT, mpt_index, false);
+ if (cqm_obj_mpt == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Can't find mpt according to index(0x%x), func_id(%d)",
+ mpt_index, rdev->glb_func_id);
+ return NULL;
+ }
+
+ cqm_mpt = container_of(cqm_obj_mpt, cqm_qpc_mpt_s, object);
+ mpt = (struct mpt *)cqm_mpt->priv;
+ if (mpt == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to convert obj to mpt.");
+ cqm5_object_put(cqm_obj_mpt);
+ }
+
+ return mpt;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h
new file mode 100644
index 000000000..3124c2470
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_MR_H
+#define ROCE_MR_H
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_umem.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+
+#include "roce.h"
+
+#include "hinic5_hmm.h"
+#include "hinic5_rdma.h"
+#include "rdma_comp.h"
+
+#include "securec.h"
+
+#define ROCE5_MR_DEFAULT_ACCESS \
+ (IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE | \
+ IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_ATOMIC)
+
+#define ROCE5_MR_PAGES_ALIGN 64U
+#define ROCE5_FRMR_UMEM_NUM 512
+#define ROCE5_SIZE_2M (2 * 1024 * 1024)
+
+struct roce5_sig_ctx {
+ struct ib_sig_attrs sig_attrs; /* recorded when submitting WR */
+ struct ib_sig_err
+ sig_err_item; /* Get information configuration from CQE when polling CQE */
+ bool sig_status_checked; /* Indicates whether the status is checked */
+ bool sig_err_exists; /* Indicates if an error exists */
+ u32 sig_err_count; /* Count the number of wrong CQEs */
+};
+
+/* uld service can extension structure space */
+#define ROCE_MR_ULD_DEF_LEN 24
+#define ROCE_NOT_USE_HUGEPAGE 0
+#define ROCE_USE_HUGEPAGE 1
+
+/* used for verify uld access whether memory stomping */
+#define ROCE_RMR_MAGIC 0x1317F73a
+struct roce5_mr_uld_def {
+ u8 self_def[ROCE_MR_ULD_DEF_LEN];
+};
+
+struct roce5_mr {
+ struct ib_mr ibmr; /* mr structure provided by ofed */
+ struct ib_umem *
+ umem; /* Maintain the structure where the user-mode allocated buffer */
+ struct ib_umem **fumem; /* used to frmr only */
+ struct hmm_rdma rdmamr; /* mr structure provided by rdma component */
+ struct roce5_sig_ctx sig; /* Record signature related information */
+ u16 signature_en; /* Whether the logo contains signature information */
+ u16 sign_val; /* MTT entry [11:0] */
+ __be64 *pages;
+ u32 npages;
+ u32 fumem_nr;
+ u32 max_pages;
+ size_t page_map_size;
+ dma_addr_t page_map;
+#ifdef __ROCE_DFX__
+ struct list_head node;
+#endif // __ROCE_DFX__
+ bool david_en; /* Indicate whether the mr is a david mr */
+ void *private_data; /* private data for driver */
+ u32 user_data_len; /* length of user data which stored in struct hmm_rdma */
+ struct roce5_mr_uld_def uld_def;
+ u32 rmr_magic; /* preventstop uld access memory stomping, support up to 24B space */
+};
+
+struct roce5_mw {
+ struct ib_mw ibmw; /* mw structure provided by ofed */
+ struct rdma_mw rdmamw; /* mw structure provided by rdma component */
+};
+
+int roce5_alloc_tpt(struct roce5_device *rdev, struct hmm_rdma *mr, u32 npages,
+ u32 page_shift, u32 mpt_xid_mode);
+void roce5_free_tpt(struct roce5_device *rdev, struct hmm_rdma *mr);
+void roce5_set_rdma_mr(struct hmm_rdma *mr, enum rdma_mr_type mr_type, u32 pdn,
+ u64 iova, u64 size, u32 access);
+void roce5_free_frmr_umem(struct ib_umem **fumem, int fumem_nr);
+struct mpt *roce5_get_mpt_by_index(struct roce5_device *rdev, u32 mpt_index);
+static inline void roce5_mpt_put(struct mpt *rmpt)
+{
+ if (rmpt == NULL) {
+ return;
+ }
+ cqm5_object_put(&((cqm_qpc_mpt_s *)(rmpt->mpt_object))->object);
+}
+
+static inline int roce5_mr_type2hmm(enum rdma_mr_type mr_type)
+{
+ int mr_type_convert_table[] = {
+ [RDMA_DMA_MR] = HMM_RDMA_DMA_MR,
+ [RDMA_USER_MR] = HMM_RDMA_USER_MR,
+ [RDMA_FRMR] = HMM_RDMA_FRMR,
+ [RDMA_FMR] = HMM_RDMA_FMR,
+ [RDMA_PHYS_MR] = HMM_RDMA_PHYS_MR,
+ [RDMA_RSVD_LKEY] = HMM_RDMA_RSVD_LKEY,
+ [RDMA_SIG_MR] = HMM_RDMA_SIG_MR,
+ [RDMA_INDIRECT_MR] = HMM_RDMA_INDIRECT_MR,
+ [RDMA_ODP_IMPLICIT_MR] = HMM_RDMA_ODP_IMPLICIT_MR,
+ [RDMA_ODP_EXPLICIT_MR] = HMM_RDMA_ODP_EXPLICIT_MR,
+ };
+
+ return mr_type_convert_table[mr_type];
+}
+
+#endif // __ROCE_MR_H_
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h
new file mode 100644
index 000000000..28a092b29
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h
@@ -0,0 +1,171 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
+
+
+#ifndef ROCE_POST_H
+#define ROCE_POST_H
+
+#include <linux/types.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "roce_wqe_format.h"
+
+#define ROCE_SQ_DOORBELL_TYPE_2 2
+#define ROCE_SQ_DOORBELL_TYPE_23 23
+#define ROCE_SQ_DOORBELL_TYPE_21 21
+
+#define ROCE_DWQE_DDB_TYPE 3 /* RoCE direct wqe of DDB type */
+
+#define ROCE_IMM_EXT_LEN 4
+#define ROCE_TASK_SEG_ALIGN 8
+
+enum roce_tsl_8_byte_aligned_size_e {
+ ROCE_SEND_LOCAL_WQE_TSL = 2,
+ ROCE_RDMA_WQE_TSL = 4,
+ ROCE_ATOMIC_CWP_WQE_TSL = 6,
+ ROCE_UD_WQE_COM_TSL = 8
+};
+
+enum {
+ ROCE_WQE_OPCODE_SEND = 0x00,
+ ROCE_WQE_OPCODE_SEND_INVAL = 0x01,
+ ROCE_WQE_OPCODE_SEND_IMM = 0x02,
+
+ ROCE_WQE_OPCODE_RDMA_WRITE = 0x04,
+ ROCE_WQE_OPCODE_RDMA_WRITE_IMM = 0x05,
+
+ ROCE_WQE_OPCODE_RDMA_READ = 0x08,
+
+ ROCE_WQE_OPCODE_ATOMIC_CMP_SWP = 0x0c,
+ ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD = 0x0d,
+ ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP = 0x0e,
+ ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD = 0x0f,
+
+ ROCE_WQE_OPCODE_FRMR = 0x10,
+ ROCE_WQE_OPCODE_LOCAL_INVAL = 0x11,
+ ROCE_WQE_OPCODE_BIND_MW = 0x12,
+ ROCE_WQE_OPCODE_REG_SIG_MR =
+ 0x13, /* Extended for further local opreation */
+};
+
+#define ROCE_KERNEL_OPCODE_NUM (ROCE_WQE_OPCODE_REG_SIG_MR + 1)
+
+enum {
+ ROCE_DWQE_DB_SUBTYPE_SEND = 0x1,
+ ROCE_DWQE_DB_SUBTYPE_SEND_IMM = 0x2,
+ ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE = 0x3,
+ ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE_IMM = 0x4,
+ ROCE_DWQE_DB_SUBTYPE_RDMA_READ = 0x5,
+ ROCE_DWQE_DB_SUBTYPE_ATOMIC_CMP_SWP = 0x6,
+ ROCE_DWQE_DB_SUBTYPE_ATOMIC_FETCH_ADD = 0x7
+};
+
+/* UD send WQE task seg1 */
+struct roce_wqe_ud_tsk_seg_cycle1 {
+ roce_wqe_tsk_com_seg_u common;
+
+ /* DW0 */
+ u32 data_len;
+
+ /* DW1 */
+ u32 immdata_invkey;
+
+ /* DW2 */
+ union {
+ struct {
+ u32 pd : 18;
+ u32 rsvd0 : 6;
+ u32 stat_rate : 4; /* 0: No limit on the static rate (100% port speed)
+ 1-6: reserved
+ 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s.
+ 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. */
+ u32 rsvd1 : 3;
+ u32 fl : 1;
+ } bs;
+ u32 value;
+ } dw2;
+
+ /* DW3 */
+ union {
+ struct {
+ u32 hop_limit : 8;
+ u32 sgid_idx : 7;
+ u32 rsvd0 : 1;
+ u32 port : 4;
+ u32 rsvd1 : 4;
+ u32 tc : 8;
+ } bs;
+ u32 value;
+ } dw3;
+
+ /* DW4 */
+ union {
+ struct {
+ u32 flow_label : 20;
+ u32 rsvd0 : 4;
+ u32 smac_index : 3;
+ u32 rsvd1 : 5;
+ } bs;
+ u32 value;
+ } dw4;
+
+ /* DW5~8 */
+ u8 dgid[16];
+
+ /* DW9 */
+ union {
+ struct {
+ u32 dst_qp : 24;
+ u32 rsvd : 8;
+ } bs;
+ u32 value;
+ } dw9;
+
+ /* DW10 */
+ u32 qkey;
+};
+
+/* UD send WQE task seg2; */
+struct roce_wqe_ud_tsk_seg_cycle2 {
+ /* DW0 */
+ union {
+ struct {
+ u32 dmac_h16 : 16;
+ u32 vlan_id : 12;
+ u32 cfi : 1;
+ u32 vlan_prio : 3;
+ } bs;
+ u32 value;
+ } dw0;
+
+ /* DW1 */
+ u32 dmac_l32;
+};
+
+struct roce5_post_send_normal_param {
+ struct roce5_wqe_ctrl_seg *ctrl_seg;
+ union roce5_wqe_tsk_com_seg *tsk_com_seg;
+ u32 wqe_size;
+ u8 *wqe;
+ struct roce_wqe_data_seg *dseg;
+ u32 wr_num; /* record posted WR numbers */
+ u32 index; /* WQEBB id */
+ int inline_flag; /* Inline flag */
+ u32 data_len;
+ u32 *data_len_addr;
+ u32 sq_rmd_size;
+ s32 opcode;
+ s32 cycle;
+ u32 sq_wqe_ssn;
+ struct roce5_wqe_ctrl_seg ctrl_seg_tmp;
+
+ unsigned long flags; /* type required by kernel API */
+};
+
+typedef struct tag_roce_opcode_attr {
+ uint32_t rsvd : 31;
+ uint32_t remote_opcode : 1;
+} roce_opcode_attr_s;
+
+#endif /* ROCE_POST_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h
new file mode 100644
index 000000000..7088cd550
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_QP_H
+#define ROCE_QP_H
+
+#include <linux/types.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "hinic5_rdma.h"
+#include "hinic5_cqm.h"
+
+#include "roce.h"
+#include "roce_cq.h"
+#include "roce_db.h"
+#include "roce_pd.h"
+#include "roce_compat.h"
+#include "roce_npu_cmd_qp_defs.h"
+#include "roce_wqe_base_format.h"
+#include "roce_post.h"
+#include "roce_mix.h"
+#include "roce_xqe_format.h"
+#include "roce_qp_ctx_format.h"
+#include "roce_cq_format.h"
+#include "roce_srq_format.h"
+#include "roce_com.h"
+#include "roce_obj_ext.h"
+#include "roce_kernel_compat_gen.h"
+
+#define ROCE_RQ_MIN_SGE 4
+#define ROCE_RQ_MID_SGE 8
+#define ROCE_RQ_MAX_SGE 16
+
+#define ROCE_WQE_NEXT_SGE_INVALID (1UL << 31)
+
+#define ROCE_RDMARC_DESTROY_CHECK_VALUE 0xcece5a5a
+
+#define RDMARC_NUM_PER_CACHELINE 8
+#define RDMARC_TABLE_ENTRY_SIZE 32
+#define ROCE_QP_MAX_PFETCH_MTT_LAYER 3
+#define ROCE_QP_MAX_TIMER_NUM 5
+
+#define ROCE_QP_SMI_QP_NUM 0x0
+#define ROCE_QP_GSI_QP_NUM 0x1
+#define ROCE_QP_INVLID_QP_NUM CQM_INDEX_INVALID
+
+#define ROCE_QP_MAX_DWQE_SIZE 256
+#define ROCE_QP_DEFAULT_WQE_SHIFT 6
+
+#define ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE 512
+
+#define ROCE_QP_STATE_MEM_INIT 0xa
+
+#define ROCE_UD_MAX_INLINE_LEN_SUB 48
+
+#define ROCE_QP_GPA_SIG_LEN 3
+#define ROCE_WR_MIN_NUM 2
+
+#define ROCE_QPC_PMTU_TRANSLATE(pmtu) (((1U << ((pmtu)-1)) - 1) & 0xf)
+
+#define ROCE_QP_OPTPAR_ALT_ADDR_PATH (1 << 0)
+#define ROCE_QP_OPTPAR_RRE (1 << 1)
+#define ROCE_QP_OPTPAR_RAE (1 << 2)
+#define ROCE_QP_OPTPAR_RWE (1 << 3)
+#define ROCE_QP_OPTPAR_PKEY_INDEX (1 << 4)
+#define ROCE_QP_OPTPAR_Q_KEY (1 << 5)
+#define ROCE_QP_OPTPAR_RNR_TIMEOUT (1 << 6)
+#define ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH (1 << 7)
+#define ROCE_QP_OPTPAR_SRA_MAX (1 << 8)
+#define ROCE_QP_OPTPAR_RRA_MAX (1 << 9)
+#define ROCE_QP_OPTPAR_PM_STATE (1 << 10)
+#define ROCE_QP_OPTPAR_RETRY_COUNT (1 << 11)
+#define ROCE_QP_OPTPAR_RNR_RETRY (1 << 12)
+#define ROCE_QP_OPTPAR_ACK_TIMEOUT (1 << 13)
+#define ROCE_QP_OPTPAR_SCHED_QUEUE (1 << 14)
+#define ROCE_QP_OPTPAR_COUNTER_INDEX (1 << 15)
+
+#define roce5_LB1_MASK_1825V100 0x7
+
+enum roce_qp_state {
+ ROCE_QP_STATE_RST = 0,
+ ROCE_QP_STATE_INIT = 1,
+ ROCE_QP_STATE_RTR = 2,
+ ROCE_QP_STATE_RTS = 3,
+ ROCE_QP_STATE_SQER = 4,
+ ROCE_QP_STATE_SQD = 5,
+ ROCE_QP_STATE_ERR = 6,
+ ROCE_QP_STATE_SQ_DRAINING = 7,
+ ROCE_QP_STATE_NUM
+};
+
+enum {
+ ROCE_QP_PM_MIGRATED = 0x3,
+ ROCE_QP_PM_ARMED = 0x0,
+ ROCE_QP_PM_REARM = 0x1
+};
+
+enum { ROCE_QP_NO_SRQ, ROCE_QP_HAS_SRQ };
+
+enum { ROCE_QP = 0, ROCE_QP_EXT };
+
+struct roce5_wq {
+ u64 *wrid;
+ spinlock_t lock;
+ u32 wqebb_cnt;
+ u32 max_post;
+ u32 max_sge;
+ u32 offset;
+ u32 wqe_shift;
+ u32 head;
+ u32 tail;
+};
+
+struct roce5_sqp_attr {
+ int pkey_index;
+ u32 qkey;
+ u32 send_psn;
+};
+
+#define ROCE_RQP_ULD_DEF_LEN 64
+#define ROCE_RQP_MAGIC 0x5a5b5c5d
+struct roce5_rqp_uld_def {
+ u8 uld_def[ROCE_RQP_ULD_DEF_LEN];
+};
+
+struct roce5_qp {
+ struct ib_qp ibqp;
+ u32 qpn;
+ struct roce5_db db;
+ cqm_queue_s *qp_buf_info;
+ cqm_qpc_mpt_s *qpc_info;
+ struct roce5_wq rq;
+ u32 sq_signal_bits;
+ unsigned int sq_next_wqe;
+ u32 sq_max_wqes_per_wr;
+ struct roce5_wq sq;
+ struct ib_umem *umem;
+ struct mtt mtt;
+ struct roce5_buf buf;
+ int buf_size;
+ struct mutex mutex;
+ u32 xrcdn;
+ u8 port;
+ u8 atomic_rd_en;
+ u8 qp_state;
+ u8 qp_ext;
+ u32 qp_type;
+ u32 max_inline_data;
+ int max_dwqe_size;
+ u32 sq_wqe_ssn;
+ struct rdma_rdmarc rdmarc;
+ u16 rsp_depth;
+ u8 sl;
+ bool has_rq;
+ u8 *sq_head_addr;
+ u8 *sq_tail_addr;
+ bool signature_en;
+ u8 pi_on_chip;
+ u8 double_sgl_mode;
+ u8 ext_mtu;
+ u8 ext_mtu_mode;
+
+ u8 db_sgid_index;
+ u8 db_path_mtu;
+ u8 db_cos;
+ u8 shadow;
+
+ struct roce5_sqp_attr *sqp_attr;
+
+ u32 qid;
+ u32 local_comm_id; /* used by NOF AA to reuse qpn when disconnection happens */
+ u32 remote_comm_id;
+ void *wb_va; /* used for xrc */
+ dma_addr_t wb_dma; /* used for xrc */
+ u32 shadow_vfid;
+
+ /* hot-plug record */
+ struct list_head qps_list;
+ struct list_head cq_recv_list;
+ struct list_head cq_send_list;
+#if defined(ROCE_VBS_EN) || defined(ROCE_EXTEND)
+ void *vbs_qp_ptr;
+#endif
+#ifdef ROCE_BONDING_EN
+ u32 tx_hash_value;
+#endif
+
+ struct roce5_obj_ext obj_ext;
+
+ struct roce5_rqp_uld_def
+ uld_def; /* must placed at qp_magic before, preventstop uld access memory stomping, support up to 64B space */
+ u32 rqp_magic;
+};
+
+typedef struct tag_roce_modify_qp_param {
+ u32 optpar;
+ roce_verbs_qp_attr_s qp_attr;
+ roce_verbs_cmd_qp_modify_rsp_s modify_rsp;
+} roce_modify_qp_param_s;
+
+void roce5_qp_async_event(struct roce5_device *rdev, struct roce5_qp *qp,
+ int type);
+
+void roce5_free_opt_rdmarc(struct roce5_qp *rqp);
+void roce5_set_opt_rdmarc(struct rdma_rdmarc *rdmarc, u16 depth,
+ roce_verbs_qp_attr_s *qp_attr);
+void roce5_get_cqs(struct roce5_qp *rqp, struct roce5_cq **send_cq,
+ struct roce5_cq **recv_cq);
+int roce5_qp_modify_2rst_cmd(struct roce5_device *rdev, u32 qpn);
+int roce5_qp_cache_out_cmd(struct roce5_device *rdev, struct roce5_qp *rqp);
+struct roce5_pd *roce5_get_pd(struct roce5_qp *rqp);
+bool roce5_sqp_check(const struct roce5_qp *rqp);
+void *roce5_get_wqe(struct roce5_qp *rqp, u32 offset);
+int roce5_wq_overflow(struct roce5_wq *wq, u32 wr_num, struct ib_cq *ibcq);
+void roce5_set_data_seg(struct roce_wqe_data_seg *dseg, struct ib_sge *sge);
+void roce5_qpc_to_be(roce_verbs_qp_attr_s *qp_attr, struct roce5_qp *rqp,
+ u32 *be_ctx);
+void roce5_qp_rst2init(struct roce5_qp *rqp, const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr);
+
+int roce5_send_qp_lb_cmd(u32 qpn, struct roce5_device *rdev, u8 cmd,
+ cqm_cmd_buf_s *buf_in, cqm_cmd_buf_s *buf_out,
+ u32 timeout);
+
+int roce5_qp_query(struct roce5_device *rdev, u32 qpn, u32 *context,
+ int qpc_size);
+int roce5_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr);
+void qpc_seg_to_le32(struct roce5_qp_context *be_ctx,
+ struct roce5_qp_context *le_ctx, u32 srq_vld);
+void roce5_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size);
+
+int roce5_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr);
+
+int roce5_post_send_ofa(struct ib_qp *ibqp, struct ib_send_wr *wr,
+ struct ib_send_wr **bad_wr);
+struct roce5_qp *roce5_get_rqp_by_qpn(struct roce5_device *rdev, u32 qpn);
+static inline void roce5_rqp_put(struct roce5_qp *rqp)
+{
+ if (rqp == NULL) {
+ return;
+ }
+ cqm5_object_put(&rqp->qpc_info->object);
+}
+
+#define ROCE_UD_MTU_SHIFT 4 /* 4K mtu */
+static inline void roce5_set_sq_db(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ struct roce_sq_db_seg *db,
+ unsigned int index)
+{
+ db->dw0.bs.qpn = rqp->qpn & 0xfffff;
+ db->dw0.bs.ctx_size = 1; /* 512B */
+ db->dw0.bs.cp_flag = 0;
+ db->dw0.bs.cos = rqp->db_cos;
+ if (ROCE5_SUPPORT_SQ_DB_TYPE_21(rdev) != 0) {
+ db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_21;
+ } else if (ROCE5_SUPPORT_SQ_DB_TYPE_MSG_V2(rdev) != 0) {
+ db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_23;
+ } else {
+ db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_2;
+ }
+
+ db->dw1.bs.pi = ((index & 0xffff) >> 8) & 0xff; /* 8bits */
+ db->dw1.bs.sub_type = 0;
+ db->dw1.bs.mtu_shift = rqp->db_path_mtu & 0x7;
+ db->dw1.bs.sgid_index = rqp->db_sgid_index;
+ if (rqp->qp_type == IB_QPT_RC) {
+ } else if (rqp->qp_type == IB_QPT_UD || rqp->qp_type == IB_QPT_GSI) {
+ db->dw1.bs.mtu_shift = ROCE_UD_MTU_SHIFT;
+ } else if (rqp->qp_type == IB_QPT_XRC_INI ||
+ rqp->qp_type == IB_QPT_XRC_TGT) {
+ db->dw1.bs.xrc_vld = 1;
+ }
+
+ db->dw0.value = roce5_convert_be32(db->dw0.value);
+ db->dw1.value = roce5_convert_be32(db->dw1.value);
+}
+
+static inline int roce5_write_sq_db(struct roce5_device *rdev,
+ struct roce5_qp *rqp, u32 index,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ union roce_sq_db sq_db;
+
+ ret = memset_s(&sq_db.sq_db_val, sizeof(u64), 0, sizeof(u64));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memset sq_db.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ roce5_set_sq_db(rdev, rqp, &sq_db.sq_db_seg, index);
+
+ if (udata == NULL) {
+ *(rqp->db.db_record) = cpu_to_be32(index & 0xffff);
+ }
+
+ wmb();
+ ret = cqm5_ring_hardware_db(rdev->hwdev, SERVICE_T_ROCE, index & 0xff,
+ sq_db.sq_db_val);
+
+ /*
+ * Make sure doorbells don't leak out of SQ spinlock
+ * and reach the HCA out of order.
+ */
+ wmb();
+ return ret;
+}
+
+#if defined(IB_QP_CREATE_FLAGS_HAVE_INTEGRITY_EN)
+#define IB_QP_CREATE_SIGNATURE_EN IB_QP_CREATE_INTEGRITY_EN
+#endif
+#endif // ROCE_QP_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c
new file mode 100644
index 000000000..6adbe787c
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c
@@ -0,0 +1,1506 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "roce_compat.h"
+
+#include "roce.h"
+#include "roce_infra.h"
+#include "roce_mix.h"
+#include "roce_mr.h"
+#include "roce_user.h"
+#include "roce_xrc.h"
+#include "roce_pd.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_qp.h"
+#include "roce_qp_extension.h"
+#include "roce_main_extension.h"
+#include "roce_aeq_format.h"
+#include "roce_restore.h"
+#include "roce_os_compat.h"
+#include "roce_cc_format.h"
+#include "roce_uld_inner.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_qp_inner.h"
+#include "roce_com.h"
+#include "roce_qp_ext.h"
+
+#include "hinic5_hmm.h"
+#ifdef __ROCE_DFX__
+#include "roce_dfx.h"
+#endif
+
+#include "securec.h"
+
+/* ****************************************************************************
+ Prototype : roce5_check_rq_size
+ Description : roce5_check_rq_size
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ struct ib_qp_cap *cap
+ bool has_rq
+ Output : None
+
+ 1.Date : 2017/4/27
+ Modification : Created function
+**************************************************************************** */
+static int roce5_check_rq_size(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_init_attr *init_attr, bool has_rq)
+{
+ u32 max_sge_num;
+ struct ib_qp_cap *cap = &init_attr->cap;
+
+ max_sge_num = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg;
+ if ((cap->max_recv_wr >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) ||
+ (cap->max_recv_sge > max_sge_num)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to check rq size, over range, func_id(%d), attr max_recv_wr(%d), cap max_wqes(%d), \
+ attr max_recv_sge(%d), cap max_sge_num(%d)",
+ rdev->glb_func_id, cap->max_recv_wr,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes,
+ cap->max_recv_sge, max_sge_num);
+ return -EINVAL;
+ }
+
+ if (has_rq && ((cap->max_recv_wr == 0) || (cap->max_recv_sge == 0))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Has rq, cap->max_recv_wr(%d), cap->max_recv_sge(%d), func_id(%d)",
+ cap->max_recv_wr, cap->max_recv_sge, rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if ((!has_rq) && (cap->max_recv_wr != 0)) {
+ ROCE_DEV_ERR(rdev,
+ "Not has rq, cap->max_recv_wr(%d), func_id(%d).",
+ cap->max_recv_wr, rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_rq_size
+ Description : roce5_set_rq_size
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ struct ib_qp_cap *cap
+ bool is_user
+ bool has_rq
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+**************************************************************************** */
+/* ****************************************************************************
+*
+* 1 there is SQ or RQ
+* SQ/RQ depth must be power of 2
+* if max_send_wr=0, no need to alloc buf for SQ/RQ
+
+* RQ WQE SIZE equals 64 or128
+* SQ WQE SIZE equals multi of wqebb size
+
+* SQ max_gs, max 8 and min 0
+* RQ max_gs, can be 4 or 8, but can not be 0.
+
+* 2 there is no SQ or RQ
+* all related params should be 0
+*
+
+**************************************************************************** */
+static int roce5_set_rq_size(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_init_attr *init_attr, bool is_user,
+ bool has_rq)
+{
+ int ret;
+ u32 wqebb_num = 0;
+ u32 sge_total_len = 0;
+ u32 page_size;
+ struct ib_qp_cap *cap = &init_attr->cap;
+
+ page_size = roce5_get_page_size(rdev, true);
+
+ ret = roce5_check_rq_size(rdev, rqp, init_attr, has_rq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to check rq size, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ if (has_rq) {
+ wqebb_num = ROCE_MAX(2U, cap->max_recv_wr);
+ rqp->rq.wqebb_cnt =
+ (u32)(ROCE_ROUNDUP_POW_OF_TWO(wqebb_num) & 0xffffffff);
+
+ /* RQ SGE range [0,4] select 4,[5,8] select 8 */
+ rqp->rq.max_sge =
+ (cap->max_recv_sge <= ROCE_RQ_MIN_SGE) ?
+ ROCE_RQ_MIN_SGE :
+ ((cap->max_recv_sge <= ROCE_RQ_MID_SGE) ?
+ ROCE_RQ_MID_SGE :
+ ROCE_RQ_MAX_SGE);
+
+ sge_total_len = (u32)((u32)rqp->rq.max_sge *
+ sizeof(struct roce_wqe_data_seg));
+ rqp->rq.wqe_shift = (u32)ROCE_ILOG2(sge_total_len);
+
+ if ((u32)((u32)rqp->rq.wqebb_cnt
+ << (unsigned)rqp->rq.wqe_shift) < page_size) {
+ rqp->rq.wqebb_cnt =
+ (page_size >> (unsigned)rqp->rq.wqe_shift);
+ }
+
+ /* leave userspace return values as they were, so as not to break ABI */
+ if (is_user) {
+ rqp->rq.max_post =
+ ROCE_MIN(rdev->rdma_cap.dev_rdma_cap
+ .roce_own_cap.max_wqes,
+ rqp->rq.wqebb_cnt);
+ cap->max_recv_wr = (u32)rqp->rq.max_post;
+ cap->max_recv_sge = (u32)rqp->rq.max_sge;
+ } else {
+ rqp->rq.max_post =
+ ROCE_MIN(rdev->rdma_cap.dev_rdma_cap
+ .roce_own_cap.max_wqes,
+ rqp->rq.wqebb_cnt);
+ cap->max_recv_wr = (u32)rqp->rq.max_post;
+ cap->max_recv_sge =
+ ROCE_MIN((u32)rqp->rq.max_sge,
+ rdev->rdma_cap.dev_rdma_cap
+ .roce_own_cap.max_rq_sg);
+ }
+ } else {
+ rqp->rq.wqebb_cnt = 0;
+ rqp->rq.max_sge = 0;
+ rqp->rq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT;
+ rqp->rq.max_post = 0;
+ cap->max_recv_sge = 0;
+ }
+
+ rqp->rq.head = 0;
+ rqp->rq.tail = 0;
+
+ return 0;
+}
+
+static int roce5_set_user_sq_size_check(struct roce5_device *rdev,
+ const struct ib_qp_cap *cap,
+ const struct roce5_qp *rqp)
+{
+ u32 ud_max_inline_size;
+
+ if (cap->max_send_wr >
+ (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes)) {
+ ROCE_DEV_ERR(rdev, "Log_sq_bb_count over range, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if (rqp->qp_type == IB_QPT_UD) {
+ ud_max_inline_size =
+ (rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_sq_inline_data_sz >
+ ROCE_UD_MAX_INLINE_LEN_SUB) ?
+ (rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_sq_inline_data_sz -
+ ROCE_UD_MAX_INLINE_LEN_SUB) :
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_sq_inline_data_sz;
+ } else {
+ ud_max_inline_size = rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_sq_inline_data_sz;
+ }
+
+ /* RC:192B inline data, UD:176B */
+ if (cap->max_inline_data > ud_max_inline_size) {
+ ROCE_DEV_ERR(rdev, "SQ max inline data over range, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+/* ****************************************************************************
+ Prototype : roce5_set_user_sq_size
+ Description : roce5_set_user_sq_size
+ Input : struct roce5_device *rdev
+ struct ib_qp_cap *cap
+ struct roce5_qp *rqp
+ struct roce5_create_qp *ucmd
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_set_user_sq_size(struct roce5_device *rdev,
+ struct ib_qp_cap *cap, struct roce5_qp *rqp,
+ struct create_qp_cmd *ucmd)
+{
+ u32 tmp_max_sq_wqe_sz = 0;
+ u32 sq_buf_sz = 0;
+ int ret = 0;
+ u32 page_size;
+
+ page_size = roce5_get_page_size(rdev, true);
+
+ ret = roce5_set_user_sq_size_check(rdev, cap, rqp);
+ if (ret != 0) {
+ return ret;
+ }
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ /* between max and min */
+ tmp_max_sq_wqe_sz =
+ (u32)(ROCE_ROUNDUP_POW_OF_TWO(
+ rdev->rdma_cap.max_sq_desc_sz) &
+ 0xffffffff);
+ if ((ucmd->log_sq_stride > ROCE_ILOG2(tmp_max_sq_wqe_sz)) ||
+ (ucmd->log_sq_stride <
+ ROCE_ILOG2(rdev->rdma_cap.wqebb_size))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "WQE size invalid, func_id(%d) log_sq_stride:%d wqebb_size:%d tmp_max_sq_wqe_sz:%d",
+ rdev->glb_func_id, ucmd->log_sq_stride,
+ rdev->rdma_cap.wqebb_size, tmp_max_sq_wqe_sz);
+ return -EINVAL;
+ }
+
+ rqp->sq.wqebb_cnt = (u32)(1U << ucmd->log_sq_bb_count);
+ rqp->sq.wqe_shift = ucmd->log_sq_stride;
+
+ if ((u32)((u64)rqp->sq.wqebb_cnt
+ << (unsigned)rqp->sq.wqe_shift) < page_size) {
+ rqp->sq.wqebb_cnt =
+ (page_size >> (unsigned)rqp->sq.wqe_shift);
+ }
+
+ sq_buf_sz = (u32)ROCE_ALIGN(
+ (u64)rqp->sq.wqebb_cnt << (unsigned)rqp->sq.wqe_shift,
+ page_size);
+ rqp->buf_size =
+ (int)ROCE_ALIGN(((unsigned)sq_buf_sz +
+ ((unsigned)rqp->rq.wqebb_cnt
+ << (unsigned)rqp->rq.wqe_shift)),
+ page_size);
+
+ rqp->sq.offset = 0;
+ rqp->rq.offset = sq_buf_sz;
+
+ rqp->max_inline_data = (u32)cap->max_inline_data;
+ } else {
+ rqp->sq.wqebb_cnt = 0;
+ rqp->sq.wqe_shift = ucmd->log_sq_stride;
+ rqp->buf_size = 0;
+ rqp->sq.offset = 0;
+ rqp->rq.offset = 0;
+ rqp->max_inline_data = 0;
+ cap->max_inline_data = 0;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_send_wqe_overhead
+ Description : roce5_send_wqe_overhead
+ Input : enum ib_qp_type qp_type
+ Output : None
+
+ 1.Date : 2015/5/7
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_send_wqe_overhead(enum ib_qp_type qp_type)
+{
+ switch (qp_type) {
+ case IB_QPT_RC:
+ return sizeof(struct roce5_wqe_ctrl_seg) +
+ sizeof(struct roce_wqe_frmr_tsk_seg);
+
+ case IB_QPT_UC:
+ return sizeof(struct roce5_wqe_ctrl_seg) +
+ sizeof(struct roce_wqe_send_tsk_seg);
+
+ case IB_QPT_UD:
+ return sizeof(struct roce5_wqe_ctrl_seg) +
+ sizeof(struct roce_wqe_ud_tsk_seg);
+
+ case IB_QPT_GSI:
+ return sizeof(struct roce5_wqe_ctrl_seg) +
+ sizeof(struct roce_wqe_ud_tsk_seg);
+
+ case IB_QPT_XRC_TGT:
+ return 0;
+
+ case IB_QPT_XRC_INI:
+ return sizeof(struct roce5_wqe_ctrl_seg) +
+ sizeof(struct roce_wqe_frmr_tsk_seg);
+
+ default:
+ ROCE_ERR("Not supported this qp_type(%d)", qp_type);
+ return -1;
+ }
+}
+
+static int roce5_set_kernel_sq_size_check(const struct roce5_device *rdev,
+ const struct ib_qp_cap *cap,
+ int *sq_max_wqe_size,
+ enum ib_qp_type qp_type)
+{
+ int wqe_overhead = 0;
+ int ret = 0;
+
+ if (cap->max_send_wr >
+ (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) {
+ ROCE_DEV_ERR(rdev, "SQ WR over range, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if (cap->max_send_sge > rdev->rdma_cap.max_sq_sg) {
+ ROCE_DEV_ERR(rdev, "SQ SGE over range, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if (cap->max_inline_data >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz) {
+ ROCE_DEV_ERR(rdev, "SQ max inline data over range, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ wqe_overhead = roce5_send_wqe_overhead(qp_type);
+ if (wqe_overhead < 0) {
+ ROCE_DEV_ERR(rdev, "Invalid service type(%d), func_id(%d)",
+ (int)qp_type, rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ *sq_max_wqe_size = ROCE_MAX((int)(cap->max_send_sge *
+ sizeof(struct roce_wqe_data_seg)),
+ (int)cap->max_inline_data) +
+ wqe_overhead;
+ if (*sq_max_wqe_size > (int)rdev->rdma_cap.max_sq_desc_sz) {
+ ROCE_DEV_ERR(
+ rdev,
+ "SQ WQE size over range, func_id(%d) max_send_sge:%u,max_inline_data:%u,"
+ "wqe_overhead:%d,sq_max_wqe_size:%d,max_sq_desc_sz:%u",
+ rdev->glb_func_id, cap->max_send_sge,
+ cap->max_inline_data, wqe_overhead, *sq_max_wqe_size,
+ rdev->rdma_cap.max_sq_desc_sz);
+ return -EINVAL;
+ }
+
+ return ret;
+}
+/* ****************************************************************************
+ Prototype : roce5_set_kernel_sq_size
+ Description : roce5_set_kernel_sq_size
+ Input : struct roce5_device *rdev
+ struct ib_qp_cap *cap
+ enum roce5_qp_type qp_type
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_set_kernel_sq_size(struct roce5_device *rdev,
+ struct ib_qp_cap *cap,
+ enum ib_qp_type qp_type,
+ struct roce5_qp *rqp)
+
+{
+ u32 sq_buf_sz = 0;
+ int sq_max_wqe_size = 0;
+ int ret = 0;
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ ret = roce5_set_kernel_sq_size_check(rdev, cap,
+ &sq_max_wqe_size, qp_type);
+ if (ret != 0) {
+ return ret;
+ }
+
+ rqp->sq.wqe_shift = (u32)ROCE_ILOG2(rdev->rdma_cap.wqebb_size);
+ rqp->sq_max_wqes_per_wr = (u32)ROCE_DIV_ROUND_UP(
+ (unsigned)sq_max_wqe_size,
+ (u32)(1UL << (unsigned)rqp->sq.wqe_shift));
+
+ /*
+ * As one wqebb ought to be reserved for invalidate wqebb, 1 is added to sq.wqebb_cnt here
+ * to satisfy the actual IB_MAD_QP_SEND_SIZE set by applications
+ */
+ rqp->sq.wqebb_cnt =
+ (u32)((u32)cap->max_send_wr * rqp->sq_max_wqes_per_wr +
+ 1);
+ rqp->sq.wqebb_cnt =
+ (u32)(ROCE_ROUNDUP_POW_OF_TWO(rqp->sq.wqebb_cnt) &
+ 0xffffffff);
+
+ if ((u32)((u64)rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift) <
+ PAGE_SIZE) {
+ rqp->sq.wqebb_cnt =
+ (PAGE_SIZE >> ((u32)rqp->sq.wqe_shift));
+ }
+
+ rqp->sq.max_sge =
+ (u32)(((u64)rqp->sq_max_wqes_per_wr
+ << (unsigned)rqp->sq.wqe_shift) -
+ (unsigned)roce5_send_wqe_overhead(qp_type));
+ rqp->sq.max_sge = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_desc_sz,
+ rqp->sq.max_sge) /
+ (u32)sizeof(struct roce_wqe_data_seg);
+
+ sq_buf_sz = (u32)ALIGN((u64)rqp->sq.wqebb_cnt
+ << (unsigned)rqp->sq.wqe_shift,
+ PAGE_SIZE);
+ rqp->buf_size = (int)ALIGN(((unsigned)sq_buf_sz +
+ ((unsigned)rqp->rq.wqebb_cnt
+ << (unsigned)rqp->rq.wqe_shift)),
+ PAGE_SIZE);
+
+ rqp->sq.offset = 0u;
+ rqp->rq.offset = sq_buf_sz;
+
+ if (((rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr) == 1) {
+ ROCE_DEV_ERR(rdev,
+ "Only one WR,please check,func_id(%d)",
+ rdev->glb_func_id);
+ rqp->sq.max_post = ROCE_WR_MIN_NUM;
+ cap->max_send_wr = rqp->sq.max_post;
+ } else {
+ rqp->sq.max_post = (rqp->sq.wqebb_cnt - 1) /
+ rqp->sq_max_wqes_per_wr;
+ cap->max_send_wr = (u32)rqp->sq.max_post;
+ }
+
+ cap->max_send_sge = (u32)ROCE_MIN(rqp->sq.max_sge,
+ rdev->rdma_cap.max_sq_sg);
+ rqp->max_inline_data = (u32)cap->max_inline_data;
+ } else {
+ rqp->sq.wqebb_cnt = 0;
+ rqp->sq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT;
+ rqp->buf_size = 0;
+ rqp->sq.offset = 0;
+ rqp->rq.offset = 0;
+ rqp->max_inline_data = 0;
+ cap->max_inline_data = 0;
+ }
+
+ rqp->sq.head = 0;
+ rqp->sq.tail = 0;
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_async_event
+ Description : roce5_qp_async_event
+ Input : struct roce5_device *rdev
+ struct roce5_qp *qp
+ int type
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_qp_async_event(struct roce5_device *rdev, struct roce5_qp *qp,
+ int type)
+{
+ struct ib_event event;
+ struct ib_qp *ibqp = &qp->ibqp;
+ int ret;
+
+ ret = memset_s(&event, sizeof(event), 0, sizeof(event));
+ if (ret != 0) {
+ ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset dmac.(ret:%d)", ret);
+ return;
+ }
+
+ if (ibqp->event_handler) {
+ event.device = ibqp->device;
+ event.element.qp = ibqp;
+
+ switch (type) {
+ case ROCE_EVENT_TYPE_COMM_EST:
+ event.event = IB_EVENT_COMM_EST;
+ break;
+
+ case ROCE_EVENT_TYPE_SQ_DRAINED:
+ event.event = IB_EVENT_SQ_DRAINED;
+ break;
+
+ case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE:
+ event.event = IB_EVENT_QP_LAST_WQE_REACHED;
+ break;
+
+ case ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+ event.event = IB_EVENT_QP_FATAL;
+ break;
+
+ case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ event.event = IB_EVENT_QP_REQ_ERR;
+ break;
+
+ case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR:
+ event.event = IB_EVENT_QP_ACCESS_ERR;
+ break;
+
+ default:
+ return;
+ }
+
+ ibqp->event_handler(&event, ibqp->qp_context);
+ }
+}
+
+static void *roce5_buf_offset(struct roce5_buf *buf, u32 offset)
+{
+ return (void *)((char *)buf->direct.buf + offset);
+}
+
+void *roce5_get_wqe(struct roce5_qp *rqp, u32 offset)
+{
+ return roce5_buf_offset(&rqp->buf, offset);
+}
+
+static void *roce5_get_send_wqe_head(struct roce5_qp *rqp)
+{
+ return roce5_get_wqe(rqp, rqp->sq.offset);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_get_send_wqe_tail
+ Description : roce5_get_send_wqe_tail
+ Input : struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2015/8/8
+ Modification : Created function
+
+**************************************************************************** */
+static void *roce5_get_send_wqe_tail(struct roce5_qp *rqp)
+{
+ return roce5_get_wqe(rqp, rqp->sq.offset + (rqp->sq.wqebb_cnt
+ << (u32)rqp->sq.wqe_shift));
+}
+
+/* ****************************************************************************
+ Prototype : roce5_sq_buf_init
+ Description : roce5_sq_buf_init
+ Input : struct roce5_qp *qp
+ Output : None
+
+ 1.Date : 2015/11/9
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_sq_buf_init(struct roce5_qp *qp)
+{
+ u32 entries = qp->rq.offset / (1U << qp->sq.wqe_shift);
+ u8 *wqe = NULL;
+ struct roce5_wqe_ctrl_seg *wqe_ctrl = NULL;
+ u32 i;
+
+ for (i = 0; i < entries; i++) {
+ wqe = (u8 *)qp->buf.direct.buf +
+ (u32)(i * (1U << ((u32)qp->sq.wqe_shift)));
+ wqe_ctrl = (struct roce5_wqe_ctrl_seg *)((void *)wqe);
+ wqe_ctrl->dw0.value = be32_to_cpu(wqe_ctrl->dw0.value);
+ wqe_ctrl->dw0.bs.owner = 1;
+ wqe_ctrl->dw0.value = cpu_to_be32(wqe_ctrl->dw0.value);
+ }
+}
+
+static int roce5_check_qp_udata(struct roce5_device *rdev,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata, struct roce5_qp *rqp,
+ struct create_qp_cmd *ucmd)
+{
+ int ret = 0;
+
+ if ((udata == NULL) ||
+ (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0)) {
+ ret = -EFAULT;
+ return ret;
+ }
+
+ ret = roce5_set_user_sq_size(rdev, &init_attr->cap, rqp, ucmd);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to set user sq_size, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ return ret;
+}
+
+static bool
+roce5_need_qpn_lb1_consistent_srqn(const struct roce5_qp *rqp,
+ const struct roce5_device *rdev,
+ const struct ib_qp_init_attr *init_attr)
+{
+ if ((init_attr->srq == NULL) || (rdev->cfg_info.lb_en == 0)) {
+ return false;
+ }
+
+ if (ROCE5_SUPPORT_QP_SRQN_LB_MSG_V2(rdev) != 0) {
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+roce5_need_qpn_lb1_consistent_scqn(struct roce5_device *rdev,
+ struct ib_qp_init_attr *init_attr)
+{
+ if ((init_attr->send_cq == NULL) || (rdev->cfg_info.lb_en == 0)) {
+ return false;
+ }
+
+ if (ROCE5_SUPPORT_ALLOC_QPC_MSG_FEATURE(rdev) != 0) {
+ return false;
+ }
+ return true;
+}
+
+static int roce5_find_free_lb_qpn(struct roce5_device *rdev)
+{
+ int idx = 0;
+ for (; idx < ROCE_LB_QPN_MAX; idx++) {
+ if (rdev->qp_lb_scqn_ctrl.qp[idx] == 0) {
+ return idx;
+ }
+ }
+ return -1;
+}
+
+static int roce5_alloc_qpc(struct roce5_qp *rqp, struct roce5_device *rdev,
+ u32 qpn, struct ib_qp_init_attr *init_attr)
+{
+ struct roce5_srq *rsrq = NULL;
+ struct roce5_cq *rcq = NULL;
+ u32 qpn_tmp = qpn;
+ u32 roce_lb_mask = 0;
+ u32 cqm_lb_mode = 0;
+ int idx = -1;
+
+ if ((rqp == NULL) || (rdev == NULL) || (init_attr == NULL)) {
+ ROCE_ERR("rqp, rdev or init_attr is NULL");
+ return -EINVAL;
+ }
+
+ if (ROCE_IS_DEV_1825V100(rdev)) {
+ cqm_lb_mode = CQM_XID_LOW_BIT_1_1_1;
+ roce_lb_mask = roce5_LB1_MASK_1825V100;
+ } else {
+ ROCE_DEV_ERR(rdev, "Unknown device type(%u), func_id(%u).",
+ rdev->device_type, rdev->glb_func_id);
+ return -EINVAL;
+ }
+ mutex_lock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex);
+
+ if (roce5_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) {
+ rsrq = to_roce5_srq(init_attr->srq);
+ qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, cqm_lb_mode,
+ rsrq->srqn);
+ } else if ((qpn != ROCE_QP_GSI_QP_NUM) &&
+ roce5_need_qpn_lb1_consistent_scqn(rdev, init_attr)) {
+ rcq = to_roce5_cq(init_attr->send_cq);
+ //if not found free of qpn, idx return -1, not need do LB
+ idx = roce5_find_free_lb_qpn(rdev);
+ if (idx >= 0) {
+ qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL,
+ cqm_lb_mode, rcq->cqn);
+ }
+ }
+
+ rqp->qpc_info = cqm5_object_qpc_mpt_create(
+ rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, rqp,
+ qpn_tmp, 0, 0);
+
+ if (rqp->qpc_info == NULL && rdev->full_qp_spec) {
+ /* For subsequent QPs, XID alignment is not mandatory. */
+ rqp->qpc_info = cqm5_object_qpc_mpt_create(
+ rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz,
+ rqp, qpn, 0, 0);
+ } else if (rqp->qpc_info != NULL) {
+ if (idx >= 0) { //idx >= 0 indicates has emptyremaining of lb qpn bit position
+ rdev->qp_lb_scqn_ctrl.qp[idx] = rqp->qpc_info->xid;
+ ROCE_DEV_INFO(
+ rdev,
+ "RoCE create lb qpn func_id(%d), qpn(%d), idx(%d)",
+ rdev->glb_func_id, rqp->qpc_info->xid, idx);
+ }
+ }
+ mutex_unlock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex);
+ if (rqp->qpc_info == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create qpc mpt by cqm object, func_id(%d), qpn(%u)",
+ rdev->glb_func_id, qpn_tmp);
+ return -ENOMEM;
+ }
+
+ rqp->qpn = rqp->qpc_info->xid;
+ if (roce5_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) {
+ if (rsrq != NULL && ((rsrq->srqn & roce_lb_mask) !=
+ (rqp->qpn & roce_lb_mask))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create qpc LB srqn, func_id(%d), qpn(%u) srqn(%u) mask:0x%x",
+ rdev->glb_func_id, rqp->qpn, rsrq->srqn,
+ roce_lb_mask);
+ }
+ }
+
+ return 0;
+}
+
+static int roce5_create_qp_user_umem_mtt(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ struct ib_pd *ibpd,
+ struct create_qp_cmd *ucmd)
+{
+ u32 npages = 0;
+ int ret;
+
+ if (rqp->shadow != 0) {
+ return 0;
+ }
+ rqp->umem = roce5_umem_get(&rdev->ib_dev, ibpd->uobject->context,
+ ucmd->buf_addr, (size_t)rqp->buf_size, 0, 0);
+ if (IS_ERR(rqp->umem)) {
+ ROCE_DEV_ERR(rdev, "Failed to get ib_umem, func_id(%d)",
+ rdev->glb_func_id);
+ ret = (int)PTR_ERR(rqp->umem);
+ return ret;
+ }
+
+ rqp->buf.page_shift = (int)roce5_umem_get_buf_page_shift(
+ rdev, rqp->umem, ucmd->buf_addr);
+ npages = roce5_umem_get_mtt_npages(rdev, rqp->umem,
+ (u32)rqp->buf.page_shift);
+
+ rqp->mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)rqp->buf.page_shift,
+ &rqp->mtt, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc rdma_mtt, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_umem;
+ }
+
+ ret = roce5_umem_write_mtt(rdev, &rqp->mtt, rqp->umem, 1);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_write_mtt;
+ }
+
+ return 0;
+
+err_write_mtt:
+ hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE);
+err_umem:
+ ib_umem_release(rqp->umem);
+ return ret;
+}
+
+void roce5_destroy_qp_user_umem_mtt(struct roce5_device *rdev,
+ struct roce5_qp *rqp)
+{
+ if (rqp->shadow != 0) {
+ return;
+ }
+ hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE);
+ ib_umem_release(rqp->umem);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_qp_user
+ Description : create qp resource for user space qp
+ Input : struct roce5_device *rdev
+ struct ib_pd *ibpd
+ struct ib_qp_init_attr *init_attr
+ struct ib_udata *udata
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2016/01/30
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_qp_user(struct roce5_device *rdev, struct ib_pd *ibpd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata, struct roce5_qp *rqp)
+{
+ int ret = 0;
+ struct roce5_ucontext *roce5_uctx = NULL;
+ struct create_qp_cmd ucmd = { 0 };
+ u32 qpn = ROCE_QP_INVLID_QP_NUM;
+
+ roce5_uctx = to_roce5_ucontext(ibpd->uobject->context);
+
+ ret = roce5_check_qp_udata(rdev, init_attr, udata, rqp, &ucmd);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = roce5_create_qp_user_umem_mtt(rdev, rqp, ibpd, &ucmd);
+ if (ret != 0) {
+ return ret;
+ }
+
+ if (init_attr->qp_type != IB_QPT_XRC_TGT) {
+ ret = roce5_db_map_user(roce5_uctx, ucmd.db_addr, &rqp->db,
+ rqp->shadow);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to map db page to user, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_map_db;
+ }
+ }
+
+ ret = roce5_alloc_qpc(rqp, rdev, qpn, init_attr);
+ if (ret != 0) {
+ goto err_alloc_qpc;
+ }
+
+ return 0;
+
+err_alloc_qpc:
+ if (init_attr->qp_type != IB_QPT_XRC_TGT) {
+ roce5_db_unmap_user(roce5_uctx, &rqp->db, rqp->shadow);
+ }
+
+err_map_db:
+ roce5_destroy_qp_user_umem_mtt(rdev, rqp);
+
+ return ret;
+}
+
+static void roce5_free_wrid(struct roce5_qp *rqp)
+{
+ if (rqp->rq.wrid) {
+ kvfree(rqp->rq.wrid);
+ }
+
+ if (rqp->sq.wrid) {
+ kvfree(rqp->sq.wrid);
+ }
+}
+
+static int roce5_get_wrid(struct roce5_qp *rqp, struct roce5_device *rdev)
+{
+ int ret = 0;
+
+ rqp->sq.wrid = (u64 *)kzalloc((unsigned)rqp->sq.wqebb_cnt * sizeof(u64),
+ GFP_KERNEL);
+ if (rqp->sq.wrid == NULL) {
+ rqp->sq.wrid = (u64 *)(void *)vzalloc(
+ (size_t)(rqp->sq.wqebb_cnt * sizeof(u64)));
+ }
+
+ rqp->rq.wrid = (u64 *)kzalloc((unsigned)rqp->rq.wqebb_cnt * sizeof(u64),
+ GFP_KERNEL);
+ if (rqp->rq.wrid == NULL) {
+ rqp->rq.wrid = (u64 *)(void *)vzalloc(
+ (size_t)(rqp->rq.wqebb_cnt * sizeof(u64)));
+ }
+
+ if ((rqp->sq.wrid == NULL) || (rqp->rq.wrid == NULL)) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc wrid of sq or rq, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -ENOMEM;
+ }
+
+ return ret;
+}
+
+static void roce5_create_qp_kernel_set_attr(struct roce5_qp *rqp,
+ cqm_queue_s *qp_buf_info)
+{
+ int ret;
+ ret = memset_s(qp_buf_info->q_room_buf_1.direct.va,
+ (size_t)((unsigned)rqp->buf_size), 0,
+ (size_t)((unsigned)rqp->buf_size));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset qp_buf_info.(ret:%d)", ret);
+ return;
+ }
+ rqp->buf.direct.buf = qp_buf_info->q_room_buf_1.direct.va;
+ rqp->buf.direct.map = qp_buf_info->q_room_buf_1.direct.pa;
+ rqp->buf.page_shift =
+ (int)ROCE_ILOG2(qp_buf_info->q_room_buf_1.buf_size);
+
+ roce5_sq_buf_init(rqp);
+
+ rqp->db.db_record = (__be32 *)(void *)(&qp_buf_info->q_header_vaddr
+ ->doorbell_record);
+ rqp->db.dma = qp_buf_info->q_header_paddr;
+
+ qp_buf_info->q_header_vaddr->doorbell_record = 0;
+
+ rqp->qp_buf_info = qp_buf_info;
+
+ rqp->sq_head_addr = (u8 *)roce5_get_send_wqe_head(rqp);
+ rqp->sq_tail_addr = (u8 *)roce5_get_send_wqe_tail(rqp);
+ rqp->max_dwqe_size = 0;
+ rqp->sq_wqe_ssn = 0;
+
+ if (rqp->max_inline_data != 0) {
+ rqp->max_dwqe_size = ROCE_QP_MAX_DWQE_SIZE;
+ }
+}
+
+static int roce5_create_qp_kernel_normal(struct roce5_device *rdev,
+ struct roce5_qp *rqp)
+{
+ int ret;
+ cqm_queue_s *qp_buf_info = NULL;
+
+ qp_buf_info = cqm5_object_rdma_queue_create(
+ rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_QP,
+ (u32)rqp->buf_size, rqp, true, ROCE_QP_INVLID_QP_NUM, 0, 0);
+ if (qp_buf_info == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create rdma queue by cqm object, func_id(%d)",
+ rdev->glb_func_id);
+ return -1;
+ }
+
+ roce5_create_qp_kernel_set_attr(rqp, qp_buf_info);
+
+ rqp->mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, qp_buf_info->q_room_buf_1.buf_number,
+ (u32)rqp->buf.page_shift, &rqp->mtt,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_alloc_mtt;
+ }
+ ret = roce5_buf_write_mtt(rdev, &rqp->mtt,
+ &rqp->qp_buf_info->q_room_buf_1);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_write_mtt;
+ }
+
+ ret = roce5_get_wrid(rqp, rdev);
+ if (ret != 0) {
+ goto err_alloc_wrid;
+ }
+
+ return 0;
+
+err_alloc_wrid:
+ roce5_free_wrid(rqp);
+
+err_write_mtt:
+ hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE);
+
+err_alloc_mtt:
+ cqm5_object_delete(&rqp->qp_buf_info->object);
+
+ return ret;
+}
+
+static void roce5_destroy_qp_kernel_normal(struct roce5_device *rdev,
+ struct roce5_qp *rqp)
+{
+ roce5_free_wrid(rqp);
+ hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE);
+ cqm5_object_delete(&rqp->qp_buf_info->object);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_qp_kernel
+ Description : create qp resource for kernel space qp
+ Input : struct roce5_device *rdev
+ struct ib_qp_init_attr *init_attr
+ struct roce5_qp *rqp
+ u32 qpn
+ Output : None
+
+ 1.Date : 2016/01/30
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_qp_kernel(struct roce5_device *rdev,
+ struct ib_qp_init_attr *init_attr,
+ struct roce5_qp *rqp, u32 qpn)
+{
+ int ret = 0;
+
+ ret = roce5_set_kernel_sq_size(rdev, &init_attr->cap,
+ (enum ib_qp_type)rqp->qp_type, rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set sq buffer size in kernel, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ ret = roce5_create_qp_kernel_normal(rdev, rqp);
+ if (ret != 0) {
+ return ret;
+ }
+ }
+
+ ret = roce5_alloc_qpc(rqp, rdev, qpn, init_attr);
+ if (ret != 0) {
+ goto err_alloc_qpc;
+ }
+
+ return 0;
+
+err_alloc_qpc:
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ roce5_destroy_qp_kernel_normal(rdev, rqp);
+ }
+
+ return ret;
+}
+
+static void roce5_qp_add_cq_list(struct ib_qp_init_attr *init_attr,
+ struct roce5_qp *rqp,
+ struct roce5_device *rdev)
+{
+ struct roce5_cq *send_rcq = to_roce5_cq(init_attr->send_cq);
+ struct roce5_cq *recv_rcq = to_roce5_cq(init_attr->recv_cq);
+ unsigned long flags = 0; /* type required by kernel API */
+
+ spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags);
+ roce5_lock_cqs(send_rcq, recv_rcq);
+ list_add_tail(&rqp->qps_list, &rdev->qp_list);
+ list_add_tail(&rqp->cq_send_list, &send_rcq->send_qp_list);
+ list_add_tail(&rqp->cq_recv_list, &recv_rcq->recv_qp_list);
+ roce5_unlock_cqs(send_rcq, recv_rcq);
+ spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags);
+}
+
+static int roce5_alloc_xrc_extra_dma(const struct roce5_device *rdev,
+ struct roce5_qp *rqp)
+{
+ dma_addr_t wb_dma = 0;
+ void *wb_va = NULL;
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ return 0;
+ }
+
+ wb_va = dma_alloc_coherent(rdev->dev, (size_t)4UL, &wb_dma, GFP_KERNEL);
+ if (wb_va == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc wb_va, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ rqp->wb_dma = wb_dma;
+ rqp->wb_va = wb_va;
+
+ return 0;
+}
+
+#define ROCE_MAX_PI_ON_CHIP_QPN 1024 /* temporarily when set PI is 1024(count word), later debug add */
+static int roce5_set_pi_on_chip(const struct roce5_device *rdev,
+ struct roce5_qp *rqp)
+{
+ /* shadow scenario next, actual use of is vf, and pi on chip function in shadow scenario not already verify, default disable */
+ if (ROCE5_SUPPORT_SET_PI_ON_CHIP_MSG_V2(rdev) != 0 &&
+ rqp->shadow == 0) {
+ /* becauseslice on store pi resource has limit, disable VF of pi_on_chip, such as need enabled, need modify configuration file in chip store store PI data, Note
审monitorslice on resource whether satisfy requirement */
+ rqp->pi_on_chip = (roce5_func_is_vf(rdev->hwdev) == false) &&
+ (rqp->qpn < ROCE_MAX_PI_ON_CHIP_QPN);
+ }
+
+ return 0;
+}
+
+static int roce5_do_create_qp(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_pd *ibpd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+
+ mutex_init(&rqp->mutex);
+
+ spin_lock_init(&rqp->sq.lock);
+ spin_lock_init(&rqp->rq.lock);
+
+ rqp->qp_state = IB_QPS_RESET;
+ rqp->qp_type = init_attr->qp_type;
+ rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);
+
+ ret = roce5_alloc_xrc_extra_dma(rdev, rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc xrc extra dma, func_id(%d), ret(%d).",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ if (ibpd != NULL && ibpd->uobject) {
+ ret = roce5_set_rq_size(rdev, rqp, init_attr, true,
+ rqp->has_rq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set rq size, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto create_qp_err;
+ }
+
+ ret = roce5_create_qp_user(rdev, ibpd, init_attr, udata, rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create qp user, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto create_qp_err;
+ }
+ } else {
+ ret = roce5_set_rq_size(rdev, rqp, init_attr, false,
+ rqp->has_rq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set rq size, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto create_qp_err;
+ }
+
+ ret = roce5_create_qp_kernel(rdev, init_attr, rqp,
+ ROCE_QP_INVLID_QP_NUM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create qp kernel, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto create_qp_err;
+ }
+
+ roce5_qp_add_cq_list(init_attr, rqp, rdev);
+ }
+
+ ret = roce5_set_pi_on_chip(rdev, rqp);
+ if (ret != 0) {
+ goto create_qp_err;
+ }
+
+ ret = roce5_create_qp_ext_init(rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init qp extension, ret(%d).",
+ ret);
+ goto create_qp_err;
+ }
+
+ return ret;
+create_qp_err:
+ if (rqp->wb_va != NULL) {
+ dma_free_coherent(rdev->dev, sizeof(u32), rqp->wb_va,
+ rqp->wb_dma);
+ }
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_qp_gsi
+ Description : roce5_create_qp_gsi
+ Input : struct roce5_device *rdev
+ struct ib_pd *ibpd
+ struct ib_qp_init_attr *init_attr
+ Output : None
+
+ 1.Date : 2017/04/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_qp_gsi(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_init_attr *init_attr)
+{
+ int ret = 0;
+
+ rqp->sqp_attr = (struct roce5_sqp_attr *)kzalloc(
+ sizeof(struct roce5_sqp_attr), GFP_KERNEL);
+ if (rqp->sqp_attr == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to allocate memory for sqp attribute.");
+ return -ENOMEM;
+ }
+
+ mutex_init(&rqp->mutex);
+
+ spin_lock_init(&rqp->sq.lock);
+ spin_lock_init(&rqp->rq.lock);
+
+ rqp->qp_state = (u8)IB_QPS_RESET;
+ rqp->qp_type = init_attr->qp_type;
+ rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);
+ rqp->has_rq = !init_attr->srq;
+
+ ret = roce5_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set rq size, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_out;
+ }
+
+ ret = roce5_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_GSI_QP_NUM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to create qp kernel, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_out;
+ }
+
+ rqp->qpn = ROCE_QP_GSI_QP_NUM;
+
+ roce5_qp_add_cq_list(init_attr, rqp, rdev);
+
+ return 0;
+
+err_out:
+ kfree(rqp->sqp_attr);
+
+ return ret;
+}
+
+static int roce5_create_check_init_attr(const struct ib_pd *ibpd,
+ const struct ib_qp_init_attr *init_attr,
+ const struct ib_udata *udata)
+{
+ if (init_attr == NULL) {
+ ROCE_ERR("init_attr is null");
+ return (-EINVAL);
+ }
+
+ if ((ibpd == NULL) && (init_attr->qp_type != IB_QPT_XRC_TGT)) {
+ ROCE_ERR(
+ "Ibpd is null and qp_type is not IB_QPT_XRC_TGT, qp_type(0x%x)",
+ init_attr->qp_type);
+ return (-EINVAL);
+ }
+
+ if ((udata) && (init_attr->qp_type == IB_QPT_GSI)) {
+ ROCE_ERR("Udata is not null and qp_type is IB_QPT_GSI");
+ return (-EINVAL);
+ }
+
+ return 0;
+}
+
+static int roce5_set_qp_attr(struct ib_qp_init_attr *init_attr,
+ struct roce5_qp *rqp, struct roce5_device *rdev,
+ struct ib_pd **ibpd, int qp_ext)
+{
+ u16 xrcdn = 0;
+
+ rqp->qp_ext = (u8)qp_ext;
+ if (init_attr->qp_type == IB_QPT_XRC_TGT) {
+ *ibpd = to_roce5_xrcd(init_attr->xrcd)->pd;
+ xrcdn = (u16)to_roce5_xrcd(init_attr->xrcd)->xrcdn;
+ init_attr->send_cq = to_roce5_xrcd(init_attr->xrcd)->cq;
+ init_attr->recv_cq = init_attr->send_cq;
+
+ rqp->has_rq = false;
+ rqp->xrcdn = xrcdn;
+ } else if (init_attr->qp_type == IB_QPT_XRC_INI) {
+ init_attr->recv_cq = init_attr->send_cq;
+ rqp->has_rq = false;
+ } else if ((init_attr->qp_type == IB_QPT_RC) ||
+ (init_attr->qp_type == IB_QPT_UC) ||
+ (init_attr->qp_type == IB_QPT_UD)) {
+ rqp->has_rq = !init_attr->srq;
+ } else {
+ ROCE_DEV_ERR(rdev, "Unknown QP Type(%u), func_id(%d)",
+ init_attr->qp_type, rdev->glb_func_id);
+ return (-EINVAL);
+ }
+
+ return 0;
+}
+
+static int roce5_create_gsi_qp_result(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ struct ib_qp_init_attr *init_attr)
+{
+ int ret;
+
+ ret = roce5_create_qp_gsi(rdev, rqp, init_attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to create gsi qp, func_id(%d), ret(%d).",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ rqp->ibqp.qp_num = rqp->qpn;
+ return 0;
+}
+
+static int roce5_create_qp_common(struct roce5_qp *rqp,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata, int qp_ext)
+{
+ int ret;
+ int uld_ret;
+ //when init_attr->qp_type == IB_QPT_XRC_TGT, ibpd is NULL.
+ struct ib_pd *ibpd = rqp->ibqp.pd;
+ struct roce5_device *rdev =
+ to_roce5_dev(ibpd ? ibpd->device : init_attr->xrcd->device);
+ roce_user_handle_t hdl = { 0 };
+
+ if (rdev->ecn_ctx.cc_algo == ROCE_CC_LDCP_ALGO &&
+ (init_attr->qp_type == IB_QPT_XRC_TGT ||
+ init_attr->qp_type == IB_QPT_XRC_INI)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "XRC QP cant be created when LDCP is enabled, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ if (init_attr->qp_type == IB_QPT_GSI) {
+ return roce5_create_gsi_qp_result(rdev, rqp, init_attr);
+ }
+
+ rqp->rqp_magic = ROCE_RQP_MAGIC;
+ ret = roce5_set_qp_attr(init_attr, rqp, rdev, &ibpd, qp_ext);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to set qp attribute, ret(%d).", ret);
+ goto err_set_attr;
+ }
+
+ ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, NULL, udata, &rqp->mtt,
+ &rqp->db.dma);
+ ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rqp->shadow, &rqp->shadow_vfid,
+ NULL);
+ ret = roce5_uld_qp_create((roce_uld_handle)&hdl);
+ if (ret != 0) {
+ ROCE_ERR("Failed to execute uld qp creation, ret(%d)", ret);
+ goto err_set_attr;
+ }
+ ret = roce5_do_create_qp(rdev, rqp, ibpd, init_attr, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to create qp in common process, ret(%d).",
+ ret);
+ goto err_create_qp;
+ }
+
+ roce5_set_qp_dif_attr(rqp, init_attr, rdev);
+ rqp->ibqp.qp_num = rqp->qpn;
+ rqp->rsp_depth = 0;
+
+ mutex_lock(&rdev->qp_cnt.cur_qps_mutex);
+ rdev->qp_cnt.alloc_qp_cnt++;
+ mutex_unlock(&rdev->qp_cnt.cur_qps_mutex);
+
+ return 0;
+
+err_create_qp:
+ uld_ret = roce5_uld_qp_destroy((roce_uld_handle)&hdl);
+ if (uld_ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to destroy uld qp in common process, uld_ret(%d).",
+ uld_ret);
+ }
+err_set_attr:
+ return ret;
+}
+
+struct ib_qp *roce5_create_qp_compat(struct ib_pd *ibpd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ struct roce5_qp *rqp = NULL;
+ int ret;
+
+ ret = roce5_create_check_init_attr(ibpd, init_attr, udata);
+ if (ret != 0) {
+ ROCE_ERR("Failed to check init attr");
+ return (struct ib_qp *)ERR_PTR((long)ret);
+ }
+
+ rqp = (struct roce5_qp *)kzalloc(sizeof(*rqp), GFP_KERNEL);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to alloc roce qp.");
+ return (struct ib_qp *)ERR_PTR((long)-ENOMEM);
+ }
+
+ rqp->ibqp.pd = ibpd;
+
+ ret = roce5_create_qp_common(rqp, init_attr, udata, ROCE_QP);
+ if (ret != 0) {
+ ROCE_ERR("Failed to execute common qp creation, ret(%d)", ret);
+ goto err_create_qp;
+ }
+
+ return &(rqp->ibqp);
+err_create_qp:
+ kfree(rqp);
+ return (struct ib_qp *)ERR_PTR((long)-ENOMEM);
+}
+
+int roce5_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_qp *rqp = NULL;
+
+ if (ibqp == NULL) {
+ ROCE_ERR("Invalid parameter, ibqp is NULL.");
+ return -EINVAL;
+ }
+
+ ret = roce5_create_check_init_attr(ibqp->pd, init_attr, udata);
+ if (ret != 0) {
+ ROCE_ERR("Failed to check init attr");
+ return ret;
+ }
+
+ rqp = to_roce5_qp(ibqp);
+ ret = roce5_create_qp_common(rqp, init_attr, udata, ROCE_QP);
+ if (ret != 0) {
+ ROCE_ERR("Failed to execute common qp creation, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h
new file mode 100644
index 000000000..c435e7090
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h
@@ -0,0 +1,278 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_QP_CTX_FOMRAT_H
+#define ROCE_QP_CTX_FOMRAT_H
+
+#include <linux/types.h>
+
+/* Align each field with 4bytes. */
+#pragma pack(push)
+#pragma pack(4)
+
+struct roce5_chip_seg_rrwc {
+ u32 dw0;
+
+ union {
+ u64 rrw_curt_sge_va;
+ struct {
+ u32 rrw_curt_sge_va_hi;
+ u32 rrw_curt_sge_va_lo;
+ } dw1;
+ };
+
+ u32 rsvd[5];
+};
+
+struct roce5_chip_seg_rcc {
+ u32 dw0;
+
+ union {
+ u64 rc_curt_sge_va;
+ struct {
+ u32 rc_curt_sge_va_hi;
+ u32 rc_curt_sge_va_lo;
+ } dw1;
+ };
+
+ u32 rsvd[5];
+};
+
+struct roce5_chip_seg_sqc {
+ u32 rsvd[9];
+
+ union {
+ u64 sq_curt_sge_va;
+ struct {
+ u32 sq_curt_sge_va_hi;
+ u32 sq_curt_sge_va_lo;
+ } dw9;
+ };
+
+ u32 rsvd1[5];
+};
+
+struct roce5_chip_seg_sqac {
+ u32 rsvd[9];
+
+ union {
+ u64 sqa_curt_sge_va;
+ struct {
+ u32 sqa_curt_sge_va_hi;
+ u32 sqa_curt_sge_va_lo;
+ } dw9;
+ };
+
+ u32 rsvd1[5];
+};
+
+struct roce5_chip_seg_rqc {
+ u32 rsvd[9];
+
+ union {
+ u64 rq_curt_sge_va;
+ struct {
+ u32 rq_curt_sge_va_hi;
+ u32 rq_curt_sge_va_lo;
+ } dw9;
+ };
+
+ u32 rsvd1[5];
+};
+
+struct roce5_chip_seg_srqc {
+ u32 rsvd[9];
+
+ union {
+ u64 srq_curt_sge_va;
+ struct {
+ u32 srq_curt_sge_va_hi;
+ u32 srq_curt_sge_va_lo;
+ } dw9;
+ };
+
+ u32 rsvd1[5];
+};
+
+struct roce5_chip_seg_qpcc {
+ union {
+ u64 sq_rq_l0mtt_gpa;
+ struct {
+ u32 sq_rq_l0mtt_gpa_hi;
+ u32 sq_rq_l0mtt_gpa_lo;
+ } dw0;
+ };
+
+ union {
+ u64 sq_rq_pi_record_gpa_at_hop_num;
+ struct {
+ u32 sq_rq_pi_record_gpa_hi;
+ u32 sq_rq_pi_record_gpa_lo_at_hop_num;
+ } dw2;
+ };
+
+ u32 dw4;
+
+ union {
+ struct {
+ u32 rsvd1 : 22;
+ u32 qp_rre : 1;
+ u32 qp_rae : 1;
+ u32 qp_rwe : 1;
+ u32 rsvd : 7;
+ } bs;
+ u32 value;
+ } dw5;
+
+ u32 rsvd[2];
+};
+
+typedef struct roce5_qpc_chip_seg {
+ struct roce5_chip_seg_rrwc rrwc;
+ struct roce5_chip_seg_rcc rcc;
+ struct roce5_chip_seg_sqc sqc;
+ struct roce5_chip_seg_sqac sqac;
+ union {
+ struct roce5_chip_seg_rqc rqc;
+ struct roce5_chip_seg_srqc srqc;
+ };
+ struct roce5_chip_seg_qpcc qpcc;
+} roce5_qpc_chip_seg_s;
+
+struct roce5_qpc_timer_seg {
+ u32 rsvd[8];
+};
+
+typedef struct roce5_qpc_drv_seg {
+ union {
+ struct {
+ u32 dest_qp : 24;
+ u32 state : 4;
+ u32 rsvd : 4;
+ } bs;
+ u32 value;
+ } dw0;
+
+ u32 dw1;
+
+ union {
+ struct {
+ u32 pmtu : 3;
+ u32 rsvd : 13;
+ u32 to_retry_limit : 3;
+ u32 rnr_retry_limit : 3;
+ u32 min_rnr_nak : 5;
+ u32 ack_to : 5;
+ } bs;
+ u32 value;
+ } dw2;
+
+ union {
+ struct {
+ u32 rsvd : 26;
+ u32 rra_max : 3;
+ u32 sra_max : 3;
+ } bs;
+ u32 value;
+ } dw3;
+} roce5_qpc_drv_seg_s;
+
+typedef struct roce5_drv_path_info {
+ u32 rsvd[2];
+
+ u8 dgid[16];
+
+ union {
+ struct {
+ u32 flow_label : 20;
+ u32 tclass : 8;
+ u32 rsvd : 4;
+ } bs;
+ u32 value;
+ } dw6;
+
+ union {
+ struct {
+ u32 hoplmt : 8;
+ u32 sgid_index : 7;
+ u32 rsvd : 14;
+ u32 sl : 3;
+ } bs;
+ u32 value;
+ } dw7;
+} roce5_drv_path_info_s;
+
+typedef struct roce5_ucode_common_ctx {
+ union {
+ struct {
+ u32 rsvd1 : 20;
+ u32 port : 2;
+ u32 rsvd : 10;
+ } bs;
+ u32 value;
+ } dw0;
+
+ u32 dw1;
+
+ union {
+ u32 qkey;
+ u32 value;
+ } dw2;
+
+ u32 dw3;
+} roce5_ucode_common_ctx_s;
+
+typedef struct roce5_ucode_sq_ctx {
+ union {
+ struct {
+ u32 next_send_psn : 24;
+ u32 rsvd : 8;
+ } bs;
+ u32 value;
+ } dw8;
+
+ u32 rsvd[12];
+} roce5_ucode_sq_ctx_s;
+
+typedef struct roce5_ucode_rq_ctx {
+ union {
+ struct {
+ u32 next_rcv_psn : 24;
+ u32 rsvd : 8;
+ } bs;
+ u32 value;
+ } dw20;
+
+ u32 rsvd[9];
+} roce5_ucode_rq_ctx_s;
+
+typedef struct roce5_drv_ucode_all_info {
+ /* DW0~DW3 */
+ roce5_ucode_common_ctx_s common;
+
+ /* DW4~DW16 */
+ roce5_ucode_sq_ctx_s sq_ctx;
+
+ /* DW17~DW26 */
+ roce5_ucode_rq_ctx_s rq_ctx;
+
+ u32 rsvd;
+} roce5_drv_ucode_all_info_s;
+
+typedef struct roce5_qpc_sw_seg {
+ roce5_qpc_drv_seg_s drv_seg;
+ roce5_drv_path_info_s path_seg;
+ roce5_drv_ucode_all_info_s ucode_seg;
+ u32 rsvd[8];
+} roce5_qpc_sw_seg_s;
+
+typedef struct roce5_qp_context {
+ struct roce5_qpc_chip_seg chip_seg;
+ struct roce5_qpc_timer_seg timer_seg;
+ struct roce5_qpc_sw_seg sw_seg;
+} roce5_qp_context_s;
+
+#pragma pack(pop)
+
+#endif /* ROCE_QP_CTX_FOMRAT_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c
new file mode 100644
index 000000000..71a77ada0
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c
@@ -0,0 +1,342 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "roce_compat.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_mr.h"
+#include "roce_xrc.h"
+#include "roce_pd.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_qp.h"
+#include "roce_qp_ext.h"
+#include "roce_qp_extension.h"
+#include "roce_restore.h"
+#ifdef __ROCE_DFX__
+#include "roce_dfx.h"
+#endif
+#include "hinic5_hmm.h"
+#include "securec.h"
+#include "roce_os_compat.h"
+#include "roce_uld_inner.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_qp_inner.h"
+
+static void roce5_clear_lb_qpn(struct roce5_device *rdev, u32 qpn)
+{
+ int idx = 0;
+ mutex_lock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex);
+ for (; idx < ROCE_LB_QPN_MAX; idx++) {
+ if (rdev->qp_lb_scqn_ctrl.qp[idx] == qpn) {
+ rdev->qp_lb_scqn_ctrl.qp[idx] = 0;
+ ROCE_DEV_INFO(
+ rdev,
+ "RoCE destroy lb qpn func_id(%d), qpn(%d), idx(%d)",
+ rdev->glb_func_id, qpn, idx);
+ break;
+ }
+ }
+ mutex_unlock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex);
+}
+
+static void roce5_clean_qp_user(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_udata *udata)
+{
+ struct roce5_ucontext *ucontext = NULL;
+
+ roce5_free_opt_rdmarc(rqp);
+
+ cqm5_object_delete(&(rqp->qpc_info->object));
+
+ roce5_clear_lb_qpn(rdev, rqp->qpn);
+
+ roce5_destroy_pcqc_ext(rqp);
+
+ ucontext = roce5_queue_to_ucontext((void *)(&rqp->ibqp), udata,
+ ROCE_Q_TYPE_QP);
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ roce5_db_unmap_user(ucontext, &rqp->db, rqp->shadow);
+ }
+
+ roce5_destroy_qp_user_umem_mtt(rdev, rqp);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_cq_clean_process
+ Description : roce5_cq_clean_process
+ Input : struct roce5_cq *cq
+ u32 qpn
+ struct roce5_srq *srq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_cq_clean_process(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq)
+{
+ u32 prod_index = 0;
+ int nfreed = 0;
+ struct roce_cqe *cqe = NULL;
+ struct roce_cqe *dest = NULL;
+ u8 owner_bit = 0;
+ int ret;
+
+ /*
+ * First we need to find the current producer index, so we
+ * know where to start cleaning from. It doesn't matter if HW
+ * adds new entries after this loop -- the QP we're worried
+ * about is already in RESET, so the new entries won't come
+ * from our QP and therefore don't need to be checked.
+ */
+ for (prod_index = cq->cons_index; roce5_get_sw_cqe(cq, prod_index);
+ ++prod_index) {
+ if (prod_index == (u32)((int)cq->cons_index + cq->ibcq.cqe)) {
+ break;
+ }
+ }
+
+ /*
+ * Now sweep backwards through the CQ, removing CQ entries
+ * that match our QP by copying older entries on top of them.
+ */
+ /* xcqe:qpn's cqe.
+ CI
+ +--------+------+------+------+----------+
+ b4_clean: | | cqe1 | xcqe2| cqe3 | |
+ +--------+------+------+------+----------+
+ \
+ \
+ +---------------+------+------+----------+
+ af_clean: | | cqe1 | cqe3 | |
+ +---------------+------+------+----------+
+ */
+ while ((int)--prod_index - (int)cq->cons_index >= 0) {
+ cqe = (struct roce_cqe *)roce5_get_cqe(
+ cq, prod_index & (u32)cq->cqe_mask);
+
+ cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value);
+ cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value);
+ cqe->dw7.value = roce5_convert_cpu32(cqe->dw7.value);
+
+ if (cqe->dw0.bs.qpn == qpn) {
+ if (srq && (cqe->dw1.bs.s_r == ROCE_CQE_RECV_COMP)) {
+ roce5_free_srq_wqe(srq, cqe->dw7.bs.wqe_cnt);
+ }
+
+ ++nfreed;
+ } else if (nfreed != 0) {
+ dest = (struct roce_cqe *)roce5_get_cqe(
+ cq, (unsigned int)((int)prod_index + nfreed) &
+ (unsigned int)cq->cqe_mask);
+ dest->dw0.value = roce5_convert_cpu32(dest->dw0.value);
+ owner_bit = dest->dw0.bs.owner;
+ ret = memcpy_s((void *)dest, sizeof(struct roce_cqe),
+ (void *)cqe, sizeof(struct roce_cqe));
+ if (ret != 0) {
+ ROCE_ERR("[Failed to memcpy dest.(ret:%d)",
+ ret);
+ return;
+ }
+ dest->dw0.bs.owner = owner_bit;
+ dest->dw0.value = roce5_convert_be32(dest->dw0.value);
+ dest->dw1.value = roce5_convert_be32(dest->dw1.value);
+ dest->dw7.value = roce5_convert_be32(dest->dw7.value);
+ } else {
+ ROCE_INFO("Nothing need to do");
+ }
+
+ cqe->dw0.value = roce5_convert_be32(cqe->dw0.value);
+ cqe->dw1.value = roce5_convert_be32(cqe->dw1.value);
+ cqe->dw7.value = roce5_convert_be32(cqe->dw7.value);
+ }
+
+ if (nfreed != 0) {
+ cq->cons_index += (u32)nfreed;
+ /*
+ * Make sure update of buffer contents is done before
+ * updating consumer index.
+ */
+ wmb();
+
+ roce5_cq_set_ci(cq);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_clean_qp_kernel
+ Description : roce5_clean_qp_kernel
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2017/04/26
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_clean_qp_kernel(struct roce5_device *rdev,
+ struct roce5_qp *rqp)
+{
+ struct roce5_cq *send_cq = NULL;
+ struct roce5_cq *recv_cq = NULL;
+ unsigned long flags = 0; /* type required by kernel API */
+
+ roce5_free_opt_rdmarc(rqp);
+
+ roce5_get_cqs(rqp, &send_cq, &recv_cq);
+
+ spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags);
+ roce5_lock_cqs(send_cq, recv_cq);
+ /* del from lists under both locks above to protect reset flow paths */
+ list_del(&rqp->qps_list);
+ list_del(&rqp->cq_send_list);
+ list_del(&rqp->cq_recv_list);
+
+ roce5_cq_clean_process(recv_cq, rqp->qpn,
+ rqp->ibqp.srq ? to_roce5_srq(rqp->ibqp.srq) :
+ NULL);
+
+ if (send_cq != recv_cq) {
+ roce5_cq_clean_process(send_cq, rqp->qpn, NULL);
+ }
+
+ roce5_unlock_cqs(send_cq, recv_cq);
+ spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags);
+
+ cqm5_object_delete(&(rqp->qpc_info->object));
+
+ roce5_clear_lb_qpn(rdev, rqp->qpn);
+
+ if (rqp->sq.wrid) {
+ kvfree(rqp->sq.wrid);
+ rqp->sq.wrid = NULL;
+ }
+
+ if (rqp->rq.wrid) {
+ kvfree(rqp->rq.wrid);
+ rqp->rq.wrid = NULL;
+ }
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ cqm5_object_delete(&(rqp->qp_buf_info->object));
+ }
+
+ hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE);
+}
+
+static int roce5_qp_destroy(struct roce5_qp *rqp, struct roce5_device *rdev)
+{
+ int ret = 0;
+
+ ret = roce5_qp_modify_2rst_cmd(rdev, rqp->qpn);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify QP(0x%06x) to RESET, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = roce5_qp_cache_out_cmd(rdev, rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "QP(0x%06x) cache invalid, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ return ret;
+ }
+
+ return 0;
+}
+
+int roce5_destroy_qp_common(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_pd *pd = roce5_get_pd(rqp);
+ roce_user_handle_t hdl = { 0 };
+
+ roce5_destroy_qp_ext_deinit(rqp);
+
+ if (rqp->wb_va != NULL) {
+ dma_free_coherent(rdev->dev, sizeof(u32), rqp->wb_va,
+ rqp->wb_dma);
+ rqp->wb_va = NULL;
+ }
+
+ if (rqp->qp_state != IB_QPS_RESET) {
+ ret = roce5_qp_destroy(rqp, rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to destroy qp, ret(%d).",
+ ret);
+ return ret;
+ }
+
+ rqp->qp_state = IB_QPS_RESET;
+ }
+
+ if (pd->ibpd.uobject) {
+ roce5_clean_qp_user(rdev, rqp, udata);
+ } else {
+ roce5_clean_qp_kernel(rdev, rqp);
+
+ if (roce5_sqp_check(rqp) != 0 && rqp->sqp_attr != NULL) {
+ kfree(rqp->sqp_attr);
+ rqp->sqp_attr = NULL;
+ }
+ }
+
+ ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, NULL, NULL, NULL, NULL);
+ ret = roce5_uld_qp_destroy((roce_uld_handle)&hdl);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to destroy uld qp, ret(%d).", ret);
+ return ret;
+ }
+
+ mutex_lock(&rdev->qp_cnt.cur_qps_mutex);
+ rdev->qp_cnt.del_qp_cnt++;
+ mutex_unlock(&rdev->qp_cnt.cur_qps_mutex);
+ return 0;
+}
+
+int roce5_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
+{
+ struct roce5_qp *rqp = NULL;
+ struct roce5_device *rdev = NULL;
+ int ret;
+
+ if (ibqp == NULL) {
+ ROCE_ERR("Ibqp is null");
+ return -EINVAL;
+ }
+
+ rqp = to_roce5_qp(ibqp);
+ rdev = to_roce5_dev(ibqp->device);
+
+ ret = roce5_destroy_qp_common(rdev, rqp, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Error occur when destroy qp! ret(%d).",
+ ret);
+ return ret;
+ }
+
+#if !defined(IB_DEVICE_OPS_HAVE_IB_QP_OBJ)
+ kfree(rqp);
+#endif
+ return 0;
+}
+
+int roce5_destroy_qp_ofa(struct ib_qp *ibqp)
+{
+ return roce5_destroy_qp(ibqp, NULL);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h
new file mode 100644
index 000000000..646717c2a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h
@@ -0,0 +1,80 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_QP_EXP_H
+#define ROCE_QP_EXP_H
+
+#include <linux/io-mapping.h>
+#include <linux/list.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "hinic5_rdma.h"
+#include "hinic5_cqm.h"
+
+#include "roce.h"
+#include "roce_db.h"
+#include "roce_cq.h"
+#include "roce_qp.h"
+
+#define EXP_OK (0)
+#define EXP_PARAM_ERR (1)
+#define EXP_FUNC_ERR (2)
+
+#define QPC_MIN_SIZE (512)
+#define CONTX_SZ (sizeof(struct roce_qp_context) / QPC_MIN_SIZE)
+
+#define BUFFER_SZ (sizeof(struct roce_qp_context) / sizeof(u32))
+
+#define TYPE_LEN (8)
+#define DEV_NUM (32)
+#define QPN_MAX (0xFFFFF)
+#define QPN_MIN (0)
+#define IB_QP_EXP_CMD (1 << 23)
+#define ROCE_OPTPAR_EXP (1 << 16)
+#define VA_OFFSET (8)
+
+struct roce5_device_list {
+ struct ib_device *ib_dev[DEV_NUM];
+ int ib_dev_num;
+ struct mutex mutex;
+};
+
+struct ib_qp_info {
+ dma_addr_t sq_pg_base_addr[64];
+ dma_addr_t hw_doorbell_addr;
+ dma_addr_t sw_doorbell_addr;
+
+ u32 sq_depth;
+ u32 qpn;
+ u8 sq_pg_cnt;
+ u8 qp_cos;
+ u8 cntx_sz;
+ u8 sq_wqe_size;
+};
+
+struct ib_qp_attr_data {
+ u32 data_type; //plog:1
+ u32 data_len;
+ u8 *data_buf;
+};
+
+struct ib_qp_attr_exp {
+ struct ib_qp_attr qp_attr;
+ struct ib_qp_attr_data qp_attr_data;
+};
+
+struct roce5_device_list *roce5_get_plog_device_info(void);
+
+int ib_get_qp_info(char *dev_name, int qpn, struct ib_qp_info *qp_info);
+struct ib_qp *ib_get_qp(const char *dev_name, int qpn);
+int ib_put_qp(struct ib_qp *ibqp);
+int roce5_qp_modify_exp(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_attr *attr, int attr_mask,
+ enum ib_qp_state cur_state, enum ib_qp_state new_state,
+ u16 vlan_id);
+int roce5_modify_extend_qp(struct ib_qp_attr *attr, int attr_mask,
+ struct ib_qp *ibqp);
+
+#endif //ROCE_QP_EXP_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c
new file mode 100644
index 000000000..a548667ee
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c
@@ -0,0 +1,39 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include "roce_qp.h"
+#include "roce_infra.h"
+#include "roce_obj_ext.h"
+#ifdef ENABLE_HYPER_ROCE
+#include "hyper_roce_qp.h"
+#endif
+
+const struct roce5_ext_reg g_rqp_exts[] = {
+#ifdef ENABLE_HYPER_ROCE
+ {
+ .ext_id = ROCE_EXT_HYPER_ROCE,
+ .data_size = sizeof(struct roce5_qp_hyper_extend),
+ .init_func = roce5_hyper_qp_init,
+ .cleanup_func = roce5_hyper_qp_cleanup,
+ },
+#endif
+};
+
+const u8 g_rqp_exts_num = sizeof(g_rqp_exts) / sizeof(struct roce5_ext_reg);
+
+int roce5_create_qp_ext_init(struct roce5_qp *rqp)
+{
+ int ret = roce5_obj_ext_init((void *)rqp, g_rqp_exts, g_rqp_exts_num,
+ &rqp->obj_ext);
+ if (ret != 0) {
+ ROCE_ERR("Failed to init qp extension, ret(%d).", ret);
+ return ret;
+ }
+ return 0;
+}
+
+void roce5_destroy_qp_ext_deinit(struct roce5_qp *rqp)
+{
+ roce5_obj_ext_deinit((void *)rqp, g_rqp_exts, g_rqp_exts_num,
+ &rqp->obj_ext);
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h
new file mode 100644
index 000000000..8b3e95565
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h
@@ -0,0 +1,19 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_QP_EXT_H
+#define ROCE_QP_EXT_H
+
+#include "roce_qp.h"
+#include "roce_obj_ext.h"
+
+static inline void *roce5_qp_get_ext_buf(struct roce5_qp *rqp,
+ enum ROCE_EXT_ID ext_id)
+{
+ return roce5_obj_ext_get_buf(&rqp->obj_ext, ext_id);
+}
+
+int roce5_create_qp_ext_init(struct roce5_qp *rqp);
+void roce5_destroy_qp_ext_deinit(struct roce5_qp *rqp);
+
+#endif /* ROCE_QP_EXT_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h
new file mode 100644
index 000000000..c757ce2ba
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h
@@ -0,0 +1,17 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+
+#ifndef ROCE_QP_INNER_H
+#define ROCE_QP_INNER_H
+
+#include <rdma/ib_verbs.h>
+#include "roce.h"
+#include "roce_qp.h"
+
+int roce5_destroy_qp_common(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_udata *udata);
+void roce5_destroy_qp_user_umem_mtt(struct roce5_device *rdev,
+ struct roce5_qp *rqp);
+
+#endif /* ROCE_QP_INNER_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c
new file mode 100644
index 000000000..5a6a4ec7b
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c
@@ -0,0 +1,2398 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <rdma/ib_verbs.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+#include <linux/sched.h>
+
+#include "comm_defs.h"
+#include "roce_compat.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_mr.h"
+#include "roce_user.h"
+#include "roce_xrc.h"
+#include "roce_pd.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_cqm_cmd.h"
+#include "roce_qp.h"
+#include "roce_qp_exp.h"
+#include "roce_qp_extension.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "securec.h"
+#include "roce_ctx_format.h"
+#include "roce_npu_cmd.h"
+#include "roce_npu_cmd_qp_defs.h"
+#include "roce_npu_cmd_ext_defs.h"
+#include "roce_cmd.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_uld_inner.h"
+#include "roce_netdev.h"
+#include "roce_os_compat.h"
+
+#ifdef ROCE_ANTI_ATTACK
+#include "roce_anti_attack.h"
+#endif
+
+#ifdef __ROCE_DFX__
+#include "roce_dfx.h"
+#endif
+
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+
+#define ROCE_MTT_GPA_VALID 0x1
+#define ROCE_WQE_PREFETCH_MAXNUM 5 //5: 8(count word) WQEBB
+#define ROCE_SQ_WQEBB_SIZE_SHIFT 4
+#define ROCE_SQ_WQEBB_SIZE_MASK 0x7
+#define ROCE_SQ_PD_MASK 0x3ffff
+#define ROCE_SQ_PAGE_SIZE_MASK 0xf
+#define ROCE_SQ_WQECNT_LTH 4
+#define ROCE_SQ_PREFETCH_MINNUM 2
+#define ROCE_SQ_PREFETCH_MAXNUM 4
+#define ROCE_SQ_SIZE_MASK 0x1f
+#define ROCE_SQ_CQN_MASK 0xfffff
+#define ROCE_RQ_WQECNT_LTH 0xe
+#define ROCE_RQ_MTT_PREFETCH_MAXLEN0 3
+#define ROCE_RQ_MTT_PREFETCH_MAXWQE 7
+#define ROCE_RQ_WQE_PREFETCH_MAXNUM 7
+#define ROCE_SRQ_INIT_STATE 0xf
+#define ROCE_MAX_PI_ON_CHIP_QPN 1024
+
+enum {
+ BIT_LENGTH_IB_MTU_256 = 8,
+ BIT_LENGTH_IB_MTU_512,
+ BIT_LENGTH_IB_MTU_1024,
+ BIT_LENGTH_IB_MTU_2048,
+ BIT_LENGTH_IB_MTU_4096,
+ BIT_LENGTH_IB_MTU_8192,
+};
+
+enum {
+ CACHE_INVLAID_ALLOC,
+ CACHE_INVLAID_CMDQ,
+ CACHE_INVLAID_RDMARC_CHECK,
+ CACHE_INVLAID_QP_CHECK,
+ MODIFY_2RST_CMDQ,
+ CMDQ_SDKCALL_BUTT
+};
+
+#define ROCE_CMDQ_TIMEOUT 1000
+#define ROCE_CMDQ_TIMEOUT_10MS 10
+
+#ifdef ROCE_SIGN_EN
+enum mtt_check_type_e { MTT_CHECK_TYPE_0 = 0, MTT_CHECK_TYPE_1 };
+
+#define ROCE_QP_QPCC_SIGN_WIDTH 5
+#define ROCE_QP_QPCC_SIGN_SPLITNUM 4
+#define ROCE_QP_QPCC_SIGN_CHECKBITS \
+ (ROCE_QP_QPCC_SIGN_WIDTH * ROCE_QP_QPCC_SIGN_SPLITNUM)
+
+#define ROCE_QP_SUB_SIGN_WIDTH 3
+#define ROCE_QP_SUB_SIGN_SPLITNUM 5
+#define ROCE_QP_SUB_SIGN_CHECKBITS \
+ (ROCE_QP_SUB_SIGN_WIDTH * ROCE_QP_SUB_SIGN_SPLITNUM)
+
+#define ROCE_QP_GPA_SIGN_WIDTH 3
+#define ROCE_QP_GPA_SIGN_SPLITNUM 11
+#define ROCE_QP_GPA_SIGN_CHECKBITS \
+ (ROCE_QP_GPA_SIGN_WIDTH * ROCE_QP_GPA_SIGN_SPLITNUM)
+
+#define ROCE_SQ_WQE_SIGN_WIDTH 8
+#define ROCE_SQ_WQE_SIGN_SPLITNUM 9
+#define ROCE_SQ_WQE_SIGN_CHECKBITS_1 32
+#define ROCE_SQ_WQE_SIGN_CHECKBITS_2 20
+#define ROCE_SQ_WQE_SIGN_CHECKBITS_12 \
+ (ROCE_SQ_WQE_SIGN_CHECKBITS_2 + ROCE_SQ_WQE_SIGN_CHECKBITS_1)
+#define ROCE_SQ_WQE_SIGN_CHECKBITS_3 20
+#define ROCE_SQ_WQE_SIGN_CHECKBITS_123 \
+ (ROCE_SQ_WQE_SIGN_CHECKBITS_3 + ROCE_SQ_WQE_SIGN_CHECKBITS_12)
+#define ROCE_SQ_WQE_SIGN_CHECKBITS \
+ (ROCE_SQ_WQE_SIGN_WIDTH * ROCE_SQ_WQE_SIGN_SPLITNUM)
+
+#define ROCE_MTT_SIGN_WIDTH 11
+#define ROCE_MTT_SIGN_SPLITNUM 5
+#define ROCE_MTT_SIGN_CHECKBITS_1 22
+#define ROCE_MTT_SIGN_CHECKBITS_2 33
+#define ROCE_MTT_SIGN_CHECKBITS_12 \
+ (ROCE_MTT_SIGN_CHECKBITS_1 + ROCE_MTT_SIGN_CHECKBITS_2)
+#define ROCE_MTT_SIGN_CHECKBITS (ROCE_MTT_SIGN_WIDTH * ROCE_MTT_SIGN_SPLITNUM)
+
+#define ROCE_MTT_BLOCK_SIGN_WIDTH 5
+#define ROCE_MTT_BLOCK_SIGN_SPLITNUM 11
+
+/* ****************************************************************************
+ Prototype : roce5_calculate_sign_bit
+ Description : roce5_calculate_signature
+ Input : u32 *check_bit
+ u32 sign_width
+ u32 sign_split
+ u32 *sign_bit
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_calculate_sign_bit(u32 *check_bit, u32 sign_width,
+ u32 sign_split, u32 *sign_bit)
+{
+ u32 i = 0;
+ u32 j = 0;
+
+ for (i = 0; i < sign_width; i++) {
+ sign_bit[i] = 0;
+ for (j = 0; j < sign_split; j++) {
+ sign_bit[i] = sign_bit[i] ^
+ check_bit[i + sign_width * j];
+ }
+
+ sign_bit[i] = (~sign_bit[i]) & 0x1;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_calculate_signature
+ Description : roce5_calculate_signature
+ Input : u32 *sign_bit
+ u32 sign_width
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_calculate_signature(u32 *sign_bit, u32 sign_width)
+{
+ u32 i = 0;
+ u32 signature = 0;
+
+ for (i = 0; i < sign_width; i++) {
+ signature |= sign_bit[i] << i;
+ }
+
+ return (signature & ((1U << sign_width) - 1));
+}
+
+/* ****************************************************************************
+ Prototype : roce5_gen_qp_rcc_sign
+ Description : Signature 3bit
+ CheckedBits = QPn[14:0]
+ Input : u32 qpn
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_gen_qp_rcc_sign(u32 qpn)
+{
+ u32 i;
+ u32 signature = 0;
+ u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 };
+ u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 };
+
+ for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) {
+ if (qpn & (1U << i)) {
+ check_bit[i] = 1;
+ }
+ }
+
+ roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH,
+ ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit);
+
+ signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH);
+
+ return signature;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_gen_qp_sqc_sign
+ Description : Signature 3bit
+ CheckedBits = QPn[14:0]
+ Input : u32 qpn
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_gen_qp_sqc_sign(u32 qpn)
+{
+ u32 i;
+ u32 signature = 0;
+ u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 };
+ u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 };
+
+ for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) {
+ if (qpn & (1U << i)) {
+ check_bit[i] = 1;
+ }
+ }
+
+ roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH,
+ ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit);
+
+ signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH);
+
+ return signature;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_gen_qp_sqac_sign
+ Description : Signature 3bit
+ CheckedBits = QPn[14:0]
+ Input : u32 qpn
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_gen_qp_sqac_sign(u32 qpn)
+{
+ u32 i;
+ u32 signature = 0;
+ u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 };
+ u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 };
+
+ for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) {
+ if (qpn & (1U << i)) {
+ check_bit[i] = 1;
+ }
+ }
+
+ roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH,
+ ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit);
+
+ signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH);
+
+ return signature;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_gen_qp_rqc_sign
+ Description : Signature 3bit
+ CheckedBits = QPn[14:0]
+ Input : u32 qpn
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_gen_qp_rqc_sign(u32 qpn)
+{
+ u32 i;
+ u32 signature = 0;
+ u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 };
+ u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 };
+
+ for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) {
+ if (qpn & (1U << i)) {
+ check_bit[i] = 1;
+ }
+ }
+
+ roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH,
+ ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit);
+
+ signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH);
+
+ return signature;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_gen_qp_rrwc_sign
+ Description : Signature 3bit
+ CheckedBits = QPn[14:0]
+ Input : u32 qpn
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_gen_qp_rrwc_sign(u32 qpn)
+{
+ u32 i;
+ u32 signature = 0;
+ u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 };
+ u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 };
+
+ for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) {
+ if (qpn & (1U << i)) {
+ check_bit[i] = 1;
+ }
+ }
+
+ roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH,
+ ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit);
+
+ signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH);
+
+ return signature;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_gen_sq_wqe_sign
+ Description : Signature 8bit
+ CheckedBits = {QPn[19:0], WQECI[19:0], WQE.Header[31:0]}
+ Input : u32 qpn
+ u32 wqe_ci
+ u32 wqe_header
+ Output : None
+
+ 1.Date : 2017/4/1
+ Modification : Created function
+
+**************************************************************************** */
+static u32 roce5_gen_sq_wqe_sign(u32 qpn, u32 wqe_ci, u32 wqe_header)
+{
+ u32 i;
+ u32 signature = 0;
+ u32 sign_bit[ROCE_SQ_WQE_SIGN_WIDTH] = { 0 };
+ u32 check_bit[ROCE_SQ_WQE_SIGN_CHECKBITS] = { 0 };
+
+ for (i = 0; i < ROCE_SQ_WQE_SIGN_CHECKBITS_1; i++) {
+ if (wqe_header & (1U << i)) {
+ check_bit[i] = 1;
+ }
+ }
+
+ for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_1;
+ i < ROCE_SQ_WQE_SIGN_CHECKBITS_12; i++) {
+ if (wqe_ci & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_1))) {
+ check_bit[i] = 1;
+ }
+ }
+
+ for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_12;
+ i < ROCE_SQ_WQE_SIGN_CHECKBITS_123; i++) {
+ if (qpn & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_12))) {
+ check_bit[i] = 1;
+ }
+ }
+
+ roce5_calculate_sign_bit(check_bit, ROCE_SQ_WQE_SIGN_WIDTH,
+ ROCE_SQ_WQE_SIGN_SPLITNUM, sign_bit);
+
+ signature = roce5_calculate_signature(sign_bit, ROCE_SQ_WQE_SIGN_WIDTH);
+
+ return signature;
+}
+#endif
+
+#define ROCE_BASE_GID_IDX \
+ 1 /* current IP mode supports precise push GID, reserved this field compatible old version firmware */
+static void roce5_qp_set_path(const struct roce5_qp *rqp,
+ struct rdma_ah_attr *ah_attr,
+ roce_verbs_qpc_attr_path_s *path_info)
+{
+ u8 *dmac = ah_attr->roce.dmac;
+ int ret;
+
+ path_info->dw7.bs.hoplmt = (u8)ah_attr->grh.hop_limit;
+ path_info->dw6.bs.tclass =
+ (u8)(ah_attr->grh.traffic_class | IP_ECN_PERMIT);
+ path_info->dw6.bs.flow_label = ah_attr->grh.flow_label &
+ 0xfffff; /* flow_label: 20bit */
+ path_info->dw7.bs.sgid_index = ah_attr->grh.sgid_index &
+ 0x7f; /* sgid_index: 7bit */
+
+ path_info->dw7.bs.base_sgid_n =
+ (path_info->dw7.bs.sgid_index != ROCE_BASE_GID_IDX);
+
+ ret = memcpy_s((void *)path_info->dgid, sizeof(path_info->dgid),
+ (void *)ah_attr->grh.dgid.raw, sizeof(path_info->dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ }
+
+ ret = memcpy_s((void *)&path_info->dmac_l32,
+ sizeof(path_info->dmac_l32),
+ (void *)&dmac[ROCE_RAH_DMAC_L32_START],
+ sizeof(path_info->dmac_l32));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dmac_l32.(ret:%d)", ret);
+ }
+ path_info->dmac_l32 = cpu_to_be32(path_info->dmac_l32);
+ path_info->dw0.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) |
+ dmac[1]; //shift dmac[0] left by 8 bits
+ path_info->dw7.bs.sl = (ah_attr->sl & 0x7); /* sl: 3bit */
+ path_info->dw0.bs.udp_src_port_h =
+ ((rqp->qpn >> ROCE_UDP_SRC_PORT_H8_SHIFT) & 0xff);
+ path_info->dw7.bs.udp_src_port = (rqp->qpn & 0xff);
+#ifdef ROCE_BONDING_EN
+ path_info->dw0.bs.bond_tx_hash_value = (rqp->tx_hash_value & 0xf);
+#endif
+}
+
+struct roce5_pd *roce5_get_pd(struct roce5_qp *rqp)
+{
+ if (rqp->qp_type == IB_QPT_XRC_TGT) {
+ return to_roce5_pd(to_roce5_xrcd(rqp->ibqp.xrcd)->pd);
+ } else {
+ return to_roce5_pd(rqp->ibqp.pd);
+ }
+}
+
+void roce5_get_cqs(struct roce5_qp *rqp, struct roce5_cq **send_cq,
+ struct roce5_cq **recv_cq)
+{
+ switch (rqp->qp_type) {
+ case IB_QPT_XRC_TGT:
+ *send_cq = to_roce5_cq(to_roce5_xrcd(rqp->ibqp.xrcd)->cq);
+ *recv_cq = *send_cq;
+ break;
+
+ case IB_QPT_XRC_INI:
+ *send_cq = to_roce5_cq(rqp->ibqp.send_cq);
+ *recv_cq = *send_cq;
+ break;
+
+ default:
+ *send_cq = to_roce5_cq(rqp->ibqp.send_cq);
+ *recv_cq = to_roce5_cq(rqp->ibqp.recv_cq);
+ break;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_access_flags
+ Description : roce5_set_access_flags
+ Input : struct roce5_qp *rqp
+ roce_verbs_qp_attr_s *qp_attr
+ struct ib_qp_attr *attr
+ int attr_mask
+ Output : None
+
+ 1.Date : 2015/5/26
+ Modification : Created function
+ 2.Date : 2016/2/26
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_set_access_flags(struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr,
+ struct ib_qp_attr *attr, int attr_mask)
+{
+ u16 dest_rd_atomic = 0;
+ u32 access_flags = 0;
+
+ if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) {
+ dest_rd_atomic = attr->max_dest_rd_atomic;
+ } else {
+ dest_rd_atomic = rqp->rsp_depth;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) {
+ access_flags = (u32)attr->qp_access_flags;
+ } else {
+ access_flags = rqp->atomic_rd_en;
+ }
+
+ if (dest_rd_atomic == 0) {
+ access_flags &= IB_ACCESS_REMOTE_WRITE;
+ }
+
+ if ((access_flags & IB_ACCESS_REMOTE_READ) != 0) {
+ qp_attr->com_info.dw0.bs.rre = 1; /* rc_rre: 1bits */
+ }
+
+ if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) != 0) {
+ qp_attr->com_info.dw0.bs.rae = 1; /* rc_rae: 1bits */
+ }
+
+ if ((access_flags & IB_ACCESS_REMOTE_WRITE) != 0) {
+ qp_attr->com_info.dw0.bs.rwe = 1; /* rrw_rwe: 1bits */
+ }
+}
+
+bool roce5_sqp_check(const struct roce5_qp *rqp)
+{
+ if (((rqp->qp_type == IB_QPT_GSI) &&
+ (rqp->qpn == ROCE_QP_SMI_QP_NUM)) ||
+ (rqp->qpn == ROCE_QP_GSI_QP_NUM)) {
+ return true;
+ }
+
+ return false;
+}
+
+/* ****************************************************************************
+ Prototype : store_sqp_attrs
+ Description : save the special
+ Input : struct roce5_qp *rqp
+ const struct ib_qp_attr *attr
+ int attr_mask
+ Output : int status
+
+ 1.Date : 2015/7/24
+ Modification : Created function
+
+**************************************************************************** */
+static int store_sqp_attrs(struct roce5_qp *rqp, const struct ib_qp_attr *attr,
+ int attr_mask)
+{
+ if (rqp->sqp_attr == NULL) {
+ ROCE_ERR("Error! sqp attribute is NULL, qpn(%u).", rqp->qpn);
+ return -EINVAL;
+ }
+
+ if (((u32)attr_mask & IB_QP_PKEY_INDEX) != 0) {
+ rqp->sqp_attr->pkey_index = attr->pkey_index;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_QKEY) != 0) {
+ rqp->sqp_attr->qkey = attr->qkey;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) {
+ rqp->sqp_attr->send_psn = attr->sq_psn;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : qpc_seg_to_be
+ Input : struct roce_qp_context *be_ctx
+ struct roce_qp_context *le_ctx
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2015/7/29
+ Author : zhaoxinteng
+ Modification : Created function
+
+**************************************************************************** */
+static void qpc_seg_to_be(roce_verbs_qp_attr_s *be_ctx,
+ roce_verbs_qp_attr_s *le_ctx, struct roce5_qp *rqp)
+{
+ /* DRV Seg */
+ int ret;
+
+ ret = memcpy_s((void *)be_ctx->path_info.dgid,
+ sizeof(le_ctx->path_info.dgid),
+ (void *)le_ctx->path_info.dgid,
+ sizeof(le_ctx->path_info.dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ }
+
+ /* CHIP Seg */
+ be_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa =
+ cpu_to_be64(le_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa);
+
+ be_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = cpu_to_be64(
+ le_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qpc_to_be
+ Description : translate big endian
+ Input : roce_verbs_qp_attr_s *qp_attr
+ struct roce5_qp *rqp
+ u32 *be_ctx
+ Output : None
+
+ 1.Date : 2015/7/29
+ Author : zhaoxinteng
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_qpc_to_be(roce_verbs_qp_attr_s *qp_attr, struct roce5_qp *rqp,
+ u32 *be_ctx)
+{
+ u32 *ctx = NULL;
+ u32 *ctx1 = NULL;
+ u32 i = 0;
+ u32 ctx_size = 0;
+
+ ctx = be_ctx;
+ ctx1 = (u32 *)qp_attr;
+ ctx_size = sizeof(roce_verbs_qp_attr_s) / sizeof(u32);
+
+ for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) {
+ *ctx = cpu_to_be32(*ctx1);
+ }
+
+ qpc_seg_to_be((roce_verbs_qp_attr_s *)((void *)be_ctx), qp_attr, rqp);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_alloc_opt_rdmarc
+ Description : alloc rdmarc when modify qp
+ Input : struct roce5_qp *rqp
+ struct ib_qp_attr *attr
+ int attr_mask
+ Output : None
+
+ 1.Date : 2016/01/30
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_alloc_opt_rdmarc(struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr, int attr_mask)
+{
+ int ret = 0;
+ struct roce5_device *rdev = NULL;
+ u32 response_depth = 0;
+
+ rdev = to_roce5_dev(rqp->ibqp.device);
+ response_depth = (attr->max_dest_rd_atomic == 0) ?
+ 1 :
+ (u32)ROCE_ROUNDUP_POW_OF_TWO(
+ (u32)attr->max_dest_rd_atomic);
+ if (rqp->rsp_depth != response_depth) {
+ if (rqp->rsp_depth > 0) {
+ roce5_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc);
+ }
+
+ ret = roce5_rdma_rdmarc_alloc(rdev->hwdev, response_depth,
+ &rqp->rdmarc);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc rdma rdmarc, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ rqp->rsp_depth = (u16)response_depth;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_free_opt_rdmarc
+ Description : free rdmarc when modify qp failed or destory qp
+ Input : struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2016/01/30
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_free_opt_rdmarc(struct roce5_qp *rqp)
+{
+ struct roce5_device *rdev = NULL;
+
+ rdev = to_roce5_dev(rqp->ibqp.device);
+
+ if (rqp->rsp_depth > 0) {
+ roce5_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc);
+ }
+
+ rqp->rsp_depth = 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_opt_rdmarc
+ Description : set opt rdmarc field
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ roce_verbs_qp_attr_s *qp_attr
+ Output : None
+
+ 1.Date : 2017/04/27
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_set_opt_rdmarc(struct rdma_rdmarc *rdmarc, u16 depth,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+ qp_attr->chip_seg.dw10.bs.rc_page_gpa_h =
+ (rdmarc->dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) &
+ 0xffffff; /* high:24bit, 64-24= 40 */
+ qp_attr->chip_seg.rc_page_gpa_l =
+ (u32)(rdmarc->dma_addr >>
+ ROCE5_RDMARC_GPA_L_SHIFT); //bit[39:8]
+ qp_attr->chip_seg.dw4.bs.rc_size = rdmarc->order;
+ qp_attr->com_info.dw1.bs.rra_max =
+ ((u8)fls(depth - 1)) &
+ ROCE5_RDMARC_RRA_MAX_MASK; /* rra_max:3bit */
+ qp_attr->chip_seg.dw4.bs.rc_max_size =
+ rdmarc->ext_order - ROCE5_RDMARC_EXT_ORDER_SHIFT;
+}
+
+static int roce5_set_opt_mtu(struct roce5_device *rdev,
+ roce_verbs_qp_attr_s *qp_attr,
+ const struct ib_qp_attr *attr)
+{
+ enum roce5_mtu rmtu = (enum roce5_mtu)attr->path_mtu;
+ if (rmtu == ROCE_MTU_8192 && ROCE5_SUPPORT_8K_MTU(rdev) == 0) {
+ ROCE_DEV_ERR(rdev, "Not support 8K MTU.");
+ return -EINVAL;
+ }
+
+ qp_attr->com_info.dw3.bs.pmtu = (u32)rmtu & 0x7; /* 1,2,3,4,5 */
+ qp_attr->com_info.dw5.bs.mtu_code =
+ ROCE_QPC_PMTU_TRANSLATE(rmtu); /* 0,1,3,7,15 */
+ if (qp_attr->com_info.dw3.bs.pmtu == IB_MTU_1024) {
+ qp_attr->com_info.dw3.bs.base_mtu_n = 0;
+ } else {
+ qp_attr->com_info.dw3.bs.base_mtu_n = 1;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_opt_sra
+ Description : set opt sra
+ Input : roce_verbs_qp_attr_s *qp_attr
+ struct ib_qp_attr *attr
+ Output : None
+
+ 1.Date : 2017/04/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_set_opt_sra(roce_verbs_qp_attr_s *qp_attr,
+ struct ib_qp_attr *attr)
+{
+ u8 initiator_depth;
+
+ initiator_depth =
+ (attr->max_rd_atomic == 0) ?
+ 1 :
+ (u8)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_rd_atomic);
+ qp_attr->com_info.dw1.bs.sra_max =
+ ((u32)ROCE_FLS(initiator_depth - 1)) & 0x7;
+}
+
+static void roce5_set_opt_attr_av(struct roce5_qp *rqp,
+ struct roce5_device *rdev,
+ struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr, u32 *optpar)
+{
+ u8 cos;
+ roce5_get_cos_inbuf_s inbuf = { 0 };
+
+ roce5_qp_set_path(rqp, &attr->ah_attr, &qp_attr->path_info);
+ qp_attr->com_info.dw3.bs.ext_mtu = rqp->ext_mtu & 0x1;
+ qp_attr->com_info.dw3.bs.ext_md = rqp->ext_mtu_mode & 0x1;
+ *optpar |= ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH;
+
+ inbuf.sl = attr->ah_attr.sl;
+ inbuf.sgid_index = attr->ah_attr.grh.sgid_index;
+ inbuf.port_num = attr->ah_attr.port_num;
+ inbuf.traffic_class = attr->ah_attr.grh.traffic_class;
+
+ (void)roce5_get_dcb_cfg_cos(rdev, &inbuf, &cos, rqp->qp_type);
+
+ qp_attr->com_info.dw5.bs.cos = cos;
+
+ rqp->db_sgid_index = attr->ah_attr.grh.sgid_index & 0x7f;
+ rqp->db_cos = cos;
+}
+
+static void roce5_set_opt_field_attr(struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr,
+ int attr_mask,
+ roce_verbs_qp_attr_s *qp_attr, u32 *optpar)
+{
+ /* RTR2RTS/SQD2SQD RETRY_CNT */
+ if ((((u32)attr_mask) & IB_QP_RETRY_CNT) != 0) {
+ qp_attr->com_info.dw1.bs.to_retry_limit =
+ attr->retry_cnt & 0x7; /* to_retry_limit:3 */
+ *optpar |= ROCE_QP_OPTPAR_RETRY_COUNT;
+ }
+
+ /* RTR2RTS/SQD2SQD RNR_RETRY */
+ if ((((u32)attr_mask) & IB_QP_RNR_RETRY) != 0) {
+ qp_attr->com_info.dw1.bs.rnr_retry_limit =
+ attr->rnr_retry & 0x7; /* rnr_retry_limit:3bits */
+ *optpar |= ROCE_QP_OPTPAR_RNR_RETRY;
+ }
+
+ /* INIT2RTR/SQD2SQD/RTR2RTS/RTS2RTS MIN_RNR_TIMER */
+ if ((((u32)attr_mask) & IB_QP_MIN_RNR_TIMER) != 0) {
+ qp_attr->com_info.dw2.bs.min_rnr_nak =
+ attr->min_rnr_timer & 0x1f; /* min_rnr_nak:5bits */
+ *optpar |= ROCE_QP_OPTPAR_RNR_TIMEOUT;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) {
+ qp_attr->com_info.dw8.bs.next_send_psn =
+ attr->sq_psn & 0xffffff; /* next_rcv_psn:24bit */
+ }
+
+ if ((((u32)attr_mask) & IB_QP_RQ_PSN) != 0) {
+ qp_attr->com_info.dw9.bs.next_rcv_psn =
+ attr->rq_psn & 0xffffff; /* next_rcv_psn:24bit */
+ }
+
+ if ((((u32)attr_mask) & IB_QP_DEST_QPN) != 0) {
+ qp_attr->com_info.dw0.bs.dest_qp = attr->dest_qp_num &
+ 0xffffff; /* dest_qp:24bit */
+ }
+
+ roce5_set_attr_ext(rqp, attr, qp_attr, attr_mask);
+}
+
+static s32 roce5_set_db_path_mtu(const struct ib_qp_attr *attr,
+ struct roce5_qp *rqp,
+ struct roce5_device *rdev)
+{
+ switch ((enum roce5_mtu)attr->path_mtu) {
+ case ROCE_MTU_256:
+ case ROCE_MTU_512:
+ ROCE_DEV_ERR(
+ rdev,
+ "Only support mtu larger than 1K.func_id(%d) mtu:%u",
+ rdev->glb_func_id, attr->path_mtu);
+ return -EINVAL;
+ case ROCE_MTU_1024:
+ rqp->db_path_mtu = BIT_LENGTH_IB_MTU_1024;
+ break;
+ case ROCE_MTU_2048:
+ rqp->db_path_mtu = BIT_LENGTH_IB_MTU_2048;
+ break;
+ case ROCE_MTU_4096:
+ rqp->db_path_mtu = BIT_LENGTH_IB_MTU_4096;
+ break;
+ case ROCE_MTU_8192:
+ if (ROCE5_SUPPORT_8K_MTU(rdev) != 0) {
+ rqp->db_path_mtu = BIT_LENGTH_IB_MTU_8192;
+ break;
+ }
+
+ ROCE_DEV_ERR(rdev, "Not support 8K MTU.");
+ return -EINVAL;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_set_opt_field(struct roce5_qp *rqp, struct ib_qp_attr *attr,
+ int attr_mask,
+ roce_modify_qp_param_s *modify_para)
+{
+ int ret = 0;
+ roce_verbs_qp_attr_s *qp_attr = &modify_para->qp_attr;
+ u32 *optpar = &modify_para->optpar;
+ struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device);
+
+ if ((((u32)attr_mask) & IB_QP_QKEY) != 0) {
+ qp_attr->com_info.q_key = attr->qkey;
+ *optpar |= ROCE_QP_OPTPAR_Q_KEY;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_AV) != 0) {
+ roce5_set_opt_attr_av(rqp, rdev, attr, qp_attr, optpar);
+ }
+
+ if ((((u32)attr_mask) & IB_QP_PORT) != 0) {
+ qp_attr->com_info.dw5.bs.port = attr->port_num &
+ 0x7; /* port: 3bits */
+ *optpar |= ROCE_QP_OPTPAR_SCHED_QUEUE;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) {
+ roce5_set_opt_rdmarc(&rqp->rdmarc, rqp->rsp_depth, qp_attr);
+ *optpar |= ROCE_QP_OPTPAR_RRA_MAX;
+ }
+
+ /* RTR2RTS/SQD2SQD SRA_MAX */
+ if ((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) {
+ roce5_set_opt_sra(qp_attr, attr);
+ *optpar |= ROCE_QP_OPTPAR_SRA_MAX;
+ }
+
+ /* RWE/RRE/RAE,RST2INIT */
+ if ((((u32)attr_mask) &
+ (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) != 0) {
+ roce5_set_access_flags(rqp, qp_attr, attr, attr_mask);
+ *optpar |= ROCE_QP_OPTPAR_RWE | ROCE_QP_OPTPAR_RRE |
+ ROCE_QP_OPTPAR_RAE;
+ }
+
+ /* RTR2RTS/SQD2SQD ACK TIMEOUT */
+ if ((((u32)attr_mask) & IB_QP_TIMEOUT) != 0) {
+ qp_attr->com_info.dw2.bs.ack_to = attr->timeout &
+ 0x1f; /* ack_to:5bits */
+ *optpar |= ROCE_QP_OPTPAR_ACK_TIMEOUT;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) {
+ ret = roce5_set_db_path_mtu(attr, rqp, rdev);
+ if (ret != 0) {
+ return ret;
+ }
+ ret = roce5_set_opt_mtu(rdev, qp_attr, attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to set MTU.");
+ return ret;
+ }
+ }
+
+ roce5_set_opt_field_attr(rqp, attr, attr_mask, qp_attr, optpar);
+ return ret;
+}
+
+static void roce5_set_qp_srq_attr(struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr,
+ struct roce5_pd *pd,
+ const struct roce5_cq *recv_cq)
+{
+ struct roce5_srq *rsrq = to_roce5_srq(rqp->ibqp.srq);
+ struct roce5_srq_context temp;
+
+ if (rsrq->container_flag != 0) {
+ struct roce5_srq_context *srqc =
+ (struct roce5_srq_context *)((void *)rsrq->cqm_srq
+ ->q_ctx_vaddr);
+ temp.dw0.value = be32_to_cpu(srqc->dw0.value);
+ qp_attr->chip_seg.dw12.bs.srq_pd = temp.dw0.bs.pdn &
+ 0x3ffff; /* srq_pd:18bits */
+ qp_attr->chip_seg.dw12.bs.srq_wqebb_size =
+ temp.dw0.bs.wqebb_size;
+ qp_attr->chip_seg.dw12.bs.srq_page_size = temp.dw0.bs.page_size;
+ qp_attr->chip_seg.dw12.bs.srq_size = temp.dw0.bs.size;
+
+ temp.dw1.value = be32_to_cpu(srqc->dw1.value);
+ qp_attr->com_info.dw5.bs.so_ro = temp.dw1.bs_c.so_ro;
+ qp_attr->com_info.dw5.bs.dma_attr_idx =
+ temp.dw1.bs_c.dma_attr_idx;
+
+ qp_attr->chip_seg.dw9.bs.container_sz =
+ MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode;
+ qp_attr->chip_seg.dw9.bs.container_en = 1;
+
+ qp_attr->com_info.dw3.bs.srq_container = 1;
+
+ if (rsrq->rqe_cnt_th > 0) {
+ qp_attr->chip_seg.dw9.bs.srq_warth_flag = 1;
+ qp_attr->chip_seg.dw8.bs.srq_rqecnt_th =
+ rsrq->rqe_cnt_th &
+ 0xf; /* srq_rqecnt_th:4bits */
+ }
+
+ qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn &
+ 0x3ffff; /* srqn:18bits */
+ } else {
+ qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn &
+ 0x3ffff; /* srqn:18bits */
+ }
+
+ qp_attr->com_info.dw2.bs.srq_en = 1;
+ qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1;
+ qp_attr->com_info.dw0.bs.rkey_en = 1;
+ qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn &
+ 0xfffff; /* rq_cqn:20bits */
+ qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1;
+}
+
+static void roce5_set_qp_rst2init_attr(struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr,
+ struct roce5_pd *pd,
+ struct roce5_cq *recv_cq)
+{
+ qp_attr->chip_seg.dw4.bs.rq_base_ci =
+ (u32)(ROCE_ILOG2(rqp->rq.offset >> rqp->rq.wqe_shift));
+ qp_attr->chip_seg.dw19.bs.qp_pd = pd->pdn & 0x3ffff; /* pd:18bits */
+ qp_attr->com_info.dw3.bs.dsgl_en = rqp->double_sgl_mode & 0x1;
+
+ if (rqp->ibqp.srq != NULL) {
+ roce5_set_qp_srq_attr(rqp, qp_attr, pd, recv_cq);
+ } else {
+ qp_attr->chip_seg.dw15.bs.rq_size =
+ (rqp->rq.wqebb_cnt > 0) ?
+ (((u32)ROCE_ILOG2(rqp->rq.wqebb_cnt)) & 0x1f) :
+ 0;
+ qp_attr->chip_seg.dw15.bs.rq_page_size =
+ ((u32)((u32)rqp->buf.page_shift - PAGE_SHIFT_4K)) &
+ ROCE_SQ_PAGE_SIZE_MASK;
+ qp_attr->chip_seg.dw15.bs.rq_pd = pd->pdn &
+ 0x3ffff; /* pd:18bits */
+ qp_attr->chip_seg.dw8.bs.rq_wqebb_size =
+ (u32)(rqp->rq.wqe_shift - BYTES_TO_16B_SHIFT) & 0x7;
+ qp_attr->com_info.dw7.bs.rq_cqn = recv_cq->cqn &
+ 0xfffff; /* rq_cqn:20bits */
+ qp_attr->chip_seg.dw15.bs.rq_rkey_en = 1;
+
+ qp_attr->chip_seg.dw8.bs.rq_pi_on_chip = 0;
+ qp_attr->chip_seg.dw7.bs.rq_wqecnt_lth = ROCE_RQ_WQECNT_LTH;
+ qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl_en = 0;
+ qp_attr->chip_seg.dw7.bs.rq_wqe_cache_thd_sel = 0;
+ qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl = 0;
+ qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen0 =
+ ROCE_RQ_MTT_PREFETCH_MAXLEN0;
+ qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen1 = 0;
+ qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxwqe =
+ ROCE_RQ_MTT_PREFETCH_MAXWQE;
+ qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_minnum = 0;
+ qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_maxnum =
+ ROCE_RQ_WQE_PREFETCH_MAXNUM;
+ }
+}
+
+static void roce5_qp_rst2init_set_qpcc(roce_verbs_qp_attr_s *qp_attr,
+ struct roce5_qp *rqp,
+ const struct roce5_device *rdev)
+{
+ qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = rqp->mtt.mtt_paddr;
+ qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num =
+ rqp->db.dma & (~0x3ULL); /* bits:63-2 */
+ qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num |=
+ rqp->mtt.mtt_layers & 0x3; /* bits:1-0 */
+ if (rqp->qp_type == IB_QPT_XRC_TGT) {
+ qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num =
+ rqp->wb_dma;
+ }
+
+ qp_attr->chip_seg.dw4.bs.sq_rq_mtt_page_size =
+ rqp->mtt.mtt_page_shift - PAGE_SHIFT_4K;
+ /* rc_entry_size:2bits */
+ qp_attr->chip_seg.dw10.bs.rc_entry_size =
+ ((u32)ROCE_ILOG2(rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .rdmarc_entry_sz >>
+ BYTES_TO_16B_SHIFT) &
+ 0x3);
+ qp_attr->chip_seg.dw14.bs.qp_rkey_en = 1;
+ qp_attr->chip_seg.dw10.bs.rc_entry_prefetch_maxnum = 0;
+}
+
+static void roce5_qp_rst2init_set_sqc_sqac(roce_verbs_qp_attr_s *qp_attr,
+ const struct roce5_qp *rqp,
+ const struct roce5_pd *pd,
+ const struct roce5_cq *send_cq)
+{
+ qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = rqp->pi_on_chip;
+ qp_attr->chip_seg.dw16.bs.sq_wqebb_size =
+ ((u32)(rqp->sq.wqe_shift - ROCE_SQ_WQEBB_SIZE_SHIFT)) &
+ ROCE_SQ_WQEBB_SIZE_MASK;
+ qp_attr->chip_seg.dw16.bs.sq_pd = ((u32)pd->pdn) &
+ 0x3ffff; /* pd:18bits */
+ qp_attr->chip_seg.dw16.bs.sq_page_size =
+ ((u32)(rqp->buf.page_shift - PAGE_SHIFT_4K)) &
+ ROCE_SQ_PAGE_SIZE_MASK;
+ qp_attr->chip_seg.dw5.bs.sq_inline_en = (rqp->max_inline_data > 0) ? 1 :
+ 0;
+
+ qp_attr->chip_seg.dw16.bs.sq_rkey_en = 1;
+
+ qp_attr->chip_seg.dw5.bs.sq_wqecnt_lth = ROCE_SQ_WQECNT_LTH;
+ qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl_en = 0;
+ qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl = 0;
+ qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_minnum =
+ ROCE_SQ_PREFETCH_MINNUM;
+ qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_maxnum =
+ ROCE_SQ_PREFETCH_MAXNUM;
+ qp_attr->chip_seg.dw5.bs.sq_wqe_cache_thd_sel = 0;
+ qp_attr->chip_seg.dw6.bs.sq_wqe_prefetch_mode = 0;
+ qp_attr->chip_seg.dw6.bs.sq_mtt_prefetch_maxlen = 0;
+
+ if (rqp->sq.wqebb_cnt != 0) {
+ qp_attr->chip_seg.dw5.bs.sq_size =
+ (u32)(ROCE_ILOG2((u32)rqp->sq.wqebb_cnt)) &
+ ROCE_SQ_SIZE_MASK;
+ }
+
+ qp_attr->chip_seg.dw17.bs.sqa_cqn = send_cq->cqn & ROCE_SQ_CQN_MASK;
+
+ qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_minnum =
+ ROCE_SQ_PREFETCH_MINNUM;
+ qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_maxnum =
+ ROCE_WQE_PREFETCH_MAXNUM;
+ qp_attr->chip_seg.dw6.bs.sqa_mtt_prefetch_maxlen = 0;
+ qp_attr->chip_seg.dw5.bs.sqa_wqe_cache_thd_sel = 0;
+}
+
+static void roce5_qp_rst2init_set_drv(roce_verbs_qp_attr_s *qp_attr,
+ struct roce5_qp *rqp,
+ struct roce5_device *rdev)
+{
+ qp_attr->com_info.dw0.bs.service_type =
+ (u32)to_roce5_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7;
+ qp_attr->com_info.dw3.bs.dif_en = (u32)rqp->signature_en;
+
+ if (!rqp->ibqp.uobject) {
+ qp_attr->com_info.dw0.bs.fre = 1;
+ }
+
+ qp_attr->com_info.dw3.bs.xrc_vld =
+ (u32)(rqp->qp_type == IB_QPT_XRC_INI);
+ qp_attr->com_info.dw1.bs.local_qp = rqp->qpn &
+ 0xffffff; /* local_qp:24bits */
+ qp_attr->com_info.dw5.bs.ep = rdev->hw_info.ep_id;
+ qp_attr->chip_seg.dw4.bs.dsgl = rqp->double_sgl_mode & 0x1;
+}
+
+static void roce5_qp_rst2init_set_common(roce_verbs_qp_attr_s *qp_attr,
+ const struct roce5_pd *pd,
+ const struct roce5_qp *rqp,
+ const struct roce5_cq *recv_cq,
+ const struct roce5_cq *send_cq)
+{
+ qp_attr->chip_seg.dw18.bs.ud_pd = pd->pdn & 0x3ffff; /* pd:18bits */
+
+ if (rqp->shadow != 0) {
+ qp_attr->com_info.dw11.bs.shadow = 1;
+ if (recv_cq != NULL) {
+ qp_attr->com_info.dw11.bs.shadow_vfid =
+ recv_cq->shadow_vfid;
+ }
+
+ if (send_cq != NULL) {
+ qp_attr->com_info.dw11.bs.shadow_cq_vfid =
+ send_cq->shadow_vfid;
+ }
+ }
+}
+
+static void roce5_qp_rst2init_set_xrc_attr(const struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr,
+ const struct roce5_device *rdev,
+ const struct roce5_cq *recv_cq)
+{
+ qp_attr->com_info.dw2.bs.xrc_srq_en = 1;
+
+ if ((((u32)attr->qp_access_flags & XRC_CONTAINER_FLAG) != 0) ||
+ (rdev->cfg_info.srq_container_en != 0)) {
+ qp_attr->chip_seg.dw9.bs.container_en = 1;
+ qp_attr->com_info.dw3.bs.srq_container = 1;
+ qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn &
+ 0xffff; /* xrcd:16bits */
+ qp_attr->com_info.dw2.bs.srq_en = 1;
+ qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = 1;
+ qp_attr->com_info.dw3.bs.xrc_vld = 1;
+ qp_attr->chip_seg.dw13.bs.srq_state = ROCE_SRQ_INIT_STATE;
+ } else {
+ qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1;
+ qp_attr->com_info.dw2.bs.srq_en = 1;
+ qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn &
+ 0xffff; /* xrcd:16bits */
+ qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1;
+ qp_attr->com_info.dw3.bs.xrc_vld = 1;
+ qp_attr->chip_seg.dw14.bs.srq_xrcd = rqp->xrcdn &
+ 0xffff; /* xrcd:16bits */
+ }
+ if (recv_cq != NULL) {
+ qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn &
+ 0xfffff; /* rq_cqn:20bits */
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_rst2init
+ Description : parameter set when QP is from RESET to INIT
+ Input : struct roce5_qp *rqp
+ const struct ib_qp_attr *attr
+ roce_verbs_qp_attr_s *qp_attr
+ Output : None
+
+ 1.Date : 2015/8/6
+ Author : zhaoxinteng
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_qp_rst2init(struct roce5_qp *rqp, const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+ struct roce5_pd *pd = NULL;
+ struct roce5_cq *send_cq = NULL;
+ struct roce5_cq *recv_cq = NULL;
+ struct roce5_device *rdev = NULL;
+
+ rdev = to_roce5_dev(rqp->ibqp.device);
+ pd = roce5_get_pd(rqp);
+ roce5_get_cqs(rqp, &send_cq, &recv_cq);
+
+ /* CHIP_SEG: QPCC */
+ roce5_qp_rst2init_set_qpcc(qp_attr, rqp, rdev);
+
+ /* CHIP_SEG: SQC & SQAC */
+ roce5_qp_rst2init_set_sqc_sqac(qp_attr, rqp, pd, send_cq);
+
+ /* RRWC */
+ qp_attr->chip_seg.dw4.bs.rrw_mtt_prefetch_maxlen = 0;
+
+ /* RCC */
+ qp_attr->chip_seg.dw10.bs.rc_mtt_prefetch_maxlen =
+ ROCE_QP_MAX_PFETCH_MTT_LAYER;
+
+ /* TIMER_SEG */
+ qp_attr->com_info.dw3.bs.tss_timer_num = ROCE_QP_MAX_TIMER_NUM;
+
+ /* DRV_SEG */
+ roce5_qp_rst2init_set_drv(qp_attr, rqp, rdev);
+
+ /* UCODE COM_SEG */
+ roce5_qp_rst2init_set_common(qp_attr, pd, rqp, recv_cq, send_cq);
+
+ if (rqp->qp_type == IB_QPT_XRC_TGT) {
+ roce5_qp_rst2init_set_xrc_attr(rqp, attr, qp_attr, rdev,
+ recv_cq);
+ } else {
+ roce5_set_qp_rst2init_attr(rqp, qp_attr, pd, recv_cq);
+ }
+
+#ifdef ROCE_BONDING_EN
+ roce5_bond_rr_set_flow(rdev, rqp, qp_attr);
+#endif
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_init2rtr
+ Description : init2rtr attributes modify
+ Input : struct roce5_qp *rqp
+ const struct ib_qp_attr *attr
+ roce_verbs_qp_attr_s *qp_attr
+ Output : None
+
+ 1.Date : 2015/8/1
+ Author : zhaoxinteng
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_qp_init2rtr(struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+ if (rqp->ibqp.srq || (rqp->qp_type == IB_QPT_XRC_TGT) ||
+ (rqp->qp_type == IB_QPT_XRC_INI) || (rqp->qp_type == IB_QPT_UD) ||
+ (rqp->qp_type == IB_QPT_UC) || (rqp->qp_type == IB_QPT_GSI)) {
+ qp_attr->com_info.dw3.bs.invalid_credit = 1;
+ }
+
+ if ((rqp->qp_type == IB_QPT_GSI) || (rqp->qp_type == IB_QPT_UD)) {
+ qp_attr->com_info.dw3.bs.pmtu = IB_MTU_4096 &
+ 0x7; /* for UD, mtu = 4096 */
+ qp_attr->com_info.dw5.bs.mtu_code = ROCE_MTU_CODE_4K;
+ }
+
+ qp_attr->com_info.dw2.bs.ack_to = 0xf; /* ack_to:5bits */
+ qp_attr->com_info.dw0.bs.service_type =
+ (u32)to_roce5_qp_type((enum ib_qp_type)rqp->qp_type) &
+ 0x7; /* service_type: 3bit */
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_rtr2rts
+ Description : roce5_qp_rtr2rts
+ Input : struct roce5_qp *rqp
+ const struct ib_qp_attr *attr
+ roce_verbs_qp_attr_s *qp_attr
+ Output : None
+
+ 1.Date : 2015/8/2
+ Author : zhaoxinteng
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_qp_rtr2rts(struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+}
+
+static void roce5_qp_2err(struct roce5_qp *rqp, const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_abs_field
+ Description : set abs field to qpc
+ Input : struct roce5_qp *rqp
+ struct ib_qp_attr *attr
+ roce_verbs_qp_attr_s *qp_attr
+ enum ib_qp_state cur_state
+ enum ib_qp_state new_state
+ Output : None
+
+ 1.Date : 2016/01/30
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_set_abs_field(struct roce5_qp *rqp, struct ib_qp_attr *attr,
+ int attr_mask, roce_verbs_qp_attr_s *qp_attr)
+{
+ enum ib_qp_state cur_state, new_state;
+
+ cur_state =
+ (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ?
+ attr->cur_qp_state :
+ rqp->qp_state);
+ new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state :
+ cur_state;
+ if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
+ roce5_qp_rst2init(rqp, attr, qp_attr);
+ }
+
+ if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR)) {
+ roce5_qp_init2rtr(rqp, attr, qp_attr);
+ }
+
+ if ((cur_state == IB_QPS_RTR) && (new_state == IB_QPS_RTS)) {
+ roce5_qp_rtr2rts(rqp, attr, qp_attr);
+ }
+
+ if (new_state == IB_QPS_ERR) {
+ roce5_qp_2err(rqp, attr, qp_attr);
+ }
+ return;
+}
+
+#ifdef ROCE_LATENCY_PROF_EN
+#define ROCE_SET_LATENCY_PROF_EN 5
+static int roce5_latency_prof_en_check(struct roce5_device *rdev,
+ u32 *latency_prof_en,
+ u32 *frequency_reduction_ratio)
+{
+ roce_dfx_set_inbuf_s buf_in = { 0 };
+ roce_dfx_set_outbuf_s buf_out = { 0 };
+ u32 out_size = sizeof(buf_out);
+ int ret = 0;
+ buf_in.show = 1;
+ buf_in.cmd_type = ROCE_CMD_ROCE_SET;
+ buf_in.sub_cmd_type = ROCE_SET_LATENCY_PROF_EN;
+ ret = roce5_adm_dfx_roce_set(rdev, &buf_in, sizeof(buf_in), &buf_out,
+ &out_size);
+ if (ret != 0) {
+ ROCE_ERR("latency prof query l2d err");
+ return ret;
+ }
+ *latency_prof_en = buf_out.sub_param.latency_prof.latency_prof_en;
+ *frequency_reduction_ratio =
+ buf_out.sub_param.latency_prof.frequency_reduction_ratio;
+ return 0;
+}
+
+static int
+roce5_latency_prof_time_get(struct roce5_device *rdev,
+ struct roce5_modify_qp_ex_resp_data *resp)
+{
+ struct hinic5_hwdev *hwdev = (struct hinic5_hwdev *)rdev->hwdev;
+ u32 latency_prof_en = 0;
+ u32 frequency_reduction_ratio = 1;
+ u64 time_diff = 0;
+ int ret = 0;
+ if (hwdev == NULL) {
+ ROCE_ERR("latency prof hwdev is null");
+ return (-EINVAL);
+ }
+ ret = roce5_latency_prof_en_check(rdev, &latency_prof_en,
+ &frequency_reduction_ratio);
+ if (ret != 0) {
+ return ret;
+ }
+ if (latency_prof_en == 0) {
+ resp->latency_prof_en = 0;
+ resp->non_ptp_time_diff = 0;
+ return 0;
+ }
+ //get clock time and non_ptp_chip time of difference value, convenient at later calculate timestamp
+ ret = hinic5_get_non_ptp_time_diff(hwdev, &time_diff);
+ if (ret != 0) {
+ ROCE_ERR("Failed to read n_ptp time, ret(%d)\n", ret);
+ return ret;
+ }
+ resp->latency_prof_en = 1;
+ resp->non_ptp_time_diff = time_diff;
+ resp->frequency_reduction_ratio = frequency_reduction_ratio;
+ return 0;
+}
+#endif
+static int roce5_record_opt_field(struct roce5_qp *rqp, struct ib_qp_attr *attr,
+ int attr_mask, struct ib_udata *udata,
+ roce_verbs_cmd_qp_modify_rsp_s *modify_rsp)
+{
+ int ret = 0;
+ int rsp_vld = 0;
+ enum ib_qp_state cur_state, new_state;
+ struct roce5_modify_qp_ex_resp_data resp = { 0 };
+ struct roce5_device *rdev = NULL;
+
+ cur_state =
+ (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ?
+ attr->cur_qp_state :
+ rqp->qp_state);
+ new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state :
+ cur_state;
+ rqp->qp_state = new_state;
+
+ if ((((u32)attr_mask) & IB_QP_AV) != 0) {
+ rqp->sl = attr->ah_attr.sl & 0x7;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) {
+ rqp->atomic_rd_en = (u8)attr->qp_access_flags;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_PORT) != 0) {
+ rqp->port = attr->port_num;
+ }
+
+ if (roce5_sqp_check(rqp) != 0) {
+ ret = store_sqp_attrs(rqp, attr, attr_mask);
+ if (ret != 0) {
+ ROCE_ERR("Failed to store sqp attribute.");
+ return ret;
+ }
+ }
+
+ rdev = to_roce5_dev(rqp->ibqp.device);
+
+ /*
+ * TGT QP no has update pi requirement, and db trigger microcode thread and driver no has strict timing disable system, one general in init status trigger microcode,
+ * but if XRC TGT QP migration
至 rts after again trigger microcode thread, SQ hardware none permission, fetch wqe will exception, because this TGT QP not next db
+ * but non- TGT QP i.e. make in qp rst status trigger, will go via wqe empty
终结, not will exception
+ */
+ if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT) &&
+ (rqp->pi_on_chip != 0) && (rqp->qp_type != IB_QPT_XRC_TGT)) {
+ /* avoid PI on chip when PI
残keep, primary dynamic clear to zero */
+ ret = roce5_write_sq_db(rdev, rqp, 0, udata);
+ if (ret != 0) {
+ ROCE_ERR("Failed to clear PI on chip");
+ return -1;
+ }
+ }
+
+ if ((udata != NULL) && (udata->outbuf != NULL)) {
+ if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR) &&
+ (((u32)attr_mask & IB_QP_AV) != 0)) {
+ resp.cos = rqp->db_cos;
+ resp.is_cos_valid = 1;
+ rsp_vld = 1;
+ }
+
+#ifdef ROCE_LATENCY_PROF_EN
+ if (ROCE_IS_DEV_1825V100(rdev) && (cur_state == IB_QPS_RTR) &&
+ (new_state == IB_QPS_RTS)) {
+ ret = roce5_latency_prof_time_get(rdev, &resp);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get time diff, ret(%d).",
+ ret);
+ return -1;
+ }
+ }
+#endif
+ /* compatible old card not will return ssn value, driver not perform update */
+ if (modify_rsp != NULL &&
+ modify_rsp->magic == ROCE_VERBS_MODIFY_RSP_MAGIC) {
+ resp.is_ssn_valid = modify_rsp->dw1.bs.is_ssn_vld;
+ resp.ssn = modify_rsp->dw1.bs.ssn;
+ rqp->sq_wqe_ssn = resp.ssn;
+ rsp_vld = 1;
+ }
+
+ if (rsp_vld != 0) {
+ ret = ib_copy_to_udata(
+ udata, &resp,
+ sizeof(struct roce5_modify_qp_ex_resp_data));
+ if (ret != 0) {
+ ROCE_ERR("Failed to copy data to user");
+ return -1;
+ }
+ }
+ }
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_modify_2rst_cmd
+ Description : cache out qpc mtt
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2016/6/17
+ Modification : Created function
+
+ 2.Date : 2017/04/26
+ Modification : Modify function
+
+**************************************************************************** */
+int roce5_qp_modify_2rst_cmd(struct roce5_device *rdev, u32 qpn)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_qp_modify2rst_s *qp_moidfy2rst_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_qp_modify2rst_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ qp_moidfy2rst_inbuf =
+ (roce_uni_cmd_qp_modify2rst_s *)cqm_cmd_inbuf->buf;
+ qp_moidfy2rst_inbuf->com.index = cpu_to_be32(qpn);
+ qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK));
+
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_2RST_QP, cqm_cmd_inbuf,
+ NULL, ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send 2RST_QP command, qpn(0x%x), func_id(%d), ret(0x%x)",
+ qpn, rdev->glb_func_id, ret);
+
+ ret = (-EINVAL);
+
+ if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) {
+ ROCE_DEV_ERR(rdev,
+ "Card not present, return OK, func_id(%d)",
+ rdev->glb_func_id);
+ ret = 0;
+ }
+ goto err_out;
+ }
+
+err_out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_check_rdmarc
+ Description : roce5_qp_check_rdmarc
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2017/04/26
+ Modification : Create function
+
+**************************************************************************** */
+static int roce5_qp_check_rdmarc(struct roce5_device *rdev,
+ struct roce5_qp *rqp, int *cost)
+{
+ int times = 0;
+ u32 rc_num = 0;
+ u32 rc_check_index = 0;
+ void *rdmarc_vaddr = NULL;
+ dma_addr_t rdmarc_paddr = 0;
+
+ rc_num =
+ (u32)(((u32)1 << rqp->rdmarc.order) / RDMARC_NUM_PER_CACHELINE);
+ rdmarc_vaddr = rqp->rdmarc.vaddr;
+ rdmarc_paddr = rqp->rdmarc.dma_addr;
+
+ times = rdev->try_times;
+ do {
+ if ((*((u32 *)rdmarc_vaddr) ==
+ ROCE_RDMARC_DESTROY_CHECK_VALUE) &&
+ (cpu_to_be32(*((u32 *)rdmarc_vaddr + 1)) == rqp->qpn) &&
+ (cpu_to_be64(*((u64 *)rdmarc_vaddr + 1)) ==
+ ((u64)rdmarc_paddr +
+ ((u64)ROCE_CACHE_LINE_SIZE * (u64)rc_check_index)))) {
+ rdmarc_vaddr = (void *)((u8 *)rdmarc_vaddr +
+ ROCE_CACHE_LINE_SIZE);
+ rc_check_index++;
+ if (rc_num == rc_check_index) {
+ break;
+ }
+ } else {
+ ROCE_DELAY();
+ times--;
+ }
+ } while (times != 0);
+
+ if (rc_num != rc_check_index) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to read rdmarc field back after try %d times, func_id(%d) rc_check_index:%u rc_num:%u",
+ ((rdev->try_times - times) + 1), rdev->glb_func_id,
+ rc_check_index, rc_num);
+ *cost = (rdev->try_times - times) + 1;
+ return -1;
+ }
+ *cost = (rdev->try_times - times) + 1;
+ return 0;
+}
+
+static int roce5_qp_check_state(struct roce5_device *rdev,
+ const struct roce5_qp *rqp, const u32 *wb_va,
+ int *cost)
+{
+ int times = 0;
+ int read_back_flag = 0;
+
+ times = rdev->try_times;
+ while ((times--) != 0) {
+ if (*(u32 *)wb_va == ROCE_QP_DESTROY_CHECK_VALUE_HIGH_4B &&
+ (*((u32 *)wb_va + 1) ==
+ ROCE_QP_DESTROY_CHECK_VALUE_LOW_4B)) {
+ read_back_flag = 1;
+ break;
+ }
+ ROCE_DELAY();
+ }
+
+ if (read_back_flag == 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to read QP field back after try %d times,qpn(0x%x), wb_data(0x%x), wb_data+1(0x%x) func_id(%d)",
+ ((rdev->try_times - times) + 1), rqp->qpn,
+ *(u32 *)wb_va, *((u32 *)wb_va + 1), rdev->glb_func_id);
+ *cost = (rdev->try_times - times) + 1;
+ return -1;
+ }
+ *cost = (rdev->try_times - times) + 1;
+ return 0;
+}
+
+static void roce5_qp_fill_cacheout_inbuf(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ dma_addr_t wb_dma,
+ cqm_cmd_buf_s *cqm_cmd_inbuf)
+{
+ roce_uni_cmd_qp_cache_invalid_s *qp_cacheout_inbuf = NULL;
+ struct rdma_service_cap *rdma_cap = NULL;
+ u32 page_size = roce5_get_page_size(rdev, true);
+
+ qp_cacheout_inbuf =
+ (roce_uni_cmd_qp_cache_invalid_s *)(cqm_cmd_inbuf)->buf;
+ rdma_cap = &rdev->rdma_cap;
+
+ qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK));
+ qp_cacheout_inbuf->com.index = cpu_to_be32(rqp->qpn);
+ qp_cacheout_inbuf->qp_cache.sq_buf_len = cpu_to_be32(
+ ALIGN(((u32)rqp->sq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift),
+ page_size));
+ qp_cacheout_inbuf->qp_cache.rq_buf_len = cpu_to_be32(
+ ALIGN(((u32)rqp->rq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift),
+ page_size));
+ qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_flags = 0;
+ qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_num = 0;
+ qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_start =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start);
+ qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_end =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end);
+ qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_size =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz);
+ qp_cacheout_inbuf->qp_cache.wb_gpa = cpu_to_be64(wb_dma);
+ qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_flags = 0;
+ qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_num = 0;
+ qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_start =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start);
+ qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_end =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end);
+ qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_size =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz);
+}
+
+static int roce5_qp_send_cache_out_cmd(struct roce5_device *rdev,
+ struct roce5_qp *rqp, void *wb_va,
+ dma_addr_t wb_dma, int *cost)
+{
+ unsigned long end;
+ int times = 0;
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_qp_cache_invalid_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ roce5_qp_fill_cacheout_inbuf(rdev, rqp, wb_dma, cqm_cmd_inbuf);
+
+ end = jiffies +
+ (unsigned long)((unsigned int)rdev->destory_flow_timeout * HZ);
+ while (time_before(jiffies, end)) {
+ times++;
+ ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev,
+ ROCE_CMD_MISC_CACHE_INVLD,
+ cqm_cmd_inbuf, NULL,
+ ROCE_CMD_TIME_CLASS_C);
+ if (ret <= 0) { /* return success or exception happened */
+ break;
+ }
+ cond_resched();
+ };
+
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed MISC_CACHE_INVLD command after try %d times, ret(%d), func_id(%d) qpn(%d)",
+ times, ret, rdev->glb_func_id, rqp->qpn);
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ *cost = times;
+
+ if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Card not present, return ERROR, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -ENODEV;
+ }
+ return ret;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ *cost = (rdev->try_times - times) + 1;
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_qp_cache_out_cmd
+ Description : cache out qpc mtt
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ Output : None
+
+ 1.Date : 2016/9/24
+ Modification : Created function
+
+ 2.Date : 2017/04/26
+ Modification : Modify function
+
+**************************************************************************** */
+/* roce5_dealloc_latch_resource() collect use construct temporary change amount rqp to reuse this function. such as newly added for rqp intomember of use, need
排 query whether need adaptation */
+int roce5_qp_cache_out_cmd(struct roce5_device *rdev, struct roce5_qp *rqp)
+{
+ int ret = 0;
+ int cost = 0;
+ void *wb_va = NULL;
+ dma_addr_t wb_dma = 0;
+
+ wb_va = dma_alloc_coherent(rdev->dev, (size_t)8UL, &wb_dma, GFP_KERNEL);
+ if (wb_va == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc wb_va, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ ret = roce5_qp_send_cache_out_cmd(rdev, rqp, wb_va, wb_dma, &cost);
+ if (ret != 0) {
+ goto err;
+ }
+
+ if (rqp->rsp_depth > 0) {
+ ret = roce5_qp_check_rdmarc(rdev, rqp, &cost);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to check qp rdmarc, func_id(%d)",
+ rdev->glb_func_id);
+ goto err;
+ }
+ }
+
+ ret = roce5_qp_check_state(rdev, rqp, (u32 *)wb_va, &cost);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to check qp state, func_id(%d)",
+ rdev->glb_func_id);
+ goto err;
+ }
+
+err:
+ dma_free_coherent(rdev->dev, sizeof(u32), wb_va, wb_dma);
+
+ /* At the end of the check, if the failure is caused by a chip link down, release resources on the host side. */
+ if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) {
+ ROCE_DEV_ERR(rdev, "Card not present, return OK, func_id(%d)",
+ rdev->glb_func_id);
+ ret = 0;
+ }
+
+ return ret;
+}
+
+static u8 roce5_get_qp_cmdq_cmd(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_attr *attr, int attr_mask)
+{
+ static u16 op[ROCE_QP_STATE_NUM][ROCE_QP_STATE_NUM];
+ enum ib_qp_state cur_state, new_state;
+
+ cur_state =
+ (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ?
+ attr->cur_qp_state :
+ rqp->qp_state);
+ new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state :
+ cur_state;
+ if ((cur_state > IB_QPS_ERR) || (new_state > IB_QPS_ERR)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "QP state modify invalid, cur_state(%d), new_state(%d), func_id(%d)",
+ cur_state, new_state, rdev->glb_func_id);
+ return 0;
+ }
+ op[IB_QPS_RESET][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+ op[IB_QPS_RESET][IB_QPS_INIT] = ROCE_CMD_RST2INIT_QP;
+
+ op[IB_QPS_INIT][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+ op[IB_QPS_INIT][IB_QPS_INIT] = ROCE_CMD_INIT2INIT_QP;
+ op[IB_QPS_INIT][IB_QPS_RTR] = ROCE_CMD_INIT2RTR_QP;
+
+ op[IB_QPS_RTR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+ op[IB_QPS_RTR][IB_QPS_RTS] = ROCE_CMD_RTR2RTS_QP;
+
+ op[IB_QPS_RTS][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+ op[IB_QPS_RTS][IB_QPS_RTS] = ROCE_CMD_RTS2RTS_QP;
+ op[IB_QPS_RTS][IB_QPS_SQD] = ROCE_CMD_RTS2SQD_QP;
+
+ op[IB_QPS_SQD][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+ op[IB_QPS_SQD][IB_QPS_RTS] = ROCE_CMD_SQD2RTS_QP;
+ op[IB_QPS_SQD][IB_QPS_SQD] = ROCE_CMD_SQD2SQD_QP;
+
+ op[IB_QPS_SQE][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+ op[IB_QPS_SQE][IB_QPS_RTS] = ROCE_CMD_SQERR2RTS_QP;
+
+ op[IB_QPS_ERR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP;
+
+ return (u8)op[cur_state][new_state];
+}
+
+static int roce5_qp_rts2sqd_cmd(struct roce5_device *rdev, struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr, int attr_mask)
+{
+ int ret = 0;
+ int sqd_event = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_qp_modify_rts2sqd_s *qp_rts2sqd_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_qp_modify_rts2sqd_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ sqd_event = ((((u32)attr_mask) & IB_QP_EN_SQD_ASYNC_NOTIFY) != 0) &&
+ (attr->en_sqd_async_notify != 0);
+ qp_rts2sqd_inbuf =
+ (roce_uni_cmd_qp_modify_rts2sqd_s *)cqm_cmd_inbuf->buf;
+ qp_rts2sqd_inbuf->com.index = cpu_to_be32(rqp->qpn);
+ qp_rts2sqd_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ qp_rts2sqd_inbuf->sqd_event_en = cpu_to_be32((u32)(sqd_event != 0));
+
+ ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_RTS2SQD_QP,
+ cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RTS2SQD_QP cmd, func_id(%d), ret(0x%x)",
+ rdev->glb_func_id, ret);
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return 0;
+}
+
+static int roce5_qp_handle_err(struct roce5_device *rdev,
+ const struct roce5_qp *rqp, u8 cmd, int ret)
+{
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send MODIFY_QP command, command(%u), func_id(%d), ret(%d)",
+ cmd, rdev->glb_func_id, ret);
+
+ if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+
+ return -1;
+}
+
+static int roce5_qp_modify_cmd(struct roce5_device *rdev, struct roce5_qp *rqp,
+ u8 cmd, roce_modify_qp_param_s *modify_para)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_modify_qpc_s *qp_modify_inbuf = NULL;
+ roce_verbs_qp_attr_s *qp_attr = &modify_para->qp_attr;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_modify_qpc_s), &cqm_cmd_outbuf,
+ ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return (-ENOMEM);
+ }
+
+ qp_modify_inbuf = (roce_uni_cmd_modify_qpc_s *)cqm_cmd_inbuf->buf;
+
+ roce5_transmit_ci_record_addr_ext(rqp, qp_attr);
+ roce5_qpc_to_be(qp_attr, rqp, (u32 *)&qp_modify_inbuf->qp_attr);
+ qp_modify_inbuf->com.index = cpu_to_be32(rqp->qpn);
+ qp_modify_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK));
+ qp_modify_inbuf->com.opt = cpu_to_be32(modify_para->optpar);
+ ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, cmd, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ret = roce5_qp_handle_err(rdev, rqp, cmd, ret);
+ } else {
+ roce5_be32_2_le32(
+ cqm_cmd_outbuf->buf, (u32 *)&modify_para->modify_rsp,
+ sizeof(roce_verbs_cmd_qp_modify_rsp_s) / sizeof(u32));
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+ return ret;
+}
+
+void roce5_cq_clean(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq)
+{
+ spin_lock_irq(&cq->lock);
+ roce5_cq_clean_process(cq, qpn, srq);
+ spin_unlock_irq(&cq->lock);
+}
+
+static int roce5_qp_clean_kernel_res(struct roce5_qp *rqp)
+{
+ int ret = 0;
+ struct roce5_cq *send_cq = NULL;
+ struct roce5_cq *recv_cq = NULL;
+
+ roce5_get_cqs(rqp, &send_cq, &recv_cq);
+
+ roce5_cq_clean(recv_cq, rqp->qpn,
+ rqp->ibqp.srq ? to_roce5_srq(rqp->ibqp.srq) : NULL);
+ if (send_cq != recv_cq) {
+ roce5_cq_clean(send_cq, rqp->qpn, NULL);
+ }
+
+ rqp->rq.head = 0;
+ rqp->rq.tail = 0;
+
+ rqp->sq.head = 0;
+ rqp->sq.tail = 0;
+
+ rqp->sq_next_wqe = 0;
+ rqp->sq_wqe_ssn = 0;
+
+ *(rqp->db.db_record) = 0;
+
+ if (rqp->qp_type != IB_QPT_XRC_TGT) {
+ if (rqp->has_rq && rqp->qp_buf_info->q_header_vaddr) {
+ ret = memset_s(rqp->qp_buf_info->q_header_vaddr,
+ sizeof(u64), 0, sizeof(u64));
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to memset QP(0x%06x) q_header_vaddr(ret:%d)",
+ rqp->qpn, ret);
+ return (-ENOMEM);
+ }
+ }
+ }
+
+ return ret;
+}
+
+static int roce5_qp_modify_2rst(struct roce5_device *rdev, struct roce5_qp *rqp)
+{
+ int ret = 0;
+
+ ret = roce5_qp_modify_2rst_cmd(rdev, rqp->qpn);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify QP(0x%06x) to RESET, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = roce5_qp_cache_out_cmd(rdev, rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "QP(0x%06x) cache invalid failed, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ return ret;
+ }
+
+ rqp->qp_state = IB_QPS_RESET;
+
+ if (rqp->ibqp.uobject == NULL) {
+ ret = roce5_qp_clean_kernel_res(rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Clean kernel QP(0x%06x) failed, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static void roce5_set_stub_field(struct roce5_device *rdev,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+ qp_attr->com_info.dw6.bs.force_local = rdev->local_en;
+}
+
+static int roce5_modify_qp_cmd_process(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ struct ib_qp_attr *attr, int attr_mask,
+ roce_modify_qp_param_s *modify_para)
+{
+ u8 cmd;
+ int ret;
+ cmd = roce5_get_qp_cmdq_cmd(rdev, rqp, attr, attr_mask);
+ if (cmd == 0) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid cmd, func_id(%u), qpn(%u), cmd(%u).",
+ rdev->glb_func_id, rqp->qpn, cmd);
+ return -EINVAL;
+ }
+ if (cmd == ROCE_CMD_RTS2SQD_QP) {
+ ret = roce5_qp_rts2sqd_cmd(rdev, rqp, attr, attr_mask);
+ } else {
+ ret = roce5_qp_modify_cmd(rdev, rqp, cmd, modify_para);
+ }
+ return ret;
+}
+
+static int roce5_qp_modify(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ roce_user_handle_t hdl;
+ roce_modify_qp_param_s modify_para;
+ roce_verbs_qp_attr_s *qp_attr = &modify_para.qp_attr;
+
+ (void)memset_s(&modify_para, sizeof(roce_modify_qp_param_s), 0,
+ sizeof(roce_modify_qp_param_s));
+ if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) {
+ ret = roce5_alloc_opt_rdmarc(rqp, attr, attr_mask);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdmarc, func(%d).",
+ rdev->glb_func_id);
+ return ret;
+ }
+ }
+
+ ret = roce5_set_opt_field(rqp, attr, attr_mask, &modify_para);
+ if (ret != 0) {
+ goto err_out;
+ }
+
+ roce5_set_abs_field(rqp, attr, attr_mask, qp_attr);
+ roce5_set_stub_field(rdev, qp_attr);
+
+ (void)memset_s(&hdl, sizeof(roce_user_handle_t), 0,
+ sizeof(roce_user_handle_t));
+ ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, qp_attr, NULL, NULL, NULL);
+ ret = roce5_uld_qp_modify((roce_uld_handle)&hdl, attr, attr_mask,
+ udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify uld qp, func(%d), qpn(%d), ret(%d).",
+ rdev->glb_func_id, rqp->qpn, ret);
+ goto err_out;
+ }
+
+ ret = roce5_modify_qp_cmd_process(rdev, rqp, attr, attr_mask,
+ &modify_para);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to modify qp, func(%d), qpn(%d), ret(%d).",
+ rdev->glb_func_id, rqp->qpn, ret);
+ goto err_out;
+ }
+
+ ret = roce5_record_opt_field(rqp, attr, attr_mask, udata,
+ &modify_para.modify_rsp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to record_opt_field, func(%d), qpn(%d).",
+ rdev->glb_func_id, rqp->qpn);
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) {
+ roce5_free_opt_rdmarc(rqp);
+ }
+
+ return ret;
+}
+
+static int roce5_qp_get_path(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct rdma_ah_attr *ah_attr)
+{
+ int ret = 0;
+ u16 vlan_id;
+
+ if ((ah_attr->ah_flags & IB_AH_GRH) == 0) {
+ ROCE_DEV_ERR(rdev, "Not supported, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ ret = roce5_resolve_grh(rdev, ah_attr, &vlan_id, NULL);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to find dmac by grh, func_id(%u)",
+ (u32)rdev->glb_func_id);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int
+roce5_qp_modify_check_rd_and_mtu(const struct roce5_qp *rqp,
+ const struct ib_qp_attr *attr, int attr_mask,
+ const struct rdma_service_cap *rdma_cap)
+{
+ struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device);
+ enum ib_qp_state cur_state =
+ (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ?
+ attr->cur_qp_state :
+ rqp->qp_state);
+ enum ib_qp_state new_state = (enum ib_qp_state)(
+ ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state :
+ cur_state);
+ enum roce5_mtu rmtu = (enum roce5_mtu)attr->path_mtu;
+
+ if (((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) &&
+ (attr->max_rd_atomic >
+ rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma)) {
+ ROCE_ERR(
+ "Qpn(0x%x), max_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)",
+ rqp->ibqp.qp_num, attr->max_rd_atomic, cur_state,
+ new_state, rqp->ibqp.qp_type);
+ return -EINVAL;
+ }
+
+ if (((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) &&
+ (attr->max_dest_rd_atomic >
+ rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma)) {
+ ROCE_ERR(
+ "Qpn(0x%x), max_dest_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)",
+ rqp->ibqp.qp_num, attr->max_dest_rd_atomic, cur_state,
+ new_state, rqp->ibqp.qp_type);
+ return -EINVAL;
+ }
+
+ if (((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) &&
+ ((rmtu < ROCE_MTU_256) ||
+ (rmtu > ((ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? ROCE_MTU_8192 :
+ ROCE_MTU_4096)))) {
+ ROCE_ERR("Input path MTU(%u) is invalid", rmtu);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_qp_modify_check(struct roce5_qp *rqp, struct ib_qp_attr *attr,
+ int attr_mask,
+ struct rdma_service_cap *rdma_cap)
+{
+ int ret;
+ enum ib_qp_state cur_state, new_state;
+ cur_state =
+ (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ?
+ attr->cur_qp_state :
+ rqp->qp_state);
+ new_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ?
+ attr->qp_state :
+ cur_state);
+
+#ifndef ROCE_VBS_EN
+ if (!roce5_check_qp_modify_ok(
+ cur_state, new_state, (enum ib_qp_type)(rqp->ibqp.qp_type),
+ (enum ib_qp_attr_mask)attr_mask, IB_LINK_LAYER_ETHERNET)) {
+ ROCE_ERR(
+ "Qpn(0x%x), invalid attribute mask specified for transition %d to %d, qp_type(%d), "
+ "attr_mask(0x%x)",
+ rqp->ibqp.qp_num, cur_state, new_state,
+ rqp->ibqp.qp_type, attr_mask);
+ return (-EINVAL);
+ }
+#endif
+
+ if (((((u32)attr_mask) & IB_QP_PORT) != 0) &&
+ (attr->port_num != ROCE_DEFAULT_PORT_NUM)) {
+ ROCE_ERR(
+ "Qpn(0x%x), invalid port number(%d) specified for transition %d to %d. qp_type(%d)",
+ rqp->ibqp.qp_num, attr->port_num, cur_state, new_state,
+ rqp->ibqp.qp_type);
+ return (-EINVAL);
+ }
+
+ if ((((u32)attr_mask) & IB_QP_PKEY_INDEX) != 0) {
+ if (attr->pkey_index > 0) {
+ ROCE_ERR(
+ "Qpn(0x%x), invalid pkey index (%d) specified for transition %d to %d. "
+ "qp_type(%d)",
+ rqp->ibqp.qp_num, attr->pkey_index, cur_state,
+ new_state, rqp->ibqp.qp_type);
+ return (-EINVAL);
+ }
+ }
+
+ ret = roce5_qp_modify_check_rd_and_mtu(rqp, attr, attr_mask, rdma_cap);
+
+ return ret;
+}
+
+#define ROCE5_QP_HANDLE 0
+#define ROCE5_QP_CONTINUE 1
+static int roce5_modify_qp_pre(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+ int attr_mask, struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_device *rdev = NULL;
+ enum ib_qp_state cur_state, new_state;
+
+ rqp = to_roce5_qp(ibqp);
+ rdev = to_roce5_dev(ibqp->device);
+ cur_state =
+ (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ?
+ attr->cur_qp_state :
+ rqp->qp_state);
+ new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state :
+ cur_state;
+
+ ret = roce5_qp_modify_check(rqp, attr, attr_mask, &rdev->rdma_cap);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed qp modify check, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ if (new_state == IB_QPS_RESET) {
+ if (cur_state != new_state) {
+ ret = roce5_qp_modify_2rst(rdev, rqp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify 2rst, func_id(%d)",
+ rdev->glb_func_id);
+ }
+ }
+ return ROCE5_QP_HANDLE;
+ }
+
+ ret = roce5_modify_qp_pre_ext(rdev, rqp, attr, attr_mask, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to modify qp pre ext, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ return ROCE5_QP_CONTINUE;
+}
+
+int roce5_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_device *rdev = NULL;
+
+ rqp = to_roce5_qp(ibqp);
+ rdev = to_roce5_dev(ibqp->device);
+
+ mutex_lock(&rqp->mutex);
+
+ ret = roce5_modify_qp_pre(ibqp, attr, attr_mask, udata);
+ if ((ret == ROCE5_QP_HANDLE) || (ret < 0)) {
+ goto out;
+ }
+
+ if ((((u32)attr_mask) & IB_QP_AV) != 0) {
+ ret = roce5_qp_get_path(rdev, rqp, &attr->ah_attr);
+ if (ret != 0) {
+ goto out;
+ }
+ }
+
+ ret = roce5_qp_modify(rdev, rqp, attr, attr_mask, udata);
+ if (ret != 0) {
+ goto out;
+ }
+
+out:
+ mutex_unlock(&rqp->mutex);
+ return ret;
+}
+
+static void roce5_verbs_shadow_set(struct roce5_device *rdev, u32 qpn,
+ cqm_cmd_buf_s *buf_in)
+{
+ roce_verbs_cmd_header_s *cmd_hdr =
+ (roce_verbs_cmd_header_s *)buf_in->buf;
+ struct roce5_qp *rqp = NULL;
+
+ if ((qpn == ROCE_QP_SMI_QP_NUM) || (qpn == ROCE_QP_GSI_QP_NUM)) {
+ return;
+ }
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ return;
+ }
+
+ if (rqp->shadow == 0 || cmd_hdr == NULL ||
+ cmd_hdr->dw0.bs.cmd_bitmask == VERBS_CMD_TYPE_VROCE_BITMASK) {
+ goto shadow_out;
+ }
+
+ cmd_hdr->dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK);
+ cmd_hdr->dw3.bs.shadow_vfid = cpu_to_be16(rqp->shadow_vfid);
+
+shadow_out:
+ roce5_rqp_put(rqp);
+}
+
+int roce5_send_qp_lb_cmd(u32 qpn, struct roce5_device *rdev, u8 cmd,
+ cqm_cmd_buf_s *buf_in, cqm_cmd_buf_s *buf_out,
+ u32 timeout)
+{
+ int ret = 0;
+
+ roce5_verbs_shadow_set(rdev, qpn, buf_in);
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, cmd, buf_in,
+ buf_out, NULL, timeout, HINIC5_CHANNEL_ROCE);
+
+ return ret;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c
new file mode 100644
index 000000000..fa48a2671
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c
@@ -0,0 +1,238 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "roce_compat.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_mr.h"
+#include "roce_xrc.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_qp.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#ifdef __ROCE_DFX__
+#include "roce_dfx.h"
+#endif
+
+/* ****************************************************************************
+ Prototype : roce5_get_recv_wqe
+ Description : roce5_get_recv_wqe
+ Input : struct roce5_qp *rqp
+ int n
+ Output : None
+
+ 1.Date : 2015/5/26
+ Modification : Created function
+
+**************************************************************************** */
+static void *roce5_get_recv_wqe(struct roce5_qp *rqp, u32 n)
+{
+ return roce5_get_wqe(rqp, rqp->rq.offset +
+ (n << (unsigned)rqp->rq.wqe_shift));
+}
+
+static int roce5_post_recv_check_qp(const struct roce5_qp *qp,
+ const struct roce5_device *rdev,
+ const struct ib_recv_wr **bad_wr,
+ const struct ib_recv_wr *wr)
+{
+ if (ROCE_UNLIKELY((qp->qp_type == IB_QPT_XRC_INI) ||
+ (qp->qp_type == IB_QPT_XRC_TGT))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Can't post WQE when TGT XRC QP, func_id(%d), qp_type(%d)",
+ rdev->glb_func_id, qp->qp_type);
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ if (ROCE_UNLIKELY(qp->qp_state == IB_QPS_RESET)) {
+ ROCE_DEV_ERR(rdev,
+ "Can't post WQE when QP is RST state, func_id(%d)",
+ rdev->glb_func_id);
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_post_recv_check_wr(struct roce5_qp *qp,
+ const struct ib_recv_wr **bad_wr,
+ const struct ib_recv_wr *wr, u32 wr_num)
+{
+ if (roce5_wq_overflow(&qp->rq, wr_num, qp->ibqp.recv_cq) != 0) {
+ ROCE_ERR_ONCE("SQ is full head:%x, tali:%x, wr_num:%d",
+ (&qp->rq)->head, (&qp->rq)->tail, wr_num);
+ *bad_wr = wr;
+ return -ENOMEM;
+ }
+
+ if (ROCE_UNLIKELY(((u32)wr->num_sge) > qp->rq.max_sge)) {
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_post_recv_check_length(struct roce5_device *rdev,
+ const struct ib_recv_wr **bad_wr,
+ const struct ib_recv_wr *wr, int i)
+{
+ u32 data_len = 0;
+
+ /* single SGEָ signs data len should less than (2G-1)B */
+ if (ROCE_UNLIKELY(wr->sg_list[i].length >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz -
+ 1)) {
+ *bad_wr = wr;
+ ROCE_DEV_ERR(
+ rdev,
+ "Sge data len is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)",
+ i, wr->sg_list[i].length,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz,
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ data_len += wr->sg_list[i].length;
+ /* all SGEָ signs data len should less than 2GB */
+ if (ROCE_UNLIKELY(
+ data_len >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) {
+ *bad_wr = wr;
+ ROCE_DEV_ERR(
+ rdev,
+ "Data_len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)",
+ data_len,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz,
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+static int roce5_post_recv_set_data_seg(const struct ib_recv_wr *wr,
+ struct roce5_device *rdev,
+ const struct ib_recv_wr **bad_wr,
+ struct roce5_qp *qp, u32 index)
+{
+ int i = 0;
+ int ret = 0;
+ struct roce_wqe_data_seg *dseg = NULL;
+
+ dseg = (struct roce_wqe_data_seg *)roce5_get_recv_wqe(qp, index);
+ for (i = 0; i < wr->num_sge; ++i) {
+ ret = roce5_post_recv_check_length(rdev, bad_wr, wr, i);
+ if (ret != 0) {
+ return ret;
+ }
+
+ roce5_set_data_seg(dseg, wr->sg_list + i);
+ dseg++;
+ }
+
+ if (wr->num_sge != 0) {
+ dseg--;
+ dseg->key =
+ cpu_to_be32((u32)(wr->sg_list[wr->num_sge - 1].lkey |
+ ROCE_WQE_NEXT_SGE_INVALID));
+ } else {
+ dseg->key = ROCE_WQE_NEXT_SGE_INVALID;
+ dseg->key = cpu_to_be32(dseg->key);
+ dseg->dw2.bs.len = 0;
+ }
+
+ return 0;
+}
+/* ****************************************************************************
+ Prototype : roce5_post_recv
+ Description : roce5_post_recv
+ Input : struct ib_qp *ibqp
+ struct ib_recv_wr *wr
+ struct ib_recv_wr **bad_wr
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr)
+{
+ struct roce5_qp *qp = NULL;
+ struct roce5_device *rdev = NULL;
+ unsigned long flags = 0; /* type required by kernel API */
+ int ret = 0;
+ u32 wr_num = 0;
+ u32 index = 0;
+ const struct ib_recv_wr *wr_tmp = wr;
+
+ if (ROCE_UNLIKELY(ibqp == NULL || bad_wr == NULL)) {
+ ROCE_ERR("ibqp(%d) or bad_wr(%d) is NULL.", !!ibqp, !!bad_wr);
+ return -EINVAL;
+ }
+
+ qp = to_roce5_qp(ibqp);
+ rdev = to_roce5_dev(ibqp->device);
+
+ ret = roce5_post_recv_check_qp(qp, rdev, bad_wr, wr_tmp);
+ if (ret != 0) {
+ return ret;
+ }
+
+ spin_lock_irqsave(&qp->rq.lock, flags);
+
+ index = (unsigned int)qp->rq.head &
+ ((unsigned int)qp->rq.wqebb_cnt - 1);
+
+ for (wr_num = 0; wr_tmp != NULL; ++wr_num, wr_tmp = wr_tmp->next) {
+ ret = roce5_post_recv_check_wr(qp, bad_wr, wr_tmp, wr_num);
+ if (ret != 0) {
+ goto out;
+ }
+
+ ret = roce5_post_recv_set_data_seg(wr_tmp, rdev, bad_wr, qp,
+ index);
+ if (ret != 0) {
+ goto out;
+ }
+
+ qp->rq.wrid[index] = wr_tmp->wr_id;
+
+ index = (index + 1) & (qp->rq.wqebb_cnt - 1);
+ }
+
+out:
+ if (ROCE_LIKELY(wr_num != 0)) {
+ qp->rq.head += (u32)wr_num;
+
+ wmb();
+
+ /* SQ need head 4B,RQ need tail 4B */
+ *(qp->db.db_record + 1) = cpu_to_be32(qp->rq.head & 0xffff);
+ }
+
+ spin_unlock_irqrestore(&qp->rq.lock, flags);
+
+ return ret;
+}
+
+int roce5_post_recv_ofa(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+ struct ib_recv_wr **bad_wr)
+{
+ return roce5_post_recv(ibqp, (const struct ib_recv_wr *)wr,
+ (const struct ib_recv_wr **)bad_wr);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c
new file mode 100644
index 000000000..ad1a8043a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c
@@ -0,0 +1,1205 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <rdma/ib_verbs.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include "roce_compat.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_mr.h"
+#include "roce_user.h"
+#include "roce_xrc.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_qp.h"
+#include "roce_post.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "roce_kernel_compat_gen.h"
+
+#include "securec.h"
+
+#ifdef __ROCE_DFX__
+#include "roce_dfx.h"
+#endif
+
+#define GOTO_LOCAL 1
+
+#define GOTO_OUT 2
+
+#define ROCE_NEED_JUMP 1
+
+#define BYTE_MASK 255
+
+#define ROCE_CTRL_SEG_SL 2
+/* ****************************************************************************
+ Prototype : roce5_get_send_wqe
+ Description : roce5_get_send_wqe
+ Input : struct roce5_qp *rqp
+ u32 n
+ Output : None
+
+ 1.Date : 2015/5/26
+ Modification : Created function
+
+**************************************************************************** */
+static void *roce5_get_send_wqe(struct roce5_qp *rqp, u32 n)
+{
+ u32 wqebb_index = (n & (((u32)rqp->sq.wqebb_cnt) - 1));
+ u32 wqebb_pos = wqebb_index << rqp->sq.wqe_shift;
+ return roce5_get_wqe(rqp, rqp->sq.offset + wqebb_pos);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_wq_overflow
+ Description : roce5_wq_overflow
+ Input : struct roce5_wq *wq
+ int wr_num
+ struct ib_cq *ibcq
+ Output : None
+
+ 1.Date : 2015/5/26
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_wq_overflow(struct roce5_wq *wq, u32 wr_num, struct ib_cq *ibcq)
+{
+ unsigned int cur = 0;
+ struct roce5_cq *cq = NULL;
+
+ cur = wq->head - wq->tail;
+ if (ROCE_LIKELY((cur + wr_num) < (unsigned int)wq->max_post)) {
+ return 0;
+ }
+
+ cq = to_roce5_cq(ibcq);
+ spin_lock(&cq->lock);
+ cur = wq->head - wq->tail;
+ spin_unlock(&cq->lock);
+
+ return ((cur + (unsigned int)wr_num) >= (unsigned int)wq->max_post);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_ib_opcode
+ Description :
+ Input :
+ Output : None
+ Return Value :
+ Calls :
+ Called By :
+
+ History :
+ 1.Date : 2015/7/30
+ Author :
+ Modification : Created function
+
+**************************************************************************** */
+static const u32 roce5_ib_opcode[] = {
+ [IB_WR_RDMA_WRITE] = ROCE_WQE_OPCODE_RDMA_WRITE,
+ [IB_WR_RDMA_WRITE_WITH_IMM] = ROCE_WQE_OPCODE_RDMA_WRITE_IMM,
+ [IB_WR_SEND] = ROCE_WQE_OPCODE_SEND,
+ [IB_WR_SEND_WITH_IMM] = ROCE_WQE_OPCODE_SEND_IMM,
+ [IB_WR_RDMA_READ] = ROCE_WQE_OPCODE_RDMA_READ,
+ [IB_WR_ATOMIC_CMP_AND_SWP] = ROCE_WQE_OPCODE_ATOMIC_CMP_SWP,
+ [IB_WR_ATOMIC_FETCH_AND_ADD] = ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD,
+ [IB_WR_SEND_WITH_INV] = ROCE_WQE_OPCODE_SEND_INVAL,
+ [IB_WR_LOCAL_INV] = ROCE_WQE_OPCODE_LOCAL_INVAL,
+ [IB_WR_REG_MR] = ROCE_WQE_OPCODE_FRMR,
+ [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] =
+ ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP,
+ [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] =
+ ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD,
+ [IB_WR_REG_MR_INTEGRITY] = ROCE_WQE_OPCODE_REG_SIG_MR,
+#if defined(IB_WR_OPCODE_HAVE_BIND_MW)
+ [IB_WR_BIND_MW] = ROCE_WQE_OPCODE_BIND_MW,
+#endif
+};
+
+static const roce_opcode_attr_s g_roce5_opcode_attr[ROCE_KERNEL_OPCODE_NUM] = {
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND = 0x0,
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND_WITH_INV,
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND_WITH_IMM,
+ { 0 }, //ROCE_WQE_OPCODE_SEND_FT_TEST
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_WRITE = 0x4,
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_WRITE_WITH_IMM,
+ { 0 },
+ { 0 },
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_READ = 0x8,
+ { 0 }, //ROCE_WQE_OPCODE_ATOMIC_WRITE,
+ { 0 }, //ROCE_WQE_OPCODE_CFLUSH,
+ { .remote_opcode =
+ 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_EXT_CMP_AND_SWP, /* support max 64B */
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_CMP_AND_SWP = 0xc,
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_FETCH_AND_ADD,
+ { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_AND_SWP,
+ { .remote_opcode =
+ 0x1 }, //ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_AND_ADD,
+ { 0 }, //ROCE_WQE_OPCODE_FAST_REG_PMR, /* 8'h10-Fast Register PMR is not used by userspace */
+ { 0 }, //ROCE_WQE_OPCODE_LOCAL_INV = 0x11,
+ { 0 }, //ROCE_WQE_OPCODE_BIND_MW,
+ { 0 } //ROCE_WQE_OPCODE_REG_SIG_MR = 0x13, /* 8'h13-Local opreation(Extended for further local opreation) */
+};
+
+static void roce5_set_send_seg(struct roce_wqe_send_tsk_seg *snd_tsk_seg,
+ const struct ib_send_wr *wr)
+{
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ snd_tsk_seg->immdata_invkey = wr->ex.imm_data;
+ }
+
+ if (wr->opcode == IB_WR_SEND_WITH_INV) {
+ snd_tsk_seg->immdata_invkey =
+ cpu_to_be32(wr->ex.invalidate_rkey);
+ }
+}
+
+static void roce5_set_atomic_seg(struct roce_wqe_atomic_tsk_seg *atomic_tsk_seg,
+ const struct ib_send_wr *wr)
+{
+ const struct ib_atomic_wr *atomic = atomic_wr(wr);
+
+ atomic_tsk_seg->key = cpu_to_be32(atomic->rkey);
+ atomic_tsk_seg->dw2.va = cpu_to_be64(atomic->remote_addr);
+
+ if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
+ atomic_tsk_seg->dw4.swap_add_data = cpu_to_be64(atomic->swap);
+ atomic_tsk_seg->dw6.cmp_data = cpu_to_be64(atomic->compare_add);
+ } else if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
+ atomic_tsk_seg->dw4.swap_add_data =
+ cpu_to_be64(atomic->compare_add);
+ atomic_tsk_seg->dw6.cmp_data = 0;
+ } else {
+ /* IB_WR_MASKED_ATOMIC_FETCH_AND_ADD */
+ atomic_tsk_seg->dw4.swap_add_data =
+ cpu_to_be64(atomic->compare_add);
+ atomic_tsk_seg->dw6.cmp_data =
+ cpu_to_be64(atomic->compare_add_mask);
+ }
+}
+
+static __always_inline void
+roce5_set_rdma_seg(struct roce_wqe_rdma_tsk_seg *rdma_tsk_seg,
+ const struct ib_send_wr *wr)
+{
+ const struct ib_rdma_wr *rdma = rdma_wr(wr);
+
+ rdma_tsk_seg->va = cpu_to_be64(rdma->remote_addr);
+ rdma_tsk_seg->rkey = cpu_to_be32(rdma->rkey);
+
+ if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
+ rdma_tsk_seg->imm_data = cpu_to_be32(wr->ex.imm_data);
+ }
+}
+
+static void
+roce5_set_local_inv_seg(struct roce_wqe_local_inv_tsk_seg *inv_tsk_seg,
+ const struct ib_send_wr *wr)
+{
+ inv_tsk_seg->inv_key = cpu_to_be32(wr->ex.invalidate_rkey);
+}
+
+static void
+roce5_set_ud_seg_cycle1(struct roce_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1,
+ const struct ib_send_wr *wr)
+{
+ const struct ib_ud_wr *ud = ud_wr(wr);
+ struct roce5_ah *rah = to_roce5_ah(ud->ah);
+ int ret;
+
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ ud_tsk_seg_cycle1->immdata_invkey = wr->ex.imm_data;
+ }
+
+ ud_tsk_seg_cycle1->dw2.value = rah->priv_ah.dw0.value;
+ ud_tsk_seg_cycle1->dw3.value = rah->priv_ah.dw1.value;
+ ud_tsk_seg_cycle1->dw4.value = rah->priv_ah.dw2.value;
+ ret = memcpy_s((void *)ud_tsk_seg_cycle1->dgid, ROCE_GID_LEN,
+ (void *)rah->priv_ah.dgid, ROCE_GID_LEN);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ return;
+ }
+ ud_tsk_seg_cycle1->dw9.value = cpu_to_be32(ud->remote_qpn);
+ ud_tsk_seg_cycle1->qkey = cpu_to_be32(ud->remote_qkey);
+}
+
+static void
+roce5_set_ud_seg_cycle2(struct roce_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2,
+ const struct ib_send_wr *wr)
+{
+ const struct ib_ud_wr *ud = ud_wr(wr);
+ struct roce5_ah *rah = to_roce5_ah(ud->ah);
+
+ ud_tsk_seg_cycle2->dw0.value = rah->priv_ah.dw7.value;
+ ud_tsk_seg_cycle2->dmac_l32 = rah->priv_ah.dmac_l32;
+}
+
+static void roce5_set_ud_seg(struct roce_wqe_ud_tsk_seg *ud_tsk_seg,
+ const struct ib_send_wr *wr)
+{
+ const struct ib_ud_wr *ud = ud_wr(wr);
+ struct roce5_ah *rah = to_roce5_ah(ud->ah);
+ int ret;
+
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ ud_tsk_seg->immdata_invkey = wr->ex.imm_data;
+ }
+
+ ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value;
+ ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value;
+ ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value;
+ ret = memcpy_s((void *)ud_tsk_seg->dgid, sizeof(ud_tsk_seg->dgid),
+ (void *)rah->priv_ah.dgid, sizeof(ud_tsk_seg->dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ return;
+ }
+ ud_tsk_seg->dw10.value = cpu_to_be32(ud->remote_qpn);
+ ud_tsk_seg->qkey = cpu_to_be32(ud->remote_qkey);
+ ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value;
+ ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32;
+}
+
+void roce5_set_data_seg(struct roce_wqe_data_seg *dseg, struct ib_sge *sge)
+{
+ dseg->addr = cpu_to_be64(sge->addr);
+
+ dseg->dw2.bs.len = sge->length & 0x7fffffff;
+ dseg->dw2.bs.rsvd = 0;
+
+ dseg->dw2.value = cpu_to_be32(dseg->dw2.value);
+
+ dseg->key = cpu_to_be32(sge->lkey);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_set_gsi_seg
+ Input : struct roce5_qp *rqp
+ struct ib_send_wr *wr
+ void *wqe
+ Output : None
+
+ 1.Date : 2015/8/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_set_gsi_seg(struct roce5_qp *rqp, const struct ib_send_wr *wr,
+ void *wqe)
+{
+ struct roce_wqe_ud_tsk_seg *ud_tsk_seg =
+ (struct roce_wqe_ud_tsk_seg *)wqe;
+ struct ib_ud_wr *ud_wr = (struct ib_ud_wr *)(void *)wr;
+ struct roce5_ah *rah = to_roce5_ah(ud_wr->ah);
+ int ret;
+
+ if (rqp->sqp_attr == NULL) {
+ ROCE_ERR("Error! sqp attribute is NULL, qpn(%u).", rqp->qpn);
+ return -EINVAL;
+ }
+
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ ud_tsk_seg->immdata_invkey = wr->ex.imm_data;
+ ud_tsk_seg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN;
+ ud_tsk_seg->dw2.value = cpu_to_be32(ud_tsk_seg->dw2.value);
+ }
+
+ ud_tsk_seg->common.bs.c = 1;
+ ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value;
+ ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value;
+ ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value;
+ ret = memcpy_s((void *)ud_tsk_seg->dgid, sizeof(ud_tsk_seg->dgid),
+ (void *)rah->priv_ah.dgid, sizeof(ud_tsk_seg->dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ return ret;
+ }
+ ud_tsk_seg->dw10.value = cpu_to_be32(ud_wr->remote_qpn);
+ ud_tsk_seg->qkey = cpu_to_be32(
+ ((ud_wr->remote_qkey & 0x80000000) != 0) ? rqp->sqp_attr->qkey :
+ ud_wr->remote_qkey);
+ ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value;
+ ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32;
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : write_invalid_wqe
+ Description : set the next wqe invalidate
+ Input : struct roce5_qp *qp
+ u32 index
+ Output : None
+
+ 1.Date : 2015/9/8
+ Author : qianyuanpan
+ Modification : Created function
+
+**************************************************************************** */
+static void write_invalid_wqe(struct roce5_qp *qp, u32 index)
+{
+ u32 invalid_wqe_dw0 = 0;
+ struct roce5_wqe_ctrl_seg *ctrl_seg = NULL;
+
+ ctrl_seg = (struct roce5_wqe_ctrl_seg *)roce5_get_send_wqe(qp, index);
+ invalid_wqe_dw0 = ((index & (u32)qp->sq.wqebb_cnt) == 0) ? 0xff000000 :
+ 0x7f000000;
+ ctrl_seg->dw0.value = cpu_to_be32(invalid_wqe_dw0);
+}
+
+#define ROCE5_PCIE_ATOMIC_EN 0x1
+static inline bool roce5_is_pcie_atomic(struct roce5_qp *rqp)
+{
+ struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device);
+ return (rdev->cfg_info.pcie_atomic_en == ROCE5_PCIE_ATOMIC_EN);
+}
+
+static inline bool roce5_is_extend_atomic(enum ib_wr_opcode opcode)
+{
+ return ((roce5_ib_opcode[opcode] ==
+ ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP) ||
+ (roce5_ib_opcode[opcode] ==
+ ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD));
+}
+
+static inline int roce5_is_vaild_wr_opcode(struct roce5_qp *rqp,
+ enum ib_wr_opcode opcode)
+{
+ if (ROCE_UNLIKELY((opcode >= ARRAY_SIZE(roce5_ib_opcode)))) {
+ return -EINVAL;
+ }
+
+ if ((roce5_ib_opcode[opcode] == 0) && (opcode != IB_WR_SEND)) {
+ return -EINVAL;
+ }
+
+ if (roce5_is_pcie_atomic(rqp) && roce5_is_extend_atomic(opcode)) {
+ ROCE_ERR(
+ "The atomic mode is pcie atomic, but extended atomic is not supported.");
+ return -EINVAL;
+ }
+
+ return 0; /* Vaild IB opcode */
+}
+
+/* ****************************************************************************
+ Prototype : roce5_validate_wr
+ Description : roce5_validate_wr
+ Input : struct roce5_qp *rqp
+ struct ib_send_wr *wr
+ int wr_num
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_validate_wr(struct roce5_qp *rqp, const struct ib_send_wr *wr,
+ u32 wr_num)
+{
+ if (roce5_wq_overflow(&rqp->sq, wr_num, rqp->ibqp.send_cq) != 0) {
+ ROCE_ERR("SQ is full");
+ return -ENOMEM;
+ }
+
+ if (ROCE_UNLIKELY(((u32)wr->num_sge) > rqp->sq.max_sge)) {
+ ROCE_ERR(
+ "Sge num is invalide, wr->num_sge(%d), rqp->sq.max_sge(%d)",
+ wr->num_sge, rqp->sq.max_sge);
+ return -EINVAL;
+ }
+
+ if (ROCE_UNLIKELY(roce5_is_vaild_wr_opcode(rqp, wr->opcode) != 0)) {
+ ROCE_ERR("wr->opcode(%d)", wr->opcode);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void
+roce5_set_rc_wqe_for_optype_send(const struct ib_send_wr *wr,
+ struct roce5_post_send_normal_param *param)
+{
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ roce5_set_send_seg((struct roce_wqe_send_tsk_seg *)param->wqe, wr);
+ param->data_len_addr =
+ &((struct roce_wqe_send_tsk_seg *)param->wqe)->data_len;
+ param->wqe = ((u8 *)param->wqe + sizeof(struct roce_wqe_send_tsk_seg));
+ param->wqe_size += (u32)sizeof(struct roce_wqe_send_tsk_seg);
+}
+
+static void
+roce5_set_rc_wqe_for_optype_atomic(const struct ib_send_wr *wr,
+ struct roce5_post_send_normal_param *param)
+{
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ roce5_set_atomic_seg((struct roce_wqe_atomic_tsk_seg *)param->wqe, wr);
+ param->wqe =
+ ((u8 *)param->wqe + sizeof(struct roce_wqe_atomic_tsk_seg));
+ param->wqe_size += (u32)sizeof(struct roce_wqe_atomic_tsk_seg);
+}
+
+static void
+roce5_set_rc_wqe_for_optype_write(const struct ib_send_wr *wr,
+ struct roce5_post_send_normal_param *param)
+{
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ roce5_set_rdma_seg((struct roce_wqe_rdma_tsk_seg *)param->wqe, wr);
+ param->data_len_addr =
+ &((struct roce_wqe_rdma_tsk_seg *)param->wqe)->data_len;
+ param->wqe = ((u8 *)param->wqe + sizeof(struct roce_wqe_rdma_tsk_seg));
+ param->wqe_size += (u32)(sizeof(struct roce_wqe_rdma_tsk_seg));
+}
+
+static void
+roce5_set_rc_wqe_for_optype_local(const struct ib_send_wr *wr,
+ struct roce5_post_send_normal_param *param)
+{
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ param->tsk_com_seg->bs.so = 1;
+ roce5_set_local_inv_seg((struct roce_wqe_local_inv_tsk_seg *)param->wqe,
+ wr);
+ param->wqe_size += (u32)sizeof(struct roce_wqe_local_inv_tsk_seg);
+ param->inline_flag = 0;
+ param->ctrl_seg_tmp.dw0.bs.df = 0;
+}
+
+static void roce5_set_wqe_last_len(u8 *wqe)
+{
+ struct roce_wqe_send_tsk_seg *snd_task =
+ (struct roce_wqe_send_tsk_seg *)(void *)wqe;
+ snd_task->dw3.value = 0;
+ snd_task->dw3.bs.last_ext_len = ROCE_IMM_EXT_LEN;
+ snd_task->dw3.value = cpu_to_be32(snd_task->dw3.value);
+}
+
+static inline u32 roce5_log2(u32 x)
+{
+ u32 shift;
+
+ for (shift = 0; (1U << shift) < x; ++shift) {
+ ;
+ }
+
+ return shift;
+}
+
+/* RoCE supports FRMR, for at privileged user, can via post send WR of way for at at free status of MR perform fast register.
+ frmr wqe contain a pbl(page buffer list) table start address, start va, length, page size etc. information.
+ 预 先 allocate of store space of start address (pa) need write pbl. only supports one level
寻 址, maximum supports 512(count word) PA. */
+static int set_frmr_wqe_section(const struct ib_send_wr *wr,
+ struct roce5_post_send_normal_param *param)
+{
+ u64 iova;
+ u32 fbo;
+ struct ib_reg_wr *frmr_reg_wr = NULL;
+ struct roce5_mr *roce5_kernel_frmr = NULL;
+ struct roce_wqe_frmr_tsk_seg *frmr_wqe_task = NULL;
+
+ frmr_reg_wr = (struct ib_reg_wr *)reg_wr((struct ib_send_wr *)wr);
+ roce5_kernel_frmr = to_roce5_mr(frmr_reg_wr->mr);
+
+ /* base section set */
+ param->wqe_size += sizeof(struct roce_wqe_frmr_tsk_seg);
+
+ frmr_wqe_task =
+ (struct roce_wqe_frmr_tsk_seg *)((void *)param->tsk_com_seg);
+ param->dseg =
+ (roce_wqe_data_seg_s *)(((u8 *)param->tsk_com_seg) +
+ sizeof(struct roce_wqe_frmr_tsk_seg));
+ param->ctrl_seg_tmp.dw0.bs.tsl =
+ sizeof(struct roce_wqe_frmr_tsk_seg) / ROCE_TASK_SEG_ALIGN;
+ /* fill wqe ctrl head section:cl bit */
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+
+ /* local invalidate associated section set */
+ param->tsk_com_seg->bs.so = 1;
+
+ iova = (u64)(void *)(uintptr_t)(frmr_reg_wr->mr->iova);
+ fbo = (u32)(iova & (FRMR_PAGE_SIZE - 1));
+ frmr_wqe_task->dw2.bs.fbo = fbo & 0x3FFFFF;
+ frmr_wqe_task->dw2.value = cpu_to_be32(frmr_wqe_task->dw2.value);
+
+ frmr_wqe_task->dw5.va =
+ (((u32)frmr_reg_wr->access & IB_ZERO_BASED) != 0) ?
+ 0 :
+ cpu_to_be64(iova);
+
+ /* fill in access permission */
+ frmr_wqe_task->dw3.bs.zbva =
+ (u32)(((u32)frmr_reg_wr->access & IB_ZERO_BASED) != 0);
+ frmr_wqe_task->dw3.bs.be =
+ (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_MW_BIND) !=
+ 0); /* bind operation enable */
+ frmr_wqe_task->dw3.bs.lre = 1; /* this place read enable */
+ frmr_wqe_task->dw3.bs.lwe =
+ (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_LOCAL_WRITE) !=
+ 0); /* this place write enable */
+ frmr_wqe_task->dw3.bs.rre =
+ (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_READ) !=
+ 0); /* remote read enable */
+ frmr_wqe_task->dw3.bs.rwe =
+ (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_WRITE) !=
+ 0); /* remote write enable */
+ frmr_wqe_task->dw3.bs.rae =
+ (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_ATOMIC) !=
+ 0); /* remote Atomic enable */
+ frmr_wqe_task->dw3.bs.block = 1;
+ frmr_wqe_task->dw3.bs.rsvd = 0;
+ frmr_wqe_task->dw3.bs.pa_num = (roce5_kernel_frmr->npages == 0) ?
+ 1 :
+ roce5_kernel_frmr->npages;
+ /* MR memory page size. equals to (2^page_size)*4KB */
+ frmr_wqe_task->dw3.bs.page_size =
+ roce5_log2(frmr_reg_wr->mr->page_size) - 12; //12 is 4K page of shift
+ if (frmr_reg_wr->mr->page_size != PAGE_SIZE) {
+ ROCE_WARN(
+ "Frmr page size doesn't match page frame of OS! It may lead to unexpected memory translation.");
+ }
+ frmr_wqe_task->dw3.value = cpu_to_be32(frmr_wqe_task->dw3.value);
+
+ frmr_wqe_task->m_key = cpu_to_be32(frmr_reg_wr->key);
+ frmr_wqe_task->dw7.len = cpu_to_be64(frmr_reg_wr->mr->length);
+ frmr_wqe_task->dw9.pbl_addr = cpu_to_be64(roce5_kernel_frmr->page_map);
+ /* set inline and direct wqe handle section */
+ frmr_wqe_task->rsvd[0] = 0;
+ frmr_wqe_task->rsvd[1] = 0;
+
+ param->inline_flag = 0;
+ return 0;
+}
+
+static int roce5_post_send_rc(struct roce5_post_send_normal_param *param,
+ struct roce5_device *rdev,
+ const struct ib_send_wr *wr)
+{
+ int ret = 0;
+
+ switch (wr->opcode) {
+ case IB_WR_SEND_WITH_IMM:
+ case IB_WR_SEND_WITH_INV:
+ roce5_set_wqe_last_len(param->wqe);
+ roce5_set_rc_wqe_for_optype_send(wr, param);
+ break;
+ case IB_WR_SEND:
+ roce5_set_rc_wqe_for_optype_send(wr, param);
+ break;
+
+ case IB_WR_ATOMIC_CMP_AND_SWP:
+ case IB_WR_ATOMIC_FETCH_AND_ADD:
+ roce5_set_rc_wqe_for_optype_atomic(wr, param);
+ break;
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ roce5_set_wqe_last_len(param->wqe);
+ roce5_set_rc_wqe_for_optype_write(wr, param);
+ break;
+
+ case IB_WR_RDMA_READ:
+ case IB_WR_RDMA_WRITE:
+ roce5_set_rc_wqe_for_optype_write(wr, param);
+ break;
+
+ case IB_WR_LOCAL_INV:
+ roce5_set_rc_wqe_for_optype_local(wr, param);
+ ret = ROCE_NEED_JUMP;
+ break;
+
+ case IB_WR_REG_MR:
+ set_frmr_wqe_section(wr, param);
+ ret = ROCE_NEED_JUMP;
+ break;
+
+ default:
+ ROCE_DEV_ERR(rdev, "Invalid opcode(%d), func_id(%d)",
+ wr->opcode, rdev->glb_func_id);
+ break;
+ }
+
+ return ret;
+}
+
+static int roce5_post_send_uc(struct roce5_post_send_normal_param *param,
+ struct roce5_device *rdev,
+ const struct ib_send_wr *wr)
+{
+ switch (wr->opcode) {
+ case IB_WR_SEND_WITH_IMM:
+ case IB_WR_SEND:
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ roce5_set_wqe_last_len(param->wqe);
+ }
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ roce5_set_send_seg((struct roce_wqe_send_tsk_seg *)param->wqe,
+ wr);
+ param->data_len_addr =
+ &((struct roce_wqe_send_tsk_seg *)param->wqe)->data_len;
+ param->wqe = ((u8 *)param->wqe +
+ sizeof(struct roce_wqe_send_tsk_seg));
+ param->wqe_size += (u32)sizeof(struct roce_wqe_send_tsk_seg);
+ break;
+
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ case IB_WR_RDMA_WRITE:
+ if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
+ roce5_set_wqe_last_len(param->wqe);
+ }
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ roce5_set_rdma_seg((struct roce_wqe_rdma_tsk_seg *)param->wqe,
+ wr);
+ param->data_len_addr =
+ &((struct roce_wqe_rdma_tsk_seg *)param->wqe)->data_len;
+ param->wqe = ((u8 *)param->wqe +
+ sizeof(struct roce_wqe_rdma_tsk_seg));
+ param->wqe_size += (u32)sizeof(struct roce_wqe_rdma_tsk_seg);
+ break;
+
+ case IB_WR_LOCAL_INV:
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ param->tsk_com_seg->bs.so = 1;
+ roce5_set_local_inv_seg(
+ (struct roce_wqe_local_inv_tsk_seg *)param->wqe, wr);
+ param->wqe_size +=
+ (u32)sizeof(struct roce_wqe_local_inv_tsk_seg);
+ return ROCE_NEED_JUMP;
+
+ default:
+ ROCE_DEV_ERR(rdev, "Invalid opcode(%d), func_id(%d)",
+ wr->opcode, rdev->glb_func_id);
+ break;
+ }
+
+ return 0;
+}
+
+static int roce5_post_send_ud(struct roce5_post_send_normal_param *param,
+ struct roce5_device *rdev, struct roce5_qp *rqp,
+ const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ struct roce_wqe_ud_tsk_seg *tsk_sg =
+ (struct roce_wqe_ud_tsk_seg *)param->wqe;
+
+ if (ROCE_UNLIKELY((wr->opcode != IB_WR_SEND) &&
+ (wr->opcode != IB_WR_SEND_WITH_IMM))) {
+ ROCE_DEV_ERR(rdev, "Wr->opcode(%d), func_id(%d)", wr->opcode,
+ rdev->glb_func_id);
+ *bad_wr = wr;
+ return ROCE_NEED_JUMP;
+ }
+
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ tsk_sg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN;
+ tsk_sg->dw2.value = cpu_to_be32(tsk_sg->dw2.value);
+ }
+
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+ param->data_len_addr =
+ &((struct roce_wqe_ud_tsk_seg *)param->wqe)->data_len;
+
+ if (param->sq_rmd_size > (1UL << (unsigned int)rqp->sq.wqe_shift)) {
+ roce5_set_ud_seg((struct roce_wqe_ud_tsk_seg *)(param->wqe),
+ wr);
+ param->wqe =
+ ((u8 *)param->wqe + sizeof(struct roce_wqe_ud_tsk_seg));
+ param->wqe_size += (u32)sizeof(struct roce_wqe_ud_tsk_seg);
+ } else {
+ roce5_set_ud_seg_cycle1(
+ (struct roce_wqe_ud_tsk_seg_cycle1 *)param->wqe, wr);
+ param->wqe = rqp->sq_head_addr;
+ param->wqe_size +=
+ (u32)sizeof(struct roce_wqe_ud_tsk_seg_cycle1);
+ roce5_set_ud_seg_cycle2(
+ (struct roce_wqe_ud_tsk_seg_cycle2 *)param->wqe, wr);
+ param->wqe = ((u8 *)param->wqe +
+ sizeof(struct roce_wqe_ud_tsk_seg_cycle2));
+ param->wqe_size +=
+ (u32)sizeof(struct roce_wqe_ud_tsk_seg_cycle2);
+ }
+
+ return 0;
+}
+
+static void roce5_post_send_set_normal_data_seg(const struct ib_send_wr *wr,
+ struct roce_wqe_data_seg **dseg,
+ struct roce5_qp *rqp)
+{
+ if (wr->num_sge != 0) {
+ if ((u8 *)(*dseg) == rqp->sq_head_addr) {
+ *dseg = (struct roce_wqe_data_seg *)((
+ void *)(rqp->sq_tail_addr -
+ sizeof(struct roce_wqe_data_seg)));
+ } else {
+ (*dseg)--;
+ }
+ (*dseg)->key = (u32)wr->sg_list[wr->num_sge - 1].lkey |
+ ROCE_WQE_NEXT_SGE_INVALID;
+ (*dseg)->key = cpu_to_be32((*dseg)->key);
+ }
+}
+
+static int
+roce5_post_send_set_normal_sge(struct roce5_post_send_normal_param *param,
+ struct roce5_device *rdev, struct roce5_qp *rqp,
+ const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ int i;
+
+ param->data_len = 0;
+ param->dseg = (struct roce_wqe_data_seg *)param->wqe;
+ param->wqe_size +=
+ (u32)(wr->num_sge * (int)sizeof(struct roce_wqe_data_seg));
+ for (i = 0; i < wr->num_sge; ++i) {
+ if (param->sq_rmd_size < sizeof(struct roce_wqe_data_seg)) {
+ param->dseg = (struct roce_wqe_data_seg
+ *)((void *)rqp->sq_head_addr);
+ }
+
+ if (ROCE_UNLIKELY(wr->sg_list[i].length >
+ (rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz))) {
+ *bad_wr = wr;
+ ROCE_DEV_ERR(
+ rdev,
+ "Sge_data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)",
+ i, wr->sg_list[i].length,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz,
+ rdev->glb_func_id);
+ return ROCE_NEED_JUMP;
+ }
+
+ param->data_len += wr->sg_list[i].length;
+ if (ROCE_UNLIKELY(param->data_len >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz)) {
+ *bad_wr = wr;
+ ROCE_DEV_ERR(
+ rdev,
+ "Data len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)",
+ param->data_len,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz,
+ rdev->glb_func_id);
+ return ROCE_NEED_JUMP;
+ }
+ roce5_set_data_seg(param->dseg, wr->sg_list + i);
+
+ (param->dseg)++;
+ param->sq_rmd_size =
+ (u32)(rqp->sq_tail_addr - (u8 *)param->dseg);
+ }
+
+ return 0;
+}
+
+static void roce5_copy_inline_data(struct roce5_post_send_normal_param *param,
+ const struct ib_send_wr *wr,
+ struct roce5_qp *rqp, int i)
+{
+ int ret = 0;
+ struct ib_sge *sge = wr->sg_list + i;
+
+ if (param->sq_rmd_size >= wr->sg_list[i].length) {
+ /* Set inline data */
+ ret = memcpy_s(param->wqe, sge->length,
+ (void *)(uintptr_t)sge->addr, sge->length);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret);
+ return;
+ }
+ param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length);
+ if ((u8 *)param->wqe == rqp->sq_tail_addr) {
+ param->wqe = rqp->sq_head_addr;
+ }
+ } else {
+ /* Set inline data cycle1 */
+ u32 size = param->sq_rmd_size;
+ ret = memcpy_s(param->wqe, size, (void *)(uintptr_t)sge->addr,
+ size);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret);
+ return;
+ }
+ param->wqe = rqp->sq_head_addr;
+
+ /* Set inline data cycle2 */
+ size = wr->sg_list[i].length - param->sq_rmd_size;
+ ret = memcpy_s(
+ param->wqe, size,
+ (void *)((u8 *)(void *)(uintptr_t)sge->addr + size),
+ size);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret);
+ return;
+ }
+ param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length -
+ param->sq_rmd_size);
+
+ param->cycle = 1;
+ }
+}
+
+static int
+roce5_post_send_set_data_seg(struct roce5_post_send_normal_param *param,
+ struct roce5_device *rdev, struct roce5_qp *rqp,
+ const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ int i;
+ int ret;
+
+ if ((((unsigned int)wr->send_flags & (unsigned int)IB_SEND_INLINE) !=
+ 0) &&
+ (wr->num_sge != 0) && (param->inline_flag != 0)) {
+ param->data_len = 0;
+
+ for (i = 0; i < wr->num_sge; ++i) {
+ param->wqe_size += wr->sg_list[i].length;
+ param->data_len += wr->sg_list[i].length;
+ if (param->data_len > rqp->max_inline_data) {
+ *bad_wr = wr;
+ ROCE_DEV_ERR(
+ rdev,
+ "Data len is too big, data_len(%d), max_inline_data(%d), func_id(%d)",
+ param->data_len, rqp->max_inline_data,
+ rdev->glb_func_id);
+ return ROCE_NEED_JUMP;
+ }
+
+ if ((u8 *)param->wqe == rqp->sq_head_addr) {
+ param->cycle = 1;
+ }
+
+ roce5_copy_inline_data(param, wr, rqp, i);
+
+ param->sq_rmd_size =
+ (u32)(rqp->sq_tail_addr - (u8 *)param->wqe);
+ }
+ param->ctrl_seg_tmp.dw0.bs.bdsl =
+ (ALIGN((u32)(param->data_len), ROCE_TASK_SEG_ALIGN) /
+ ROCE_TASK_SEG_ALIGN) &
+ 0x7ff;
+ param->ctrl_seg_tmp.dw0.bs.df = 1;
+ param->inline_flag = 1;
+ } else {
+ ret = roce5_post_send_set_normal_sge(param, rdev, rqp, wr,
+ bad_wr);
+ if (ret != 0) {
+ return ret;
+ }
+
+ roce5_post_send_set_normal_data_seg(wr, ¶m->dseg, rqp);
+
+ param->ctrl_seg_tmp.dw0.bs.bdsl =
+ (u32)((wr->num_sge *
+ (int)sizeof(struct roce_wqe_data_seg)) /
+ ROCE_TASK_SEG_ALIGN) &
+ 0x7ff;
+ param->ctrl_seg_tmp.dw0.bs.df = 0;
+ }
+
+ return 0;
+}
+
+static void roce5_fill_task_data_len(u32 *data_len_addr, u32 data_len)
+{
+ if (data_len_addr != NULL) {
+ *data_len_addr = cpu_to_be32(data_len);
+ }
+}
+
+static int roce5_construct_wqe(struct roce5_post_send_normal_param *param,
+ struct roce5_device *rdev, struct roce5_qp *rqp,
+ const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ int need_goto = 0;
+ int ret;
+
+ switch (rqp->qp_type) {
+ case IB_QPT_RC:
+ need_goto = roce5_post_send_rc(param, rdev, wr);
+ if (need_goto == ROCE_NEED_JUMP) {
+ return GOTO_LOCAL;
+ }
+ break;
+
+ case IB_QPT_UC:
+ need_goto = roce5_post_send_uc(param, rdev, wr);
+ if (need_goto == ROCE_NEED_JUMP) {
+ return GOTO_LOCAL;
+ }
+ break;
+
+ case IB_QPT_UD:
+ need_goto = roce5_post_send_ud(param, rdev, rqp, wr, bad_wr);
+ if (need_goto == ROCE_NEED_JUMP) {
+ return GOTO_OUT;
+ }
+ break;
+
+ case IB_QPT_GSI:
+ param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL;
+ param->ctrl_seg_tmp.dw1.bs.cl = 1;
+
+ ret = roce5_set_gsi_seg(rqp, wr, param->wqe);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to set gsi seg, ret(%d).",
+ ret);
+ return GOTO_OUT;
+ }
+
+ param->data_len_addr =
+ &((struct roce_wqe_ud_tsk_seg *)param->wqe)->data_len;
+ param->wqe =
+ ((u8 *)param->wqe + sizeof(struct roce_wqe_ud_tsk_seg));
+ param->wqe_size += (u32)sizeof(struct roce_wqe_ud_tsk_seg);
+ break;
+
+ default:
+ ROCE_DEV_ERR(rdev, "Unknown qp type, func_id(%d)",
+ rdev->glb_func_id);
+ break;
+ }
+
+ return 0;
+}
+
+static void roce5_init_ctrl_seg(struct roce5_post_send_normal_param *param,
+ const struct ib_send_wr *wr,
+ const struct roce5_qp *rqp)
+{
+ u32 wqe_opcode = roce5_ib_opcode[wr->opcode] & 0x1f;
+ param->ctrl_seg_tmp.dw0.bs.owner =
+ ((param->index & (u32)rqp->sq.wqebb_cnt) != 0) ? 1 : 0;
+ param->ctrl_seg_tmp.dw0.bs.drvsl = 0;
+ param->ctrl_seg_tmp.dw0.bs.wf = 0;
+ param->ctrl_seg_tmp.dw0.bs.cf = 0;
+ param->ctrl_seg_tmp.dw0.bs.va = 1;
+ param->ctrl_seg_tmp.dw0.bs.df = 0; /* va:1bits, 0-sge,1-inline */
+ param->ctrl_seg_tmp.dw0.bs.cr =
+ (u32)(!(((((u32)wr->send_flags) & IB_SEND_SIGNALED) |
+ rqp->sq_signal_bits) == 0));
+ param->ctrl_seg_tmp.dw0.bs.difsl = 0;
+ param->ctrl_seg_tmp.dw0.bs.csl = 0;
+ param->ctrl_seg_tmp.dw0.bs.ctrlsl = ROCE_CTRL_SEG_SL;
+ param->ctrl_seg_tmp.dw0.bs.bdsl = 0;
+
+ param->ctrl_seg_tmp.dw0.bs.wqe_ssn = (param->sq_wqe_ssn & 0x3);
+ param->sq_wqe_ssn += (u8)g_roce5_opcode_attr[wqe_opcode].remote_opcode;
+}
+
+static void roce5_init_task_com_seg(union roce5_wqe_tsk_com_seg **tsk_com_seg,
+ u8 *wqe, const struct ib_send_wr *wr,
+ const struct roce5_qp *rqp)
+{
+ struct roce_wqe_send_tsk_seg *send_task =
+ (struct roce_wqe_send_tsk_seg *)(void *)wqe;
+ *tsk_com_seg = (union roce5_wqe_tsk_com_seg *)(void *)wqe;
+
+ send_task->dw3.value = 0;
+ (*tsk_com_seg)->bs.c = (u32)(!(
+ (((unsigned)wr->send_flags & (unsigned)IB_SEND_SIGNALED) |
+ rqp->sq_signal_bits) == 0));
+ (*tsk_com_seg)->bs.se = (u32)(!(
+ ((unsigned)wr->send_flags & (unsigned)IB_SEND_SOLICITED) == 0));
+ (*tsk_com_seg)->bs.f = (u32)(!(
+ ((unsigned)wr->send_flags & (unsigned)IB_SEND_FENCE) == 0));
+ (*tsk_com_seg)->bs.op_type = roce5_ib_opcode[wr->opcode] & 0x1f;
+}
+
+static void
+roce5_post_send_local_operation(struct roce5_post_send_normal_param *param,
+ struct roce5_qp *rqp, struct roce5_device *rdev)
+{
+ param->opcode = param->tsk_com_seg->bs.op_type;
+ param->tsk_com_seg->value = cpu_to_be32(param->tsk_com_seg->value);
+ param->ctrl_seg->dw1.value = cpu_to_be32(param->ctrl_seg_tmp.dw1.value);
+ param->index += (u32)ROCE_DIV_ROUND_UP(param->wqe_size,
+ 1U << (u32)rqp->sq.wqe_shift);
+
+ write_invalid_wqe(rqp, param->index);
+
+ /*
+ * Make sure descriptor is fully written before
+ * setting ownership bit (because HW can start
+ * executing as soon as we do).
+ */
+ wmb();
+
+ param->ctrl_seg->dw0.value = cpu_to_be32(param->ctrl_seg_tmp.dw0.value);
+}
+/* ****************************************************************************
+ Prototype : roce5_post_send_normal
+ Description : roce5_post_send_normal
+ Input : struct roce5_device *rdev
+ struct roce5_qp *rqp
+ struct ib_send_wr *wr
+ struct ib_send_wr **bad_wr
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+
+ 2.Date : 2015/8/8
+ Modification : Modify function
+
+**************************************************************************** */
+static int roce5_post_send_normal(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ int need_goto = 0;
+ int ret = 0;
+ struct roce5_post_send_normal_param param = { 0 };
+ const struct ib_send_wr *wr_tmp = wr;
+
+ spin_lock_irqsave(&rqp->sq.lock, param.flags);
+
+ param.index = rqp->sq_next_wqe;
+ param.sq_wqe_ssn = rqp->sq_wqe_ssn;
+
+ for (param.wr_num = 0; wr_tmp != NULL;
+ ++(param.wr_num), wr_tmp = wr_tmp->next) {
+ ret = roce5_validate_wr(rqp, wr_tmp, param.wr_num);
+ if (ret != 0) {
+ *bad_wr = wr_tmp;
+ ROCE_DEV_ERR(rdev,
+ " Failed to validate wr, func(%d) qpn(%u)",
+ rdev->glb_func_id, rqp->qpn);
+ goto out;
+ }
+
+ param.wqe = (u8 *)roce5_get_send_wqe(rqp, param.index);
+ param.ctrl_seg = (struct roce5_wqe_ctrl_seg *)param.wqe;
+
+ param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe);
+
+ rqp->sq.wrid[(rqp->sq.head + param.wr_num) &
+ (u32)(rqp->sq.wqebb_cnt - 1)] = wr_tmp->wr_id;
+
+ roce5_init_ctrl_seg(¶m, wr_tmp, rqp);
+
+ param.wqe = ((u8 *)param.wqe + sizeof(*(param.ctrl_seg)));
+ param.wqe_size = (u32)sizeof(*param.ctrl_seg);
+ roce5_init_task_com_seg(¶m.tsk_com_seg, param.wqe, wr_tmp,
+ rqp);
+
+ ret = roce5_construct_wqe(¶m, rdev, rqp, wr_tmp, bad_wr);
+ if (ret == GOTO_LOCAL) {
+ ret = 0;
+ goto local;
+ } else if (ret == GOTO_OUT) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe);
+
+ need_goto = roce5_post_send_set_data_seg(¶m, rdev, rqp,
+ wr_tmp, bad_wr);
+ if (need_goto == ROCE_NEED_JUMP) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ roce5_fill_task_data_len(param.data_len_addr, param.data_len);
+
+local:
+ roce5_post_send_local_operation(¶m, rqp, rdev);
+ }
+
+out:
+ if (ROCE_LIKELY(param.wr_num != 0)) {
+ rqp->sq.head += param.wr_num;
+
+ wmb();
+
+ ret = roce5_write_sq_db(rdev, rqp, param.index, NULL);
+ rqp->sq_next_wqe = param.index;
+ rqp->sq_wqe_ssn = param.sq_wqe_ssn;
+ }
+
+ spin_unlock_irqrestore(&rqp->sq.lock, param.flags);
+
+ return ret;
+}
+
+static int roce5_post_send_check_qp_type(const struct roce5_qp *rqp)
+{
+ if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_TGT)) {
+ ROCE_ERR("Can't post WQE when TGT XRC QP");
+ return -EINVAL;
+ }
+
+ if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_INI)) {
+ ROCE_ERR("not support xrc in kernel space");
+ return -EINVAL;
+ }
+
+ if (ROCE_UNLIKELY((rqp->qp_state == IB_QPS_RESET) ||
+ (rqp->qp_state == IB_QPS_INIT) ||
+ (rqp->qp_state == IB_QPS_RTR))) {
+ ROCE_ERR("Can't post WQE when QP is RST/INIT/RTR state");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_post_send
+ Description : roce5_post_send
+ Input : struct ib_qp *ibqp
+ struct ib_send_wr *wr
+ const struct ib_send_wr **bad_wr
+ Output : None
+**************************************************************************** */
+int roce5_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ int ret = 0;
+ struct roce5_qp *rqp = to_roce5_qp(ibqp);
+ struct roce5_device *rdev = to_roce5_dev(ibqp->device);
+
+ ret = roce5_post_send_check_qp_type(rqp);
+ if (ret != 0) {
+ *bad_wr = wr;
+ ROCE_DEV_ERR(rdev, "Failed to check qp.ret(%d)", ret);
+ return ret;
+ }
+
+ ret = roce5_post_send_normal(rdev, rqp, wr, bad_wr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to post send normal wr, ret(%d)",
+ ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+int roce5_post_send_ofa(struct ib_qp *ibqp, struct ib_send_wr *wr,
+ struct ib_send_wr **bad_wr)
+{
+ return roce5_post_send(ibqp, (const struct ib_send_wr *)wr,
+ (const struct ib_send_wr **)bad_wr);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c
new file mode 100644
index 000000000..260eca49e
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c
@@ -0,0 +1,543 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "roce_compat.h"
+
+#include "hinic5_crm.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_mr.h"
+#include "roce_xrc.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_cqm_cmd.h"
+#include "roce_qp.h"
+#include "roce_restore.h"
+#include "roce_npu_cmd.h"
+#include "roce_main_extension.h"
+#include "roce_com.h"
+#include "securec.h"
+#ifdef __ROCE_DFX__
+#include "roce_dfx.h"
+#endif
+
+/**
+ * @brief Get roce qp by qpn. When querying succeeded, you have to call roce5_rqp_put() to free
+ the cqm_obj_qp. But there is no need to call roce5_rqp_put() if querying failed.
+ * @return Success: return the roce qp. Failure: return NULL.
+ */
+struct roce5_qp *roce5_get_rqp_by_qpn(struct roce5_device *rdev, u32 qpn)
+{
+ cqm_object_s *cqm_obj_qp = NULL;
+ struct roce5_qp *rqp = NULL;
+
+ cqm_obj_qp = cqm5_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn,
+ false);
+ if (cqm_obj_qp == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Can't find rqp according to qpn(0x%x), func_id(%d)",
+ qpn, rdev->glb_func_id);
+ return NULL;
+ }
+
+ rqp = cqmobj_to_roce_qp(cqm_obj_qp);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to convert obj to rqp.");
+ cqm5_object_put(cqm_obj_qp);
+ }
+
+ return rqp;
+}
+
+static enum ib_mig_state to_ib_mig_state(int roce5_mig_state)
+{
+ switch (roce5_mig_state) {
+ case ROCE_QP_PM_ARMED:
+ return IB_MIG_ARMED;
+
+ case ROCE_QP_PM_REARM:
+ return IB_MIG_REARM;
+
+ case ROCE_QP_PM_MIGRATED:
+ return IB_MIG_MIGRATED;
+
+ default:
+ return (enum ib_mig_state)(-1);
+ }
+}
+
+static int to_ib_qp_access_flags(const roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ unsigned int ib_flags = 0;
+ u32 tmp = 0;
+
+ /* RRE */
+ if (verbs_qp_attr->com_info.dw0.bs.rre != 0) {
+ tmp = ib_flags | IB_ACCESS_REMOTE_READ;
+ ib_flags = tmp;
+ }
+
+ /* RWE */
+ if (verbs_qp_attr->com_info.dw0.bs.rwe != 0) {
+ tmp = ib_flags | IB_ACCESS_REMOTE_WRITE;
+ ib_flags = tmp;
+ }
+
+ /* RAE */
+ if (verbs_qp_attr->com_info.dw0.bs.rae != 0) {
+ tmp = ib_flags | IB_ACCESS_REMOTE_ATOMIC;
+ ib_flags = tmp;
+ }
+
+ return (int)ib_flags;
+}
+
+/* ****************************************************************************
+ Prototype : to_ib_ah_attr
+ Description : to_ib_ah_attr
+ Input : struct roce5_device *rdev
+ struct ib_ah_attr *ah_attr(struct rdma_ah_attr *ah_attr)
+ struct roce5_qp_context *context
+ Output : None
+
+ 1.Date : 2015/5/26
+ Modification : Created function
+
+**************************************************************************** */
+static void to_ib_ah_attr(struct roce5_device *rdev,
+ struct rdma_ah_attr *ah_attr,
+ roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ int ret;
+
+ ret = memset_s(ah_attr, sizeof(*ah_attr), 0, sizeof(*ah_attr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ah_attr.(ret:%d)", ret);
+ return;
+ }
+
+ ah_attr->port_num = verbs_qp_attr->com_info.dw5.bs.port;
+
+ ah_attr->sl = 0x7 - verbs_qp_attr->path_info.dw7.bs.sl;
+
+ ah_attr->static_rate = 0;
+
+ ah_attr->ah_flags = IB_AH_GRH;
+
+ ah_attr->grh.sgid_index = verbs_qp_attr->path_info.dw7.bs.sgid_index;
+ ah_attr->grh.hop_limit = verbs_qp_attr->path_info.dw7.bs.hoplmt;
+ ah_attr->grh.traffic_class = verbs_qp_attr->path_info.dw6.bs.tclass;
+ ah_attr->grh.flow_label = verbs_qp_attr->path_info.dw6.bs.flow_label;
+
+ ret = memcpy_s((void *)ah_attr->grh.dgid.raw,
+ sizeof(ah_attr->grh.dgid.raw),
+ (void *)verbs_qp_attr->path_info.dgid,
+ sizeof(ah_attr->grh.dgid.raw));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy dgid.(ret:%d)", ret);
+ }
+}
+
+int roce5_uni_cmd_qp_query(struct roce5_device *rdev, u32 qpn,
+ u32 *verbs_qp_attr, int qp_attr_size)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_qp_query_s *qp_query_inbuf = NULL;
+ roce_uni_cmd_qp_query_outbuf_s *qp_query_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_qp_query_s), &cqm_cmd_outbuf,
+ (u16)sizeof(roce_uni_cmd_qp_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret=%d",
+ ret);
+ return ret;
+ }
+
+ qp_query_inbuf = (roce_uni_cmd_qp_query_s *)cqm_cmd_inbuf->buf;
+ qp_query_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK));
+ qp_query_inbuf->com.index = cpu_to_be32(qpn);
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_QUERY_QP, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_B);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to send QUERY_QP command, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -1;
+ goto free_cqm_buf;
+ }
+
+ qp_query_outbuf = (roce_uni_cmd_qp_query_outbuf_s *)cqm_cmd_outbuf->buf;
+ ret = (int)memcpy_s((void *)verbs_qp_attr, (size_t)qp_attr_size,
+ (void *)&qp_query_outbuf->qp_info.qp_attr,
+ (size_t)qp_attr_size);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy verbs_qp_attr, ret(%d).",
+ ret);
+ ret = -EINVAL;
+ }
+free_cqm_buf:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+ return ret;
+}
+
+int roce5_qp_query(struct roce5_device *rdev, u32 qpn, u32 *context,
+ int qpc_size)
+{
+ if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) {
+ return roce5_uni_cmd_qp_query(rdev, qpn, context, qpc_size);
+ }
+ return -EINVAL;
+}
+
+void qpc_seg_to_le32(struct roce5_qp_context *be_ctx,
+ struct roce5_qp_context *le_ctx, u32 srq_vld)
+{
+ /* DRV Seg */
+ int ret;
+
+ ret = memcpy_s((void *)le_ctx->sw_seg.path_seg.dgid,
+ sizeof(be_ctx->sw_seg.path_seg.dgid),
+ (void *)be_ctx->sw_seg.path_seg.dgid,
+ sizeof(be_ctx->sw_seg.path_seg.dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ return;
+ }
+
+ /* CHIP Seg */
+ le_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa =
+ be64_to_cpu(be_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa);
+ le_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num = cpu_to_be64(
+ be_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num);
+ le_ctx->chip_seg.rcc.rc_curt_sge_va =
+ be64_to_cpu(be_ctx->chip_seg.rcc.rc_curt_sge_va);
+ le_ctx->chip_seg.sqc.sq_curt_sge_va =
+ be64_to_cpu(be_ctx->chip_seg.sqc.sq_curt_sge_va);
+ le_ctx->chip_seg.sqac.sqa_curt_sge_va =
+ be64_to_cpu(be_ctx->chip_seg.sqac.sqa_curt_sge_va);
+
+ if (srq_vld != 0) {
+ le_ctx->chip_seg.srqc.srq_curt_sge_va =
+ be64_to_cpu(be_ctx->chip_seg.srqc.srq_curt_sge_va);
+ } else {
+ le_ctx->chip_seg.rqc.rq_curt_sge_va =
+ be64_to_cpu(be_ctx->chip_seg.rqc.rq_curt_sge_va);
+ }
+
+ le_ctx->chip_seg.rrwc.rrw_curt_sge_va =
+ be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va);
+}
+
+void roce5_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size)
+{
+ u32 *ctx = NULL, *ctx1 = NULL;
+ u32 i = 0;
+
+ ctx = le_ctx;
+ ctx1 = (u32 *)context;
+
+ for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) {
+ *ctx = be32_to_cpu(*ctx1);
+ }
+
+ return;
+}
+
+static void roce5_get_ah_attr(const struct roce5_qp *rqp,
+ struct ib_qp_attr *qp_attr,
+ struct roce5_device *rdev,
+ roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ if ((rqp->qp_type == IB_QPT_RC) || (rqp->qp_type == IB_QPT_UC)) {
+ to_ib_ah_attr(rdev, &qp_attr->ah_attr, verbs_qp_attr);
+ }
+}
+
+static void roce5_query_set_common_attr(struct ib_qp_attr *qp_attr,
+ const struct ib_qp *ibqp,
+ struct roce5_qp *rqp,
+ struct ib_qp_init_attr *qp_init_attr)
+{
+ qp_attr->cur_qp_state = qp_attr->qp_state;
+ qp_attr->cap.max_recv_wr = (u32)rqp->rq.wqebb_cnt;
+ qp_attr->cap.max_recv_sge = (u32)rqp->rq.max_sge;
+
+ if (!ibqp->uobject) {
+ qp_attr->cap.max_send_wr = (u32)rqp->sq.max_post;
+ qp_attr->cap.max_send_sge = (u32)rqp->sq.max_sge;
+ } else {
+ qp_attr->cap.max_send_wr = 0;
+ qp_attr->cap.max_send_sge = 0;
+ }
+
+ qp_attr->cap.max_inline_data = rqp->max_inline_data;
+
+ qp_init_attr->cap = qp_attr->cap;
+
+ qp_init_attr->create_flags = (enum ib_qp_create_flags)0;
+
+ qp_init_attr->sq_sig_type = (rqp->sq_signal_bits == 1) ?
+ IB_SIGNAL_ALL_WR :
+ IB_SIGNAL_REQ_WR;
+}
+
+static void roce5_query_set_attr(struct ib_qp_attr *qp_attr, u8 tmp_qp_state,
+ struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *verbs_qp_attr,
+ struct roce5_device *rdev)
+{
+ qp_attr->qp_state = (enum ib_qp_state)rqp->qp_state;
+ qp_attr->path_mtu = (enum ib_mtu)verbs_qp_attr->com_info.dw3.bs.pmtu;
+ qp_attr->path_mig_state =
+ (enum ib_mig_state)to_ib_mig_state(ROCE_QP_PM_MIGRATED);
+ qp_attr->qkey = verbs_qp_attr->com_info.q_key;
+ qp_attr->rq_psn = verbs_qp_attr->com_info.dw9.bs.next_rcv_psn;
+ qp_attr->sq_psn = verbs_qp_attr->com_info.dw8.bs.next_send_psn;
+ qp_attr->dest_qp_num = verbs_qp_attr->com_info.dw0.bs.dest_qp;
+ qp_attr->qp_access_flags = to_ib_qp_access_flags(verbs_qp_attr);
+
+ roce5_get_ah_attr(rqp, qp_attr, rdev, verbs_qp_attr);
+
+ qp_attr->pkey_index = 0;
+
+ if (qp_attr->qp_state == IB_QPS_INIT) {
+ qp_attr->port_num = rqp->port;
+ } else {
+ qp_attr->port_num = verbs_qp_attr->com_info.dw5.bs.port;
+ }
+
+ /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
+
+ /* SQ Draining FLAG */
+ qp_attr->sq_draining =
+ (tmp_qp_state ==
+ (u8)(enum roce_qp_state)ROCE_QP_STATE_SQ_DRAINING);
+
+ qp_attr->max_rd_atomic = (1 << verbs_qp_attr->com_info.dw1.bs.sra_max);
+ qp_attr->max_dest_rd_atomic =
+ (1 << verbs_qp_attr->com_info.dw1.bs.rra_max);
+
+ qp_attr->min_rnr_timer = verbs_qp_attr->com_info.dw2.bs.min_rnr_nak;
+ qp_attr->timeout = verbs_qp_attr->com_info.dw2.bs.ack_to;
+ qp_attr->retry_cnt = verbs_qp_attr->com_info.dw1.bs.to_retry_limit;
+ qp_attr->rnr_retry = verbs_qp_attr->com_info.dw1.bs.rnr_retry_limit;
+}
+
+static int roce5_query_qp_struct_clean(struct ib_qp_attr *qp_attr,
+ struct ib_qp_init_attr *qp_init_attr,
+ roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ int ret;
+
+ ret = memset_s(qp_attr, sizeof(struct ib_qp_attr), 0,
+ sizeof(struct ib_qp_attr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = memset_s(qp_init_attr, sizeof(struct ib_qp_init_attr), 0,
+ sizeof(struct ib_qp_init_attr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset qp_init_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = memset_s(verbs_qp_attr, sizeof(roce_verbs_qp_attr_s), 0,
+ sizeof(roce_verbs_qp_attr_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset verbs_qp_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void roce_qp_attr_com_info_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ verbs_qp_attr->com_info.dw0.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw0.value);
+ verbs_qp_attr->com_info.dw1.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw1.value);
+ verbs_qp_attr->com_info.dw2.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw2.value);
+ verbs_qp_attr->com_info.dw3.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw3.value);
+ verbs_qp_attr->com_info.q_key =
+ be32_to_cpu(verbs_qp_attr->com_info.q_key);
+ verbs_qp_attr->com_info.dw5.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw5.value);
+ verbs_qp_attr->com_info.dw6.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw6.value);
+ verbs_qp_attr->com_info.dw7.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw7.value);
+ verbs_qp_attr->com_info.dw8.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw8.value);
+ verbs_qp_attr->com_info.dw9.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw9.value);
+ verbs_qp_attr->com_info.lsn = be32_to_cpu(verbs_qp_attr->com_info.lsn);
+ verbs_qp_attr->com_info.dw11.value =
+ be32_to_cpu(verbs_qp_attr->com_info.dw11.value);
+ verbs_qp_attr->com_info.rsvd =
+ be32_to_cpu(verbs_qp_attr->com_info.rsvd);
+}
+
+static void roce_qp_attr_path_info_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ verbs_qp_attr->path_info.dw0.value =
+ be32_to_cpu(verbs_qp_attr->path_info.dw0.value);
+ verbs_qp_attr->path_info.dmac_l32 =
+ be32_to_cpu(verbs_qp_attr->path_info.dmac_l32);
+ verbs_qp_attr->path_info.dw6.value =
+ be32_to_cpu(verbs_qp_attr->path_info.dw6.value);
+ verbs_qp_attr->path_info.dw7.value =
+ be32_to_cpu(verbs_qp_attr->path_info.dw7.value);
+}
+
+static void roce_qp_attr_chip_seg_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ verbs_qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa =
+ be64_to_cpu(verbs_qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa);
+ verbs_qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = be64_to_cpu(
+ verbs_qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num);
+ verbs_qp_attr->chip_seg.dw4.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw4.value);
+ verbs_qp_attr->chip_seg.dw5.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw5.value);
+ verbs_qp_attr->chip_seg.dw6.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw6.value);
+ verbs_qp_attr->chip_seg.dw7.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw7.value);
+ verbs_qp_attr->chip_seg.dw8.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw8.value);
+ verbs_qp_attr->chip_seg.dw9.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw9.value);
+ verbs_qp_attr->chip_seg.dw10.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw10.value);
+ verbs_qp_attr->chip_seg.rc_page_gpa_l =
+ be32_to_cpu(verbs_qp_attr->chip_seg.rc_page_gpa_l);
+ verbs_qp_attr->chip_seg.dw12.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw12.value);
+ verbs_qp_attr->chip_seg.dw13.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw13.value);
+ verbs_qp_attr->chip_seg.dw14.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw14.value);
+ verbs_qp_attr->chip_seg.dw15.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw15.value);
+ verbs_qp_attr->chip_seg.dw16.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw16.value);
+ verbs_qp_attr->chip_seg.dw17.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw17.value);
+ verbs_qp_attr->chip_seg.dw18.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw18.value);
+ verbs_qp_attr->chip_seg.dw19.value =
+ be32_to_cpu(verbs_qp_attr->chip_seg.dw19.value);
+}
+
+static void roce5_qp_attr_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr)
+{
+ roce_qp_attr_com_info_to_le32(verbs_qp_attr);
+ roce_qp_attr_path_info_to_le32(verbs_qp_attr);
+ roce_qp_attr_chip_seg_to_le32(verbs_qp_attr);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_query_qp
+ Description : roce5_query_qp
+ Input : struct ib_qp *ibqp
+ struct ib_qp_attr *qp_attr
+ int qp_attr_mask
+ struct ib_qp_init_attr *qp_init_attr
+ Output : None
+
+ 1.Date : 2015/4/29
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+ int ret = 0;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_device *rdev = NULL;
+ roce_verbs_qp_attr_s verbs_qp_attr;
+ u32 srq_vld;
+ u8 tmp_qp_state = 0;
+
+ if ((ibqp == NULL) || (qp_attr == NULL) || (qp_init_attr == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ ret = roce5_query_qp_struct_clean(qp_attr, qp_init_attr,
+ &verbs_qp_attr);
+ if (ret != 0) {
+ return ret;
+ }
+
+ rqp = to_roce5_qp(ibqp);
+ rdev = to_roce5_dev(ibqp->device);
+
+ mutex_lock(&rqp->mutex);
+
+ if (rqp->qp_state == (u8)(enum ib_qp_state)IB_QPS_RESET) {
+ qp_attr->qp_state = IB_QPS_RESET;
+ goto done;
+ }
+
+ if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) {
+ ret = roce5_qp_query(rdev, rqp->qpn,
+ (u32 *)(void *)(&verbs_qp_attr),
+ sizeof(verbs_qp_attr));
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to query qp with 25V100 or 23V200, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ mutex_unlock(&rqp->mutex);
+ return ret;
+ }
+ }
+
+ srq_vld = (!(rqp->ibqp.srq)) ? ROCE_QP_NO_SRQ : ROCE_QP_HAS_SRQ;
+
+ if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) {
+ roce5_qp_attr_to_le32(&verbs_qp_attr);
+ rqp->qp_state = verbs_qp_attr.com_info.dw6.bs
+ .state; /* 25V100 or 23V200 qp state */
+ }
+
+ tmp_qp_state = rqp->qp_state;
+ if (rqp->qp_state ==
+ (u8)(enum roce_qp_state)ROCE_QP_STATE_SQ_DRAINING) {
+ rqp->qp_state = (u8)(enum ib_qp_state)IB_QPS_SQD;
+ }
+
+ if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) {
+ /* 25V100 or 23V200 set attr */
+ roce5_query_set_attr(qp_attr, tmp_qp_state, rqp, &verbs_qp_attr,
+ rdev);
+ }
+
+done:
+ roce5_query_set_common_attr(qp_attr, ibqp, rqp, qp_init_attr);
+
+ mutex_unlock(&rqp->mutex);
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c
new file mode 100644
index 000000000..257e9a8aa
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c
@@ -0,0 +1,22 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_hw.h"
+#include "rdma_comp.h"
+
+struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev)
+{
+ struct rdma_comp_priv *comp_private = NULL;
+ comp_private = (struct rdma_comp_priv *)hinic5_get_service_adapter(
+ hwdev, SERVICE_T_ROCE);
+ return comp_private;
+}
+
+void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv)
+{
+ rdma_bitmap_cleanup(&comp_priv->pd_bitmap);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h
new file mode 100644
index 000000000..8a1a8d665
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h
@@ -0,0 +1,166 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef RDMA_COMP_H
+#define RDMA_COMP_H
+
+/* x86 */
+#ifndef BIG_ENDIAN
+#define BIG_ENDIAN 0x4321
+#endif
+
+#ifndef LITTLE_ENDIAN
+#define LITTLE_ENDIAN 0x1234
+#endif
+
+#ifndef BYTE_ORDER
+#define BYTE_ORDER LITTLE_ENDIAN
+#endif
+
+#include <linux/delay.h>
+#include <linux/types.h>
+#ifdef ROCE_KNL_DT
+#include <linux/compiler_types.h>
+#endif
+
+#include "hinic5_crm.h"
+#include "hinic5_cqm.h"
+#include "hinic5_rdma.h"
+
+#include "roce_gid.h"
+#include "roce_npu_cmd_defs.h"
+#include "roce.h"
+#include "roce_compat.h"
+
+#include "rdma_bitmap.h"
+#include "hmm_em_inner.h"
+#include "hmm_common.h"
+
+#ifndef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME ": [RDMA]" fmt
+#endif
+
+#define RDMA_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL
+
+#define RDMA_INVALID_GUID 0
+
+#define RDMA_SERVICE_TYPE_ROCE SERVICE_T_ROCE
+
+#define RDMA_BIT_SHIFT_1 1
+#define RDMA_BIT_SHIFT_2 2
+#define RDMA_BIT_SHIFT_4 4
+#define RDMA_BIT_SHIFT_8 8
+#define RDMA_BIT_SHIFT_16 16
+
+#define ROCE5_RDMARC_MIN_DEPTH 1
+#define ROCE5_RDMARC_MAX_DEPTH 512
+#define ROCE5_RDMARC_MIN_ENTRY 8 /* min entry aligned required by engine */
+#define ROCE5_RDMARC_MAX_ENTRY 128 /* max entry supported by engine */
+#define ROCE5_RDMARC_EXT_ENTRY 384 //384 /* ext tbl 12K. 12k/32 = 384 */
+
+#define ROCE5_RDMARC_GPA_H_SHIFT 40 /* high:24bit, 64-24= 40 */
+#define ROCE5_RDMARC_GPA_L_SHIFT 8 /* bit[39:8] */
+#define ROCE5_RDMARC_RRA_MAX_MASK 0x7 /* rra_max:3bit */
+#define ROCE5_RDMARC_EXT_ORDER_SHIFT 3
+
+extern u32 g_mtt_page_size;
+
+#define RDMA_EM_MIN_ORDER 2 /* rdma rc extend table at least 4 page */
+
+#define ROCE_MTT_PAGE_SIZE_4K 4096
+#define ROCE_MTT_PAGE_SIZE_64K (64 * 1024)
+#define ROCE_MTT_PAGE_SIZE_2M (2 * 1024 * 1024)
+#define ROCE_MTT_PAGE_SIZE_4K_SHIFT 12
+#define ROCE_MTT_PAGE_SIZE_64K_SHIFT 16
+#define ROCE_MTT_PAGE_SIZE_2M_SHIFT 21
+
+enum {
+ ROCE5_RDMA_MTT_PAGE_SIZE_4K = 0,
+ ROCE5_RDMA_MTT_PAGE_SIZE_64K = 1,
+ ROCE5_RDMA_MTT_PAGE_SIZE_2M = 2
+};
+
+enum {
+ ROCE5_RDMA_CMD_TIME_OUT_A = 30000,
+ ROCE5_RDMA_CMD_TIME_OUT_B = 40000,
+ ROCE5_RDMA_CMD_TIME_OUT_C = 50000
+};
+
+/**
+ * @brief struct rdma_comp_resource
+ * @details RDMA component resource structure, used to store RDMA component of related resources
+ */
+struct rdma_comp_resource {
+ struct mutex mutex; /**< gid_entry mutex used by */
+ __be64 node_guid; /**< and ibdev in of node_guid same as */
+ struct rdma_gid_entry *
+ *gid_table; /**< gid_entry in rdma component initialize when allocate memory */
+};
+
+/**
+ * @brief enum rdma_mw_type
+ * @details define an enum type, used to represent RDMA memory window of type
+ */
+enum rdma_mw_type { RDMA_MW_TYPE_1 = 1, RDMA_MW_TYPE_2 = 2 };
+
+/**
+ * @brief struct rdma_mw
+ * @details define a rdma_mw structure, used to represent a RDMA memory window
+ */
+struct rdma_mw {
+ struct mpt mpt;
+ u32 key; /**< mw corresponding to key */
+ u32 pdn; /**< mw bind of pdn */
+ enum rdma_mw_type type; /**< mw of type,type1,type2 */
+ int enabled; /**< mw of status */
+};
+
+struct rdma_comp_priv {
+ struct hmm_comp_priv hmm_priv;
+ struct rdma_comp_resource rdma_comp_res; /* gid & guid */
+ struct hmm_rdma rsvd_lkey;
+ struct hmm_rdma fixed_mr;
+ struct rdma_service_cap rdma_cap;
+
+ struct rdma_bitmap pd_bitmap;
+ struct rdma_bitmap xrcd_bitmap;
+
+ struct hmm_buddy rdmarc_buddy;
+ struct hmm_em_table rdmarc_em_table;
+};
+
+/* Only V100 use */
+struct rdma_gid_update_inbuf {
+ roce_verbs_cmd_com_s com;
+ __be32 port;
+ __be32 rsvd;
+ struct rdma_gid_entry gid_entry;
+};
+
+struct rdma_gid_clear_inbuf {
+ __be32 port;
+ __be32 gid_num;
+};
+
+typedef struct tag_roce_clear_gid {
+ roce_verbs_cmd_com_s com;
+
+ u32 port;
+ u32 gid_num;
+} roce_cmd_clear_gid_s;
+
+/* for llt to set stub */
+void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv);
+struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev);
+int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry,
+ struct rdma_gid_entry *gid_entry);
+int roce5_update_gid_mac(struct roce5_device *rdev, u32 port, int *index,
+ struct rdma_gid_entry *gid_entry);
+int roce5_update_gid(struct roce5_device *rdev, u32 port, u32 update_index,
+ struct rdma_gid_entry *gid_entry);
+int roce5_reset_gid_table(struct roce5_device *rdev, u32 port);
+int roce5_get_gid(struct roce5_device *rdev, u32 port, u32 gid_index,
+ struct rdma_gid_entry *gid);
+
+#endif //RDMA_COMP_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c
new file mode 100644
index 000000000..b7919b409
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c
@@ -0,0 +1,489 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "comm_defs.h"
+#include "hinic5_hw.h"
+#include "rdma_comp.h"
+#include "roce.h"
+#include "roce_cqm_cmd.h"
+#include "roce_npu_cmd.h"
+#include "roce_mix.h"
+#include "roce_gid.h"
+#include "securec.h"
+#include "roce_npu_cmd_gid_defs.h"
+
+#define ROCE_GID_IFID_DW_INDEX \
+ 2 /* Interface ID starts at dword index 2 in u32 gid[4] (indices 0-1: subnet_prefix, 2-3: interface_id) */
+
+static int roce5_uni_cmd_update_gid(struct roce5_device *rdev,
+ struct rdma_comp_priv *comp_priv,
+ struct rdma_gid_entry *new_gid_entry,
+ int port, int gid_index)
+{
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_update_gid_s *gid_update_inbuf = NULL;
+ int ret;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_update_gid_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_ERR("Failed to alloc cqm_cmd_inoutbuf, ret(%d)", ret);
+ return -ENOMEM;
+ }
+
+ gid_update_inbuf = (roce_uni_cmd_update_gid_s *)cqm_cmd_inbuf->buf;
+ *(u64 *)(&gid_update_inbuf->gid_attr.gid[0]) =
+ new_gid_entry->global.subnet_prefix;
+ *(u64 *)(&gid_update_inbuf->gid_attr.gid[ROCE_GID_IFID_DW_INDEX]) =
+ new_gid_entry->global.interface_id;
+ gid_update_inbuf->gid_attr.dw4.value =
+ cpu_to_be32(new_gid_entry->dw4.value);
+ gid_update_inbuf->gid_attr.dw5.value =
+ cpu_to_be32(new_gid_entry->hdr_len_value);
+ gid_update_inbuf->gid_attr.dw6.bs1.ctrl_val =
+ new_gid_entry->dw6_h.value;
+ gid_update_inbuf->gid_attr.dw6.bs1.smac_hi16 =
+ be16_to_cpu(*((u16 *)(&new_gid_entry->smac[0])));
+ gid_update_inbuf->gid_attr.dw6.value =
+ cpu_to_be32(gid_update_inbuf->gid_attr.dw6.value);
+ gid_update_inbuf->gid_attr.smac_lo32 =
+ *((u32 *)(&new_gid_entry->smac[2])); //4=lo32
+
+ gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index);
+ gid_update_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK);
+ gid_update_inbuf->com.opt = cpu_to_be32((u32)port);
+ if (rdev->shadow != 0) {
+ gid_update_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK);
+ gid_update_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(rdev->shadow_vfid);
+ gid_update_inbuf->gid_attr.dw10.shadow.shadow_vf_num =
+ rdev->shadow_vf_num;
+ gid_update_inbuf->gid_attr.dw10.value =
+ cpu_to_be32(gid_update_inbuf->gid_attr.dw10.value);
+ }
+
+ ROCE_PRINT_GID(rdev, new_gid_entry, "gid_index(0x%x)",
+ gid_update_inbuf->com.index);
+ ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL,
+ ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Send cmd update_gid failed, ret(%d)", ret);
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf,
+ NULL);
+
+ return ret;
+}
+
+static int roce5_cmd_update_gid(struct roce5_device *rdev,
+ struct rdma_comp_priv *comp_priv,
+ struct rdma_gid_entry *new_gid_entry, int port,
+ int gid_index)
+{
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ struct rdma_gid_update_inbuf *gid_update_inbuf = NULL;
+ int ret;
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ return roce5_uni_cmd_update_gid(rdev, comp_priv, new_gid_entry,
+ port, gid_index);
+ }
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(struct rdma_gid_update_inbuf), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret(%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ gid_update_inbuf = (struct rdma_gid_update_inbuf *)cqm_cmd_inbuf->buf;
+ gid_update_inbuf->port = cpu_to_be32((u32)port);
+ gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index);
+ gid_update_inbuf->gid_entry.global.subnet_prefix =
+ new_gid_entry->global.subnet_prefix;
+ gid_update_inbuf->gid_entry.global.interface_id =
+ new_gid_entry->global.interface_id;
+ gid_update_inbuf->gid_entry.dw4.value =
+ cpu_to_be32(new_gid_entry->dw4.value);
+ gid_update_inbuf->gid_entry.dw6_h.value =
+ cpu_to_be16(new_gid_entry->dw6_h.value);
+ gid_update_inbuf->gid_entry.hdr_len_value =
+ cpu_to_be32(new_gid_entry->hdr_len_value);
+ gid_update_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK);
+
+ ret = memcpy_s(&gid_update_inbuf->gid_entry.smac[0], ETH_ALEN,
+ &new_gid_entry->smac[0], ETH_ALEN);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy_s smac.(ret:%d)", ret);
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ ROCE_PRINT_GID(rdev, new_gid_entry, "gid_index(0x%x) ",
+ gid_update_inbuf->com.index);
+ ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL,
+ ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Send cmd update_gid failed, ret(%d)", ret);
+ ret = -1;
+ }
+
+err:
+ roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf,
+ NULL);
+
+ return ret;
+}
+
+int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry,
+ struct rdma_gid_entry *gid_entry)
+{
+ struct rdma_gid_entry entry1;
+ struct rdma_gid_entry entry2;
+ int ret;
+
+ ret = memcpy_s(&entry1, sizeof(struct rdma_gid_entry), gid_tbl_entry,
+ sizeof(struct rdma_gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy entry1.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = memcpy_s(&entry2, sizeof(struct rdma_gid_entry), gid_entry,
+ sizeof(struct rdma_gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy entry2.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ /* compare ip+vlan, compare after clear smac */
+ ret = memset_s((void *)entry1.smac, sizeof(entry1.smac), 0,
+ sizeof(entry1.smac));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset entry1 smac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = memset_s((void *)entry2.smac, sizeof(entry2.smac), 0,
+ sizeof(entry2.smac));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset entry2.smac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ return memcmp((void *)&entry1, (void *)&entry2,
+ sizeof(struct rdma_gid_entry));
+}
+
+static int roce5_uni_cmd_reset_gid(struct roce5_device *rdev,
+ struct rdma_comp_priv *comp_priv, u32 port,
+ u32 gid_num)
+{
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_clear_gid_s *gid_clear_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_clear_gid_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_ERR("Failed to alloc cqm_cmd_inoutbuf, ret(%d)", ret);
+ return -ENOMEM;
+ }
+
+ gid_clear_inbuf = (roce_uni_cmd_clear_gid_s *)cqm_cmd_inbuf->buf;
+ gid_clear_inbuf->gid_clear.gid_num = cpu_to_be32(gid_num);
+ gid_clear_inbuf->com.index = 0;
+ gid_clear_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK);
+ gid_clear_inbuf->com.opt = cpu_to_be32((u32)port);
+
+ if (rdev->shadow != 0) {
+ gid_clear_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK);
+ gid_clear_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(rdev->shadow_vfid);
+ gid_clear_inbuf->gid_clear.dw1.shadow.shadow_vf_num =
+ rdev->shadow_vf_num;
+ gid_clear_inbuf->gid_clear.dw1.value =
+ cpu_to_be32(gid_clear_inbuf->gid_clear.dw1.value);
+ }
+
+ ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL,
+ ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Send cmd clear_gid failed, ret(%d)", ret);
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf,
+ NULL);
+
+ return ret;
+}
+
+static int roce5_cmd_reset_gid(struct roce5_device *rdev,
+ struct rdma_comp_priv *comp_priv, u32 port,
+ u32 gid_num)
+{
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_cmd_clear_gid_s *gid_clear_inbuf = NULL;
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ return roce5_uni_cmd_reset_gid(rdev, comp_priv, port, gid_num);
+ }
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(comp_priv->hmm_priv.hwdev,
+ &cqm_cmd_inbuf,
+ (u16)sizeof(roce_cmd_clear_gid_s),
+ NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret(%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ gid_clear_inbuf = (roce_cmd_clear_gid_s *)cqm_cmd_inbuf->buf;
+ gid_clear_inbuf->port = cpu_to_be32(port);
+ gid_clear_inbuf->gid_num = cpu_to_be32(gid_num);
+ gid_clear_inbuf->com.index = 0;
+ gid_clear_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK);
+
+ ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL,
+ ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Send cmd clear_gid failed, ret(%d)", ret);
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf,
+ NULL);
+
+ return ret;
+}
+
+int roce5_update_gid_mac(struct roce5_device *rdev, u32 port, int *index,
+ struct rdma_gid_entry *gid_entry)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+ struct rdma_gid_entry **gid_table = NULL;
+ u32 gid_index = 0;
+ int update_index =
+ -1; /* -1 is initvalue, indicating no need to update */
+ int ret = 0;
+ if ((rdev->hwdev == NULL) || (gid_entry == NULL)) {
+ ROCE_ERR("Hwdev or gid_tbl is null");
+ return -EINVAL;
+ }
+ comp_priv = get_rdma_comp_priv(rdev->hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+ /* port num index from zero */
+ if (port >= comp_priv->rdma_cap.num_ports) {
+ ROCE_ERR("Input port(%d) is invalid", port);
+ return -EINVAL;
+ }
+ /* invalid gid cause of with the head fe80:: of IPV6 */
+ if (cpu_to_be64(gid_entry->global.subnet_prefix) ==
+ RDMA_DEFAULT_GID_SUBNET_PREFIX) {
+ return 0;
+ }
+ gid_table = comp_priv->rdma_comp_res.gid_table;
+ mutex_lock(&comp_priv->rdma_comp_res.mutex);
+ /* loop all gid, ensure all gid cleared not exist */
+ for (gid_index = 1; gid_index < comp_priv->rdma_cap.max_gid_per_port;
+ gid_index++) {
+ if (rdma_gid_entry_cmp(&gid_table[port][gid_index],
+ gid_entry) == 0) {
+ update_index = (int)gid_index;
+ break;
+ }
+ }
+
+ if (update_index > 0) {
+ ret = roce5_cmd_update_gid(rdev, comp_priv, gid_entry,
+ (int)port, update_index);
+ if (ret != 0) {
+ ROCE_ERR("Update gid tbl failed, ret(%d)", ret);
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+ return ret;
+ }
+ /* update gid table after CMDQ */
+ ret = memcpy_s((void *)&gid_table[port][update_index],
+ sizeof(*gid_entry), (void *)gid_entry,
+ sizeof(*gid_entry));
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to memcpy gid_entry to gid_table.(ret:%d)",
+ ret);
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+ return (-ENOMEM);
+ }
+
+ *index = update_index;
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+ return 0;
+ }
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+ return 0;
+}
+
+int roce5_update_gid(struct roce5_device *rdev, u32 port, u32 update_index,
+ struct rdma_gid_entry *gid_entry)
+{
+ int ret = 0;
+ u32 port_num = 0;
+ struct rdma_comp_priv *comp_priv = NULL;
+ struct rdma_gid_entry **gid_table = NULL;
+
+ if ((rdev->hwdev == NULL) || (gid_entry == NULL)) {
+ ROCE_ERR("Hwdev or gid_tbl is null");
+ return -EINVAL;
+ }
+
+ comp_priv = get_rdma_comp_priv(rdev->hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+
+ gid_table = comp_priv->rdma_comp_res.gid_table;
+
+ /* port start from zero */
+ port_num = comp_priv->rdma_cap.num_ports;
+ if (port >= port_num) {
+ ROCE_DEV_ERR(rdev, "Input port(%d) is invalid", port);
+ return -EINVAL;
+ }
+
+ mutex_lock(&comp_priv->rdma_comp_res.mutex);
+ ret = roce5_cmd_update_gid(rdev, comp_priv, gid_entry, (int)port,
+ (int)update_index);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Rdma_update_gid failed");
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+ return ret;
+ }
+
+ ret = memcpy_s((void *)&gid_table[port][update_index],
+ sizeof(*gid_entry), (void *)gid_entry,
+ sizeof(*gid_entry));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy gid_entry.(ret:%d)", ret);
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+ return -ENOMEM;
+ }
+ mutex_unlock(&comp_priv->rdma_comp_res.mutex);
+
+ return 0;
+}
+
+int roce5_reset_gid_table(struct roce5_device *rdev, u32 port)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+ struct rdma_gid_entry **gid_table = NULL;
+ int ret = 0;
+ u32 gids_per_port = 0;
+
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return -EINVAL;
+ }
+
+ comp_priv = get_rdma_comp_priv(rdev->hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+
+ gid_table = comp_priv->rdma_comp_res.gid_table;
+ if (gid_table == NULL) {
+ ROCE_ERR("Gid_table is null");
+ return -EINVAL;
+ }
+
+ gids_per_port = comp_priv->rdma_cap.max_gid_per_port;
+
+ if (gids_per_port == 0) {
+ ROCE_ERR("Gids_per_port(%d) is invalid", gids_per_port);
+ return -EINVAL;
+ }
+
+ ret = memset_s(&gid_table[port][0],
+ gids_per_port * sizeof(struct rdma_gid_entry), 0,
+ gids_per_port * sizeof(struct rdma_gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset gid_table.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_cmd_reset_gid(rdev, comp_priv, port, gids_per_port);
+ if (ret != 0) {
+ ROCE_ERR("Reset gid table failed, gid_num(%d), ret(%d)",
+ gids_per_port, ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+int roce5_get_gid(struct roce5_device *rdev, u32 port, u32 gid_index,
+ struct rdma_gid_entry *gid)
+{
+ struct rdma_service_cap *rdma_cap = NULL;
+ struct rdma_comp_priv *comp_priv = NULL;
+ int ret;
+
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return -EINVAL;
+ }
+
+ if (gid == NULL) {
+ ROCE_ERR("Gid is null");
+ return -EINVAL;
+ }
+
+ comp_priv = get_rdma_comp_priv(rdev->hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+
+ rdma_cap = &comp_priv->rdma_cap;
+ if (port < 1 || port > rdma_cap->num_ports) {
+ ROCE_ERR("Port(%d) invalid", port);
+ return -EINVAL;
+ }
+
+ if (gid_index >= rdma_cap->max_gid_per_port) {
+ ROCE_ERR("gid_index(%d) invalid", gid_index);
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(
+ (void *)gid, sizeof(*gid),
+ (void *)&comp_priv->rdma_comp_res.gid_table[port - 1][gid_index],
+ sizeof(*gid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dmac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c
new file mode 100644
index 000000000..ca5e88aff
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c
@@ -0,0 +1,443 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_hw.h"
+#include "rdma_comp.h"
+#include "roce_infra.h"
+#include "hinic5_hmm.h"
+#include "securec.h"
+
+#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 12K */
+
+static int rdma_init_pd_table(struct rdma_comp_priv *comp_priv)
+{
+ int ret = 0;
+ u32 num = 0;
+ u32 reserved_bot = 0;
+
+ num = comp_priv->rdma_cap.num_pds;
+ reserved_bot = comp_priv->rdma_cap.reserved_pds;
+
+ ret = rdma_bitmap_init(&comp_priv->pd_bitmap, num, num - 1,
+ reserved_bot, 0);
+ if (ret != 0) {
+ ROCE_ERR("Can't initialize pd's bitmap, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void rdma_cleanup_gid_table(struct rdma_comp_priv *comp_priv)
+{
+ struct rdma_gid_entry **gid_table = NULL;
+ int i = 0;
+ int port_num = 0;
+
+ gid_table = comp_priv->rdma_comp_res.gid_table;
+ if (gid_table == NULL) {
+ ROCE_ERR("Gid_table is null");
+ return;
+ }
+
+ port_num = (int)comp_priv->rdma_cap.num_ports;
+ for (i = 0; i < port_num; i++) {
+ if (gid_table[i]) {
+ kfree(gid_table[i]);
+ gid_table[i] = NULL;
+ }
+ }
+
+ kfree(gid_table);
+ comp_priv->rdma_comp_res.gid_table = NULL;
+
+ return;
+}
+
+static int roce5_get_gid_nums(struct rdma_comp_priv *comp_priv)
+{
+ struct hinic5_hwdev *dev = comp_priv->hmm_priv.hwdev;
+ struct service_cap *cap = &(dev->cfg_mgmt->svc_cap);
+ struct dev_roce_svc_own_cap *roce_cap =
+ &(cap->rdma_cap.dev_rdma_cap.roce_own_cap);
+
+ /* 23v100 and earlier generations: The configuration file does not contain max gid. */
+ return (roce_cap->max_gid != 0 ? roce_cap->max_gid :
+ comp_priv->rdma_cap.max_gid_per_port);
+}
+
+static int rdma_init_gid_table(struct rdma_comp_priv *comp_priv)
+{
+ struct rdma_gid_entry **gid_table = NULL;
+ u32 i = 0;
+ u32 port_num = 0;
+ u32 gids_per_port = 0;
+
+ port_num = comp_priv->rdma_cap.num_ports;
+ gids_per_port = comp_priv->rdma_cap.max_gid_per_port;
+
+ if ((port_num == 0) || (gids_per_port == 0)) {
+ ROCE_ERR(
+ "Alloc memory for gid_tbl failed, port_num(%d), gids_per_ports(%d)",
+ port_num, gids_per_port);
+ return -EINVAL;
+ }
+
+ gid_table = (struct rdma_gid_entry **)kzalloc(
+ port_num * sizeof(struct rdma_gid_entry *), GFP_KERNEL);
+ if (gid_table == NULL) {
+ ROCE_ERR(
+ "Alloc memory for gid_tbl failed, ret(%d), port_num(%d)",
+ -ENOMEM, port_num);
+ return -ENOMEM;
+ }
+
+ comp_priv->rdma_comp_res.gid_table = gid_table;
+
+ for (i = 0; i < port_num; i++) {
+ gid_table[i] = (struct rdma_gid_entry *)kzalloc(
+ gids_per_port * sizeof(struct rdma_gid_entry),
+ GFP_KERNEL);
+ if (gid_table[i] == NULL) {
+ ROCE_ERR(
+ "Alloc memory for gid_tbl[%d] failed, ret(%d), gids_per_ports(%d)",
+ i, -ENOMEM, gids_per_port);
+ goto err_out;
+ }
+ }
+ return 0;
+err_out:
+ for (i = 0; i < port_num; i++) {
+ if (gid_table[i]) {
+ kfree(gid_table[i]);
+ gid_table[i] = NULL;
+ }
+ }
+ kfree(gid_table);
+ comp_priv->rdma_comp_res.gid_table = NULL;
+
+ return -ENOMEM;
+}
+
+static int rdma_init_xrcd_table(struct rdma_comp_priv *comp_priv)
+{
+ int ret = 0;
+ u32 num = 0;
+ u32 reserved_bot = 0;
+
+ num = comp_priv->rdma_cap.max_xrcds;
+ reserved_bot = comp_priv->rdma_cap.reserved_xrcds;
+
+ ret = rdma_bitmap_init(&comp_priv->xrcd_bitmap, num, num - 1,
+ reserved_bot, 0);
+ if (ret != 0) {
+ ROCE_ERR("Can't initialize xrcd's bitmap, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void rdma_cleanup_xrcd_table(struct rdma_comp_priv *comp_priv)
+{
+ rdma_bitmap_cleanup(&comp_priv->xrcd_bitmap);
+}
+
+static int rdma_init_rdmarc_table(struct rdma_comp_priv *comp_priv)
+{
+ int ret = 0;
+ u32 i = 0;
+ u32 max_order = 0; /* rdmarc buddy max priority num */
+ u32 qp_num = 0;
+ u32 rdmarc_per_qp = 0; /* rdmarc num per qp */
+ u32 rdmarc_size = 0;
+ u32 log_rdmarc_per_seg = 0; /* min num of entry, 2^log_rdmarc_per_seg */
+ u32 rdmarc_pow_of_two = 0;
+
+ qp_num = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qps;
+ rdmarc_per_qp =
+ comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qp_dest_rdma +
+ ROCE_MAX_RDMA_RC_EXTEND;
+ rdmarc_size =
+ comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz;
+ log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg;
+ for (i = 1; i < qp_num * rdmarc_per_qp; i <<= 1) {
+ max_order++;
+ }
+
+ max_order = (max_order > log_rdmarc_per_seg) ?
+ (max_order - log_rdmarc_per_seg) :
+ 0;
+
+ ret = hmm_buddy_init(&comp_priv->rdmarc_buddy, max_order);
+ if (ret != 0) {
+ ROCE_ERR("Initialize rdmarc's buddy failed, ret(%d)", -ENOMEM);
+ return -ENOMEM;
+ }
+ rdmarc_pow_of_two =
+ roce_roundup_pow_of_two((u32)(qp_num * rdmarc_per_qp));
+ ret = hmm_em_init_table(comp_priv->hmm_priv.dev,
+ &comp_priv->rdmarc_em_table, rdmarc_size,
+ rdmarc_pow_of_two, 0, RDMA_EM_MIN_ORDER);
+ if (ret != 0) {
+ ROCE_ERR("Initialize rdmarc's em_table failed, ret(%d)", ret);
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ hmm_buddy_cleanup(&comp_priv->rdmarc_buddy);
+
+ return -ENOMEM;
+}
+
+static int roce5_rdma_init_pd_table(struct rdma_comp_priv *comp_priv)
+{
+ int ret = 0;
+ ret = rdma_init_pd_table(comp_priv);
+ if (ret != 0) {
+ ROCE_ERR("Initialize pd's table failed, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int roce5_rdma_init_mtt_table(struct rdma_comp_priv *comp_priv)
+{
+ int ret = 0;
+ ret = hmm_mtt_init((struct hmm_comp_priv *)(void *)comp_priv);
+ if (ret != 0) {
+ ROCE_ERR("Initialize mtt's table failed, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int roce5_rdma_init_bitmap(struct rdma_comp_priv *comp_priv)
+{
+ int ret;
+ ret = roce5_rdma_init_pd_table(comp_priv);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = roce5_rdma_init_mtt_table(comp_priv);
+ if (ret != 0) {
+ rdma_cleanup_pd_table(comp_priv);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void roce5_rdma_unit_bitmap(struct rdma_comp_priv *comp_priv)
+{
+ hmm_mtt_cleanup((struct hmm_comp_priv *)(void *)comp_priv);
+ rdma_cleanup_pd_table(comp_priv);
+}
+
+static int roce5_rdma_init_table(void *hwdev, struct rdma_comp_priv *comp_priv)
+{
+ int ret = 0;
+
+ ret = roce5_rdma_init_bitmap(comp_priv);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = rdma_init_gid_table(comp_priv);
+ if (ret != 0) {
+ ROCE_ERR("Initialize gid table failed, ret(%d)", ret);
+ goto err_init_gid_table;
+ }
+
+ ret = rdma_init_xrcd_table(comp_priv);
+ if (ret != 0) {
+ ROCE_ERR("Initialize xrcd's table failed, ret(%d)", ret);
+ goto err_init_xrcd;
+ }
+
+ ret = rdma_init_rdmarc_table(comp_priv);
+ if (ret != 0) {
+ ROCE_ERR("Initialize rdmarc's table failed, ret(%d)", ret);
+ goto err_init_rdmarc_table;
+ }
+
+ ROCE_INFO("Rdma init resource successful.");
+ return 0;
+
+err_init_rdmarc_table:
+ rdma_cleanup_xrcd_table(comp_priv);
+err_init_xrcd:
+ rdma_cleanup_gid_table(comp_priv);
+err_init_gid_table:
+ roce5_rdma_unit_bitmap(comp_priv);
+ return ret;
+}
+
+static void rdma_cleanup_rdmarc_table(struct rdma_comp_priv *comp_priv)
+{
+ hmm_em_cleanup_table(comp_priv->hmm_priv.dev,
+ &comp_priv->rdmarc_em_table);
+
+ hmm_buddy_cleanup(&comp_priv->rdmarc_buddy);
+
+ return;
+}
+
+static void roce5_rdma_unit_table(void *hwdev, struct rdma_comp_priv *comp_priv)
+{
+ if (hinic5_support_roce(hwdev, NULL)) {
+ rdma_cleanup_rdmarc_table(comp_priv);
+ rdma_cleanup_xrcd_table(comp_priv);
+ rdma_cleanup_gid_table(comp_priv);
+ }
+
+ roce5_rdma_unit_bitmap(comp_priv);
+}
+
+void roce5_rdma_cleanup_resource(void *hwdev)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return;
+ }
+
+ if (!hinic5_support_rdma(hwdev, NULL)) {
+ ROCE_ERR("Not support rdma service");
+ return;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return;
+ }
+
+ roce5_rdma_unit_table(hwdev, comp_priv);
+
+ kfree(comp_priv);
+
+ hinic5_unregister_service_adapter((void *)hwdev, SERVICE_T_ROCE);
+
+ ROCE_INFO("Rdma cleanup resource successful.");
+
+ return;
+}
+
+static int roce5_rdma_init_comp_priv(struct rdma_comp_priv *comp_priv,
+ void *hwdev,
+ struct rdma_service_cap *rdma_cap)
+{
+ int ret;
+
+ mutex_init(&comp_priv->rdma_comp_res.mutex);
+ comp_priv->hmm_priv.hwdev = hwdev;
+
+ comp_priv->hmm_priv.dev =
+ (struct device *)(((struct hinic5_hwdev *)hwdev)->dev_hdl);
+ if (comp_priv->hmm_priv.dev == NULL) {
+ ROCE_ERR("Failed to init, device is NULL.");
+ return -EINVAL;
+ }
+
+ ret = memcpy_s((void *)&comp_priv->rdma_cap,
+ sizeof(struct rdma_service_cap), (void *)rdma_cap,
+ sizeof(struct rdma_service_cap));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy rdma_cap.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ //to adapt hmm struct
+ comp_priv->hmm_priv.dev_cap.log_mtt = rdma_cap->log_mtt;
+ comp_priv->hmm_priv.dev_cap.num_mtts = rdma_cap->num_mtts;
+ comp_priv->hmm_priv.dev_cap.log_mtt_seg = rdma_cap->log_mtt_seg;
+ comp_priv->hmm_priv.dev_cap.mtt_entry_sz = rdma_cap->mtt_entry_sz;
+ comp_priv->hmm_priv.dev_cap.mpt_entry_sz = rdma_cap->mpt_entry_sz;
+ comp_priv->hmm_priv.dev_cap.dmtt_cl_start = rdma_cap->dmtt_cl_start;
+ comp_priv->hmm_priv.dev_cap.dmtt_cl_end = rdma_cap->dmtt_cl_end;
+ comp_priv->hmm_priv.dev_cap.dmtt_cl_sz = rdma_cap->dmtt_cl_sz;
+
+ switch (g_mtt_page_size) {
+ case ROCE5_RDMA_MTT_PAGE_SIZE_4K:
+ comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_4K;
+ comp_priv->hmm_priv.mtt_page_shift =
+ ROCE_MTT_PAGE_SIZE_4K_SHIFT;
+ break;
+ case ROCE5_RDMA_MTT_PAGE_SIZE_64K:
+ comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_64K;
+ comp_priv->hmm_priv.mtt_page_shift =
+ ROCE_MTT_PAGE_SIZE_64K_SHIFT;
+ break;
+ case ROCE5_RDMA_MTT_PAGE_SIZE_2M:
+ comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_2M;
+ comp_priv->hmm_priv.mtt_page_shift =
+ ROCE_MTT_PAGE_SIZE_2M_SHIFT;
+ break;
+ default:
+ comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_4K;
+ comp_priv->hmm_priv.mtt_page_shift =
+ ROCE_MTT_PAGE_SIZE_4K_SHIFT;
+ break;
+ }
+
+ /* Recalculate the gid num. */
+ comp_priv->rdma_cap.max_gid_per_port =
+ (u32)roce5_get_gid_nums(comp_priv);
+
+ ret = roce5_rdma_init_table(hwdev, comp_priv);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = hinic5_register_service_adapter((void *)hwdev, (void *)comp_priv,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ roce5_rdma_unit_table(hwdev, comp_priv);
+ ROCE_ERR("put rdma_comp_res failed, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+int roce5_rdma_init_resource(void *hwdev)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+ struct rdma_service_cap rdma_cap;
+ int ret = 0;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return -EINVAL;
+ }
+
+ if (hinic5_support_rdma(hwdev, &rdma_cap) == false) {
+ ROCE_INFO("Neither ROCE nor IWARP");
+ return 0;
+ }
+
+ comp_priv = (struct rdma_comp_priv *)kzalloc(
+ sizeof(struct rdma_comp_priv), GFP_KERNEL);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Alloc memory for comp_priv failed, ret(%d)", -ENOMEM);
+ return -ENOMEM;
+ }
+
+ ret = roce5_rdma_init_comp_priv(comp_priv, hwdev, &rdma_cap);
+ if (ret != 0) {
+ kfree(comp_priv);
+ return ret;
+ }
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c
new file mode 100644
index 000000000..aa252e2db
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_hw.h"
+#include "rdma_comp.h"
+#include "roce_cqm_cmd.h"
+#include "hmm_common.h"
+#include "securec.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_main_extension.h"
+#include "roce_compat.h"
+#include "roce_mr.h"
+#include "rdma_comp_mw_mr.h"
+
+static int rdma_mpt_alloc_rsvd_lkey(struct rdma_comp_priv *comp_priv,
+ struct mpt *mpt)
+{
+ u32 mpt_entry_size = 0;
+
+ mpt_entry_size = comp_priv->hmm_priv.dev_cap.mpt_entry_sz;
+
+ /* call cqm interface allocate mpt, and mpt_index and vaddr save in rdma_mpt structure */
+ mpt->mpt_object = (void *)cqm5_object_qpc_mpt_create(
+ comp_priv->hmm_priv.hwdev, RDMA_SERVICE_TYPE_ROCE,
+ CQM_OBJECT_MPT, mpt_entry_size, mpt, mpt->mpt_index, 0, 0);
+ if (mpt->mpt_object == NULL) {
+ ROCE_ERR("Alloc mpt_object failed, err(%d)", -ENOMEM);
+ return -ENOMEM;
+ }
+
+ mpt->vaddr = (void *)((cqm_qpc_mpt_s *)mpt->mpt_object)->vaddr;
+
+ return 0;
+}
+
+int roce5_rdma_init_rsvd_lkey(struct roce5_device *rdev)
+{
+ struct hmm_rdma *mr = NULL;
+ struct rdma_service_cap *rdma_cap = NULL;
+ struct rdma_comp_priv *comp_priv = NULL;
+ int ret = 0;
+
+ if (rdev == NULL) {
+ ROCE_ERR("rdev is null");
+ return -EINVAL;
+ }
+
+ comp_priv = get_rdma_comp_priv(rdev->hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+
+ rdma_cap = &comp_priv->rdma_cap;
+ mr = &comp_priv->rsvd_lkey;
+ mr->mpt.mpt_index = rdma_cap->reserved_lkey >> MR_KEY_LEFT_SHIFT_OFS;
+ if (mr->mpt.mpt_index >= rdma_cap->reserved_mrws) {
+ ROCE_ERR("Cfg err, reserved_lkey(0x%x), reserved_mrws(0x%x)",
+ rdma_cap->reserved_lkey, rdma_cap->reserved_mrws);
+ return -EINVAL;
+ }
+ /* Alloc MPT */
+ ret = rdma_mpt_alloc_rsvd_lkey(comp_priv, &mr->mpt);
+ if (ret != 0) {
+ ROCE_ERR("Rdma_mpt_alloc_rsvd_lkey failed, ret(%d)", ret);
+ return ret;
+ }
+ mr->mtt.mtt_layers = 0;
+ mr->iova = 0;
+ mr->size = ~0ULL;
+ mr->key = comp_priv->rdma_cap.reserved_lkey;
+ mr->type = roce5_mr_type2hmm(RDMA_RSVD_LKEY);
+ mr->access = RDMA_IB_ACCESS_LOCAL_WRITE;
+ /* Enable MPT */
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, mr,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Rdma_enable_mr_mpt failed, ret(%d)", ret);
+ goto err_out;
+ }
+ return 0;
+err_out:
+ hmm_mpt_free(rdev->hwdev, &mr->mpt);
+ return ret;
+}
+
+void roce5_rdma_free_rsvd_lkey(struct roce5_device *rdev)
+{
+ struct hmm_rdma *mr = NULL;
+ struct rdma_comp_priv *comp_priv = NULL;
+ int ret = 0;
+
+ if (rdev == NULL) {
+ ROCE_ERR("rdev is null");
+ return;
+ }
+
+ comp_priv = get_rdma_comp_priv(rdev->hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return;
+ }
+
+ mr = &comp_priv->rsvd_lkey;
+ ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, mr,
+ RDMA_SERVICE_TYPE_ROCE, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Disable mpt of mr failed, ret(%d)", ret);
+ return;
+ }
+
+ hmm_mpt_free(rdev->hwdev, &mr->mpt);
+
+ return;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h
new file mode 100644
index 000000000..1e9c1132d
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h
@@ -0,0 +1,12 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef RDMA_COMP_MW_MR_H
+#define RDMA_COMP_MW_MR_H
+
+#include "roce.h"
+
+int roce5_rdma_init_rsvd_lkey(struct roce5_device *rdev);
+void roce5_rdma_free_rsvd_lkey(struct roce5_device *rdev);
+
+#endif /* RDMA_COMP_MW_MR_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c
new file mode 100644
index 000000000..ee3486e97
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c
@@ -0,0 +1,51 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_hw.h"
+#include "rdma_comp.h"
+
+int roce5_rdma_pd_alloc(void *hwdev, u32 *pdn)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+
+ if ((hwdev == NULL) || (pdn == NULL)) {
+ ROCE_ERR("Hwdev or pdn is null");
+ return -EINVAL;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+
+ *pdn = rdma_bitmap_alloc(&comp_priv->pd_bitmap);
+ if (*pdn == RDMA_INVALID_INDEX) {
+ ROCE_ERR("Can't get valid pdn, err(%d)", -ENOMEM);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+void roce5_rdma_pd_free(void *hwdev, u32 pdn)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return;
+ }
+
+ rdma_bitmap_free(&comp_priv->pd_bitmap, pdn);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c
new file mode 100644
index 000000000..d1265e54e
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c
@@ -0,0 +1,140 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_hw.h"
+#include "rdma_comp.h"
+#include "hmm_em_inner.h"
+#include "hmm_buddy.h"
+
+static int roce5_rdma_rdmarc_alloc_check(struct rdma_comp_priv **comp_priv,
+ void *hwdev,
+ const struct rdma_rdmarc *rdmarc,
+ u32 *log_rdmarc_per_seg, u32 num)
+{
+ if ((hwdev == NULL) || (rdmarc == NULL)) {
+ ROCE_ERR("Hwdev or rdmarc is null");
+ return -EINVAL;
+ }
+
+ *comp_priv = get_rdma_comp_priv(hwdev);
+ if (*comp_priv == NULL || (*comp_priv)->hmm_priv.dev == NULL) {
+ ROCE_ERR("Comp_priv(%d) or PCI device is NULL.",
+ !!(*comp_priv));
+ return -EINVAL;
+ }
+
+ *log_rdmarc_per_seg = (*comp_priv)->rdma_cap.log_rdmarc_seg;
+
+ if ((num < ROCE5_RDMARC_MIN_DEPTH) || (num > ROCE5_RDMARC_MAX_DEPTH)) {
+ ROCE_ERR("Num is invalid, ret(%d)", -EINVAL);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void roce5_rdma_rdmarc_order_set(struct rdma_rdmarc *rdmarc, u32 order,
+ u32 ext_order, u32 log_rdmarc_per_seg)
+{
+ rdmarc->order = order;
+ rdmarc->ext_order = ext_order + log_rdmarc_per_seg;
+ return;
+}
+
+int roce5_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+ int ret = 0;
+ u32 i = 0;
+ u32 order = 0;
+ u32 ext_order = 0;
+ u32 ext_num = num + ROCE5_RDMARC_EXT_ENTRY;
+ u32 offset = 0;
+ void *vaddr = NULL;
+ u32 log_rdmarc_per_seg = 0;
+ u32 chip_num = (num < ROCE5_RDMARC_MIN_ENTRY) ? ROCE5_RDMARC_MIN_ENTRY :
+ num;
+ ret = roce5_rdma_rdmarc_alloc_check(&comp_priv, hwdev, rdmarc,
+ &log_rdmarc_per_seg, num);
+ if (ret != 0) {
+ return ret;
+ }
+ for (i = 1; i < chip_num; i <<= 1) {
+ order++;
+ }
+ for (i = 1; i < ext_num; i <<= 1) {
+ ext_order++;
+ }
+ ext_order = (ext_order > log_rdmarc_per_seg) ?
+ (ext_order - log_rdmarc_per_seg) :
+ 0;
+ roce5_rdma_rdmarc_order_set(rdmarc, order, ext_order,
+ log_rdmarc_per_seg);
+ offset = hmm_buddy_alloc(&comp_priv->rdmarc_buddy, ext_order);
+ if (offset == RDMA_INVALID_INDEX) {
+ ROCE_ERR("Alloc rdmarc index failed, ret(%d)", -ENOMEM);
+ return -ENOMEM;
+ }
+ rdmarc->offset = offset << log_rdmarc_per_seg;
+
+ ret = hmm_em_table_get_range(
+ comp_priv->hmm_priv.dev, &comp_priv->rdmarc_em_table,
+ rdmarc->offset, rdmarc->offset + (1U << rdmarc->ext_order) - 1);
+ if (ret != 0) {
+ ROCE_ERR("Alloc rdmarc entry failed, ret(%d)", -ENOMEM);
+ goto err_table_get;
+ }
+ vaddr = hmm_em_table_find(&comp_priv->rdmarc_em_table, rdmarc->offset,
+ &rdmarc->dma_addr);
+ if (vaddr == NULL) {
+ ret = -ENOMEM;
+ ROCE_ERR("Can't find va and pa of rdmarc entry, ret(%d)",
+ -ENOMEM);
+ goto err_rdmarc_find;
+ }
+ rdmarc->vaddr = vaddr;
+ return 0;
+
+err_rdmarc_find:
+ hmm_em_table_put_range(comp_priv->hmm_priv.dev,
+ &comp_priv->rdmarc_em_table, rdmarc->offset,
+ rdmarc->offset + (1U << rdmarc->ext_order) - 1);
+err_table_get:
+ hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, ext_order);
+ return ret;
+}
+
+void roce5_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+ u32 order = 0;
+ u32 offset = 0;
+ u32 log_rdmarc_per_seg = 0;
+
+ if ((hwdev == NULL) || (rdmarc == NULL)) {
+ ROCE_ERR("Hwdev or rdmarc is null");
+ return;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL || comp_priv->hmm_priv.dev == NULL) {
+ ROCE_ERR("Comp_priv(%d) or comp_priv->hmm_priv.dev is NULL",
+ !!comp_priv);
+ return;
+ }
+
+ hmm_em_table_put_range(comp_priv->hmm_priv.dev,
+ &comp_priv->rdmarc_em_table, rdmarc->offset,
+ rdmarc->offset + (1U << rdmarc->ext_order) - 1);
+
+ log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg;
+
+ order = rdmarc->ext_order - log_rdmarc_per_seg;
+ offset = rdmarc->offset >> log_rdmarc_per_seg;
+
+ hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, order);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h
new file mode 100644
index 000000000..f268757a4
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h
@@ -0,0 +1,78 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_COM_H
+#define ROCE_COM_H
+
+#include "roce_cfg_format.h"
+
+/* kernel mode basic capability negotiation support macro */
+#define ROCE5_SUPPORT_IB_FEATURE(rdev, feature) \
+ ((rdev)->cfg_info.feature_cap.roce_kernel_ability & \
+ (ROCE_SUPPORT_IB_##feature))
+
+#define ROCE5_SUPPORT_IB_CMD_RESIZE_CQ_MSG_V2(rdev) \
+ ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_RESIZE_CQ_MSG_V2)
+#define ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) \
+ ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_QUERY_QP_MSG_V2)
+#define ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) \
+ ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_MODIFY_SRQ_MSG_V2)
+#define ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) \
+ ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_DESTROY_SRQ_MSG_V2)
+
+/* kernel mode extended capability negotiation support macro */
+#define ROCE5_SUPPORT_FEATURE(rdev, feature) \
+ ((rdev)->cfg_info.feature_cap.roce_kernel_ext_ability & \
+ (ROCE_SUPPORT_##feature))
+
+#define ROCE5_SUPPORT_CQ_MODIFY_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CQ_MODIFY_MSG_V2)
+#define ROCE5_SUPPORT_CQ_CHECK_DATA_STATE_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CQ_CHECK_DATA_STATE_MSG_V2)
+#define ROCE5_SUPPORT_CQ_CACHE_OUT_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CQ_CACHE_OUT_MSG_V2)
+#define ROCE5_SUPPORT_QP_SRQN_LB_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, QP_SRQN_LB_MSG_V2)
+#define ROCE5_SUPPORT_SET_PI_ON_CHIP_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, SET_PI_ON_CHIP_MSG_V2)
+#define ROCE5_SUPPORT_SQ_DB_TYPE_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, SQ_DB_TYPE_MSG_V2)
+#define ROCE5_SUPPORT_GET_QP_RX_PORT_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, GET_QP_RX_PORT_MSG_V2)
+#define ROCE5_SUPPORT_MODIFY_UDP_SRC_PORT_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, MODIFY_UDP_SRC_PORT_MSG_V2)
+#define ROCE5_SUPPORT_GET_QP_UDP_SRC_PORT_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, GET_QP_UDP_SRC_PORT_MSG_V2)
+#define ROCE5_SUPPORT_DFX_CMD_QUERY_MPT_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_MPT_MSG_V2)
+#define ROCE5_SUPPORT_DFX_CMD_QUERY_GID_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_GID_MSG_V2)
+#define ROCE5_SUPPORT_DFX_CMD_QUERY_PI_CI_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_PI_CI_MSG_V2)
+#define ROCE5_SUPPORT_DFX_CMD_QUERY_CQ_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_CQ_MSG_V2)
+#define ROCE5_SUPPORT_ALLOC_QPC_MSG_FEATURE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, ALLOC_QPC_MSG_FEATURE)
+#define ROCE5_SUPPORT_GET_OQID_MSG_FEATURE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, GET_OQID_MSG_FEATURE)
+#define ROCE5_SUPPORT_RETRY_LOAD_CQC_MSG_FEATURE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, RETRY_LOAD_CQC_MSG_FEATURE)
+#define ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, SET_HASH_VALUE_MSG_V2)
+#define ROCE5_SUPPORT_CC_GET_DCQCN_ENABLE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CC_GET_DCQCN_ENABLE)
+#define ROCE5_SUPPORT_CC_GET_LDCP_ENABLE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CC_GET_LDCP_ENABLE)
+#define ROCE5_SUPPORT_CC_GET_IPQCN_ENABLE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CC_GET_IPQCN_ENABLE)
+#define ROCE5_SUPPORT_CC_GET_USER_ALGO_ENABLE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, CC_GET_USER_ALGO_ENABLE)
+#define ROCE5_SUPPORT_CC_PARAM_V1(rdev) ROCE5_SUPPORT_FEATURE(rdev, CC_PARAM_V1)
+#define ROCE5_SUPPORT_CC_PARAM_V2(rdev) ROCE5_SUPPORT_FEATURE(rdev, CC_PARAM_V2)
+#define ROCE5_SUPPORT_8K_MTU(rdev) ROCE5_SUPPORT_FEATURE(rdev, KERNEL_8K_MTU)
+#define ROCE5_SUPPORT_SML_MAC_TABLE(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, SML_MAC_TABLE)
+#define ROCE5_SUPPORT_SQ_DB_TYPE_21(rdev) \
+ ROCE5_SUPPORT_FEATURE(rdev, SQ_DB_TYPE_21)
+#endif /* ROCE_COM_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c
new file mode 100644
index 000000000..a15e865ec
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c
@@ -0,0 +1,1594 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_verbs.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+
+#include "securec.h" /* Huawei secure functions; */
+
+#include "hinic5_crm.h"
+#include "hinic5_srv_nic.h"
+#include "hinic5_lld.h"
+
+#include "roce.h"
+#include "roce_compat.h"
+#include "roce_user.h"
+#include "roce_pd.h"
+#include "roce_qp.h"
+#include "roce_cmd.h"
+#include "roce_gid.h"
+#include "roce_netdev.h"
+#include "roce_main_extension.h"
+#include "roce_qp_extension.h"
+#include "roce_restore.h"
+#include "roce_infra.h"
+#include "roce_ctx_format.h"
+#include "roce_kernel_compat_gen.h"
+#include "roce5_gid_type.h"
+
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+#include "roce_mix.h"
+
+uint8_t roce5_get_db_cos_from_vlan_pri(struct roce5_device *rdev,
+ uint8_t vlan_pri)
+{
+ uint8_t db_cos;
+ int ret;
+
+ ret = hinic5_get_cos_by_pri(rdev->hwdev, vlan_pri, &db_cos);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "ret:%d vlan_pri:%u", ret, vlan_pri);
+ db_cos = 0;
+ }
+
+ return db_cos;
+}
+
+struct net_device *roce5_ib_get_netdev(struct ib_device *ibdev, u32 port_num)
+{
+ struct roce5_device *rdev = NULL;
+ struct net_device *netdev = NULL;
+
+ if (ibdev == NULL) {
+ ROCE_ERR("Ibdev is null");
+ return NULL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+#ifdef ROCE_BONDING_EN
+ netdev = roce5_bond_get_netdev(rdev);
+ if (netdev != NULL) {
+ return netdev;
+ }
+#endif
+
+ netdev = rdev->ndev;
+
+ dev_hold(netdev);
+
+ return netdev;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+struct net_device *roce5_ib_get_netdev_compat(struct ib_device *ibdev,
+ u8 port_num)
+{
+ return roce5_ib_get_netdev(ibdev, (u32)port_num);
+}
+
+static void roce5_parse_fw_version(struct roce5_device *rdev, u64 *fw_ver)
+{
+ int ret;
+ int i = 0;
+ struct hinic5_fw_version fw_version;
+ char *fw_str = (char *)fw_version.microcode_ver;
+ char *fw_temp = NULL;
+ u64 fw_verion[ROCE_FW_VERSION_LEN] = { 0 };
+
+ ret = hinic5_get_fw_version(rdev->hwdev, &fw_version,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_WARN("get fw version failed");
+ *fw_ver = ROCE_FW_VER;
+ return;
+ }
+ ROCE_INFO("fw ver:%s - %s - %s", fw_version.boot_ver,
+ fw_version.mgmt_ver, fw_version.microcode_ver);
+
+ while (((fw_temp = strsep(&fw_str, ".")) != NULL) &&
+ (i < ROCE_FW_VERSION_LEN)) {
+ ret = sscanf_s(fw_temp, "%llu", &fw_verion[i]);
+ if (ret != 1) {
+ ROCE_WARN("parse fw version failed ret(%d)", ret);
+ *fw_ver = ROCE_FW_VER;
+ return;
+ }
+
+ i++;
+ }
+
+ *fw_ver = (((fw_verion[0] & 0xffffffff)
+ << 32) | // 0 is fw_version array idx, 32 is offset
+ ((fw_verion[1] & 0xffff)
+ << 16) | // 1 is fw_version array idx, 16 is offset
+ ((fw_verion[2] & 0xff)
+ << 8) | // 2 is fw_version array idx, 8 is offset
+ (fw_verion[3] & 0xff)); // 3 is fw_version array idx
+}
+
+static void roce5_set_local_cap_flag(const struct rdma_service_cap *rdma_cap,
+ struct ib_device_attr *props)
+{
+ if (((rdma_cap->flags & RDMA_BMME_FLAG_LOCAL_INV) != 0) &&
+ ((rdma_cap->flags & RDMA_BMME_FLAG_REMOTE_INV) != 0) &&
+ ((rdma_cap->flags & RDMA_BMME_FLAG_FAST_REG_WR) != 0)) {
+ props->device_cap_flags = props->device_cap_flags |
+ IB_DEVICE_MEM_MGT_EXTENSIONS;
+ }
+}
+
+static void roce5_set_bmme_cap_flag(const struct rdma_service_cap *rdma_cap,
+ struct ib_device_attr *props)
+{
+ if ((rdma_cap->flags & RDMA_BMME_FLAG_TYPE_2_WIN) != 0) {
+ if ((rdma_cap->flags & RDMA_BMME_FLAG_WIN_TYPE_2B) != 0) {
+ props->device_cap_flags = props->device_cap_flags |
+ IB_DEVICE_MEM_WINDOW_TYPE_2B;
+ } else {
+ props->device_cap_flags = props->device_cap_flags |
+ IB_DEVICE_MEM_WINDOW_TYPE_2A;
+ }
+ }
+}
+
+static void roce5_query_device_props_set(struct roce5_device *rdev,
+ struct rdma_service_cap *rdma_cap,
+ struct ib_device_attr *props)
+{
+ if (rdev->lld_dev->dev_type == HINIC5_DEVICE_T_PCI) {
+ struct pci_dev *pdev =
+ container_of(rdev->dev, struct pci_dev, dev);
+ props->vendor_id = pdev->vendor;
+ props->vendor_part_id = pdev->device;
+ }
+ roce5_parse_fw_version(rdev, &props->fw_ver);
+ props->hw_ver = ROCE_HW_VER;
+
+ /* sys_image_guid equal GID */
+ props->sys_image_guid = rdev->ib_dev.node_guid;
+
+ props->max_mr_size = ~0ULL;
+ props->page_size_cap = rdma_cap->page_size_cap;
+ props->max_qp = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_qps -
+ rdma_cap->reserved_qps);
+ props->max_qp_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes;
+ /* We should ensure sq's max size(max_sq_sg) as the max_sge */
+ props->max_send_sge = (int)rdma_cap->max_sq_sg;
+ props->max_recv_sge =
+ (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg;
+ props->max_cq = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs -
+ rdma_cap->reserved_cqs);
+ props->max_cqe = (int)rdma_cap->max_cqes;
+ props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts -
+ rdma_cap->reserved_mrws);
+ props->max_mw = props->max_mr;
+ props->max_pd = (int)(rdma_cap->num_pds - rdma_cap->reserved_pds);
+ props->max_qp_rd_atom =
+ (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma;
+ props->max_qp_init_rd_atom =
+ (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma;
+ props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
+ props->max_srq =
+ (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs -
+ rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs);
+ props->max_srq_wr =
+ (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_wqes;
+ props->max_srq_sge =
+ (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_sge;
+ props->max_fast_reg_page_list_len = rdma_cap->max_frpl_len;
+ props->local_ca_ack_delay = (u8)rdma_cap->local_ca_ack_delay;
+ props->atomic_cap =
+ ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_ATOMIC) != 0) ?
+ IB_ATOMIC_HCA :
+ IB_ATOMIC_NONE;
+ props->masked_atomic_cap = props->atomic_cap;
+ props->max_pkeys = (u16)rdma_cap->max_pkeys;
+ props->max_sge_rd = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg;
+#if defined(IB_DEV_ATTR_HAVE_MAX_FMR)
+ props->max_fmr = props->max_map_per_fmr;
+#endif
+#if defined(IB_DEV_ATTR_HAVE_MAX_MAP_PER_FMR)
+ props->max_map_per_fmr = (int)rdma_cap->max_fmr_maps;
+#endif
+ props->max_ah = INT_MAX;
+ return;
+}
+/* ****************************************************************************
+ Prototype : roce5_query_device
+ Description : query device attribute
+ Input : struct ib_device *ibdev
+ struct ib_device_attr *props
+ struct ib_udata *uhw
+ Output : struct ib_device_attr *props
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
+ struct ib_udata *uhw)
+{
+ struct roce5_device *rdev = NULL;
+ struct rdma_service_cap *rdma_cap = NULL;
+ int ret;
+
+ if ((ibdev == NULL) || (props == NULL)) {
+ ROCE_ERR("Ibdev or props is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ ret = memset_s(props, sizeof(*props), 0, sizeof(*props));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset props.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ rdma_cap = &rdev->rdma_cap;
+
+ props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT |
+ IB_DEVICE_RC_RNR_NAK_GEN;
+
+ /* APM */
+ if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_APM) != 0) {
+ props->device_cap_flags = props->device_cap_flags |
+ IB_DEVICE_AUTO_PATH_MIG;
+ }
+
+ /* rsvd_lKey */
+ if ((rdma_cap->flags & RDMA_BMME_FLAG_RESERVED_LKEY) != 0) {
+#if defined(HAVE_IB_DEVICE_LOCAL_DMA_LKEY)
+ props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
+#elif defined(HAVE_IB_KERNEL_CAP_FLAGS)
+ props->kernel_cap_flags |= IBK_LOCAL_DMA_LKEY;
+#endif
+ }
+
+ roce5_set_local_cap_flag(rdma_cap, props);
+
+ /* support XRC */
+ if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_XRC) != 0) {
+ props->device_cap_flags = props->device_cap_flags |
+ IB_DEVICE_XRC;
+ }
+
+ /* support MW */
+ if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_MEM_WINDOW) != 0) {
+ props->device_cap_flags = props->device_cap_flags |
+ IB_DEVICE_MEM_WINDOW;
+ }
+
+ roce5_set_bmme_cap_flag(rdma_cap, props);
+
+ roce5_query_device_props_set(rdev, rdma_cap, props);
+ return 0;
+}
+
+static void roce5_eth_link_get_speed(struct ib_port_attr *props,
+ enum mag_cmd_port_speed speed)
+{
+ switch (speed) {
+ /* 10G <==> 1X x 10G */
+ case PORT_SPEED_10GB:
+ props->active_width = IB_WIDTH_1X;
+ props->active_speed = IB_SPEED_QDR;
+ break;
+
+ /* 25G <==> 1X x 25G */
+ case PORT_SPEED_25GB:
+ props->active_width = IB_WIDTH_1X;
+ props->active_speed = IB_SPEED_EDR;
+ break;
+
+ /* 40G <==> 4X x 10G */
+ case PORT_SPEED_40GB:
+ props->active_width = IB_WIDTH_4X;
+ props->active_speed = IB_SPEED_QDR;
+ break;
+
+ /* 100G <==> 4X x 25G */
+ case PORT_SPEED_100GB:
+ props->active_width = IB_WIDTH_4X;
+ props->active_speed = IB_SPEED_EDR;
+ break;
+
+ /* 200G <==> 8X x 25G */
+ case PORT_SPEED_200GB:
+ props->active_width = IB_WIDTH_8X;
+ props->active_speed = IB_SPEED_EDR;
+ break;
+
+ /* 400G <==> 8X x 50G */
+ case PORT_SPEED_400GB:
+ props->active_width = IB_WIDTH_8X;
+ props->active_speed = IB_SPEED_HDR;
+ break;
+
+ default:
+ props->active_width = 0;
+ props->active_speed = 0;
+ break;
+ }
+}
+
+static void roce5_set_ib_port_attr(struct ib_port_attr *props,
+ struct roce5_device *rdev)
+{
+ struct rdma_comp_priv *comp_priv = get_rdma_comp_priv(rdev->hwdev);
+
+ if (comp_priv == NULL) {
+ ROCE_ERR("comp_priv is NULL");
+ return;
+ }
+
+ props->port_cap_flags = IB_PORT_CM_SUP;
+ props->gid_tbl_len = (int)comp_priv->rdma_cap.max_gid_per_port;
+ props->max_msg_sz = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz;
+ props->pkey_tbl_len = (u16)rdev->rdma_cap.max_pkeys;
+ props->max_mtu = (enum ib_mtu)((ROCE5_SUPPORT_8K_MTU(rdev) != 0) ?
+ ROCE_MTU_8192 :
+ ROCE_MTU_4096);
+ props->state = IB_PORT_DOWN;
+ props->phys_state = ROCE_PORT_PHYS_STATE_DISABLED;
+ props->active_mtu = IB_MTU_256;
+}
+
+static inline int roce5_get_port_speed_by_rdev(struct roce5_device *rdev,
+ enum mag_cmd_port_speed *speed)
+{
+ enum mag_cmd_port_speed speed_table[] = {
+ [ROCE5_4_25G_HCA] = PORT_SPEED_25GB,
+ [ROCE5_2_25G_HCA] = PORT_SPEED_25GB,
+ [ROCE5_1_100G_HCA] = PORT_SPEED_100GB,
+ [ROCE5_2_100G_HCA] = PORT_SPEED_100GB,
+ [ROCE5_2_200G_HCA] = PORT_SPEED_200GB,
+ [ROCE5_2_400G_HCA] = PORT_SPEED_400GB,
+ [ROCE5_4_200G_HCA] = PORT_SPEED_200GB,
+ };
+ int hca_type = rdev->hw_info.hca_type;
+
+ if (hca_type >= ARRAY_SIZE(speed_table) ||
+ speed_table[hca_type] == PORT_SPEED_NOT_SET) {
+ return -EINVAL;
+ }
+
+ *speed = speed_table[hca_type];
+ return 0;
+}
+
+static void roce5_eth_link_query_port(struct ib_device *ibdev,
+ struct ib_port_attr *props)
+{
+ struct roce5_device *rdev = NULL;
+ struct net_device *netdev = NULL;
+#ifdef ROCE_BONDING_EN
+ struct net_device *upper = NULL;
+#endif
+ enum roce5_mtu rmtu;
+ enum mag_cmd_port_speed speed = PORT_SPEED_10GB;
+ int ret = 0;
+
+ rdev = to_roce5_dev(ibdev);
+
+ roce5_set_ib_port_attr(props, rdev);
+
+ ret = roce5_get_port_speed_by_rdev(rdev, &speed);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to get speed, func_id(%d), hca_type(%d)",
+ rdev->glb_func_id, rdev->hw_info.hca_type);
+ props->active_width = 0;
+ props->active_speed = 0;
+ return;
+ }
+
+ roce5_eth_link_get_speed(props, speed);
+
+ netdev = roce5_ib_get_netdev(ibdev, ROCE_DEFAULT_PORT_NUM);
+ if (netdev == NULL) {
+ return;
+ }
+
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ rcu_read_lock();
+ upper = netdev_master_upper_dev_get_rcu(netdev);
+ if (upper != NULL) {
+ dev_put(netdev);
+ netdev = upper;
+ dev_hold(netdev);
+ }
+ rcu_read_unlock();
+ }
+#endif
+
+ if (netif_running(netdev) && netif_carrier_ok(netdev)) {
+ props->state = IB_PORT_ACTIVE;
+ props->phys_state = ROCE_PORT_PHYS_STATE_LINKUP;
+ }
+
+ rmtu = (ROCE5_SUPPORT_8K_MTU(rdev) != 0) ?
+ roce5_get_mtu((int)netdev->mtu) :
+ (enum roce5_mtu)iboe_get_mtu((int)netdev->mtu);
+
+ dev_put(netdev);
+
+ props->active_mtu =
+ (enum ib_mtu)ROCE_MIN((enum roce5_mtu)props->max_mtu, rmtu);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_query_port
+ Description : query port attribute
+ Input : struct ib_device *ibdev
+ u8 port
+ struct ib_port_attr *props
+ Output : struct ib_port_attr *props
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_query_port(struct ib_device *ibdev, u32 port,
+ struct ib_port_attr *props)
+{
+ int ret;
+ if ((ibdev == NULL) || (props == NULL)) {
+ ROCE_ERR("Ibdev or props is null");
+ return -EINVAL;
+ }
+
+ ret = memset_s(props, sizeof(*props), 0, sizeof(*props));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset props.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ roce5_eth_link_query_port(ibdev, props);
+
+ return 0;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+int roce5_query_port_compat(struct ib_device *ibdev, u8 port,
+ struct ib_port_attr *props)
+{
+ return roce5_query_port(ibdev, (u32)port, props);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_query_gid
+ Description : query gid
+ Input : struct ib_device *ibdev
+ u8 port
+ int index
+ union ib_gid *gid
+ Output : union ib_gid *gid
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+ 2.Date : 2015/6/8
+ Modification : Modify function
+
+**************************************************************************** */
+int roce5_query_gid(struct ib_device *ibdev, u32 port, int index,
+ union ib_gid *gid)
+{
+ int ret = 0;
+ struct roce5_device *rdev = NULL;
+ struct rdma_gid_entry gid_entry;
+
+ if ((ibdev == NULL) || (gid == NULL)) {
+ ROCE_ERR("Ibdev or gid is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ ret = roce5_get_gid(rdev, port, (u32)index, &gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = memcpy_s((void *)gid->raw, sizeof(*gid), (void *)gid_entry.raw,
+ sizeof(*gid));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy gid.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ // per per OFED of gid generation method convert GID, only IPv4 scenario need convert
+ if (gid_entry.dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) {
+ // not add of gid returns directly, no need to convert
+ if ((gid->global.subnet_prefix == 0) &&
+ (gid->global.interface_id == 0)) {
+ return 0;
+ }
+ ipv6_addr_set_v4mapped(*((u32 *)(void *)gid + ROCE_GID_IP_IDX),
+ (struct in6_addr *)gid);
+ }
+
+ return 0;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+int roce5_query_gid_compat(struct ib_device *ibdev, u8 port, int index,
+ union ib_gid *gid)
+{
+ return roce5_query_gid(ibdev, (u32)port, index, gid);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_query_pkey
+ Description : query pkey
+ Input : struct ib_device *ibdev
+ u8 port
+ u16 index
+ u16 *pkey
+ Output : u16 *pkey
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
+{
+ struct roce5_device *rdev = NULL;
+ if ((ibdev == NULL) || (pkey == NULL)) {
+ ROCE_ERR("Ibdev or pkey is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ if (pkey == NULL) {
+ ROCE_ERR("Pkey is null");
+ return -EINVAL;
+ }
+
+ *pkey = 0xffff;
+
+ return 0;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+int roce5_query_pkey_compat(struct ib_device *ibdev, u8 port, u16 index,
+ u16 *pkey)
+{
+ return roce5_query_pkey(ibdev, (u32)port, index, pkey);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_modify_device
+ Description : modify device attribute
+ Input : struct ib_device *ibdev
+ int mask
+ struct ib_device_modify *props
+ Output : None
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_modify_device(struct ib_device *ibdev, int mask,
+ struct ib_device_modify *props)
+{
+ unsigned long flags = 0; /* type required by kernel API */
+ struct roce5_device *rdev = NULL;
+ int ret;
+
+ if ((ibdev == NULL) || (props == NULL)) {
+ ROCE_ERR("Ibdev or props is null");
+ return -EINVAL;
+ }
+
+ if (((unsigned int)mask & ~IB_DEVICE_MODIFY_NODE_DESC) != 0) {
+ ROCE_ERR("Not supported to modify node description");
+ return -EOPNOTSUPP;
+ }
+
+ if ((((u32)mask) & IB_DEVICE_MODIFY_NODE_DESC) == 0) {
+ ROCE_INFO("No need to modify node description");
+ return 0;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ spin_lock_irqsave(&rdev->node_desc_lock, flags);
+ ret = memcpy_s((void *)ibdev->node_desc, IB_DEVICE_NODE_DESC_MAX,
+ (void *)props->node_desc, IB_DEVICE_NODE_DESC_MAX);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy node_desc.(ret:%d)", ret);
+ spin_unlock_irqrestore(&rdev->node_desc_lock, flags);
+ return -ENOMEM;
+ }
+ spin_unlock_irqrestore(&rdev->node_desc_lock, flags);
+
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_modify_port
+ Description : modify port attribute
+ Input : struct ib_device *ibdev
+ u8 port
+ int mask
+ struct ib_port_modify *props
+ Output : None
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_modify_port(struct ib_device *ibdev, u32 port, int mask,
+ struct ib_port_modify *props)
+{
+ int ret = 0;
+ struct ib_port_attr attr;
+ struct roce5_device *rdev = NULL;
+
+ if (ibdev == NULL) {
+ ROCE_ERR("Ibdev is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ ret = memset_s(&attr, sizeof(struct ib_port_attr), 0,
+ sizeof(struct ib_port_attr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ mutex_lock(&rdev->cap_mask_mutex);
+
+ ret = roce5_query_port(ibdev, port, &attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query port, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ }
+
+ mutex_unlock(&rdev->cap_mask_mutex);
+
+ return ret;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+int roce5_modify_port_compat(struct ib_device *ibdev, u8 port, int mask,
+ struct ib_port_modify *props)
+{
+ return roce5_modify_port(ibdev, (u32)port, mask, props);
+}
+
+static int roce5_alloc_ucontext_set(struct roce5_device *rdev,
+ struct roce5_alloc_ucontext_resp *resp)
+{
+ struct rdma_service_cap *rdma_cap = NULL;
+
+ rdma_cap = &rdev->rdma_cap;
+
+ resp->num_qps = rdma_cap->dev_rdma_cap.roce_own_cap.max_qps;
+ resp->num_xsrqs = rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs;
+ resp->cqe_size = rdma_cap->cqe_size;
+ resp->wqebb_size = rdma_cap->wqebb_size;
+ resp->dwqe_size = rdma_cap->direct_wqe_size;
+ resp->max_msg_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_msg_sz;
+ resp->max_comp_vector = rdma_cap->num_comp_vectors;
+ resp->max_inline_size =
+ rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz;
+
+ resp->storage_aa_en = 0;
+ resp->phy_port = rdev->hw_info.phy_port;
+ resp->srq_container_en = rdev->cfg_info.srq_container_en;
+ resp->srq_container_mode = rdev->cfg_info.srq_container_mode;
+ resp->xrc_srq_container_mode = rdev->cfg_info.xrc_srq_container_mode;
+ resp->warn_th = rdev->cfg_info.warn_th;
+ resp->func_id = rdev->glb_func_id;
+ resp->pcie_atomic_en = rdev->cfg_info.pcie_atomic_en;
+ resp->roce_user_ext_ability =
+ rdev->cfg_info.feature_cap.roce_user_ext_ability;
+ resp->roce_uld_ext_ability =
+ rdev->cfg_info.feature_cap.uld_feature.roce_uld_user_ext_ability;
+ resp->max_send_sge = (uint8_t)rdev->rdma_cap.max_sq_sg;
+ resp->max_recv_sge =
+ (uint8_t)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg;
+ resp->max_sq_wqe_len = rdev->rdma_cap.max_sq_desc_sz;
+ resp->huge_page_en = rdev->huge_page_en;
+ return hinic5_get_chip_node_id(rdev->lld_dev, &(resp->card_id));
+}
+
+static int roce5_alloc_ucontext_pre_check(struct ib_device *ibdev,
+ const struct ib_udata *udata)
+{
+ struct roce5_device *rdev = NULL;
+
+ if ((ibdev == NULL) || (udata == NULL)) {
+ ROCE_ERR("Ibdev or udata is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+ if (!rdev->ib_active) {
+ ROCE_DEV_ERR(rdev, "Device is abnormal, func_id(%d)",
+ rdev->glb_func_id);
+ return -EAGAIN;
+ }
+
+ return 0;
+}
+
+static int roce5_alloc_ucontext_return(struct roce5_device *rdev,
+ struct ib_udata *udata,
+ struct roce5_ucontext *context,
+ struct roce5_alloc_ucontext_resp *resp)
+{
+ int ret;
+
+ resp->db_offset = context->db_dma_addr & ((1 << PAGE_SHIFT) - 1);
+#ifdef ROCE_DWQE_EN
+ resp->dwqe_offset = context->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1);
+
+ if (context->dwqe_dma_addr == 0) {
+ ROCE_DEV_ERR(rdev, "dwqe_dma_addr = 0");
+ resp->dwqe_size = 0;
+ }
+#else
+ resp->dwqe_size = 0;
+#endif
+
+ INIT_LIST_HEAD(&context->db_page_list);
+ mutex_init(&context->db_page_mutex);
+
+ /* Copy data to user space */
+ ret = ib_copy_to_udata(udata, resp,
+ sizeof(struct roce5_alloc_ucontext_resp));
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to copy data to user space, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+int roce5_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
+{
+ int ret;
+
+ struct roce5_ucontext *context = to_roce5_ucontext(ibucontext);
+ struct roce5_device *rdev = to_roce5_dev(ibucontext->device);
+ struct roce5_alloc_ucontext_resp *resp = NULL;
+
+ ret = roce5_alloc_ucontext_pre_check(ibucontext->device, udata);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "pre check err, ret(%d), func_id(%d)", ret,
+ rdev->glb_func_id);
+ goto err;
+ }
+
+ resp = kzalloc(sizeof(struct roce5_alloc_ucontext_resp), GFP_KERNEL);
+ if (resp == NULL) {
+ ret = (-ENOMEM);
+ ROCE_DEV_ERR(rdev, "Failed alloc ucontext, func_id(%d)",
+ rdev->glb_func_id);
+ goto err;
+ }
+
+ ret = roce5_alloc_ucontext_set(rdev, resp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get chip node id, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_db;
+ }
+ /* Alloc user space context Doorbell and DWQE */
+ ret = hinic5_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr,
+ &context->dwqe_dma_addr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc DB pa, ret(%d), func_id(%d)", ret,
+ rdev->glb_func_id);
+ goto err_db;
+ }
+
+ /* Copy data to user space */
+ ret = roce5_alloc_ucontext_return(rdev, udata, context, resp);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to return to user, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto err_return;
+ }
+
+ kfree(resp);
+#ifdef __ROCE_DFX__
+ rdev->ucontext = context;
+#endif
+ return 0;
+
+err_return:
+ hinic5_free_db_phy_addr(rdev->hwdev, context->db_dma_addr,
+ context->dwqe_dma_addr);
+err_db:
+ kfree(resp);
+err:
+ return ret;
+}
+
+void roce5_dealloc_ucontext(struct ib_ucontext *ibcontext)
+{
+ struct roce5_ucontext *context = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if (ibcontext == NULL) {
+ ROCE_ERR("Ibcontext is null");
+ return;
+ }
+
+ context = to_roce5_ucontext(ibcontext);
+ rdev = to_roce5_dev(ibcontext->device);
+
+ hinic5_free_db_phy_addr(rdev->hwdev, context->db_dma_addr,
+ context->dwqe_dma_addr);
+
+ return;
+}
+
+struct ib_ucontext *roce5_alloc_ucontext_ofa(struct ib_device *ibdev,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_ucontext *context = NULL;
+
+ if (ibdev == NULL) {
+ ROCE_ERR("Ibdev is null");
+ return ERR_PTR(-EINVAL);
+ }
+
+ context = kzalloc(sizeof(struct roce5_ucontext), GFP_KERNEL);
+ if (context == NULL) {
+ ROCE_ERR("Failed to alloc ucontext");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ context->ibucontext.device = ibdev;
+ ret = roce5_alloc_ucontext(&context->ibucontext, udata);
+ if (ret != 0) {
+ kfree(context);
+ return ERR_PTR(ret);
+ }
+
+ return (struct ib_ucontext *)&context->ibucontext;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_dealloc_ucontext_ofa
+ Description : dealloc ucontext
+ Input : struct ib_ucontext *ibcontext
+ Output : None
+
+ 1.Date : 2023/11/9
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_dealloc_ucontext_ofa(struct ib_ucontext *ibcontext)
+{
+ struct roce5_ucontext *context = NULL;
+
+ if (ibcontext == NULL) {
+ ROCE_ERR("Ibcontext is null");
+ return -EINVAL;
+ }
+
+ roce5_dealloc_ucontext(ibcontext);
+ context = to_roce5_ucontext(ibcontext);
+
+ kfree(context);
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_mmap
+ Description : memory map
+ Input : struct ib_ucontext *ibcontext
+ struct vm_area_struct *vma
+ Output : None
+
+ 1.Date : 2015/5/8
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
+{
+ struct roce5_device *rdev = NULL;
+ struct roce5_ucontext *ucontext = NULL;
+ unsigned long db_pfn = 0;
+ unsigned long dwqe_pfn = 0;
+
+ if ((ibcontext == NULL) || (vma == NULL)) {
+ ROCE_ERR("Ibcontext or vma is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibcontext->device);
+ ucontext = to_roce5_ucontext(ibcontext);
+ db_pfn = ucontext->db_dma_addr >> PAGE_SHIFT;
+ dwqe_pfn = ucontext->dwqe_dma_addr >> PAGE_SHIFT;
+
+ if ((vma->vm_end - vma->vm_start) != PAGE_SIZE) {
+ ROCE_DEV_ERR(
+ rdev,
+ "(Vm_end(%lu) - vm_start(%lu)) is not equal to PAGE_SIZE, func_id(%d)",
+ vma->vm_end, vma->vm_start, rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ /* map hw DB to physical page from user */
+ if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) {
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ /* construct vm_start~vm_start+PAGE_SIZE page table
+ * db_pfn is page number
+ * vm_page_prot means attr
+ */
+ if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE,
+ vma->vm_page_prot) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to do db io remap, func_id(%d)",
+ rdev->glb_func_id);
+ return -EAGAIN;
+ }
+ return 0;
+ }
+
+#ifdef ROCE_DWQE_EN
+ if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) &&
+ (rdev->rdma_cap.direct_wqe_size != 0)) { // DWQE mmap
+#ifdef __aarch64__
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+#else
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+#endif
+
+ if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE,
+ vma->vm_page_prot) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to do dwqe io remap, func_id(%d)",
+ rdev->glb_func_id);
+ return -EAGAIN;
+ }
+ return 0;
+ }
+#endif
+
+ ROCE_DEV_ERR(rdev, "Not support, func_id(%d)", rdev->glb_func_id);
+ return -EINVAL;
+}
+
+enum rdma_link_layer roce5_port_link_layer(struct ib_device *ibdev,
+ u32 port_num)
+{
+ struct roce5_device *rdev = NULL;
+
+ if (ibdev == NULL) {
+ return IB_LINK_LAYER_UNSPECIFIED;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ if (port_num != ROCE_DEFAULT_PORT_NUM) {
+ return IB_LINK_LAYER_UNSPECIFIED;
+ }
+
+ return IB_LINK_LAYER_ETHERNET;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+enum rdma_link_layer roce5_port_link_layer_compat(struct ib_device *ibdev,
+ u8 port_num)
+{
+ return roce5_port_link_layer(ibdev, (u32)port_num);
+}
+
+static void roce5_resolve_cb(int status, struct sockaddr *src_addr,
+ struct rdma_dev_addr *addr, void *context)
+{
+ ((struct roce5_resolve_cb_context *)context)->status = status;
+ complete(&((struct roce5_resolve_cb_context *)context)->comp);
+}
+
+static int roce5_rdma_addr_find_l2_eth_by_grh(
+ const union ib_gid *sgid, const union ib_gid *dgid, u8 *dmac,
+ const struct net_device *ndev, int *hoplimit, struct roce5_device *rdev,
+ const struct ib_gid_attr *sgid_attr)
+{
+ struct rdma_dev_addr dev_addr;
+ struct roce5_resolve_cb_context ctx;
+ union {
+ struct sockaddr_in _sockaddr_in;
+ struct sockaddr_in6 _sockaddr_in6;
+ } sgid_addr, dgid_addr;
+ int ret;
+
+ rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid);
+ rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid);
+
+ ret = memset_s(&dev_addr, sizeof(dev_addr), 0, sizeof(dev_addr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset dev_addr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ if (ndev) {
+ dev_addr.bound_dev_if = ndev->ifindex;
+ dev_addr.net = dev_net(ndev);
+ } else {
+ dev_addr.net = &init_net;
+ }
+
+ init_completion(&ctx.comp);
+#if defined(RDMA_RESOLVE_IP_HAVE_GID_ATTR)
+ dev_addr.sgid_attr = sgid_attr;
+ ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr,
+ (struct sockaddr *)&dgid_addr, &dev_addr,
+ RESOLVE_IP_TIME_OUT, roce5_resolve_cb, true,
+ &ctx);
+#else
+ ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr,
+ (struct sockaddr *)&dgid_addr, &dev_addr,
+ RESOLVE_IP_TIME_OUT, roce5_resolve_cb, &ctx);
+#endif
+ if (ret != 0) {
+ ROCE_ERR("rdma_resolve_ip failed. Igonore the err(%d).", ret);
+ roce5_resolve_cb(0, (struct sockaddr *)&sgid_addr, &dev_addr,
+ &ctx);
+ }
+
+ wait_for_completion(&ctx.comp);
+
+ ret = memcpy_s(dmac, ETH_ALEN, dev_addr.dst_dev_addr, ETH_ALEN);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dmac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ if (hoplimit) {
+ *hoplimit = dev_addr.hoplimit;
+ }
+
+ return 0;
+}
+
+static int roce5_ah_valid_check(struct ib_global_route *grh, u16 *vlan_id,
+ u8 *dmac)
+{
+ u8 unicast_gid0[ROCE_GID_LEN] = { 0 };
+ u8 unicast_gid1[ROCE_GID_LEN] = { 0 };
+
+ /* check gid(unicast gid can not be 0 or 1) */
+ unicast_gid0[ROCE_GID_HIGHEST_BYTE] = 0;
+ unicast_gid1[ROCE_GID_HIGHEST_BYTE] = 1;
+
+ if ((ROCE_MEMCMP(grh->dgid.raw, unicast_gid0, sizeof(union ib_gid)) ==
+ 0) ||
+ (ROCE_MEMCMP(grh->dgid.raw, unicast_gid1, sizeof(union ib_gid)) ==
+ 0)) {
+ ROCE_ERR(
+ "Invalid unicast dgid, The ROCE_GID_HIGHEST_BYTE of unicast_dgid is: 0x%02x",
+ *(grh->dgid.raw + ROCE_GID_HIGHEST_BYTE));
+ return (-EINVAL);
+ }
+
+ if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) != 0) {
+ rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, dmac);
+ *vlan_id = ROCE_DEFAULT_VLAN_ID;
+ }
+
+ return 0;
+}
+
+static int roce5_fill_gid_attr(struct roce5_device *rdev,
+ struct rdma_ah_attr *ah_attr, union ib_gid *sgid,
+ const struct ib_gid_attr **sgid_attr)
+{
+ int ret = 0;
+
+ ret = rdma_query_gid(&rdev->ib_dev, ah_attr->port_num,
+ ah_attr->grh.sgid_index, sgid);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to query gid func_id(%hu),port_num(%d),gid_index(%d),ret(%d)",
+ rdev->glb_func_id, ah_attr->port_num,
+ ah_attr->grh.sgid_index, ret);
+ return ret;
+ }
+
+ *sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, ah_attr->port_num,
+ ah_attr->grh.sgid_index);
+ if (IS_ERR_OR_NULL(*sgid_attr)) {
+ ret = (int)PTR_ERR(*sgid_attr);
+ ROCE_PRINT_GID(rdev, sgid->raw,
+ "Failed to get sgid_attr, func_id(%hu), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ return ret;
+}
+
+static void roce5_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr)
+{
+ rdma_put_gid_attr(sgid_attr);
+}
+
+static struct net_device *roce5_fill_netdev(struct roce5_device *rdev,
+ union ib_gid *sgid)
+{
+ struct net_device *netdev = NULL;
+ union {
+ struct sockaddr_in _sockaddr_in;
+ struct sockaddr_in6 _sockaddr_in6;
+ } socket_addr;
+
+ rdma_gid2ip((struct sockaddr *)&socket_addr, sgid);
+
+ /* find netdev,rdev->ndevis not valid in vlan scenario */
+ netdev = ip_dev_find(&init_net,
+ socket_addr._sockaddr_in.sin_addr.s_addr);
+ if (netdev) {
+ dev_put(netdev);
+ }
+
+ return netdev;
+}
+
+int roce5_resolve_grh(struct roce5_device *rdev, struct rdma_ah_attr *ah_attr,
+ u16 *vlan_id, struct ib_udata *udata)
+{
+ int ret = 0;
+ u8 zero_mac[ETH_ALEN] = { 0 };
+ u8 *dmac = NULL;
+ union ib_gid sgid = { 0 };
+ const struct ib_gid_attr *sgid_attr = NULL;
+ struct net_device *netdev = NULL;
+
+ if ((rdev == NULL) || (ah_attr == NULL) || (vlan_id == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ dmac = ah_attr->roce.dmac;
+ ret = roce5_ah_valid_check(&ah_attr->grh, vlan_id, dmac);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to check grh input, func_id(%d), ret(%d),"
+ "The grh inputs are: vlan_id(%d), dmac:%02x-%02x-%02x-%02x-%02x-%02x",
+ rdev->glb_func_id, ret, *vlan_id,
+ *((u8 *)dmac + MAC_INDEX_0),
+ *((u8 *)dmac + MAC_INDEX_1),
+ *((u8 *)dmac + MAC_INDEX_2),
+ *((u8 *)dmac + MAC_INDEX_3),
+ *((u8 *)dmac + MAC_INDEX_4),
+ *((u8 *)dmac + MAC_INDEX_5));
+ return ret;
+ }
+ if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) != 0) {
+ return 0;
+ }
+
+ ret = roce5_fill_gid_attr(rdev, ah_attr, &sgid, &sgid_attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to fill gid attr, func_id(%hu), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ netdev = roce5_fill_netdev(rdev, &sgid);
+
+ /* reparse dmac avoiding invalid damc from OFED */
+ ret = roce5_rdma_addr_find_l2_eth_by_grh(
+ &sgid, &ah_attr->grh.dgid, dmac, netdev, NULL, rdev, sgid_attr);
+ if (ret != 0) {
+ ROCE_PRINT_GID(
+ rdev, sgid.raw,
+ "Failed to find dmac by grh, func_id(%hu), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto resolve_grh_end;
+ }
+
+ if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) == 0) {
+ ROCE_DEV_ERR(rdev, "Failed to find valid dmac, func_id(%hu)",
+ rdev->glb_func_id);
+ ret = (-EINVAL);
+ goto resolve_grh_end;
+ }
+
+ if (sgid_attr != NULL && sgid_attr->ndev != NULL) {
+ *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev);
+ }
+
+resolve_grh_end:
+ roce5_release_gid_ref_cnt(sgid_attr);
+
+ return ret;
+}
+
+static int roce5_ah_get_vlan_id(struct roce5_device *rdev, struct ib_pd *pd,
+ struct rdma_ah_attr *ah_attr, u32 *vlan_id)
+{
+ u32 ret = 0;
+ struct net_device *ndev = NULL;
+ rcu_read_lock();
+ ndev = rcu_dereference(ah_attr->grh.sgid_attr->ndev);
+ if (ndev == NULL) {
+ rcu_read_unlock();
+ ROCE_DEV_ERR(rdev, "Net device is NULL, func_id(%hu)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+ ret = rdma_vlan_dev_vlan_id(ndev);
+ if (ret != 0xffff) {
+ *vlan_id = ret;
+ }
+ rcu_read_unlock();
+ return 0;
+}
+
+static int roce5_create_ib_ah_kernel(struct roce5_device *rdev,
+ struct ib_pd *pd, struct roce5_ah *rah,
+ struct rdma_ah_attr *ah_attr)
+{
+ int ret;
+ u8 *dmac = ah_attr->roce.dmac;
+ u32 vlan_id = 0;
+
+ ret = roce5_ah_get_vlan_id(rdev, pd, ah_attr, &vlan_id);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get vlan_id (ret:%d)", ret);
+ return -EFAULT;
+ }
+
+ if (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) != 0) {
+ ret = memcpy_s((void *)rah->priv_ah.dgid,
+ sizeof(rah->priv_ah.dgid),
+ (void *)ah_attr->grh.dgid.raw,
+ sizeof(rah->priv_ah.dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ rah->priv_ah.dw2.bs.flow_label = ah_attr->grh.flow_label &
+ 0xfffff;
+ rah->priv_ah.dw1.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f;
+ rah->priv_ah.dw1.bs.hoplimit = ah_attr->grh.hop_limit;
+ rah->priv_ah.dw1.bs.tclass =
+ (u8)(ah_attr->grh.traffic_class | IP_ECN_PERMIT);
+ }
+ rah->priv_ah.dw0.bs.pd = to_roce5_pd(pd)->pdn & 0x3ffff;
+ rah->priv_ah.dw0.bs.wqe_cos =
+ roce5_get_db_cos_from_vlan_pri(rdev, ah_attr->sl);
+ rah->priv_ah.dw0.value = cpu_to_be32(rah->priv_ah.dw0.value);
+
+ rah->priv_ah.dw1.bs.port = ah_attr->port_num & 0xf;
+ rah->priv_ah.dw2.bs.smac_index =
+ rdev->glb_func_id; /* set global Function ID */
+ rah->priv_ah.dw2.value = cpu_to_be32(rah->priv_ah.dw2.value);
+
+ rah->priv_ah.dw1.bs.resv = 0;
+ rah->priv_ah.dw7.bs.vlan_id = vlan_id & 0xfff;
+ rah->priv_ah.dw7.bs.vlan_pri = ah_attr->sl & 0x7;
+
+ rah->priv_ah.dw1.value = cpu_to_be32(rah->priv_ah.dw1.value);
+
+ rah->priv_ah.dw7.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) |
+ dmac[1];
+ rah->priv_ah.dw7.value = cpu_to_be32(rah->priv_ah.dw7.value);
+
+ ret = memcpy_s((void *)&rah->priv_ah.dmac_l32,
+ sizeof(rah->priv_ah.dmac_l32),
+ (void *)&dmac[ROCE_RAH_DMAC_L32_START],
+ sizeof(rah->priv_ah.dmac_l32));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dmac_l32.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static int roce5_create_ib_ah_user(struct roce5_device *rdev,
+ struct rdma_ah_attr *ah_attr,
+ struct ib_udata *udata)
+{
+ struct roce5_create_ah_resp resp;
+ roce5_get_cos_inbuf_s dcb_inbuf;
+ int ret;
+
+ ret = memset_s(ah_attr->roce.dmac, sizeof(ah_attr->roce.dmac), 0,
+ sizeof(ah_attr->roce.dmac));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset dmac.");
+ return (-ENOMEM);
+ }
+
+ ret = roce5_resolve_grh(rdev, ah_attr, &resp.vlan_id, udata);
+ if (ret != 0) {
+ ROCE_ERR("Failed to resolve grh");
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(resp.dmac, ETH_ALEN, ah_attr->roce.dmac, ETH_ALEN);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy_s ah dmac.");
+ return (-ENOMEM);
+ }
+
+ dcb_inbuf.sl = ah_attr->sl;
+ dcb_inbuf.sgid_index = ah_attr->grh.sgid_index;
+ dcb_inbuf.port_num = ah_attr->port_num;
+ dcb_inbuf.traffic_class = ah_attr->grh.traffic_class;
+ ret = roce5_get_dcb_cfg_cos(rdev, &dcb_inbuf, &resp.cos, IB_QPT_UD);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get cos from dcb info, ret:%d", ret);
+ return ret;
+ }
+
+ ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
+ if (ret != 0) {
+ ROCE_ERR("Failed to copy data to user");
+ return ret;
+ }
+
+ return 0;
+}
+
+int roce5_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_ah *rah = NULL;
+ struct roce5_device *rdev = NULL;
+ struct rdma_ah_attr *ah_attr = NULL;
+ enum rdma_ah_attr_type ah_type = 0;
+
+ if ((ibah == NULL) || (init_attr == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ rah = to_roce5_ah(ibah);
+ rdev = to_roce5_dev(ibah->device);
+ ah_attr = init_attr->ah_attr;
+ ah_type = ah_attr->type;
+
+ if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) &&
+ (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) == 0)) {
+ ROCE_DEV_ERR(rdev, "Invalid ah_type(%d)", ah_type);
+ return -EINVAL;
+ }
+
+ if (udata != NULL) {
+ ret = roce5_create_ib_ah_user(rdev, ah_attr, udata);
+ } else {
+ ret = roce5_create_ib_ah_kernel(rdev, ibah->pd, rah, ah_attr);
+ }
+ return ret;
+}
+
+struct ib_ah *roce5_create_ah_ofa(struct ib_pd *pd,
+ struct rdma_ah_attr *ah_attr,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_ah *rah = NULL;
+ struct roce5_device *rdev = NULL;
+ struct rdma_ah_init_attr init_attr;
+
+ if ((pd == NULL) || (ah_attr == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ rdev = to_roce5_dev(pd->device);
+ init_attr.ah_attr = ah_attr;
+
+ rah = kzalloc(sizeof(*rah), GFP_ATOMIC);
+ if (rah == NULL) {
+ ret = -ENOMEM;
+ ROCE_DEV_ERR(rdev, "Failed to alloc ah, func_id(%d)",
+ rdev->glb_func_id);
+ goto out;
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ rah->ibah.device = pd->device;
+ rah->ibah.pd = pd;
+ ret = roce5_create_ah(&rah->ibah, &init_attr, udata);
+ if (ret != 0) {
+ kfree(rah);
+ }
+
+out:
+ return (ret != 0) ? ERR_PTR(ret) : &rah->ibah;
+}
+
+int roce5_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
+{
+ struct roce5_ah *ah = NULL;
+ struct roce5_priv_ah priv_ah;
+ int ret;
+
+ if ((ibah == NULL) || (ah_attr == NULL)) {
+ ROCE_ERR("Ibah or ah_attr is null");
+ return -EINVAL;
+ }
+
+ ah = to_roce5_ah(ibah);
+ ret = memset_s(ah_attr, sizeof *ah_attr, 0, sizeof *ah_attr);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ah_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ priv_ah.dw1.value = be32_to_cpu(ah->priv_ah.dw1.value);
+ priv_ah.dw2.value = be32_to_cpu(ah->priv_ah.dw2.value);
+ priv_ah.dw7.value = be32_to_cpu(ah->priv_ah.dw7.value);
+
+ ah_attr->ah_flags = IB_AH_GRH;
+ ah_attr->sl = priv_ah.dw7.bs.vlan_pri;
+ ah_attr->port_num = priv_ah.dw1.bs.port;
+ ah_attr->grh.traffic_class = priv_ah.dw1.bs.tclass;
+ ah_attr->grh.hop_limit = priv_ah.dw1.bs.hoplimit;
+ ah_attr->grh.sgid_index = priv_ah.dw1.bs.sgid_index;
+ ah_attr->grh.flow_label = priv_ah.dw2.bs.flow_label;
+
+ ret = memcpy_s((void *)ah_attr->grh.dgid.raw, sizeof(ah->priv_ah.dgid),
+ (void *)ah->priv_ah.dgid, sizeof(ah->priv_ah.dgid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+int roce5_destroy_ah(struct ib_ah *ibah, u32 flags)
+{
+ return 0;
+}
+
+int roce5_destroy_ah_ofa(struct ib_ah *ibah)
+{
+ struct roce5_ah *rah = NULL;
+
+ if (ibah == NULL) {
+ ROCE_ERR("Ibah is null");
+ return -EINVAL;
+ }
+
+ rah = to_roce5_ah(ibah);
+
+ kfree(rah);
+
+ return 0;
+}
+
+int roce5_port_immutable(struct ib_device *ibdev, u32 port_num,
+ struct ib_port_immutable *immutable)
+{
+ struct ib_port_attr attr;
+ int err;
+
+ if ((ibdev == NULL) || (immutable == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ immutable->core_cap_flags =
+ RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; // only rocev2
+
+#if defined(IB_QUERY_PORT_HAVE_U8_PORT)
+ err = ib_query_port(ibdev, (u8)port_num, &attr);
+#elif defined(IB_QUERY_PORT_HAVE_U32_PORT)
+ err = ib_query_port(ibdev, port_num, &attr);
+#endif
+ if (err != 0) {
+ ROCE_ERR("dev(%s) query ib port(%d) failed, err(%d)",
+ ibdev->name, port_num, err);
+ return err;
+ }
+
+ immutable->pkey_tbl_len = attr.pkey_tbl_len;
+ immutable->gid_tbl_len = attr.gid_tbl_len;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+
+ return 0;
+}
+
+/* compatible version for kernel 5.10 and kernel 4.19 */
+int roce5_port_immutable_compat(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_immutable *immutable)
+{
+ return roce5_port_immutable(ibdev, (u32)port_num, immutable);
+}
+
+#define ROCE_DEFAULT_COS 3
+
+int roce5_get_dcb_cfg_cos(struct roce5_device *rdev,
+ roce5_get_cos_inbuf_s *inbuf, u8 *cos,
+ enum ib_qp_type qp_type)
+{
+ int ret;
+ u8 pri;
+ struct rdma_gid_entry gid;
+ struct hinic5_dcb_state dcb = { 0 };
+
+#if !defined(UVERBS_IOCTL_HAVE_DECLARE_UVERBS_WRITE_EX)
+ /* Note: low version of ofed in modify qp when unused udate->outbuf, cause cannot from kernel mode get db cos, therefore stub uses a fixed value. */
+ *cos = ROCE_DEFAULT_COS;
+ goto cos_success;
+#endif
+
+ ret = roce5_get_gid(rdev, inbuf->port_num, inbuf->sgid_index, &gid);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init gid info, ret(%d).", ret);
+ return -EINVAL;
+ }
+
+ ret = hinic5_get_dcb_state(rdev->hwdev, &dcb);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "hinic5_get_dcb_state failed, ret(%d).",
+ ret);
+ return -EINVAL;
+ }
+
+ *cos = dcb.default_cos;
+ gid.dw6_h.value = cpu_to_le16(gid.dw6_h.value);
+ if ((dcb.trust == ROCE5_DCB_PCP) &&
+ (gid.dw6_h.bs.tag == ROCE_GID_VLAN_INVALID)) {
+ // pcp cfg & no vlan should use default cos
+ goto cos_success;
+ }
+
+ pri = (dcb.trust == ROCE5_DCB_PCP) ?
+ inbuf->sl :
+ (inbuf->traffic_class >> ROCE5_DSCP_IDX);
+ ret = hinic5_get_cos_by_pri(rdev->hwdev, pri, cos);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "get cos failed, ret(%d), pri(%u), dcb_on(%u), trust(%u).",
+ ret, pri, dcb.dcb_on, dcb.trust);
+ return -EINVAL;
+ }
+
+cos_success:
+ roce5_cos_update_ext(rdev, cos, qp_type);
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h
new file mode 100644
index 000000000..2fb0eb2b9
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h
@@ -0,0 +1,269 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_MIX_H
+#define ROCE_MIX_H
+
+#include <net/arp.h>
+#include <net/neighbour.h>
+#include <net/route.h>
+#include <net/addrconf.h>
+#include <net/ip6_route.h>
+#include <linux/mutex.h>
+#include <linux/inetdevice.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include <linux/module.h>
+
+#include <rdma/ib_addr.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_pack.h>
+
+#include "roce.h"
+
+#define ROCE_HW_VER 0ULL
+#define ROCE_FW_VER_0 1ULL
+#define ROCE_FW_VER_1 4ULL
+#define ROCE_FW_VER_2 0ULL
+#define ROCE_FW_VER_3 0ULL
+#define ROCE_FW_VER \
+ ((ROCE_FW_VER_0 << 32) | (ROCE_FW_VER_1 << 16) | \
+ (ROCE_FW_VER_2 << 8) | ROCE_FW_VER_3)
+
+#define RESOLVE_IP_TIME_OUT 1000
+#define ROCE_GID_HIGHEST_BYTE 15
+#define ROCE_FW_VERSION_LEN 4
+#define ROCE_DEFAULT_VLAN_ID 0xFFFF
+#define ROCE_RAH_TC_SHIFT 2
+#define ROCE_RAH_DMAC_H16_SHIFT 8
+#define ROCE_RAH_DMAC_L32_START 2
+#define ROCE_UDP_SRC_PORT_H8_SHIFT 8
+
+#define ROCE5_DCB_PCP 0
+#define ROCE5_DCB_DSCP 1
+
+#define ROCE5_DSCP_IDX 2 //dscp begin from bit 2 in traffic_class
+
+#define ROCE_GID_IP_IDX 3
+
+#define MAC_INDEX_0 0
+#define MAC_INDEX_1 1
+#define MAC_INDEX_2 2
+#define MAC_INDEX_3 3
+#define MAC_INDEX_4 4
+#define MAC_INDEX_5 5
+
+#define IP_INDEX_0 0
+#define IP_INDEX_1 1
+#define IP_INDEX_2 2
+#define IP_INDEX_3 3
+#define IP_INDEX_4 4
+#define IP_INDEX_5 5
+#define IP_INDEX_6 6
+#define IP_INDEX_7 7
+
+#define ROCE_MTU_256_INT 256
+#define ROCE_MTU_512_INT 512
+#define ROCE_MTU_1024_INT 1024
+#define ROCE_MTU_2048_INT 2048
+#define ROCE_MTU_4096_INT 4096
+#define ROCE_MTU_8192_INT 8192
+
+enum {
+ USR_MMAP_DB_OFFSET = 0,
+ USR_MMAP_DWQE_OFFSET,
+ USR_MMAP_DFX_OFFSET,
+ USR_MMAP_DBAR3_OFFSET
+};
+
+enum roce_port_state_e {
+ ROCE_PORT_PHYS_STATE_NO_CHANGE = 0,
+ ROCE_PORT_PHYS_STATE_SLEEP,
+ ROCE_PORT_PHYS_STATE_POLLING,
+ ROCE_PORT_PHYS_STATE_DISABLED = 3,
+ ROCE_PORT_PHYS_STATE_PORTCONFTRAIN,
+ ROCE_PORT_PHYS_STATE_LINKUP = 5,
+ ROCE_PORT_PHYS_STATE_LINKERRRECOVER,
+ ROCE_PORT_PHYS_STATE_PHYTEST
+};
+
+enum ROCE_MBOX_CMD { ROCE_MBOX_CMD_SEND_MAIL_BOX, ROCE_MBOX_CMD_MAX };
+
+enum roce5_mtu {
+ ROCE_MTU_256 = IB_MTU_256,
+ ROCE_MTU_512 = IB_MTU_512,
+ ROCE_MTU_1024 = IB_MTU_1024,
+ ROCE_MTU_2048 = IB_MTU_2048,
+ ROCE_MTU_4096 = IB_MTU_4096,
+ ROCE_MTU_8192,
+};
+
+static inline int roce5_mtu_enum_to_int(enum roce5_mtu mtu)
+{
+ switch (mtu) {
+ case ROCE_MTU_256:
+ return ROCE_MTU_256_INT;
+ case ROCE_MTU_512:
+ return ROCE_MTU_512_INT;
+ case ROCE_MTU_1024:
+ return ROCE_MTU_1024_INT;
+ case ROCE_MTU_2048:
+ return ROCE_MTU_2048_INT;
+ case ROCE_MTU_4096:
+ return ROCE_MTU_4096_INT;
+ case ROCE_MTU_8192:
+ return ROCE_MTU_8192_INT;
+ default:
+ return -1;
+ }
+}
+
+static inline enum roce5_mtu roce5_get_mtu(int mtu)
+{
+ int local_mtu = mtu - (IB_GRH_BYTES + IB_UDP_BYTES + IB_BTH_BYTES +
+ IB_EXT_XRC_BYTES + IB_EXT_ATOMICETH_BYTES +
+ IB_ICRC_BYTES);
+
+ if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_8192))
+ return ROCE_MTU_8192;
+ else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_4096))
+ return ROCE_MTU_4096;
+ else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_2048))
+ return ROCE_MTU_2048;
+ else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_1024))
+ return ROCE_MTU_1024;
+ else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_512))
+ return ROCE_MTU_512;
+ else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_256))
+ return ROCE_MTU_256;
+ else
+ return 0;
+}
+
+struct roce5_priv_ah {
+ /* DW0 */
+ union {
+ struct {
+#if (BYTE_ORDER != BIG_ENDIAN)
+ u32 pd : 18;
+ u32 rsvd0 : 6;
+ u32 stat_rate : 4;
+ u32 wqe_cos : 3;
+ u32 fl : 1; /* forcelock flag, it is used in multicast communication */
+#else
+ u32 f1 : 1;
+ u32 wqe_cos : 3;
+ u32 stat_rate : 4;
+ u32 rsvd0 : 6;
+ u32 pd : 18;
+#endif
+ } bs;
+ u32 value;
+ } dw0;
+
+ /* DW1 */
+ union {
+ struct {
+#if (BYTE_ORDER != BIG_ENDIAN)
+ u32 hoplimit : 8;
+ u32 sgid_index : 7;
+ u32 resv : 1;
+ u32 port : 4;
+ u32 rsvd1 : 4;
+ u32 tclass : 8;
+#else
+ u32 tclass : 8;
+ u32 rsvd1 : 4;
+ u32 port : 4;
+ u32 resv : 1;
+ u32 sgid_index : 7;
+ u32 hoplimit : 8;
+#endif
+ } bs;
+ u32 value;
+ } dw1;
+
+ /* DW2 */
+ union {
+ struct {
+#if (BYTE_ORDER != BIG_ENDIAN)
+ u32 flow_label : 20;
+ u32 smac_index : 10;
+ u32 rsvd : 2;
+#else
+ u32 rsvd : 2;
+ u32 smac_index : 10;
+ u32 flow_label : 20;
+#endif
+ } bs;
+ u32 value;
+ } dw2;
+
+ /* DW3~6 */
+ u8 dgid[16];
+
+ /* DW7 */
+ union {
+ struct {
+#if (BYTE_ORDER != BIG_ENDIAN)
+ u32 dmac_h16 : 16;
+ u32 vlan_id : 12;
+ u32 rsvd : 1;
+ u32 vlan_pri : 3;
+#else
+ u32 vlan_pri : 3;
+ u32 rsvd : 1;
+ u32 vlan_id : 12;
+ u32 dmac_h16 : 16;
+#endif
+ } bs;
+ u32 value;
+ } dw7;
+
+ /* DW8 */
+ u32 dmac_l32;
+};
+
+struct roce5_ah {
+ struct ib_ah ibah;
+ struct roce5_priv_ah priv_ah;
+};
+
+union gid_addr {
+ struct sockaddr _sockaddr;
+ struct sockaddr_in _sockaddr_in;
+ struct sockaddr_in6 _sockaddr_in6;
+};
+
+struct mailbox_header {
+ u32 dest_host_id;
+ u32 dest_global_function_id;
+};
+
+struct roce5_mail_box_buffer {
+ struct mailbox_header header;
+ u32 data[0];
+};
+
+struct roce5_resolve_cb_context {
+ struct completion comp;
+ int status;
+};
+
+typedef struct roce5_get_cos_inbuf {
+ u8 sl;
+ u8 sgid_index;
+ u8 port_num;
+ u8 traffic_class;
+} roce5_get_cos_inbuf_s;
+
+int roce5_resolve_grh(struct roce5_device *rdev, struct rdma_ah_attr *ah_attr,
+ u16 *vlan_id, struct ib_udata *udata);
+int roce5_get_dcb_cfg_cos(struct roce5_device *rdev,
+ roce5_get_cos_inbuf_s *inbuf, u8 *cos,
+ enum ib_qp_type qp_type);
+uint8_t roce5_get_db_cos_from_vlan_pri(struct roce5_device *rdev,
+ uint8_t vlan_pri);
+
+#endif /* ROCE_MIX_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c
new file mode 100644
index 000000000..92a3ada97
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c
@@ -0,0 +1,138 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+
+#include "roce.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "roce_pd.h"
+
+int roce5_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if (CHECK_2LAYER_IS_NULL(ibpd, device)) {
+ ROCE_ERR(
+ "Invalid NULL parameter, ibpd or ibpd->device is NULL.");
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ pd = to_roce5_pd(ibpd);
+ rdev = to_roce5_dev(ibpd->device);
+
+ ret = roce5_rdma_pd_alloc(rdev->hwdev, &pd->pdn);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc pdn, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto err_out;
+ }
+
+ if (udata != NULL) {
+ if (udata->outbuf == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid NULL parameter, udata->outbuf is NULL.");
+ ret = -EINVAL;
+ goto err_copy_to_user;
+ }
+
+ if (ib_copy_to_udata(udata, &pd->pdn, sizeof(u32)) != 0) {
+ ret = -EFAULT;
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to copy data to user space, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_copy_to_user;
+ }
+ }
+
+ return 0;
+
+err_copy_to_user:
+ roce5_rdma_pd_free(rdev->hwdev, pd->pdn);
+err_out:
+ return ret;
+}
+
+struct ib_pd *roce5_alloc_pd_ofa(struct ib_device *ibdev,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if (ibdev == NULL) {
+ ret = -EINVAL;
+ ROCE_ERR("Ibdev is null");
+ goto err_out;
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ pd = (struct roce5_pd *)kmalloc(sizeof *pd, GFP_KERNEL);
+ if (pd == NULL) {
+ ret = -ENOMEM;
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc memory for roce pd, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_out;
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ pd->ibpd.device = ibdev;
+ ret = roce5_alloc_pd(&pd->ibpd, udata);
+ if (ret != 0) {
+ goto err_alloc_pdn;
+ }
+
+ return &pd->ibpd;
+
+err_alloc_pdn:
+ kfree(pd);
+
+err_out:
+ return (struct ib_pd *)ERR_PTR((long)ret);
+}
+
+int roce5_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
+{
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+
+ if (ibpd == NULL) {
+ ROCE_ERR("Ibpd is null");
+ return -EINVAL;
+ }
+
+ pd = to_roce5_pd(ibpd);
+ rdev = to_roce5_dev(ibpd->device);
+
+ roce5_rdma_pd_free(rdev->hwdev, pd->pdn);
+
+ return 0;
+}
+
+int roce5_dealloc_pd_ofa(struct ib_pd *ibpd)
+{
+ int ret;
+ struct roce5_pd *pd = NULL;
+
+ if (ibpd == NULL) {
+ ROCE_ERR("Ibpd is null");
+ return -EINVAL;
+ }
+
+ pd = to_roce5_pd(ibpd);
+ ret = roce5_dealloc_pd(ibpd, NULL);
+ if (ret == 0) {
+ kfree(pd);
+ }
+
+ return ret;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h
new file mode 100644
index 000000000..e40c5a476
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h
@@ -0,0 +1,15 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_PD_H
+#define ROCE_PD_H
+
+#include <rdma/ib_verbs.h>
+
+struct roce5_pd {
+ struct ib_pd ibpd;
+ u32 pdn;
+};
+
+#endif //ROCE_PD_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h
new file mode 100644
index 000000000..1886af73f
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h
@@ -0,0 +1,191 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_SRQ_H
+#define ROCE_SRQ_H
+
+#include <linux/types.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "hinic5_rdma.h"
+#include "hinic5_cqm.h"
+
+#include "roce.h"
+#include "roce_pd.h"
+#include "roce_db.h"
+#include "roce_compat.h"
+#include "roce_wqe_format.h"
+
+/* definition SRQ SGE maximum max and minimum value */
+#define ROCE_SRQ_CONT_MAX_SGE 16
+#define ROCE_SRQ_MAX_SGE 15
+#define ROCE_SRQ_MID_SGE 7
+#define ROCE_SRQ_MIN_SGE 3
+
+#define ROCE_SRQN_INVLD CQM_INDEX_INVALID
+#define ROCE_SRQ_CONTAINER_LWM_MASK 0xFFFF
+#define ROCE_SRQ_CONTAINER_WARTH_MASK 0xF
+
+#define MAX_SUPPORT_CONTAINER_MODE 3
+#define DWORD_LEN 32
+#define XRC_CQN_FIRST_LEN 10
+#define XRC_CQN_SECOND_LEN 3
+#define SRQ_GPA_SIG_LEN 3
+#define SRQ_WQEBB_SIZE_CAL_SECTTOR \
+ 4 /* (2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 */
+#define RDMA_PREFETCH_WQE_MAX 7
+#define RDMA_PREFETCH_MTT_LEN_MAX 3
+#define ROCE_WQE_BB_SIZE_MIN 64
+
+#define LAST_SGE_NO_PRESENT \
+ 0x80000000UL /* Lbit:0 - The next SGE is present in the list \
+ * Lbit:1 - The last SGE, no SGE is present in the list */
+#define NORMAL_FMT_AND_NEXT_SGE_PRESENT \
+ 0x3FFFFFFFUL /* Ebit:b0 - Normal format, without extension. \
+ * Ebit:b1 - The pointer inside of SGE points to the next SGL. \
+ * "Length" and "Key" fields are */
+#define NORMAL_FMT_AND_LAST_SGE_NO_PRESENT 0xBFFFFFFFUL
+
+/**
+ * CHIP container_mode:
+ * chip mode: 0 -> Not container
+ * chip mode: 1 -> container_size: 2
+ * chip mode: 2 -> container_size: 4
+ * chip mode: 3 -> container_size: 8
+ * chip mode: 4 -> container_size: 16
+ */
+enum roce5_chip_srq_mode {
+ ROCE_CHIP_SRQ_MODE_N = 0,
+ ROCE_CHIP_SRQ_MODE_1,
+ ROCE_CHIP_SRQ_MODE_2,
+ ROCE_CHIP_SRQ_MODE_3,
+ ROCE_CHIP_SRQ_MODE_4
+};
+
+enum srq_state {
+ ROCE_SRQ_STATE_INVALID = 0x0,
+ ROCE_SRQ_STATE_ERR = 0x1,
+ ROCE_SRQ_STATE_VALID = 0xf,
+ ROCE_SRQ_STATE_MEM_INIT = 0xa
+};
+
+#define ROCE_SRQ_STATE_CHECK_VALUE 0x0
+
+struct roce5_wqe_srq_next_seg {
+ u16 reserved1;
+ __be16 pcnt; /* indicate the pi */
+ u8 signature;
+ u8 reserved2;
+ __be16 next_wqe_index;
+ u32 reserved3[2];
+};
+
+struct roce5_wqe_container_srq_next_seg {
+ u32 next_gpa_h;
+
+ struct {
+ u32 rsvd : 11;
+ u32 next_gpa_vd : 1;
+ u32 next_gpa_h : 20; /* indicate the pi */
+ } dw1;
+
+ struct {
+ u32 next_idx : 16;
+ u32 rsvd : 16;
+ } dw2;
+
+ struct {
+ u32 rsvd2 : 30;
+ u32 link_flag : 1;
+ u32 rsvd : 1;
+ } dw3;
+
+ struct {
+ u32 osd_next_idx : 16;
+ u32 osd_cur_idx : 16;
+ } dw4;
+};
+
+#define ROCE_SRQ_SGE_LAST 1
+#define ROCE_SRQ_SGE_NLAST 0
+
+#define ROCE_SRQ_SGE_LKEY_NOEXT 0
+#define ROCE_SRQ_SGE_LKEY_EXT 1
+
+struct roce5_srq {
+ struct ib_srq ibsrq; /* ibsrq */
+ cqm_queue_s *cqm_srq;
+
+ u32 srqn;
+ int max_depth;
+ int max_gs;
+ int wqe_shift;
+
+ cqm_buf_s *buf;
+ struct roce5_db db;
+ u64 *wrid;
+ spinlock_t lock;
+ int head;
+ int tail;
+ u16 wqe_ctr;
+ u8 xrc_en;
+ u8 rsvd;
+ struct ib_umem *umem;
+ struct mtt mtt;
+ struct mutex mutex;
+ u32 rqe_cnt_th;
+ u8 container_flag;
+ u8 container_size;
+ u8 container_mode;
+ u8 container_warn_th;
+
+ int buf_sz;
+};
+
+static inline u8 roce5_srq_get_max_sge(struct roce5_srq *rsrq)
+{
+ if (rsrq->container_flag != 0) {
+ return ROCE_SRQ_CONT_MAX_SGE;
+ }
+ return ROCE_SRQ_MAX_SGE;
+}
+
+static inline int roce5_srq_init_wqe_size(struct roce5_srq *rsrq)
+{
+ int desc_size = 0;
+
+ if (rsrq->container_flag != 0) {
+ desc_size = rsrq->max_gs * sizeof(struct roce_wqe_srq_data_seg);
+ } else {
+ desc_size = sizeof(struct roce5_wqe_srq_next_seg) +
+ rsrq->max_gs * sizeof(struct roce_wqe_srq_data_seg);
+ }
+ desc_size = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)desc_size));
+ desc_size = ROCE_MAX(ROCE_WQE_BB_SIZE_MIN, desc_size);
+
+ rsrq->wqe_shift = (int)ROCE_ILOG2((unsigned)desc_size);
+ return desc_size;
+}
+
+void roce5_free_srq_wqe(struct roce5_srq *srq, int wqe_index);
+void roce5_srq_async_event(struct roce5_device *rdev, struct roce5_srq *srq,
+ int type);
+void *roce5_srq_get_wqe(struct roce5_srq *srq, int n);
+u8 roce5_get_container_sz(u32 container_mode);
+u32 roce5_srq_max_avail_wr_set(struct roce5_srq *rsrq);
+int roce5_create_srq_common(struct roce5_device *rdev, struct roce5_srq *rsrq,
+ struct roce5_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata, u32 index);
+struct roce5_srq *roce5_get_rsrq_by_srqn(struct roce5_device *rdev, u32 srqn);
+static inline void roce5_rsrq_put(struct roce5_srq *rsrq)
+{
+ if (rsrq == NULL) {
+ return;
+ }
+ cqm5_object_put(&rsrq->cqm_srq->object);
+}
+
+#endif //ROCE_SRQ_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c
new file mode 100644
index 000000000..0a3beb338
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c
@@ -0,0 +1,116 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include "roce.h"
+#include "roce_compat.h"
+#include "roce_xrc.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_aeq_format.h"
+#include "roce_srq_extension.h"
+#include "roce_restore.h"
+
+static void *roce5_srq_buf_offset(struct tag_cqm_buf *buf, int offset)
+{
+ return (void *)((char *)buf->direct.va + offset);
+}
+
+void *roce5_srq_get_wqe(struct roce5_srq *srq, int n)
+{
+ return roce5_srq_buf_offset(srq->buf,
+ (int)((u32)n << (unsigned)srq->wqe_shift));
+}
+
+void roce5_srq_async_event(struct roce5_device *rdev, struct roce5_srq *srq,
+ int type)
+{
+ struct ib_srq *ibsrq = &srq->ibsrq;
+ struct ib_event event;
+ int ret;
+
+ ret = memset_s(&event, sizeof(event), 0, sizeof(event));
+ if (ret != 0) {
+ ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset event.(ret:%d)",
+ ret);
+ }
+ if (ibsrq->event_handler) {
+ event.device = ibsrq->device;
+ event.element.srq = ibsrq;
+ switch (type) {
+ case ROCE_EVENT_TYPE_SRQ_LIMIT:
+ event.event = IB_EVENT_SRQ_LIMIT_REACHED;
+ break;
+
+ case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
+ event.event = IB_EVENT_SRQ_ERR;
+ break;
+
+ default:
+ ROCE_DEV_WARN_LIMIT(
+ rdev,
+ "unexpected event type(%d) on SRQ(%06x), func_id(%d)",
+ type, srq->srqn, rdev->glb_func_id);
+ return;
+ }
+
+ ibsrq->event_handler(&event, ibsrq->srq_context);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_free_srq_wqe
+ Description : roce5_free_srq_wqe
+ Input : struct roce5_srq *srq
+ int wqe_index
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+void roce5_free_srq_wqe(struct roce5_srq *srq, int wqe_index)
+{
+ struct roce5_wqe_srq_next_seg *next = NULL;
+
+ spin_lock(&srq->lock);
+
+ next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(srq,
+ srq->tail);
+ next->next_wqe_index = (u16)wqe_index;
+ srq->tail = wqe_index;
+
+ spin_unlock(&srq->lock);
+}
+
+/**
+ * @brief Get roce srq by srqn. When querying succeeded, you have to call roce5_rsrq_put() to free
+ the cqm_obj_srq. But there is no need to call roce5_rsrq_put() if querying failed.
+ * @return Success: return the roce srq. Failure: return NULL.
+ */
+struct roce5_srq *roce5_get_rsrq_by_srqn(struct roce5_device *rdev, u32 srqn)
+{
+ cqm_object_s *cqm_obj_srq = NULL;
+ struct roce5_srq *rsrq = NULL;
+
+ cqm_obj_srq =
+ cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false);
+ if (cqm_obj_srq == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Can't find srq according to srqn(%u), func_id(%d).",
+ srqn, rdev->glb_func_id);
+ return NULL;
+ }
+
+ rsrq = cqmobj_to_roce5_srq(cqm_obj_srq);
+ if (rsrq == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to convert obj to rsrq.");
+ cqm5_object_put(cqm_obj_srq);
+ }
+
+ return rsrq;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c
new file mode 100644
index 000000000..47c86b2fa
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c
@@ -0,0 +1,712 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/kernel.h>
+
+#include "comm_defs.h"
+#include "roce.h"
+#include "roce_cq.h"
+#include "roce_cqm_cmd.h"
+#include "roce_user.h"
+#include "roce_xrc.h"
+#include "roce_pd.h"
+#include "roce_srq.h"
+#include "roce_restore.h"
+#include "roce_npu_cmd_srq_defs.h"
+#include "roce_npu_cmd_defs.h"
+
+#include "roce_srq_extension.h"
+#include "roce_main_extension.h"
+#include "hinic5_hmm.h"
+#include "roce_npu_cmd.h"
+#include "securec.h"
+#include "roce_os_compat.h"
+
+/* ****************************************************************************
+ Prototype : roce5_srq_sw2hw
+ Description : roce5_srq_sw2hw
+ Input : struct roce5_device *rdev
+ struct roce5_srq *rsrq
+ u32 pdn
+ u32 cqn
+ u16 xrcdn
+ int page_shift
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+ 1.Date : 2015/7/27
+ Modification : modify function
+**************************************************************************** */
+static void roce5_srq_sw2hw_set(roce_verbs_srq_attr_s *srq_attr,
+ struct roce5_srq *rsrq, u32 pdn, u32 cqn,
+ u16 xrcdn, int page_shift)
+{
+ srq_attr->dw0.bs.size = (u32)(ROCE_ILOG2((unsigned)rsrq->max_depth));
+ srq_attr->dw0.bs.page_size = (u32)page_shift;
+ srq_attr->dw0.bs.wqebb_size =
+ (u32)rsrq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR;
+ srq_attr->dw0.bs.mtt_page_size =
+ rsrq->mtt.mtt_page_shift - PAGE_SHIFT_4K;
+ srq_attr->dw0.bs.xrcd = xrcdn;
+
+ srq_attr->dw1.bs.dma_attr_idx = 0;
+ srq_attr->dw1.bs.so_ro = 1;
+ srq_attr->dw1.bs.state = ROCE_SRQ_STATE_VALID;
+
+ srq_attr->dw3.bs.container = rsrq->container_flag;
+ srq_attr->dw3.bs.lth_pre_en =
+ (rsrq->container_flag == 0); //rdma engine prefetch
+ srq_attr->dw3.bs.pcnt_on_chip = 0;
+ srq_attr->dw3.bs.rkey_en = 1;
+ srq_attr->dw3.bs.pd = pdn;
+
+ srq_attr->srqn = cpu_to_be32(rsrq->srqn);
+ srq_attr->xrc_cqn = cpu_to_be32(cqn);
+
+ if (rsrq->container_flag == 0) { //normal srq
+ srq_attr->dw1.bs.ep = 0;
+ srq_attr->dw1.bs.cos = 0;
+
+ srq_attr->dw2.value = 0;
+ srq_attr->dw2.bs.wqe_prefetch_max_num =
+ RDMA_PREFETCH_WQE_MAX; //rdma engine wqe prefetch max num
+ srq_attr->dw2.bs.wqecnt_lth = 0xe;
+ srq_attr->dw2.bs.mtt_prefetch_maxlen =
+ 0; //rdma engine mtt prefetch max len
+
+ srq_attr->dw3.bs.lth_pre_en = 1; //rdma engine prefetch
+ srq_attr->dw3.bs.lth_gap = 0;
+ } else {
+ srq_attr->cont.dw2.bs.head_idx = 0;
+ srq_attr->cont.dw2.bs.warn_th =
+ 0; //Do NOT init warn_th until user arm srq
+ srq_attr->cont.dw2.bs.cont_size =
+ (MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode) &
+ 0x3;
+ srq_attr->cont.dw2.value =
+ cpu_to_be32(srq_attr->cont.dw2.value);
+ srq_attr->record_gpa_at_hop_num =
+ cpu_to_be64((rsrq->mtt.mtt_layers & 0x3));
+ }
+
+ srq_attr->dw0.value = cpu_to_be32(srq_attr->dw0.value);
+ srq_attr->dw1.value = cpu_to_be32(srq_attr->dw1.value);
+ srq_attr->dw2.value = cpu_to_be32(srq_attr->dw2.value);
+ srq_attr->dw3.value = cpu_to_be32(srq_attr->dw3.value);
+
+ srq_attr->l0mtt_gpa = rsrq->mtt.mtt_paddr;
+
+ srq_attr->l0mtt_gpa = cpu_to_be64(srq_attr->l0mtt_gpa);
+
+ srq_attr->record_gpa_at_hop_num =
+ cpu_to_be64(rsrq->db.dma | (rsrq->mtt.mtt_layers & 0x3));
+
+ return;
+}
+
+static int roce5_srq_sw2hw(struct roce5_device *rdev, struct roce5_srq *rsrq,
+ u32 pdn, u32 cqn, u16 xrcdn, int page_shift)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_verbs_srq_attr_s *srq_attr = NULL;
+ roce_uni_cmd_create_srq_s *srq_sw2hw_inbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_create_srq_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ srq_sw2hw_inbuf = (roce_uni_cmd_create_srq_s *)cqm_cmd_inbuf->buf;
+ srq_sw2hw_inbuf->com.index = cpu_to_be32((u32)rsrq->srqn);
+ srq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK));
+ srq_attr = &srq_sw2hw_inbuf->srq_attr;
+
+ roce5_srq_sw2hw_set(srq_attr, rsrq, pdn, cqn, xrcdn, page_shift);
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_SW2HW_SRQ, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send SW2HW_SRQ command, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+
+ /* update dev status after cmdq timeout or exception, notify PCIe reset */
+ if ((ret == -ETIMEDOUT) || (ret == -EPERM)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ return ret;
+}
+
+static int roce5_create_srq_check(const struct ib_pd *ibpd,
+ const struct ib_srq_init_attr *init_attr,
+ const struct roce5_srq *rsrq)
+{
+ struct roce5_device *rdev = NULL;
+
+ if ((ibpd == NULL) || (init_attr == NULL)) {
+ ROCE_ERR("Ibpd or init_attr is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibpd->device);
+ if ((init_attr->attr.max_wr >
+ (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes) ||
+ (init_attr->attr.max_sge >
+ ((u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge +
+ rsrq->container_flag))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid input parms, max_wr(%d), max_srq_wqes(%d), max_sge(%d), max_srq_sge(%d)",
+ init_attr->attr.max_wr,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes,
+ init_attr->attr.max_sge,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_create_srq_write_mtt_and_db(struct roce5_device *rdev,
+ struct roce5_srq *rsrq,
+ struct roce5_create_srq_cmd *ucmd,
+ struct roce5_pd *pd)
+{
+ int ret = 0;
+
+ ret = roce5_umem_write_mtt(rdev, &rsrq->mtt, rsrq->umem, 1);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = roce5_db_map_user(to_roce5_ucontext(pd->ibpd.uobject->context),
+ ucmd->db_addr, &rsrq->db, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to map db to user space, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ return ret;
+}
+
+static int roce5_create_srq_get_umem(struct roce5_create_srq_cmd *ucmd,
+ struct ib_udata *udata,
+ struct roce5_srq *rsrq,
+ struct roce5_device *rdev,
+ struct roce5_pd *pd)
+{
+ int ret = 0;
+
+ if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0) {
+ ret = -EFAULT;
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy from user space, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ rsrq->umem = roce5_umem_get(&rdev->ib_dev, pd->ibpd.uobject->context,
+ ucmd->buf_addr, (size_t)rsrq->buf_sz, 0, 0);
+ if (IS_ERR(rsrq->umem)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to get ib_umem, func_id(%d) rsrq->buf_sz:%u",
+ rdev->glb_func_id, rsrq->buf_sz);
+ ret = (int)PTR_ERR(rsrq->umem);
+ return ret;
+ }
+
+ return ret;
+}
+
+static int roce5_create_user_srq_update(struct roce5_device *rdev,
+ struct roce5_srq *rsrq,
+ struct roce5_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata, int page_shift)
+{
+ u32 cqn = 0;
+ u16 xrcdn = 0;
+ int ret = 0;
+
+ cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ?
+ to_roce5_cq(init_attr->ext.cq)->cqn :
+ 0);
+ xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
+ (u16)(to_roce5_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) :
+ (u16)rdev->rdma_cap.reserved_xrcds;
+
+ /* set SRQC */
+ ret = roce5_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn,
+ (page_shift - PAGE_SHIFT_4K));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to handle srq_sw2hw, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_unmap;
+ }
+
+ rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn;
+
+ if (ib_copy_to_udata(udata, &rsrq->srqn, sizeof(u32)) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy data to user space, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -EFAULT;
+ goto err_unmap;
+ }
+
+ return 0;
+
+err_unmap:
+ roce5_db_unmap_user(to_roce5_ucontext(pd->ibpd.uobject->context),
+ &rsrq->db, 0);
+ return ret;
+}
+
+u32 roce5_srq_max_avail_wr_set(struct roce5_srq *rsrq)
+{
+ u32 link_wqe_count = 0;
+
+ if (rsrq->container_flag != 0) {
+ link_wqe_count = (u32)rsrq->max_depth / rsrq->container_size;
+ return ((u32)rsrq->max_depth - link_wqe_count) -
+ (rsrq->container_size - 1);
+ } else {
+ return (u32)rsrq->max_depth - 1;
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_user_srq
+ Description : roce5_create_user_srq
+ Input : struct roce5_device *rdev
+ struct roce5_srq *rsrq
+ struct roce5_pd *pd
+ struct ib_srq_init_attr *init_attr
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_user_srq(struct roce5_device *rdev,
+ struct roce5_srq *rsrq, struct roce5_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata, u32 index)
+{
+ int ret = 0;
+ u32 npages = 0;
+ int page_shift = 0;
+ struct roce5_create_srq_cmd ucmd = { 0 };
+
+ /* dispatch SRQN and SRQC */
+ rsrq->cqm_srq = cqm5_object_rdma_queue_create(rdev->hwdev,
+ SERVICE_T_ROCE,
+ CQM_OBJECT_RDMA_SRQ, 0,
+ rsrq, false, index, 0, 0);
+ if (rsrq->cqm_srq == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create rdma queue by cqm_object, func_id(%d) index(%d)",
+ rdev->glb_func_id, index);
+ return (-ENOMEM);
+ }
+
+ rsrq->srqn = rsrq->cqm_srq->index;
+
+ ret = roce5_create_srq_get_umem(&ucmd, udata, rsrq, rdev, pd);
+ if (ret != 0) {
+ goto err_free_cqm_srq;
+ }
+
+ page_shift =
+ roce5_umem_get_buf_page_shift(rdev, rsrq->umem, ucmd.buf_addr);
+ npages = roce5_umem_get_mtt_npages(rdev, rsrq->umem, (u32)page_shift);
+
+ rsrq->mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &rsrq->mtt,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, " Failed to alloc rdma_mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_release_umem;
+ }
+
+ ret = roce5_create_srq_write_mtt_and_db(rdev, rsrq, &ucmd, pd);
+ if (ret != 0) {
+ goto err_free_mtt;
+ }
+
+ ret = roce5_create_user_srq_update(rdev, rsrq, pd, init_attr, udata,
+ page_shift);
+ if (ret != 0) {
+ goto err_free_mtt;
+ }
+
+ init_attr->attr.max_wr = roce5_srq_max_avail_wr_set(rsrq);
+
+ return 0;
+
+err_free_mtt:
+ hmm_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE);
+
+err_release_umem:
+ ib_umem_release(rsrq->umem);
+
+err_free_cqm_srq:
+ cqm5_object_delete(&rsrq->cqm_srq->object);
+
+ return ret;
+}
+
+static void roce5_get_cqn_xrcdn(u32 *cqn, u16 *xrcdn,
+ struct ib_srq_init_attr *init_attr,
+ struct roce5_device *rdev)
+{
+ *cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ?
+ to_roce5_cq(init_attr->ext.cq)->cqn :
+ 0);
+ *xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
+ (u16)(to_roce5_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) :
+ (u16)rdev->rdma_cap.reserved_xrcds;
+}
+
+static int roce5_create_kernel_srq_update(struct roce5_device *rdev,
+ struct roce5_srq *rsrq,
+ struct roce5_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ int page_shift)
+{
+ u32 cqn = 0;
+ u16 xrcdn = 0;
+ int ret;
+ ret = roce5_buf_write_mtt(rdev, &rsrq->mtt, rsrq->buf);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_mtt;
+ }
+
+ rsrq->wrid = (u64 *)kmalloc(
+ (size_t)((u32)rsrq->max_depth * sizeof(u64)), GFP_KERNEL);
+ if (rsrq->wrid == NULL) {
+ rsrq->wrid = (u64 *)vzalloc(
+ (size_t)((u32)rsrq->max_depth * sizeof(u64)));
+ if (rsrq->wrid == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc srq wrid, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -ENOMEM;
+ goto err_free_mtt;
+ }
+ }
+
+ roce5_get_cqn_xrcdn(&cqn, &xrcdn, init_attr, rdev);
+
+ ret = roce5_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn,
+ (page_shift - PAGE_SHIFT_4K));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to handle srq sw2hw, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_wrid;
+ }
+
+ return 0;
+
+err_free_wrid:
+ kvfree(rsrq->wrid);
+ rsrq->wrid = NULL;
+
+err_free_mtt:
+ hmm_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_create_kernel_srq
+ Description : roce5_create_kernel_srq
+ Input : struct roce5_device *rdev
+ struct roce5_srq *rsrq
+ struct roce5_pd *pd
+ struct ib_srq_init_attr *init_attr
+ Output : None
+
+ 1.Date : 2017/5/4
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_create_kernel_srq(struct roce5_device *rdev,
+ struct roce5_srq *rsrq, struct roce5_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ u32 index)
+{
+ int i = 0;
+ int ret = 0;
+ int page_shift = 0;
+ struct roce5_wqe_srq_next_seg *next = NULL;
+ struct roce_wqe_srq_data_seg *scatter = NULL;
+
+ /* alloc queue Buf/ Soft DB/SRQN/SRQC */
+ rsrq->cqm_srq = cqm5_object_rdma_queue_create(
+ rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_SRQ,
+ (u32)rsrq->buf_sz, rsrq, true, index, 0, 0);
+ if (rsrq->cqm_srq == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create rdma queue by cqm_object, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ rsrq->srqn = rsrq->cqm_srq->index;
+ rsrq->buf = &rsrq->cqm_srq->q_room_buf_1;
+
+ /* set Soft DB */
+ rsrq->db.db_record = (__be32 *)(void *)(&rsrq->cqm_srq->q_header_vaddr
+ ->doorbell_record);
+ rsrq->db.dma = rsrq->cqm_srq->q_header_paddr;
+ *rsrq->db.db_record = 0;
+
+ rsrq->head = 0;
+ rsrq->tail = rsrq->max_depth - 1;
+ rsrq->wqe_ctr = 0;
+
+ for (i = 0; i < rsrq->max_depth; ++i) {
+ next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(rsrq,
+ i);
+ next->next_wqe_index = cpu_to_be16(
+ (u16)((u32)(i + 1) & (u32)(rsrq->max_depth - 1)));
+
+ scatter = (struct roce_wqe_srq_data_seg *)(next + 1);
+ scatter->dw2.length = 0;
+
+ /* first SGE = last SGE */
+ scatter->dw3.lkey = LAST_SGE_NO_PRESENT;
+ scatter->dw3.lkey = cpu_to_be32(scatter->dw3.lkey);
+ }
+
+ page_shift = (int)ROCE_ILOG2(rsrq->buf->buf_size);
+ rsrq->mtt.mtt_type = MTT_CMTT_TYPE;
+ ret = hmm_mtt_alloc(rdev->hwdev, rsrq->buf->buf_number, (u32)page_shift,
+ &rsrq->mtt, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdma_mtt, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free_cqm_srq;
+ }
+
+ ret = roce5_create_kernel_srq_update(rdev, rsrq, pd, init_attr,
+ page_shift);
+ if (ret != 0) {
+ goto err_free_cqm_srq;
+ }
+
+ rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn;
+ init_attr->attr.max_wr = (u32)rsrq->max_depth - 1;
+ return 0;
+
+err_free_cqm_srq:
+ cqm5_object_delete(&rsrq->cqm_srq->object);
+
+ return ret;
+}
+
+static void roce5_set_srq_depth(struct ib_srq_init_attr *init_attr,
+ struct roce5_srq *rsrq)
+{
+ u32 link_wqe_count = 0;
+ u32 remain_wqe_count = 0;
+
+ /*
+ * +---+---+---+---+---+---+---+---+
+ * |wqe|wqe|wqe|wqe|wqe|wqe|wqe|nil|
+ * +---+---+---+---+---+---+---+---+
+ * ^
+ * |
+ * full condition:head==tail==nil
+ */
+ if (rsrq->container_flag != 0) {
+ link_wqe_count =
+ init_attr->attr.max_wr / (rsrq->container_size - 1u);
+ remain_wqe_count =
+ init_attr->attr.max_wr % (rsrq->container_size - 1u);
+ link_wqe_count +=
+ ((remain_wqe_count == 0) ?
+ 0 :
+ (rsrq->container_size - remain_wqe_count));
+ rsrq->max_depth = (int)(init_attr->attr.max_wr +
+ link_wqe_count + rsrq->container_size);
+ } else {
+ rsrq->max_depth = (int)(init_attr->attr.max_wr + 1);
+ }
+ rsrq->max_depth =
+ (int)((ROCE_ROUNDUP_POW_OF_TWO((u32)rsrq->max_depth)) &
+ 0xffffffff);
+}
+
+static void roce5_rsrq_init(struct ib_srq_init_attr *init_attr,
+ struct roce5_srq *rsrq, struct roce5_device *rdev)
+{
+ int buf_size = 0;
+ int desc_size = 0;
+
+ roce5_srq_container_init(init_attr, rsrq, rdev);
+
+ mutex_init(&rsrq->mutex);
+ spin_lock_init(&rsrq->lock);
+
+ roce5_set_srq_depth(init_attr, rsrq);
+
+ rsrq->max_gs = (init_attr->attr.max_sge <= ROCE_SRQ_MIN_SGE) ?
+ ROCE_SRQ_MIN_SGE :
+ ((init_attr->attr.max_sge <= ROCE_SRQ_MID_SGE) ?
+ ROCE_SRQ_MID_SGE :
+ roce5_srq_get_max_sge(rsrq));
+ init_attr->attr.max_sge = (u32)rsrq->max_gs;
+
+ /* 0 <= max_gs <= 3, desc_size = 64
+ * 4 <= max_gs <= 7, desc_size = 128
+ */
+ desc_size = roce5_srq_init_wqe_size(rsrq);
+ buf_size = rsrq->max_depth * desc_size;
+
+ /* WQEBB align 1 PAGE */
+ if (buf_size < (int)PAGE_SIZE) {
+ rsrq->max_depth =
+ (int)((u64)PAGE_SIZE >> (unsigned int)rsrq->wqe_shift);
+ }
+
+ /* align with PAGE SIZE */
+ buf_size = (int)ROCE_ALIGN((u32)buf_size, PAGE_SIZE);
+ rsrq->buf_sz = buf_size;
+
+ ROCE_DEV_INFO(
+ rdev,
+ "RoCE srq mode: container_EN(%d), container_mode(%d), XRC_EN(%d), MAX_DEPTH(%d), MAX_SGE(%d), desc_size(%d)",
+ rsrq->container_flag, rsrq->container_mode, rsrq->xrc_en,
+ rsrq->max_depth, rsrq->max_gs, desc_size);
+}
+
+int roce5_create_srq_common(struct roce5_device *rdev, struct roce5_srq *rsrq,
+ struct roce5_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata, u32 index)
+{
+ int ret;
+ roce5_rsrq_init(init_attr, rsrq, rdev);
+
+ ret = roce5_create_srq_check(&pd->ibpd, init_attr, rsrq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to check srq info");
+ return ret;
+ }
+
+ if (rsrq->ibsrq.uobject != NULL) {
+ ret = roce5_create_user_srq(rdev, rsrq, pd, init_attr, udata,
+ index);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to create user srq, func_id(%d), index(%d)",
+ rdev->glb_func_id, index);
+ }
+ } else {
+ ret = roce5_create_kernel_srq(rdev, rsrq, pd, init_attr, index);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to create kernel srq, func_id(%d)",
+ rdev->glb_func_id);
+ }
+ }
+
+ ROCE_DEV_INFO(
+ rdev,
+ "SRQ create:srqn:%d,container_en_mode:%d_%d,xrc_en:%d,max_depth:%d",
+ rsrq->srqn, rsrq->container_flag, rsrq->container_mode,
+ rsrq->xrc_en, rsrq->max_depth);
+
+ return ret;
+}
+
+int roce5_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ struct roce5_pd *pd = NULL;
+ struct roce5_device *rdev = NULL;
+ struct roce5_srq *rsrq = NULL;
+
+ if ((ibsrq == NULL) || (init_attr == NULL)) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ pd = to_roce5_pd(ibsrq->pd);
+ rdev = to_roce5_dev(ibsrq->device);
+ rsrq = to_roce5_srq(ibsrq);
+
+ return roce5_create_srq_common(rdev, rsrq, pd, init_attr, udata,
+ ROCE_SRQN_INVLD);
+}
+
+struct ib_srq *roce5_create_srq_ofa(struct ib_pd *ibpd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_srq *rsrq = NULL;
+
+ if (ibpd == NULL) {
+ ROCE_ERR("Ibpd is null");
+ return ERR_PTR(-EINVAL);
+ }
+
+ rsrq = (struct roce5_srq *)kzalloc(sizeof(struct roce5_srq),
+ GFP_KERNEL);
+ if (rsrq == NULL) {
+ ROCE_ERR("Failed to alloc srq");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ rsrq->ibsrq.pd = ibpd;
+ rsrq->ibsrq.device = ibpd->device;
+#if defined(IB_SRQ_HAVE_UOBJECT)
+ rsrq->ibsrq.uobject =
+ (udata ? ibpd->uobject :
+ NULL); /* udata not NULL means user srq create */
+#endif
+ ret = roce5_create_srq(&rsrq->ibsrq, init_attr, udata);
+ if (ret != 0) {
+ kfree(rsrq);
+ return ERR_PTR(ret);
+ }
+
+ return &rsrq->ibsrq;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c
new file mode 100644
index 000000000..887a857ad
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c
@@ -0,0 +1,635 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include "comm_defs.h"
+#include "roce.h"
+#include "roce_compat.h"
+#include "roce_user.h"
+#include "roce_xrc.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_mix.h"
+#include "roce_cqm_cmd.h"
+#include "roce_restore.h"
+#include "hinic5_hmm.h"
+#include "roce_main_extension.h"
+#include "roce_srq_extension.h"
+#include "roce_npu_cmd.h"
+#include "roce_com.h"
+#include "securec.h"
+#include "roce_os_compat.h"
+
+/* and above next is V100 compatible data structure */
+typedef struct tag_roce_cmd_srq_arm {
+ roce_verbs_cmd_com_s com;
+ union {
+ u32 limitwater;
+ struct {
+#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN))
+ u32 lwm : 16;
+ u32 warth : 4;
+ u32 th_up_en : 1;
+ u32 cont_en : 1;
+ u32 rsvd : 10;
+#else
+ u32 rsvd : 10;
+ u32 cont_en : 1;
+ u32 th_up_en : 1;
+ u32 warth : 4;
+ u32 lwm : 16;
+#endif
+ } bs;
+ };
+} roce_cmd_srq_arm_s;
+
+typedef struct tag_roce_cmd_srq_hw2sw {
+ roce_verbs_cmd_com_s com;
+ roce_xq_mtt_info_s mtt_info;
+ u32 srq_buf_len;
+ u32 wqe_cache_line_start;
+ u32 wqe_cache_line_end;
+ u32 wqe_cache_line_size;
+} roce_cmd_srq_hw2sw_s;
+
+static u8 roce5_cal_srq_container_num(u32 avail_wr, struct roce5_srq *srq)
+{
+ u32 container_num = 0;
+ u32 srqe_num = 0;
+
+ /* cal container_num by avail_wr from user */
+ srqe_num = srq->container_size - 1;
+
+ /* explain the formula: round up by srqe_num */
+ container_num = (avail_wr + srqe_num - 1) / srqe_num;
+ return roce5_calculate_cont_th(container_num);
+}
+
+static void roce5_srq_arm_config_cqm_cmd(cqm_cmd_buf_s *cqm_cmd_inbuf,
+ struct roce5_srq *srq, u32 srq_limit,
+ u8 *warth)
+{
+ roce_uni_cmd_srq_arm_s *srq_arm_inbuf = NULL;
+
+ srq_arm_inbuf = (roce_uni_cmd_srq_arm_s *)cqm_cmd_inbuf->buf;
+ srq_arm_inbuf->com.index = cpu_to_be32((u32)(srq->srqn));
+ srq_arm_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK);
+ if (srq->container_flag != 0) {
+ srq_arm_inbuf->srq_arm.bs.cont_en = srq->container_flag;
+ srq_arm_inbuf->srq_arm.bs.th_up_en =
+ 1; //Valid for setting up container warn_threshold
+ *warth = roce5_cal_srq_container_num(srq_limit, srq);
+ srq_arm_inbuf->srq_arm.bs.warth = *warth &
+ ROCE_SRQ_CONTAINER_WARTH_MASK;
+ srq_arm_inbuf->srq_arm.limitwater =
+ cpu_to_be32(srq_arm_inbuf->srq_arm.limitwater);
+ } else {
+ srq_arm_inbuf->srq_arm.bs.lwm = srq_limit;
+ srq_arm_inbuf->srq_arm.limitwater =
+ cpu_to_be32(srq_arm_inbuf->srq_arm.limitwater);
+ }
+}
+
+/* ****************************************************************************
+ Prototype : roce5_srq_arm
+ Description : roce5_srq_arm
+ Input : struct roce5_device *rdev
+ struct roce5_srq *srq
+ u32 srq_limit
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_srq_arm(struct roce5_device *rdev, struct roce5_srq *srq,
+ u32 srq_limit)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ uint8_t warth = 0;
+ uint16_t inbuf_size = 0;
+
+ if (ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) != 0) {
+ inbuf_size = sizeof(roce_uni_cmd_srq_arm_s);
+ }
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf,
+ inbuf_size, NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ if (ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) != 0) {
+ roce5_srq_arm_config_cqm_cmd(cqm_cmd_inbuf, srq, srq_limit,
+ &warth);
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_ARM_SRQ,
+ cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "send ARM_SRQ command Failed, ret(%d), func_id(%d), warth(%d)",
+ ret, rdev->glb_func_id, warth);
+
+ if ((ret == -ETIMEDOUT) || (ret == -EPERM)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_modify_srq
+ Description : roce5_modify_srq
+ Input : struct ib_srq *ibsrq
+ struct ib_srq_attr *attr
+ enum ib_srq_attr_mask attr_mask
+ struct ib_udata *udata
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
+{
+ int ret = 0;
+ struct roce5_device *rdev = NULL;
+ struct roce5_srq *srq = NULL;
+ struct roce5_modify_srq_cmd ucmd = { 0 };
+ u32 max_avail_wr = 0;
+
+ if ((ibsrq == NULL) || (attr == NULL)) {
+ ROCE_ERR("Ibsrq or attr is null");
+ return (-EINVAL);
+ }
+
+ if (((unsigned int)attr_mask & IB_SRQ_MAX_WR) != 0) {
+ ROCE_ERR("Not support resizing SRQs");
+ return (-EINVAL);
+ }
+
+ rdev = to_roce5_dev(ibsrq->device);
+ srq = to_roce5_srq(ibsrq);
+
+ if ((udata != NULL) &&
+ (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "SRQN:(%d), Failed to copy from user space, func_id(%d)",
+ srq->srqn, rdev->glb_func_id);
+ return (-EFAULT);
+ }
+
+ if (((unsigned int)attr_mask & IB_SRQ_LIMIT) != 0) {
+ /* calculate max_wr */
+ max_avail_wr = roce5_srq_max_avail_wr_set(srq);
+ if (attr->srq_limit > max_avail_wr) {
+ ROCE_DEV_ERR(
+ rdev,
+ "srq_limit > max_wr, func_id(%d), max_avail_wr(%d)",
+ rdev->glb_func_id, max_avail_wr);
+ return (-EINVAL);
+ }
+
+ mutex_lock(&srq->mutex);
+ ret = roce5_srq_arm(rdev, srq, attr->srq_limit);
+ mutex_unlock(&srq->mutex);
+
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "[Failed to handle srq arm, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static int roce5_cmd_srq_query(struct roce5_device *rdev, struct roce5_srq *srq,
+ u32 *srq_limit)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_srq_query_s *srq_query_inbuf = NULL;
+ roce_uni_cmd_srq_query_outbuf_s *srq_query_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_srq_query_s), &cqm_cmd_outbuf,
+ (u16)sizeof(roce_uni_cmd_srq_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ srq_query_inbuf = (roce_uni_cmd_srq_query_s *)cqm_cmd_inbuf->buf;
+ srq_query_outbuf =
+ (roce_uni_cmd_srq_query_outbuf_s *)cqm_cmd_outbuf->buf;
+ srq_query_inbuf->com.index = cpu_to_be32((u32)srq->srqn);
+ srq_query_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK));
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_QUERY_SRQ, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_A,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send QUERY_SRQ command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ if ((ret == -ETIMEDOUT) || (ret == -EPERM)) {
+ rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT;
+ }
+
+ ret = -1;
+ goto err_cmd;
+ }
+
+ *srq_limit = be32_to_cpu(srq_query_outbuf->com.srq_limit);
+
+ if (srq->container_flag != 0) {
+ /* parse srqc to get warn_th */
+ u32 warn_th = be32_to_cpu(srq_query_outbuf->com.warn_th);
+ *srq_limit = (warn_th == 0) ? 0 :
+ ((u32)((srq->container_size - 1) *
+ (1U << warn_th)));
+ }
+
+err_cmd:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+
+ return ret;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_query_srq
+ Description : roce5_query_srq
+ Input : struct ib_srq *ibsrq
+ struct ib_srq_attr *srq_attr
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+int roce5_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
+{
+ int ret = 0;
+ u32 limit_water = 0;
+ struct roce5_device *rdev = NULL;
+ struct roce5_srq *rsrq = NULL;
+
+ if ((ibsrq == NULL) || (srq_attr == NULL)) {
+ ROCE_ERR("Ibsrq or srq_attr is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibsrq->device);
+ rsrq = to_roce5_srq(ibsrq);
+
+ ret = roce5_cmd_srq_query(rdev, rsrq, &limit_water);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query srq, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ srq_attr->srq_limit = limit_water;
+ srq_attr->max_wr = roce5_srq_max_avail_wr_set(rsrq);
+ srq_attr->max_sge = (u32)rsrq->max_gs;
+
+ return 0;
+}
+
+static void roce5_srq_hw2sw_config_cqm_cmd(struct roce5_device *rdev,
+ cqm_cmd_buf_s *cqm_cmd_inbuf,
+ struct rdma_service_cap *rdma_cap,
+ struct roce5_srq *srq)
+{
+ roce_uni_cmd_srq_hw2sw_s *srq_hw2sw_inbuf = NULL;
+
+ srq_hw2sw_inbuf = (roce_uni_cmd_srq_hw2sw_s *)cqm_cmd_inbuf->buf;
+ srq_hw2sw_inbuf->com.index = cpu_to_be32((u32)srq->srqn);
+ srq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK));
+ srq_hw2sw_inbuf->srq_cache.srq_buf_len = cpu_to_be32((u32)srq->buf_sz);
+
+ srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_flags = 0;
+ srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_num = 0;
+ srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_cache_line_start =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start);
+ srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_cache_line_end =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end);
+ srq_hw2sw_inbuf->srq_cache.wqe_cache_line_size =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz);
+
+ srq_hw2sw_inbuf->srq_cache.wqe_cache_line_start =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start);
+ srq_hw2sw_inbuf->srq_cache.wqe_cache_line_end =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end);
+ srq_hw2sw_inbuf->srq_cache.wqe_cache_line_size =
+ cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_srq_hw2sw
+ Description : roce5_srq_hw2sw
+ Input : struct roce5_device *rdev
+ struct roce5_srq *srq
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_srq_hw2sw(struct roce5_device *rdev, struct roce5_srq *srq)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ uint16_t inbuf_size = 0;
+
+ if (ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) != 0) {
+ inbuf_size = sizeof(roce_uni_cmd_srq_hw2sw_s);
+ }
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf,
+ inbuf_size, NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ if (ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) != 0) {
+ roce5_srq_hw2sw_config_cqm_cmd(rdev, cqm_cmd_inbuf,
+ &rdev->rdma_cap, srq);
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_HW2SW_SRQ, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to send HW2SW_SRQ command, func_id(%d)",
+ rdev->glb_func_id);
+
+ ret = -1;
+
+ if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) {
+ ROCE_DEV_ERR(rdev,
+ "Card not present, return OK, func_id(%d)",
+ rdev->glb_func_id);
+ ret = 0;
+ }
+
+ goto err_out;
+ }
+
+err_out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+
+ return ret;
+}
+
+static void roce5_free_srq(struct ib_srq *ibsrq, struct roce5_device *rdev,
+ struct roce5_srq *srq, struct ib_udata *udata)
+{
+ hmm_mtt_free(rdev->hwdev, &srq->mtt, SERVICE_T_ROCE);
+
+ if (ibsrq->uobject) {
+ struct roce5_ucontext *ucontext = roce5_queue_to_ucontext(
+ (void *)ibsrq, udata, ROCE_Q_TYPE_SRQ);
+ roce5_db_unmap_user(ucontext, &srq->db, 0);
+ ib_umem_release(srq->umem);
+ } else {
+ kfree(srq->wrid);
+ }
+
+ cqm5_object_delete(&srq->cqm_srq->object);
+}
+
+int roce5_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
+{
+ int ret = 0;
+ int times = 0;
+ struct roce5_device *rdev = NULL;
+ struct roce5_srq *rsrq = NULL;
+
+ if (ibsrq == NULL) {
+ ROCE_ERR("Ibsrq is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibsrq->device);
+ rsrq = to_roce5_srq(ibsrq);
+ times = rdev->try_times;
+
+ ret = roce5_srq_hw2sw(rdev, rsrq);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to handle srq hw2sw, func_id(%d), ret:%d",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ roce5_free_srq(ibsrq, rdev, rsrq, udata);
+ return 0;
+}
+
+int roce5_destroy_srq_ofa(struct ib_srq *ibsrq)
+{
+ int ret = roce5_destroy_srq(ibsrq, NULL);
+ if (ret == 0) {
+ kfree(ibsrq);
+ }
+
+ return ret;
+}
+
+static int roce5_check_sge_length(const struct ib_recv_wr *wr,
+ struct roce5_device *rdev,
+ const struct ib_recv_wr **bad_wr,
+ struct roce_wqe_srq_data_seg *scat)
+{
+ int i = 0;
+ int ret = 0;
+ u32 data_len = 0;
+
+ for (i = 0; i < wr->num_sge; ++i) {
+ if (ROCE_UNLIKELY(
+ wr->sg_list[i].length >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz -
+ 1)) {
+ ret = -EINVAL;
+ *bad_wr = wr;
+ ROCE_DEV_ERR(
+ rdev,
+ "Sge data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)",
+ i, wr->sg_list[i].length,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz,
+ rdev->glb_func_id);
+
+ scat[0].dw2.length = 0;
+ scat[0].dw3.lkey = LAST_SGE_NO_PRESENT;
+ scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey);
+ return ret;
+ }
+
+ data_len += wr->sg_list[i].length;
+ if (ROCE_UNLIKELY(data_len >
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz)) {
+ ret = -EINVAL;
+ *bad_wr = wr;
+ scat[0].dw2.length = 0;
+ scat[0].dw3.lkey = LAST_SGE_NO_PRESENT;
+ scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey);
+
+ ROCE_DEV_ERR(
+ rdev,
+ "Data length is over range, data_len(0x%x), max_msg_sz(0x%x), func_id(%d)",
+ data_len,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap
+ .max_msg_sz,
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ scat[i].dw2.length = cpu_to_be32(wr->sg_list[i].length);
+ scat[i].dw3.lkey =
+ cpu_to_be32((u32)(wr->sg_list[i].lkey &
+ NORMAL_FMT_AND_NEXT_SGE_PRESENT));
+ scat[i].addr = cpu_to_be64(wr->sg_list[i].addr);
+ }
+
+ if (ROCE_UNLIKELY(i == 0)) {
+ scat[0].dw3.lkey = LAST_SGE_NO_PRESENT;
+ scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey);
+ scat[0].dw2.length = 0;
+ } else {
+ scat[i - 1].dw3.lkey = cpu_to_be32(
+ (u32)((wr->sg_list[i - 1].lkey | LAST_SGE_NO_PRESENT) &
+ NORMAL_FMT_AND_LAST_SGE_NO_PRESENT));
+ }
+
+ return ret;
+}
+
+static int roce5_post_srq_recv_precheck(struct ib_srq *ibsrq,
+ const struct ib_recv_wr **bad_wr)
+{
+ if (ibsrq == NULL) {
+ ROCE_ERR("Ibsrq is null");
+ return -EINVAL;
+ }
+
+ if (bad_wr == NULL) {
+ ROCE_ERR("Bad_wr is null");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int roce5_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr)
+{
+ int ret = 0;
+ int nreq = 0;
+ unsigned long flags = 0; /* type required by kernel API */
+ struct roce5_srq *srq = NULL;
+ struct roce5_device *rdev = NULL;
+ struct roce5_wqe_srq_next_seg *next = NULL;
+ struct roce_wqe_srq_data_seg *scat = NULL;
+ const struct ib_recv_wr *wr_tmp = wr;
+
+ ret = roce5_post_srq_recv_precheck(ibsrq, bad_wr);
+ if (ret != 0) {
+ return -EINVAL;
+ }
+
+ srq = to_roce5_srq(ibsrq);
+ rdev = to_roce5_dev(ibsrq->device);
+
+ spin_lock_irqsave(&srq->lock, flags);
+
+ for (nreq = 0; wr_tmp; ++nreq, wr_tmp = wr_tmp->next) {
+ if (ROCE_UNLIKELY((wr_tmp->num_sge > srq->max_gs) ||
+ (srq->head == srq->tail))) {
+ ret = (wr_tmp->num_sge > srq->max_gs) ? (-EINVAL) :
+ (-ENOMEM);
+ *bad_wr = wr_tmp;
+ ROCE_DEV_ERR(
+ rdev,
+ "Num_sge(%d) > max_gs(%d) or srq is empty(head(%d) == tail(%d)), func_id(%d)",
+ wr_tmp->num_sge, srq->max_gs, rdev->glb_func_id,
+ srq->head, srq->tail);
+ break;
+ }
+
+ next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(
+ srq, srq->head);
+ scat = (struct roce_wqe_srq_data_seg *)(next + 1);
+
+ ret = roce5_check_sge_length(wr_tmp, rdev, bad_wr, scat);
+ if (ret != 0) {
+ goto sge_len_err;
+ }
+
+ next->pcnt = (u16)cpu_to_be16(srq->wqe_ctr + (u16)nreq);
+ srq->wrid[srq->head] = wr_tmp->wr_id;
+ srq->head = be16_to_cpu(next->next_wqe_index);
+ }
+
+sge_len_err:
+ if (ROCE_UNLIKELY(nreq != 0)) {
+ srq->wqe_ctr += (u16)nreq;
+
+ /*
+ * Make sure that descriptors are written before
+ * doorbell record.
+ */
+ wmb();
+
+ *srq->db.db_record = cpu_to_be32(srq->wqe_ctr);
+ }
+
+ spin_unlock_irqrestore(&srq->lock, flags);
+
+ return ret;
+}
+
+int roce5_post_srq_recv_ofa(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
+ struct ib_recv_wr **bad_wr)
+{
+ return roce5_post_srq_recv(ibsrq, (const struct ib_recv_wr *)wr,
+ (const struct ib_recv_wr **)bad_wr);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h
new file mode 100644
index 000000000..12d501b4c
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h
@@ -0,0 +1,36 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_SRQ_FORMAT_H
+#define ROCE_SRQ_FORMAT_H
+
+#include <linux/types.h>
+
+typedef struct roce5_srq_context {
+ /* DW0 */
+ union {
+ struct {
+ u32 size : 5;
+ u32 page_size : 4;
+ u32 wqebb_size : 3;
+ u32 rsvd : 2;
+ u32 pdn : 18;
+ } bs;
+ u32 value;
+ } dw0;
+
+ /* DW1 */
+ union {
+ struct {
+ u32 rsvd : 24;
+ u32 dma_attr_idx : 6;
+ u32 so_ro : 2;
+ } bs_c;
+ u32 value;
+ } dw1;
+
+ u32 rsvd[14];
+} roce5_srq_context_s;
+
+#endif /* ROCE_SRQ_FORMAT_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c
new file mode 100644
index 000000000..e66a99f39
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c
@@ -0,0 +1,729 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+
+#include "comm_defs.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_bond.h"
+#include "roce_uld_inner.h"
+#include "roce_mpu_cmd.h"
+#include "roce_cqm_cmd.h"
+
+#define PLUGIN __attribute__((__used__, __weak__, optimize("O0")))
+#define ROCE_ULD_PRINT_BUF_SIZE 256
+
+PLUGIN int roce5_uld_reg_user_mr(roce_uld_handle handle)
+{
+ return -1;
+}
+
+PLUGIN int roce5_uld_dereg_user_mr(roce_uld_handle handle)
+{
+ return 0;
+}
+
+PLUGIN int roce5_adm_dfx_uld_extend(roce_uld_handle handle, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ return 0;
+}
+
+PLUGIN int roce5_uld_qp_modify(roce_uld_handle handle, struct ib_qp_attr *attr,
+ int attr_mask, struct ib_udata *udata)
+{
+ return 0;
+}
+
+PLUGIN void roce5_uld_rdev_remove(roce_uld_handle handle)
+{
+ return;
+}
+
+PLUGIN int roce5_uld_rdev_add(roce_uld_handle handle)
+{
+ return 0;
+}
+
+PLUGIN int roce5_uld_qp_destroy(roce_uld_handle handle)
+{
+ return 0;
+}
+
+PLUGIN int roce5_uld_qp_create(roce_uld_handle handle)
+{
+ return 0;
+}
+
+PLUGIN int roce5_uld_cq_create(roce_uld_handle handle)
+{
+ return 0;
+}
+
+PLUGIN void roce5_uld_cq_destroy(roce_uld_handle handle)
+{
+ return;
+}
+
+PLUGIN int32_t roce5_uld_user2kernel_msg_handle(roce_uld_handle handle,
+ void *buf_in, u16 buf_in_len,
+ void *buf_out, u16 buf_out_len,
+ u16 *buf_out_size)
+{
+ return 0;
+}
+
+PLUGIN int roce5_uld_service_init(void)
+{
+ return 0;
+}
+
+PLUGIN void roce5_uld_service_exit(void)
+{
+ return;
+}
+
+u64 roce5_mr_virt_addr_get(roce_uld_handle handle)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) {
+ ROCE_ERR("handle is invalid");
+ return 0;
+ }
+
+ return hdl->virt_addr;
+}
+
+int roce5_mr_virt_addr_set(roce_uld_handle handle, u64 virt_addr)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) {
+ ROCE_ERR("handle is invalid");
+ return -EINVAL;
+ }
+
+ hdl->virt_addr = virt_addr;
+
+ return 0;
+}
+
+int roce5_mr_user_data_get(roce_uld_handle handle, u8 *user_data,
+ u32 user_data_len)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ u32 cpy_len;
+
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ ROCE_ERR("handle is invalid");
+ return -EINVAL;
+ }
+
+ if (user_data == NULL) {
+ ROCE_ERR("user_data is NULL");
+ return -EINVAL;
+ }
+
+ cpy_len = (user_data_len > hdl->mr->user_data_len) ?
+ hdl->mr->user_data_len :
+ user_data_len;
+ (void)memcpy_s(user_data, user_data_len, hdl->mr->rdmamr.user_data,
+ cpy_len);
+
+ return cpy_len;
+}
+
+int roce5_mpt_alloc_and_init(roce_uld_handle handle, int mode, u8 *user_data,
+ u32 user_data_len, u32 mpt_xid_mode)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ struct roce5_mr *mr = NULL;
+ struct roce5_device *rdev = NULL;
+ int ret = 0;
+ u32 npages = 0;
+ u32 page_shift = PAGE_SHIFT;
+ u32 mpt_xid_create_mode;
+
+ if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) {
+ ROCE_ERR("handle is invalid");
+ return -EINVAL;
+ }
+
+ mr = hdl->mr;
+ rdev = hdl->rdev;
+ if (rdev == NULL) {
+ ROCE_ERR("rdev is NULL");
+ return -EINVAL;
+ }
+
+ if (user_data_len > HMM_USER_DATA_MAX_BYTE) {
+ ROCE_ERR(
+ "user_data_len(%u) is invalid, max user_data_len is %u",
+ user_data_len, HMM_USER_DATA_MAX_BYTE);
+ return -EINVAL;
+ }
+
+ if (mode == ROCE_ULD_MPT_NORMAL) {
+ mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE;
+ npages = (u32)ib_umem_num_pages(mr->umem);
+#if defined(UMEM_HAVE_PAGE_SHIFT)
+ page_shift = (u32)(mr->umem->page_shift);
+#endif
+ }
+
+ mpt_xid_create_mode = CQM_DYNAMIC_XID_MOD(
+ CQM_XID_SEARCH_ALL, CQM_XID_LOW_BIT_0_0_1, mpt_xid_mode);
+ ret = roce5_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift,
+ mpt_xid_create_mode);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc mpt and mtt, func_id(%d), npages(%u), page_shift(%u)",
+ hinic5_global_func_id(rdev->hwdev), npages, page_shift);
+ goto err_alloc_tpt;
+ }
+
+ roce5_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, hdl->pdn, hdl->virt_addr,
+ hdl->length, hdl->access);
+
+ if (mode == ROCE_ULD_MPT_NORMAL) {
+ ret = roce5_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)",
+ hinic5_global_func_id(rdev->hwdev));
+ goto err_write_mtt;
+ }
+ }
+
+ if (user_data != NULL) {
+ (void)memcpy_s(hdl->mr->rdmamr.user_data, user_data_len,
+ user_data, user_data_len);
+ hdl->mr->user_data_len = user_data_len;
+ } else {
+ hdl->mr->user_data_len = 0;
+ }
+
+ return 0;
+
+err_write_mtt:
+ roce5_free_tpt(rdev, &mr->rdmamr);
+
+err_alloc_tpt:
+ return ret;
+}
+
+void roce5_mpt_destroy(roce_uld_handle handle)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ struct roce5_device *rdev = NULL;
+
+ if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) {
+ ROCE_ERR("handle is invalid");
+ return;
+ }
+
+ if (hdl->mr == NULL) {
+ ROCE_ERR("mr is NULL");
+ return;
+ }
+
+ rdev = hdl->rdev;
+ if (rdev == NULL) {
+ ROCE_ERR("rdev is NULL");
+ return;
+ }
+
+ if (hdl->mr->rdmamr.enabled != RDMA_MPT_DISABLED) {
+ roce5_free_tpt(rdev, &hdl->mr->rdmamr);
+ }
+
+ return;
+}
+
+int roce5_mpt_enable(roce_uld_handle handle)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ struct roce5_device *rdev = NULL;
+ int ret;
+
+ if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) {
+ ROCE_ERR("handle is invalid");
+ return -EINVAL;
+ }
+
+ rdev = hdl->rdev;
+ if (rdev == NULL) {
+ ROCE_ERR("rdev is NULL");
+ return -EINVAL;
+ }
+
+ /* here hmm will userdata send to npu */
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &hdl->mr->rdmamr,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to enable mpt of user mr, func_id(%d)",
+ hinic5_global_func_id(rdev->hwdev));
+ }
+
+ return ret;
+}
+
+int roce5_uld_get_func_id(roce_uld_handle handle)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) {
+ ROCE_ERR("handle is invalid");
+ return -1;
+ }
+
+ if (hdl->rdev == NULL) {
+ ROCE_ERR("rdev is NULL");
+ return -1;
+ }
+
+ return (int)hdl->rdev->glb_func_id;
+}
+
+int roce5_uld_send_msg_to_mgmt(roce_uld_handle handle, u16 cmd, void *buf_in,
+ u16 in_size, void *buf_out, u16 *out_size)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((cmd < ROCE_MPU_CMD_ULP_START) || (cmd >= ROCE_MPU_CMD_MAX)) {
+ ROCE_ERR("cmd is invalid");
+ return -1;
+ }
+
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (hdl->rdev == NULL) || (buf_in == NULL) || (buf_out == NULL) ||
+ (out_size == NULL)) {
+ ROCE_ERR(
+ "handle is invalid, or buf_in(%d), buf_out(%d), out_size(%d) is invalid",
+ (buf_in == NULL), (buf_out == NULL),
+ (out_size == NULL));
+ return -1;
+ }
+
+ return hinic5_msg_to_mgmt_sync(hdl->rdev->hwdev, HINIC5_MOD_ROCE, cmd,
+ buf_in, in_size, buf_out, out_size, 0,
+ HINIC5_CHANNEL_ROCE);
+}
+
+int roce5_uld_get_qp_info(roce_uld_handle handle, roce_uld_qp_info *qp_info)
+{
+ struct roce5_qp *rqp = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (hdl->rqp == NULL) || (qp_info == NULL)) {
+ ROCE_ERR("param invalid");
+ return -1;
+ }
+
+ rqp = (struct roce5_qp *)hdl->rqp;
+ qp_info->qpn = rqp->qpn;
+ qp_info->qp_type = rqp->qp_type;
+ if (hdl->qp_attr != NULL) {
+ qp_info->dgid = (u8 *)&hdl->qp_attr->path_info.dgid[0];
+ }
+ return 0;
+}
+
+int roce5_uld_modify_qp_tx_bond_hash(roce_uld_handle handle, u32 tx_bond_hash)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (hdl->qp_attr == NULL)) {
+ ROCE_ERR("param invalid");
+ return -1;
+ }
+
+ hdl->qp_attr->path_info.dw0.bs.bond_tx_hash_value = tx_bond_hash & 0xf;
+ return 0;
+}
+
+u8 *roce5_get_rdev_uld_def(roce_uld_handle handle)
+{
+ struct roce5_device *rdev = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ return NULL;
+ }
+
+ rdev = hdl->rdev;
+ if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic);
+ return NULL;
+ }
+ return (u8 *)&rdev->uld_def.self_def[0];
+}
+
+void *roce5_uld_get_rdev_hwdev(roce_uld_handle handle)
+{
+ struct roce5_device *rdev = NULL;
+ void *hwdev = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ return NULL;
+ }
+
+ rdev = hdl->rdev;
+ if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic);
+ return NULL;
+ }
+ hwdev = hdl->rdev->hwdev;
+ if (hwdev == NULL) {
+ ROCE_ERR("handle is invalid, hwdev is NULL");
+ return NULL;
+ }
+ return hwdev;
+}
+
+struct ib_device *roce5_uld_get_rdev_ibdev(roce_uld_handle handle)
+{
+ struct roce5_device *rdev = NULL;
+ struct ib_device *ib_dev = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ return NULL;
+ }
+
+ rdev = hdl->rdev;
+ if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic);
+ return NULL;
+ }
+ ib_dev = &(hdl->rdev->ib_dev);
+ return ib_dev;
+}
+
+struct hinic5_lld_dev *roce5_uld_get_rdev_lld_dev(roce_uld_handle handle)
+{
+ struct roce5_device *rdev = NULL;
+ struct hinic5_lld_dev *lld_dev = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ return NULL;
+ }
+
+ rdev = hdl->rdev;
+ if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic);
+ return NULL;
+ }
+ lld_dev = hdl->rdev->lld_dev;
+ if (lld_dev == NULL) {
+ ROCE_ERR("handle is invalid, lld_dev is NULL");
+ return NULL;
+ }
+ return lld_dev;
+}
+
+#ifdef ROCE_BONDING_EN
+int roce5_uld_get_rdev_bond_dev_slave_cnt(roce_uld_handle handle,
+ u32 *slave_cnt, uint8_t *bond_en)
+{
+ struct roce5_device *rdev = NULL;
+ struct roce5_bond_device *bond_dev = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (slave_cnt == NULL) || (bond_en == NULL)) {
+ return -EINVAL;
+ }
+ rdev = hdl->rdev;
+ if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic);
+ return -EINVAL;
+ }
+ bond_dev = rdev->bond_dev;
+ if (bond_dev == NULL) {
+ *slave_cnt = 0;
+ *bond_en = 0;
+ return 0;
+ }
+ *slave_cnt = hdl->rdev->bond_dev->slave_cnt;
+ *bond_en = 1;
+ return 0;
+}
+#endif
+
+u8 *roce5_get_rqp_uld_def(roce_uld_handle handle)
+{
+ struct roce5_qp *rqp = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ return NULL;
+ }
+
+ rqp = (struct roce5_qp *)hdl->rqp;
+ if ((rqp == NULL) || (rqp->rqp_magic != ROCE_RQP_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rqp == NULL) ? ROCE_MAGIC_NULL : rqp->rqp_magic);
+ return NULL;
+ }
+ return (u8 *)&rqp->uld_def.uld_def[0];
+}
+
+u8 *roce5_get_rcq_uld_def(roce_uld_handle handle)
+{
+ struct roce5_cq *rcq = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ return NULL;
+ }
+
+ rcq = (struct roce5_cq *)hdl->rcq;
+ if ((rcq == NULL) || (rcq->rcq_magic != ROCE_RCQ_MAGIC)) {
+ ROCE_ERR("handle is invalid, magic:0x%x",
+ (rcq == NULL) ? ROCE_MAGIC_NULL : rcq->rcq_magic);
+ return NULL;
+ }
+ return (u8 *)&rcq->uld_def.uld_def[0];
+}
+
+int roce5_uld_cos_set(roce_uld_handle handle, struct ib_qp_attr *attr, u8 type,
+ u8 value)
+{
+ u8 cos;
+ roce5_get_cos_inbuf_s inbuf = { 0 };
+ struct roce5_qp *rqp = NULL;
+ roce_verbs_qp_attr_s *qp_attr = NULL;
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ int ret;
+
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (hdl->rqp == NULL) || (hdl->qp_attr == NULL) ||
+ (hdl->rdev == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ rqp = hdl->rqp;
+ qp_attr = hdl->qp_attr;
+
+ if (type == ROCE5_COS_SET_TYPE_CLASS) {
+ inbuf.sl = attr->ah_attr.sl;
+ inbuf.sgid_index = attr->ah_attr.grh.sgid_index;
+ inbuf.port_num = attr->ah_attr.port_num;
+ inbuf.traffic_class = value;
+
+ ret = roce5_get_dcb_cfg_cos(hdl->rdev, &inbuf, &cos,
+ rqp->qp_type);
+ if (ret != 0) {
+ ROCE_ERR("get dcb cfg cos failed");
+ return ret;
+ }
+
+ /* update tclass and cos */
+ qp_attr->path_info.dw6.bs.tclass = value;
+ qp_attr->com_info.dw5.bs.cos = cos;
+ rqp->db_cos = cos;
+ } else if (type == ROCE5_COS_SET_TYPE_COS) {
+ if (value > 7) { /* cos range is 0-7 */
+ ROCE_ERR("invalid cos(%u)", value);
+ return -EINVAL;
+ }
+ qp_attr->com_info.dw5.bs.cos = value;
+ rqp->db_cos = value;
+ } else {
+ ROCE_ERR("set type(%u) invalid", type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int roce5_support_uld_kernel_feature(roce_uld_handle handle, u64 feature)
+{
+ roce_user_handle_t *user_handle = (roce_user_handle_t *)handle;
+ struct roce5_device *rdev = NULL;
+
+ if ((!user_handle) ||
+ (user_handle->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ ROCE_ERR("Invalid RoCE handle\n");
+ return -EINVAL;
+ }
+
+ rdev = user_handle->rdev;
+
+ if (rdev == NULL) {
+ ROCE_ERR("rdev is NULL in handle\n");
+ return -EINVAL;
+ }
+
+ //return capability bit AND result
+ return (rdev)->cfg_info.feature_cap.uld_feature
+ .roce_uld_kernel_ext_ability &
+ feature;
+}
+
+int roce5_uld_set_mpt_xid_mode(roce_uld_handle handle, u8 mode)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ struct roce5_device *rdev = NULL;
+
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (hdl->rdev == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ rdev = hdl->rdev;
+ rdev->mpt_xid_mode = mode;
+
+ return 0;
+}
+
+int roce5_uld_get_dgid(roce_uld_handle handle, u8 **dgid)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->qp_attr == NULL) || (dgid == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+ *dgid = hdl->qp_attr->path_info.dgid;
+ return 0;
+}
+
+int roce5_uld_set_bond_tx_hash_value(roce_uld_handle handle,
+ u32 bond_tx_hash_value)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->qp_attr == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ hdl->qp_attr->path_info.dw0.bs.bond_tx_hash_value = bond_tx_hash_value;
+ return 0;
+}
+
+int roce5_uld_get_mr_uld_self_def(roce_uld_handle handle, u8 **self_def,
+ u32 *self_def_len)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) ||
+ (hdl->mr == NULL) || (self_def == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ *self_def = &hdl->mr->uld_def.self_def[0];
+ if (self_def_len != NULL) {
+ *self_def_len = ROCE_MR_ULD_DEF_LEN;
+ }
+ return 0;
+}
+
+int roce5_uld_get_rdmamr(roce_uld_handle handle, struct hmm_rdma **rdmamr_ptr)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->mr == NULL) || (rdmamr_ptr == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ *rdmamr_ptr = &(hdl->mr->rdmamr);
+ return 0;
+}
+
+int roce5_uld_set_qp_attr_ack_to(roce_uld_handle handle, u32 ack_to)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->qp_attr == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ hdl->qp_attr->com_info.dw2.bs.ack_to = ack_to;
+ return 0;
+}
+
+int roce5_uld_get_qp_state(roce_uld_handle handle, u8 *out_qp_state)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->rqp == NULL) || (out_qp_state == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ *out_qp_state = hdl->rqp->qp_state;
+ return 0;
+}
+
+int roce5_uld_get_config_num_ports(roce_uld_handle handle,
+ int *out_cfg_num_ports)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (hdl->rdev == NULL) ||
+ (out_cfg_num_ports == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ *out_cfg_num_ports = hdl->rdev->hw_info.config_num_ports;
+ return 0;
+}
+
+int roce5_uld_get_handle_length(roce_uld_handle handle, u64 *length)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+
+ if ((hdl == NULL) || (length == NULL)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ *length = hdl->length;
+ return 0;
+}
+
+void roce5_uld_print_kernel_err_impl(roce_uld_handle handle, const char *func,
+ int line, const char *fmt, ...)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ va_list args;
+ char buf[ROCE_ULD_PRINT_BUF_SIZE];
+
+ va_start(args, fmt);
+ if (vsnprintf(buf, sizeof(buf), fmt, args) < 0) {
+ va_end(args);
+ return;
+ }
+ va_end(args);
+
+ if ((hdl != NULL) && (hdl->magic == ROCE_ULD_HANDLE_MAGIC_NUM) &&
+ (hdl->rdev != NULL)) {
+ ROCE_DEV_ERR(hdl->rdev, "[ULP][%s,%d]%s", func, line, buf);
+ } else {
+ ROCE_ERR("[ULP][%s,%d]%s", func, line, buf);
+ }
+}
+
+int roce5_uld_qp_get_port(roce_uld_handle handle, u8 *port)
+{
+ roce_user_handle_t *hdl = (roce_user_handle_t *)handle;
+ if ((hdl == NULL) || (hdl->rqp == NULL) || (port == NULL) ||
+ (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) {
+ ROCE_ERR("param invalid");
+ return -EINVAL;
+ }
+
+ *port = hdl->rqp->port;
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h
new file mode 100644
index 000000000..e61160c2f
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+
+#ifndef ROCE_ULD_INNER_H
+#define ROCE_ULD_INNER_H
+
+#include "roce.h"
+#include "roce_mr.h"
+#include "roce_qp.h"
+#include "roce_cq.h"
+#include "hmm_common.h"
+#include "roce_npu_cmd_mr_defs.h"
+#include "roce_npu_cmd_qp_defs.h"
+
+#define ROCE_ULD_HANDLE_MAGIC_NUM 0x524F4345 /* "ROCE" string of ascii code */
+#define ROCE_MAGIC_NULL 0x12345678 /* used to distinguish null pointer and unassigned */
+
+typedef struct {
+ u32 magic; /* magic number, determine handle whether valid */
+ struct roce5_device *rdev;
+ struct roce5_mr *mr;
+ roce_verbs_mr_attr_s *mr_attr;
+ u64 virt_addr;
+ u64 length;
+ u32 pdn;
+ int access;
+
+ struct roce5_qp *rqp;
+ roce_verbs_qp_attr_s *qp_attr;
+
+ struct roce5_cq *rcq;
+ int cqe;
+
+ struct ib_udata *udata;
+ struct mtt *
+ mtt; /* hmm module of mtt management structure, shadow scenario page_shift and paddr mandatory, mtt_layers constraint is 0 */
+ dma_addr_t *db_dma_handle; /* software db of dma address */
+
+ u8 *shadow;
+ u32 *shadow_vfid;
+ u16 *shadow_vf_num;
+} roce_user_handle_t;
+
+#define ROCE_CONSTUCT_ULD_HDL_RQP(hdl, roce_dev, roce_qp, roce_qp_attr, \
+ roce_udata, roce_mtt, roce_db_dma_handle) \
+ do { \
+ (hdl)->magic = ROCE_ULD_HANDLE_MAGIC_NUM; \
+ (hdl)->rdev = (roce_dev); \
+ (hdl)->rqp = (roce_qp); \
+ (hdl)->qp_attr = (roce_qp_attr); \
+ (hdl)->udata = (roce_udata); \
+ (hdl)->mtt = (roce_mtt); \
+ (hdl)->db_dma_handle = (roce_db_dma_handle); \
+ } while (0)
+
+#define ROCE_CONSTUCT_ULD_HDL_RCQ(hdl, roce_dev, roce_cq, roce_udata, \
+ roce_mtt, roce_db_dma_handle, roce_cqe) \
+ do { \
+ (hdl)->magic = ROCE_ULD_HANDLE_MAGIC_NUM; \
+ (hdl)->rdev = (roce_dev); \
+ (hdl)->rcq = (roce_cq); \
+ (hdl)->udata = (roce_udata); \
+ (hdl)->mtt = (roce_mtt); \
+ (hdl)->db_dma_handle = (roce_db_dma_handle); \
+ (hdl)->cqe = (roce_cqe); \
+ } while (0)
+
+#define ROCE_CONSTUCT_ULD_HDL_SHADOW(hdl, roce_shadow, roce_shadow_vfid, \
+ roce_shadow_vf_num) \
+ do { \
+ (hdl)->shadow = (roce_shadow); \
+ (hdl)->shadow_vfid = (roce_shadow_vfid); \
+ (hdl)->shadow_vf_num = (roce_shadow_vf_num); \
+ } while (0)
+
+#endif
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c
new file mode 100644
index 000000000..7530a655b
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c
@@ -0,0 +1,169 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/slab.h>
+
+#include "roce.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "roce_xrc.h"
+
+static int roce5_rdma_xrcd_alloc(void *hwdev, u32 *xrcdn)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+
+ if ((hwdev == NULL) || (xrcdn == NULL)) {
+ ROCE_ERR("Hwdev or xrcdn is null");
+ return -EINVAL;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -EINVAL;
+ }
+
+ *xrcdn = rdma_bitmap_alloc(&comp_priv->xrcd_bitmap);
+ if (*xrcdn == RDMA_INVALID_INDEX) {
+ ROCE_ERR("Can't get valid xrcdn, err(%d)", -ENOMEM);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void roce5_rdma_xrcd_free(void *hwdev, u32 xrcdn)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return;
+ }
+
+ rdma_bitmap_free(&comp_priv->xrcd_bitmap, xrcdn);
+}
+
+static int roce5_init_xrcd(struct ib_device *ibdev, struct roce5_device *rdev,
+ struct roce5_xrcd *xrcd)
+{
+ int ret = 0;
+ struct ib_cq_init_attr cq_attr = { 0 };
+
+ ret = roce5_rdma_xrcd_alloc(rdev->hwdev, &xrcd->xrcdn);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma xrcd, ret(%d)", ret);
+ goto err_alloc_xrcdn;
+ }
+
+ xrcd->pd = ib_alloc_pd(ibdev, 0);
+
+ if (IS_ERR(xrcd->pd)) {
+ ret = (int)PTR_ERR(xrcd->pd);
+ ROCE_DEV_ERR(rdev, "Failed to alloc pd, ret(%d)", ret);
+ goto err_alloc_pd;
+ }
+
+ cq_attr.cqe = 1;
+ xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr);
+ if (IS_ERR(xrcd->cq)) {
+ ret = (int)PTR_ERR(xrcd->cq);
+ ROCE_DEV_ERR(rdev, "Failed to create cq, ret(%d)", ret);
+ goto err_create_cq;
+ }
+ return 0;
+
+err_create_cq:
+ ib_dealloc_pd(xrcd->pd);
+
+err_alloc_pd:
+ roce5_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn);
+
+err_alloc_xrcdn:
+ return ret;
+}
+
+int roce5_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
+{
+ struct roce5_device *rdev = NULL;
+ struct roce5_xrcd *xrcd = NULL;
+
+ if (ibxrcd == NULL) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ rdev = to_roce5_dev(ibxrcd->device);
+ xrcd = to_roce5_xrcd(ibxrcd);
+
+ return roce5_init_xrcd(ibxrcd->device, rdev, xrcd);
+}
+
+struct ib_xrcd *roce5_alloc_xrcd_ofa(struct ib_device *ibdev,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_device *rdev = NULL;
+ struct roce5_xrcd *xrcd = NULL;
+
+ if (ibdev == NULL) {
+ ROCE_ERR("Ibdev is null");
+ return ERR_PTR(-EINVAL);
+ }
+
+ rdev = to_roce5_dev(ibdev);
+
+ xrcd = (struct roce5_xrcd *)kzalloc(sizeof(*xrcd), GFP_KERNEL);
+ if (xrcd == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc memory for xrcd");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */
+ xrcd->ibxrcd.device = ibdev;
+
+ ret = roce5_alloc_xrcd(&xrcd->ibxrcd, NULL);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init xrcd, ret(%d)", ret);
+ kfree(xrcd);
+ return ERR_PTR(ret);
+ }
+ return &xrcd->ibxrcd;
+}
+
+int roce5_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
+{
+ struct roce5_device *rdev = NULL;
+ struct roce5_xrcd *xrcd = NULL;
+ if (ibxrcd == NULL) {
+ ROCE_ERR("Ibxrcd is null");
+ return -EINVAL;
+ }
+
+ rdev = to_roce5_dev(ibxrcd->device);
+ xrcd = to_roce5_xrcd(ibxrcd);
+ ib_destroy_cq(xrcd->cq);
+
+ ib_dealloc_pd(xrcd->pd);
+ roce5_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn);
+ return 0;
+}
+
+int roce5_dealloc_xrcd_ofa(struct ib_xrcd *ibxrcd)
+{
+ struct roce5_xrcd *xrcd = to_roce5_xrcd(ibxrcd);
+ int ret = roce5_dealloc_xrcd(ibxrcd, NULL);
+ if (ret == 0) {
+ kfree(xrcd);
+ }
+
+ return ret;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h
new file mode 100644
index 000000000..1a4e603ab
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h
@@ -0,0 +1,17 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_XRC_H
+#define ROCE_XRC_H
+
+#include <rdma/ib_verbs.h>
+
+struct roce5_xrcd {
+ struct ib_xrcd ibxrcd;
+ u32 xrcdn;
+ struct ib_pd *pd;
+ struct ib_cq *cq;
+};
+
+#endif //ROCE_XRC_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c
new file mode 100644
index 000000000..d67dd0844
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c
@@ -0,0 +1,162 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+#include <rdma/ib_user_verbs.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+
+#include "roce.h"
+#include "hinic5_lld.h"
+#include "roce_infra.h"
+#include "roce_card.h"
+
+static struct roce5_cards g_roce5_cards[MAX_DPU_CARD_NUM];
+
+void roce5_card_init(u16 idx, u64 card_id)
+{
+ g_roce5_cards[idx].card_status = RUNNING;
+ g_roce5_cards[idx].card_id = card_id;
+ INIT_LIST_HEAD(&g_roce5_cards[idx].rdev_list);
+ mutex_init(&g_roce5_cards[idx].card_mutex);
+}
+
+void roce5_card_deinit(u16 idx)
+{
+ g_roce5_cards[idx].card_status = INVALID;
+ g_roce5_cards[idx].card_id = 0;
+ mutex_destroy(&g_roce5_cards[idx].card_mutex);
+}
+
+int roce5_g_rdev_add(struct roce5_device *rdev)
+{
+ struct roce5_rdev_list_node *rdev_node = NULL;
+ u64 card_id;
+ u16 i;
+
+ if (hinic5_get_chip_node_id(rdev->lld_dev, &card_id) != 0) {
+ ROCE_ERR("Failed to get chip node id, func_id(%d)",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ rdev_node = kzalloc(sizeof(struct roce5_rdev_list_node), GFP_KERNEL);
+ if (rdev_node == NULL) {
+ ROCE_ERR("Failed to alloc memory.");
+ return -ENOMEM;
+ }
+ rdev_node->rdev = rdev;
+
+ for (i = 0; i < MAX_DPU_CARD_NUM; i++) {
+ if (g_roce5_cards[i].card_status == RUNNING &&
+ g_roce5_cards[i].card_id == card_id) {
+ break;
+ }
+ }
+
+ if (i >= MAX_DPU_CARD_NUM) {
+ for (i = 0; i < MAX_DPU_CARD_NUM; i++) {
+ if (g_roce5_cards[i].card_status == INVALID) {
+ roce5_card_init(i, card_id);
+ break;
+ }
+ }
+
+ if (i >= MAX_DPU_CARD_NUM) {
+ ROCE_ERR("Exceeded the MAX_DPU_CARD_NUM:0x%x",
+ MAX_DPU_CARD_NUM);
+ kfree(rdev_node);
+ return -ENOMEM;
+ }
+ }
+
+ mutex_lock(&g_roce5_cards[i].card_mutex);
+ list_add_tail(&rdev_node->node, &g_roce5_cards[i].rdev_list);
+ mutex_unlock(&g_roce5_cards[i].card_mutex);
+ return 0;
+}
+
+void roce5_g_rdev_remove(struct roce5_device *rdev)
+{
+ struct roce5_rdev_list_node *rdev_node = NULL, *rdev_next = NULL;
+ u64 card_id;
+ int i;
+
+ if (hinic5_get_chip_node_id(rdev->lld_dev, &card_id) != 0) {
+ ROCE_ERR("Failed to get chip node id, func_id(%d)",
+ rdev->glb_func_id);
+ return;
+ }
+
+ for (i = 0; i < MAX_DPU_CARD_NUM; i++) {
+ if (g_roce5_cards[i].card_status == RUNNING &&
+ g_roce5_cards[i].card_id == card_id) {
+ break;
+ }
+ }
+
+ if (i < MAX_DPU_CARD_NUM) {
+ mutex_lock(&g_roce5_cards[i].card_mutex);
+ list_for_each_entry_safe(rdev_node, rdev_next,
+ &g_roce5_cards[i].rdev_list, node) {
+ if (rdev_node->rdev == rdev) {
+ list_del(&rdev_node->node);
+ kfree(rdev_node);
+ break;
+ }
+ }
+ mutex_unlock(&g_roce5_cards[i].card_mutex);
+ /* deinit card if rdev list is empty */
+ if (list_empty(&g_roce5_cards[i].rdev_list)) {
+ roce5_card_deinit(i);
+ }
+ }
+}
+
+struct roce5_device *roce5_get_rdev_by_card_func_id(uint64_t card_id,
+ uint16_t func_id)
+{
+ struct roce5_rdev_list_node *rdev_node = NULL;
+ int i;
+
+ for (i = 0; i < MAX_DPU_CARD_NUM; i++) {
+ if (g_roce5_cards[i].card_status == RUNNING &&
+ g_roce5_cards[i].card_id == card_id) {
+ break;
+ }
+ }
+
+ if (i < MAX_DPU_CARD_NUM) {
+ list_for_each_entry(rdev_node, &g_roce5_cards[i].rdev_list,
+ node) {
+ if (rdev_node->rdev->glb_func_id == func_id) {
+ return rdev_node->rdev;
+ }
+ }
+ }
+ return NULL;
+}
+
+struct roce5_device *roce5_get_rdev_by_name(const char *dev_name)
+{
+ struct roce5_rdev_list_node *rdev_node = NULL;
+ int i;
+
+ if (dev_name == NULL) {
+ return NULL;
+ }
+
+ for (i = 0; i < MAX_DPU_CARD_NUM; i++) {
+ if (g_roce5_cards[i].card_status == INVALID) {
+ continue;
+ }
+
+ list_for_each_entry(rdev_node, &g_roce5_cards[i].rdev_list,
+ node) {
+ if (strncmp(rdev_node->rdev->ib_dev.name, dev_name,
+ IB_DEVICE_NAME_MAX) == 0) {
+ return rdev_node->rdev;
+ }
+ }
+ }
+ return NULL;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h
new file mode 100644
index 000000000..5ceabdd1f
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h
@@ -0,0 +1,27 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+#ifndef ROCE_CARD_H
+#define ROCE_CARD_H
+
+#include <linux/slab.h>
+#include <linux/list.h>
+
+#include "roce.h"
+
+#define MAX_DPU_CARD_NUM \
+ 1024 //current SDK PCIe on current cannot identifies VM on and bare metal on is same one
张 card, temporary add specification supports start multi VM
+struct roce5_rdev_list_node {
+ struct list_head node;
+ struct roce5_device *rdev;
+};
+
+typedef enum roce5_card_status { INVALID = 0, RUNNING } roce5_card_status_e;
+
+struct roce5_cards {
+ u64 card_id;
+ roce5_card_status_e card_status;
+ struct mutex card_mutex;
+ struct list_head rdev_list;
+};
+#endif /* ROCE_CARD_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c
new file mode 100644
index 000000000..2ad39f97f
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c
@@ -0,0 +1,1270 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/errno.h>
+#include <linux/if_ether.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+
+#include "rdma_comp.h"
+#include "roce5_gid_type.h"
+#include "roce_compat.h"
+#include "roce_com.h"
+#include "roce_cqm_cmd.h"
+#include "roce_mpu_cmd.h"
+#include "roce_ctx_format.h"
+#include "roce_infra.h"
+#include "roce_main_extension.h"
+#include "roce_mix.h"
+#include "roce_npu_cmd.h"
+#include "roce_npu_cmd_ext_defs.h"
+#include "roce_srq_extension.h"
+#include "roce_uld_inner.h"
+#include "roce_uld_kernel_api.h"
+#include "securec.h"
+#include "roce_cmd.h"
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+
+#define OVS_MPU_CMD_QOS_SET_VM_POLICING 30
+#define OVS_MPU_CMD_VPORT_SET 13
+#define OVS_MPU_VPORT_SET_CMD_ATTR 2
+
+uint8_t roce5_srq_mode_chip_adapt(uint8_t cfg_mode)
+{
+ switch (cfg_mode) {
+ case ROCE_CHIP_SRQ_MODE_1:
+ return ROCE_SRQ_MODE_3;
+ case ROCE_CHIP_SRQ_MODE_2:
+ return ROCE_SRQ_MODE_2;
+ case ROCE_CHIP_SRQ_MODE_3:
+ return ROCE_SRQ_MODE_1;
+ case ROCE_CHIP_SRQ_MODE_4:
+ return ROCE_SRQ_MODE_0;
+ default:
+ return ROCE_SRQ_MODE_3;
+ }
+}
+
+int roce5_add_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr,
+ size_t mac_addr_len, u32 vlan_id, u16 func_id)
+{
+ void *hwdev = rdev->hwdev;
+ u16 er_fwd_id = rdev->glb_func_id;
+ roce_cfg_mac_cmd_s add_mac_entry_info;
+ u16 out_size = (u16)sizeof(add_mac_entry_info);
+ int ret;
+
+ if (ROCE5_SUPPORT_SML_MAC_TABLE(rdev) == 0) {
+ return 0;
+ }
+
+ if (rdev->hca_vf_mode != 0) {
+ ROCE_INFO("No need to add mac table entry.");
+ return 0;
+ }
+
+ if ((hwdev == NULL) || (mac_addr == NULL)) {
+ ROCE_ERR("Hwdev or mac_addr is null");
+ return -EINVAL;
+ }
+
+ if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) {
+ ROCE_ERR("Invalid vlan_id(%u)", vlan_id);
+ return -EINVAL;
+ }
+
+ ret = memset_s(&add_mac_entry_info, sizeof(add_mac_entry_info), 0,
+ sizeof(add_mac_entry_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset add_mac_entry_info.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ add_mac_entry_info.func_id = func_id;
+ add_mac_entry_info.vlan_id = vlan_id;
+ add_mac_entry_info.er_fwd_id = er_fwd_id;
+ ret = memcpy_s(add_mac_entry_info.mac, ETH_ALEN, mac_addr, ETH_ALEN);
+ if (ret != 0) {
+ ROCE_ERR("memory copy failed. ret%d", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_MAC,
+ &add_mac_entry_info,
+ sizeof(add_mac_entry_info),
+ &add_mac_entry_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (add_mac_entry_info.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to add mac entry, err(%d), status(0x%x), out size(0x%x)",
+ ret, add_mac_entry_info.head.status, out_size);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int roce5_del_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr,
+ size_t mac_addr_len, u32 vlan_id, u16 func_id)
+{
+ void *hwdev = rdev->hwdev;
+ u16 er_fwd_id = rdev->glb_func_id;
+ roce_cfg_mac_cmd_s del_mac_tbl_info;
+ u16 out_size = (u16)sizeof(del_mac_tbl_info);
+ int ret;
+
+ if (ROCE5_SUPPORT_SML_MAC_TABLE(rdev) == 0) {
+ return 0;
+ }
+
+ if (rdev->hca_vf_mode != 0) {
+ ROCE_DEV_INFO(rdev, "No need to delete mac table entry.");
+ return 0;
+ }
+
+ if ((hwdev == NULL) || (mac_addr == NULL)) {
+ ROCE_ERR("Hwdev or mac_addr is null");
+ return -EINVAL;
+ }
+
+ if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) {
+ ROCE_ERR("Invalid vlan_id(%u)", vlan_id);
+ return -EINVAL;
+ }
+
+ ret = memset_s(&del_mac_tbl_info, sizeof(del_mac_tbl_info), 0,
+ sizeof(del_mac_tbl_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset del_mac_tbl_info.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ del_mac_tbl_info.func_id = func_id;
+ del_mac_tbl_info.vlan_id = vlan_id;
+ del_mac_tbl_info.er_fwd_id = er_fwd_id;
+ ret = memcpy_s(del_mac_tbl_info.mac, mac_addr_len, mac_addr,
+ mac_addr_len);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy_s dmac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_MAC,
+ &del_mac_tbl_info,
+ sizeof(del_mac_tbl_info),
+ &del_mac_tbl_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (del_mac_tbl_info.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)",
+ ret, del_mac_tbl_info.head.status, out_size);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void roce5_add_ipsu_tbl_mac_entry(struct roce5_device *rdev,
+ u8 mac_addr[ETH_ALEN], u32 vlan_id,
+ u16 func_id, u8 er_id, u16 src_func_id)
+{
+ void *hwdev = rdev->hwdev;
+ roce_cfg_ipsu_mac_cmd_s ipsu_add_mac_entry_info;
+ u16 out_size = (u16)sizeof(ipsu_add_mac_entry_info);
+ int ret;
+
+ if (rdev->hca_vf_mode != 0) {
+ ROCE_INFO("No need to add ipsu tbl mac entry");
+ return;
+ }
+
+#ifdef ROCE_BONDING_EN
+ if (!roce5_get_bond_ipsurx_en()) {
+ ROCE_ERR("No need to add ipsu tbl mac entry");
+ return;
+ }
+#endif
+
+ if ((hwdev == NULL) || (mac_addr == NULL)) {
+ ROCE_ERR("Hwdev or mac_addr is null");
+ return;
+ }
+
+ if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) {
+ ROCE_ERR("Invalid vlan_id(%u)", vlan_id);
+ }
+
+ ret = memset_s(&ipsu_add_mac_entry_info,
+ sizeof(ipsu_add_mac_entry_info), 0,
+ sizeof(ipsu_add_mac_entry_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ipsu_add_mac_entry_info.(ret:%d)",
+ ret);
+ return;
+ }
+ ipsu_add_mac_entry_info.func_id = func_id;
+ ipsu_add_mac_entry_info.er_id = er_id;
+ ipsu_add_mac_entry_info.src_func_id = src_func_id;
+ ipsu_add_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK;
+ ipsu_add_mac_entry_info.vni_en = 0;
+ ret = memcpy_s(ipsu_add_mac_entry_info.mac, ETH_ALEN, mac_addr,
+ ETH_ALEN);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to memcpy ipsu_add_mac_entry_info.mac (ret:%d)",
+ ret);
+ return;
+ }
+
+ ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_IPSU_MAC,
+ &ipsu_add_mac_entry_info,
+ sizeof(ipsu_add_mac_entry_info),
+ &ipsu_add_mac_entry_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ ((ipsu_add_mac_entry_info.head.status != 0) &&
+ (ipsu_add_mac_entry_info.head.status != ERR_EXIST))) {
+ ROCE_ERR(
+ "Failed to add mac entry to IPSU, err(%d), status(0x%x), out size(0x%x)",
+ ret, ipsu_add_mac_entry_info.head.status, out_size);
+ return;
+ }
+
+ if (ipsu_add_mac_entry_info.head.status == ERR_EXIST) {
+ ROCE_INFO("This mac entry has been added to IPSU");
+ }
+
+ return;
+}
+
+static int32_t roce5_del_mac_entry_precheck(struct roce5_device *rdev,
+ void *hwdev, uint8_t *mac_addr,
+ uint32_t vlan_id)
+{
+ if (rdev->hca_vf_mode != 0) {
+ ROCE_DEV_INFO(rdev, "No need to del ipsu tbl mac entry.");
+ return -EINVAL;
+ }
+
+ if ((hwdev == NULL) || (mac_addr == NULL)) {
+ ROCE_DEV_ERR(rdev, "hwdev or mac_addr is null.");
+ return -EINVAL;
+ }
+
+#ifdef ROCE_BONDING_EN
+ if (!roce5_get_bond_ipsurx_en()) {
+ ROCE_DEV_INFO(rdev, "No need to del ipsu tbl mac entry.");
+ return -EINVAL;
+ }
+#endif
+
+ if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) {
+ ROCE_DEV_ERR(rdev, "Invalid vlan_id(%d).", vlan_id);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void roce5_del_ipsu_tbl_mac_entry(struct roce5_device *rdev,
+ u8 mac_addr[ETH_ALEN], size_t eth_alen,
+ u32 vlan_id, u8 er_id, u16 src_func_id)
+{
+ void *hwdev = rdev->hwdev;
+ u16 func_id = rdev->glb_func_id;
+ roce_cfg_ipsu_mac_cmd_s ipsu_del_mac_entry_info;
+ u16 out_size = (u16)sizeof(ipsu_del_mac_entry_info);
+ int ret;
+
+ ret = roce5_del_mac_entry_precheck(rdev, hwdev, mac_addr, vlan_id);
+ if (ret != 0) {
+ return;
+ }
+
+ ret = memset_s(&ipsu_del_mac_entry_info,
+ sizeof(ipsu_del_mac_entry_info), 0,
+ sizeof(ipsu_del_mac_entry_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ipsu_del_mac_entry_info.(ret:%d)",
+ ret);
+ return;
+ }
+ ipsu_del_mac_entry_info.func_id = func_id;
+ ipsu_del_mac_entry_info.er_id = er_id;
+ ipsu_del_mac_entry_info.src_func_id = src_func_id;
+ ipsu_del_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK;
+ ipsu_del_mac_entry_info.vni_en = 0;
+ ret = memcpy_s(ipsu_del_mac_entry_info.mac, eth_alen, mac_addr,
+ eth_alen);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy_s dmac.(ret:%d)", ret);
+ return;
+ }
+
+ ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_IPSU_MAC,
+ &ipsu_del_mac_entry_info,
+ sizeof(ipsu_del_mac_entry_info),
+ &ipsu_del_mac_entry_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ ((ipsu_del_mac_entry_info.head.status != 0) &&
+ (ipsu_del_mac_entry_info.head.status != ERR_NOT_FOUND))) {
+ ROCE_ERR(
+ "Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)",
+ ret, ipsu_del_mac_entry_info.head.status, out_size);
+ return;
+ }
+
+ if (ipsu_del_mac_entry_info.head.status == ERR_NOT_FOUND) {
+ ROCE_INFO("This mac entry of ipsu has been deleted");
+ }
+
+ return;
+}
+
+int roce5_add_ipsu_tbl_ip_entry(struct roce5_device *rdev,
+ struct roce_cmd_mpu_addr_info *addr_info)
+{
+ roce_cfg_ipsu_ip_cmd_s ipsu_add_ip_entry_info;
+ u16 out_size = sizeof(ipsu_add_ip_entry_info);
+ union roce_addr_info *ip = NULL;
+ int ret;
+
+ if ((rdev == NULL) || (rdev->hwdev == NULL) || (addr_info == NULL)) {
+ ROCE_ERR("rdev, hwdev or addr_info is null");
+ return -EINVAL;
+ }
+
+ ip = &addr_info->ip_addr;
+
+ ret = memset_s(&ipsu_add_ip_entry_info, sizeof(ipsu_add_ip_entry_info),
+ 0, sizeof(ipsu_add_ip_entry_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ipsu_add_ip_entry_info.(ret:%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ ipsu_add_ip_entry_info.func_id = rdev->glb_func_id;
+ ipsu_add_ip_entry_info.ip_ver = addr_info->ip_ver;
+ ipsu_add_ip_entry_info.port_id = hinic5_physical_port_id(rdev->hwdev);
+ ret = memcpy_s(&ipsu_add_ip_entry_info.ip, sizeof(union roce_addr_info),
+ ip, sizeof(union roce_addr_info));
+ if (ret) {
+ ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ ROCE_INFO("funcid=%u,ipver=%u,portid=%u,ip=0x%x-%x-%x-%x",
+ ipsu_add_ip_entry_info.func_id, ipsu_add_ip_entry_info.ip_ver,
+ ipsu_add_ip_entry_info.port_id, ip->dw[0x0], ip->dw[0x1],
+ ip->dw[0x2], ip->dw[0x3]);
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_ADD_IPSU_IP,
+ &ipsu_add_ip_entry_info,
+ sizeof(ipsu_add_ip_entry_info),
+ &ipsu_add_ip_entry_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ ((ipsu_add_ip_entry_info.head.status != 0) &&
+ (ipsu_add_ip_entry_info.head.status != ERR_EXIST))) {
+ ROCE_ERR(
+ "Failed to add ip tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x)",
+ ret, ipsu_add_ip_entry_info.head.status, out_size);
+ return -EINVAL;
+ }
+
+ if (ipsu_add_ip_entry_info.head.status == ERR_EXIST) {
+ ROCE_INFO("This ip tbl entry has been added to IPSU");
+ }
+
+ return 0;
+}
+
+int roce5_del_ipsu_tbl_ip_entry(struct roce5_device *rdev,
+ struct roce_cmd_mpu_addr_info *addr_info)
+{
+ roce_cfg_ipsu_ip_cmd_s ipsu_del_ip_entry_info;
+ u16 out_size = sizeof(ipsu_del_ip_entry_info);
+ union roce_addr_info *ip = NULL;
+ int ret;
+
+ if ((rdev == NULL) || (rdev->hwdev == NULL) || (addr_info == NULL)) {
+ ROCE_ERR("rdev, hwdev or addr_info is null");
+ return -EINVAL;
+ }
+
+ ip = &addr_info->ip_addr;
+
+ ret = memset_s(&ipsu_del_ip_entry_info, sizeof(ipsu_del_ip_entry_info),
+ 0, sizeof(ipsu_del_ip_entry_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ipsu_add_ip_entry_info.(ret:%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ ipsu_del_ip_entry_info.func_id = rdev->glb_func_id;
+ ipsu_del_ip_entry_info.ip_ver = addr_info->ip_ver;
+ ipsu_del_ip_entry_info.port_id = hinic5_physical_port_id(rdev->hwdev);
+ ret = memcpy_s(&ipsu_del_ip_entry_info.ip, sizeof(union roce_addr_info),
+ ip, sizeof(union roce_addr_info));
+ if (ret) {
+ ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ ROCE_INFO("funcid=%u,ipver=%u,portid=%u,ip=0x%x-%x-%x-%x",
+ ipsu_del_ip_entry_info.func_id, ipsu_del_ip_entry_info.ip_ver,
+ ipsu_del_ip_entry_info.port_id, ip->dw[0x0], ip->dw[0x1],
+ ip->dw[0x2], ip->dw[0x3]);
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_IPSU_IP,
+ &ipsu_del_ip_entry_info,
+ sizeof(ipsu_del_ip_entry_info),
+ &ipsu_del_ip_entry_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ ((ipsu_del_ip_entry_info.head.status != 0) &&
+ (ipsu_del_ip_entry_info.head.status != ERR_NOT_FOUND))) {
+ ROCE_ERR(
+ "Failed to add ip tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x)",
+ ret, ipsu_del_ip_entry_info.head.status, out_size);
+ return -EINVAL;
+ }
+
+ if (ipsu_del_ip_entry_info.head.status == ERR_NOT_FOUND) {
+ ROCE_INFO("This ip tbl entry of ipsu has been deleted");
+ }
+
+ return 0;
+}
+
+int roce5_modify_ipsu_tbl_ip_entry(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry, u8 type)
+{
+ struct roce_cmd_mpu_addr_info addr_info = { 0 };
+ union roce_addr_info *ip = NULL;
+ int ret;
+
+ if ((rdev == NULL) || (rdev->hwdev == NULL) || (gid_entry == NULL)) {
+ ROCE_ERR("rdev, hwdev or gid_entry is null");
+ return -EINVAL;
+ }
+
+ ip = (union roce_addr_info *)&gid_entry->raw;
+
+ if (rdev->hca_vf_mode == 0) {
+ ROCE_INFO("No need to modify ipsu tbl ip entry");
+ return 0;
+ }
+
+ if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) {
+ ROCE_ERR("gid_type:%d is unsupported",
+ gid_entry->dw6_h.bs.gid_type);
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(&addr_info.ip_addr, sizeof(union roce_addr_info), ip,
+ sizeof(union roce_addr_info));
+ if (ret) {
+ ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ if (gid_entry->dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) {
+ addr_info.ip_addr.dw[0x0] = 0x0;
+ addr_info.ip_addr.dw[0x1] = 0x0;
+ addr_info.ip_addr.dw[0x2] = 0x0;
+ addr_info.ip_ver = 0;
+ } else {
+ addr_info.ip_ver = 1;
+ }
+
+ if (type == ROCE_MPU_CMD_ADD_IPSU_IP) {
+ return roce5_add_ipsu_tbl_ip_entry(rdev, &addr_info);
+ }
+
+ return roce5_del_ipsu_tbl_ip_entry(rdev, &addr_info);
+}
+
+static int32_t
+roce5_add_ipsu_tcam_tbl_entry_precheck(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry)
+{
+ if ((rdev->hwdev == NULL) || (gid_entry == NULL)) {
+ ROCE_DEV_ERR(rdev, "hwdev or gid_entry is null.");
+ return -EINVAL;
+ }
+
+ if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) {
+ ROCE_DEV_ERR(rdev, "gid_type:%d is unsupported.",
+ gid_entry->dw6_h.bs.gid_type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int32_t roce5_ipsu_tcam_tbl_set_entry(
+ struct roce5_device *rdev, struct rdma_gid_entry *gid_entry,
+ uint32_t gid_index, roce_cfg_ipsu_tcam_cmd_s *ipsu_add_tcam_entry_info)
+{
+ union roce_addr_info *ip = (union roce_addr_info *)&gid_entry->raw;
+ int32_t i;
+ int32_t ret;
+
+ ipsu_add_tcam_entry_info->func_id = rdev->glb_func_id;
+ ipsu_add_tcam_entry_info->ip_ver = gid_entry->dw6_h.bs.gid_type;
+ ipsu_add_tcam_entry_info->gid_idx = gid_index;
+ ret = memcpy_s(&ipsu_add_tcam_entry_info->ip,
+ sizeof(union roce_addr_info), ip,
+ sizeof(union roce_addr_info));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy ip info.(ret:%d).", ret);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < 0x4; i++) {
+ ipsu_add_tcam_entry_info->ip.dw[i] =
+ htonl(ipsu_add_tcam_entry_info->ip.dw[i]);
+ }
+
+ if (gid_entry->dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) {
+ ipsu_add_tcam_entry_info->ip.dw[0x0] = 0x0;
+ ipsu_add_tcam_entry_info->ip.dw[0x1] = 0x0;
+ ipsu_add_tcam_entry_info->ip.dw[0x2] = 0x0;
+ }
+
+ return 0;
+}
+
+void roce5_add_ipsu_tcam_tbl_entry(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry,
+ u32 gid_index)
+{
+ roce_cfg_ipsu_tcam_cmd_s ipsu_add_tcam_entry_info;
+ u16 out_size = sizeof(roce_cfg_ipsu_tcam_cmd_s);
+ int ret;
+
+ ret = roce5_add_ipsu_tcam_tbl_entry_precheck(rdev, gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to precheck params(ret:%d).", ret);
+ return;
+ }
+
+ ret = memset_s(&ipsu_add_tcam_entry_info,
+ sizeof(ipsu_add_tcam_entry_info), 0,
+ sizeof(ipsu_add_tcam_entry_info));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to memset ipsu_tcam_entry_info(ret:%d).",
+ ret);
+ return;
+ }
+
+ ret = roce5_ipsu_tcam_tbl_set_entry(rdev, gid_entry, gid_index,
+ &ipsu_add_tcam_entry_info);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set ipsu_tcam_entry_info(ret:%d).",
+ ret);
+ return;
+ }
+
+ ROCE_DEV_INFO(rdev, "funcid=%u,ipver=%u,gid_idx=%u,ip=0x%x-%x-%x-%x.",
+ ipsu_add_tcam_entry_info.func_id,
+ ipsu_add_tcam_entry_info.ip_ver,
+ ipsu_add_tcam_entry_info.gid_idx,
+ ipsu_add_tcam_entry_info.ip.dw[0x0],
+ ipsu_add_tcam_entry_info.ip.dw[0x1],
+ ipsu_add_tcam_entry_info.ip.dw[0x2],
+ ipsu_add_tcam_entry_info.ip.dw[0x3]);
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_ADD_IPSU_TCAM,
+ &ipsu_add_tcam_entry_info,
+ sizeof(ipsu_add_tcam_entry_info),
+ &ipsu_add_tcam_entry_info, &out_size);
+ if (ipsu_add_tcam_entry_info.head.status == ERR_FULL) {
+ ROCE_DEV_INFO(rdev, "TCAM Table is full.");
+ } else if ((ret == 0) && (out_size != 0) &&
+ (ipsu_add_tcam_entry_info.head.status == ERR_EXIST)) {
+ ROCE_DEV_INFO(rdev, "This tcam entry has been added to IPSU.");
+ } else if ((ret != 0) || (out_size == 0) ||
+ ((ipsu_add_tcam_entry_info.head.status != 0))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to add tcam tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x).",
+ ret, ipsu_add_tcam_entry_info.head.status, out_size);
+ return;
+ }
+
+ return;
+}
+
+static int32_t
+roce5_del_ipsu_tcam_tbl_entry_precheck(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry)
+{
+ if ((rdev->hwdev == NULL) || (gid_entry == NULL)) {
+ ROCE_DEV_ERR(rdev, "hwdev or gid_entry is null.");
+ return -EINVAL;
+ }
+
+ if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) {
+ ROCE_DEV_ERR(rdev, "gid_type:%d is unsupported",
+ gid_entry->dw6_h.bs.gid_type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void roce5_del_ipsu_tcam_tbl_entry(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry)
+{
+ roce_cfg_ipsu_tcam_cmd_s ipsu_del_tcam_entry_info;
+ u16 out_size = sizeof(roce_cfg_ipsu_tcam_cmd_s);
+ int ret;
+
+ ret = roce5_del_ipsu_tcam_tbl_entry_precheck(rdev, gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to precheck params(ret:%d).", ret);
+ return;
+ }
+
+ ret = memset_s(&ipsu_del_tcam_entry_info,
+ sizeof(ipsu_del_tcam_entry_info), 0,
+ sizeof(ipsu_del_tcam_entry_info));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to memset ipsu_tcam_entry_info(ret:%d).",
+ ret);
+ return;
+ }
+
+ ret = roce5_ipsu_tcam_tbl_set_entry(rdev, gid_entry, 0,
+ &ipsu_del_tcam_entry_info);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set ipsu_tcam_entry_info(ret:%d).",
+ ret);
+ return;
+ }
+
+ ROCE_DEV_INFO(rdev, "funcid=%u,ipver=%u,gid_idx=%u,ip=0x%x-%x-%x-%x",
+ ipsu_del_tcam_entry_info.func_id,
+ ipsu_del_tcam_entry_info.ip_ver,
+ ipsu_del_tcam_entry_info.gid_idx,
+ ipsu_del_tcam_entry_info.ip.dw[0x0],
+ ipsu_del_tcam_entry_info.ip.dw[0x1],
+ ipsu_del_tcam_entry_info.ip.dw[0x2],
+ ipsu_del_tcam_entry_info.ip.dw[0x3]);
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_IPSU_TCAM,
+ &ipsu_del_tcam_entry_info,
+ sizeof(ipsu_del_tcam_entry_info),
+ &ipsu_del_tcam_entry_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ ((ipsu_del_tcam_entry_info.head.status != 0) &&
+ (ipsu_del_tcam_entry_info.head.status != ERR_NOT_FOUND))) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to del tcam entry, err(%d), status(0x%x), out size(0x%x).",
+ ret, ipsu_del_tcam_entry_info.head.status, out_size);
+ return;
+ }
+
+ if (ipsu_del_tcam_entry_info.head.status == ERR_NOT_FOUND) {
+ ROCE_DEV_INFO(rdev,
+ "This tcam entry of ipsu has been deleted.");
+ }
+ return;
+}
+
+int roce5_set_sysfs_cfg_param(void *hwdev, u16 func_id, u32 *param,
+ u32 param_len, u16 type)
+{
+ roce_cc_cfg_param_cmd_s cfg_param;
+ u16 out_size = (u16)sizeof(cfg_param);
+ int ret;
+ int param_idx = 0;
+ int param_dw_num;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return -EINVAL;
+ }
+
+ param_dw_num = (int)(param_len / sizeof(u32));
+ if (param_dw_num > ROCE_CC_CFG_PARAM_MAX_DW_NUM) {
+ ROCE_ERR("cfg_param len :%d dw_num:%d exceed max, type:0x%x",
+ param_len, param_dw_num, type);
+ return -EINVAL;
+ }
+
+ ret = memset_s(&cfg_param, sizeof(cfg_param), 0, sizeof(cfg_param));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset cfg_param.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ cfg_param.func_id = func_id;
+ for (; param_idx < param_dw_num; param_idx++) {
+ cfg_param.param[param_idx] = param[param_idx];
+ }
+
+ ret = roce5_msg_to_mgmt_sync(hwdev, type, &cfg_param, sizeof(cfg_param),
+ &cfg_param, &out_size);
+ if ((ret != 0) || (out_size == 0) || (cfg_param.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to cfg param, err(%d), status(0x%x), out size(0x%x), type(0x%x)",
+ ret, cfg_param.head.status, out_size, type);
+
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int roce5_set_func_tbl_func_state(void *hwdev, u8 func_state, u16 func_id)
+{
+ roce_set_func_state_cmd_s set_func_state_info;
+ u16 out_size = (u16)sizeof(set_func_state_info);
+ int ret;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&set_func_state_info, sizeof(set_func_state_info), 0,
+ sizeof(set_func_state_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset set_func_state_info.(ret:%d)", ret);
+ return (-ENOMEM);
+ }
+ set_func_state_info.func_id = func_id;
+ set_func_state_info.func_en = func_state;
+ ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_FUNC_STATE,
+ &set_func_state_info,
+ sizeof(set_func_state_info),
+ &set_func_state_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (set_func_state_info.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set func state(%d), err(%d), status(0x%x), out size(0x%x), func_id(%d)",
+ func_state, ret, set_func_state_info.head.status,
+ out_size, func_id);
+ return (-EINVAL);
+ }
+
+ return 0;
+}
+
+int roce5_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value)
+{
+ roce_get_func_table_cmd_s get_func_table;
+ roce_get_func_table_rsp_s get_func_table_rsp;
+ u16 out_size = (u16)sizeof(get_func_table_rsp);
+ int ret;
+
+ if (hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&get_func_table, sizeof(get_func_table), 0,
+ sizeof(get_func_table));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset get_func_table.(ret:%d)", ret);
+ return (-ENOMEM);
+ }
+ ret = memset_s(&get_func_table_rsp, out_size, 0, out_size);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset get_func_table_rsp.(ret:%d)", ret);
+ return (-ENOMEM);
+ }
+ get_func_table.func_id = func_id;
+ ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_FUNC_TABLE,
+ &get_func_table, sizeof(get_func_table),
+ &get_func_table_rsp, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (get_func_table_rsp.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to get func table, err(%d), status(0x%x), out size(0x%x), func_id(%d)",
+ ret, get_func_table_rsp.head.status, out_size, func_id);
+ return (-EINVAL);
+ }
+
+ *func_tbl_value = get_func_table_rsp.func_tbl_value;
+
+ return 0;
+}
+
+int roce5_set_func_tbl_cpu_endian(struct roce5_device *rdev)
+{
+ roce_set_cpu_endian_cmd_s set_cpu_endian_info;
+ u16 out_size = (u16)sizeof(set_cpu_endian_info);
+ int ret;
+
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return -EINVAL;
+ }
+
+ ret = memset_s(&set_cpu_endian_info, sizeof(set_cpu_endian_info), 0,
+ sizeof(set_cpu_endian_info));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to memset set_cpu_endian_info.(ret:%d).",
+ ret);
+ return -ENOMEM;
+ }
+ set_cpu_endian_info.func_id = rdev->glb_func_id;
+ set_cpu_endian_info.cpu_endian = rdev->hw_info.cpu_endian;
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_SET_CPU_ENDIAN,
+ &set_cpu_endian_info,
+ sizeof(set_cpu_endian_info),
+ &set_cpu_endian_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (set_cpu_endian_info.head.status != 0)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set func state, err(%d), status(0x%x), out size(0x%x).",
+ ret, set_cpu_endian_info.head.status, out_size);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void roce5_init_cfg_info_parse_resp(struct roce5_device *rdev,
+ roce_get_cfg_info_resp_s *resp)
+{
+ rdev->cfg_info.scence_id = resp->scence_id;
+ rdev->cfg_info.lb_en = resp->lb_en;
+ rdev->cfg_info.lb_mode = resp->lb_mode;
+ rdev->cfg_info.srq_container_en =
+ (resp->container_mode != ROCE_CHIP_SRQ_MODE_N);
+ rdev->cfg_info.srq_container_mode =
+ roce5_srq_mode_chip_adapt(resp->container_mode);
+ rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3;
+ rdev->cfg_info.warn_th = 0;
+
+ rdev->cfg_info.fake_en = resp->fake_en;
+ rdev->cfg_info.pf_start_bit = resp->pf_start_bit;
+ rdev->cfg_info.pf_end_bit = resp->pf_end_bit;
+ rdev->cfg_info.page_bit = resp->page_bit;
+
+ rdev->cfg_info.port_num = resp->port_num;
+ rdev->cfg_info.host_num = resp->host_num;
+ rdev->cfg_info.pcie_atomic_en = resp->pcie_atomic_en;
+
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes = resp->max_wqe;
+ rdev->rdma_cap.max_cqes = resp->max_cqe;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes =
+ resp->max_srq_wqe;
+ rdev->rdma_cap.max_sq_sg = resp->max_sq_sge;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg = resp->max_rq_sge;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge =
+ resp->max_srq_sge;
+ rdev->rdma_cap.max_sq_desc_sz = resp->max_sq_wqe_len;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_desc_sz =
+ resp->max_rq_wqe_len;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz =
+ resp->max_sq_inline;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_pd = resp->max_pd;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_xrcd = resp->max_xrcd;
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_gid = resp->max_gid;
+
+ rdev->hca_vf_mode = resp->ip2func;
+
+ roce5_cfg_info_update_ext(rdev, resp);
+}
+
+int roce5_get_feature(struct roce5_device *rdev,
+ roce_get_chip_ability_s *roce_feature)
+{
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_verbs_cmd_header_s *cmd_hdr = NULL;
+ roce_get_chip_ability_s *resp_buf = NULL;
+ int ret = 0;
+
+ ret = memset_s(roce_feature, sizeof(*roce_feature), 0,
+ sizeof(*roce_feature));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset roce_feature, ret(%d).", ret);
+ return (-ENOMEM);
+ }
+
+ // allocate input and output buffer
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf,
+ sizeof(roce_verbs_cmd_header_s),
+ &cqm_cmd_outbuf,
+ sizeof(roce_get_chip_ability_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to allocate cqm buffers, ret = %d.",
+ ret);
+ return ret;
+ }
+
+ // fill input command
+ cmd_hdr = (roce_verbs_cmd_header_s *)cqm_cmd_inbuf->buf;
+ cmd_hdr->dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_COMM_BITMASK);
+
+ // send command and wait response
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_GET_CHIP_ABILITY, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_C,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send feature cmd, ret = %d.",
+ ret);
+ goto free_buf;
+ }
+
+ // get response data
+ resp_buf = (roce_get_chip_ability_s *)cqm_cmd_outbuf->buf;
+ roce_feature->roce_kernel_ability =
+ be64_to_cpu(resp_buf->roce_kernel_ability);
+ roce_feature->roce_kernel_ext_ability =
+ be64_to_cpu(resp_buf->roce_kernel_ext_ability);
+ roce_feature->roce_user_ext_ability =
+ be64_to_cpu(resp_buf->roce_user_ext_ability);
+
+ // get on layer service supported attribute
+ roce_feature->uld_feature.roce_uld_kernel_ext_ability =
+ be64_to_cpu(resp_buf->uld_feature.roce_uld_kernel_ext_ability);
+ roce_feature->uld_feature.roce_uld_user_ext_ability =
+ be64_to_cpu(resp_buf->uld_feature.roce_uld_user_ext_ability);
+free_buf:
+ // free buffer
+ if (cqm_cmd_inbuf || cqm_cmd_outbuf) {
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+ }
+ return ret;
+}
+
+int roce5_uni_cmd_init_cfg_info(struct roce5_device *rdev)
+{
+ roce_get_cfg_info_cmd_s get_cfg_info;
+ roce_get_cfg_info_resp_s resp;
+ u16 out_size = (u16)sizeof(resp);
+ int ret = 0;
+
+ ret = memset_s(&get_cfg_info, sizeof(get_cfg_info), 0,
+ sizeof(get_cfg_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset get_cfg_info, ret(%d).", ret);
+ return (-ENOMEM);
+ }
+ ret = memset_s(&resp, sizeof(resp), 0, sizeof(resp));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset resp, ret(%d).", ret);
+ return (-ENOMEM);
+ }
+
+ get_cfg_info.func_id = rdev->glb_func_id;
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO,
+ &get_cfg_info, sizeof(get_cfg_info), &resp,
+ &out_size);
+ if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) {
+ ROCE_DEV_ERR(rdev,
+ "Failed, err(%d), status(0x%x), out size(0x%x)",
+ ret, get_cfg_info.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ roce5_init_cfg_info_parse_resp(rdev, &resp);
+
+ ROCE_DEV_INFO(
+ rdev,
+ "init, func_id(%d), lb_en(%d), lb_mode(%d), srq_mode(%d), scence_id(%d), pcie_atomic_en(%d).",
+ rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode,
+ rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id,
+ rdev->cfg_info.pcie_atomic_en);
+ ROCE_DEV_INFO(
+ rdev,
+ "max_wqes(0x%x), max_cqes(0x%x), max_srq_wqes(0x%x), max_sq_sg(0x%x), max_rq_sg(0x%x), "
+ "max_srq_sge(0x%x), max_sq_desc_sz(0x%x), max_rq_desc_sz(0x%x), max_sq_inline_data_sz(0x%x).",
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes,
+ rdev->rdma_cap.max_cqes,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes,
+ rdev->rdma_cap.max_sq_sg,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge,
+ rdev->rdma_cap.max_sq_desc_sz,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_desc_sz,
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz);
+
+ return 0;
+}
+
+int roce5_init_cfg_info(struct roce5_device *rdev)
+{
+ roce_get_cfg_info_cmd_s get_cfg_info;
+ roce_get_cfg_info_comp_resp_s resp;
+ u16 out_size = (u16)sizeof(resp);
+ int ret = 0;
+
+ if (rdev == NULL || rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ return roce5_uni_cmd_init_cfg_info(rdev);
+ }
+
+ (void)memset_s(&get_cfg_info, sizeof(get_cfg_info), 0,
+ sizeof(get_cfg_info));
+ (void)memset_s(&resp, sizeof(resp), 0, sizeof(resp));
+
+ get_cfg_info.func_id = rdev->glb_func_id;
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO,
+ &get_cfg_info, sizeof(get_cfg_info), &resp,
+ &out_size);
+ if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) {
+ ROCE_DEV_ERR(rdev,
+ "Failed, err(%d), status(0x%x), out size(0x%x)",
+ ret, get_cfg_info.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ rdev->cfg_info.scence_id = resp.scence_id;
+ rdev->cfg_info.lb_en = resp.lb_en;
+ rdev->cfg_info.lb_mode = resp.lb_mode;
+ rdev->cfg_info.srq_container_en =
+ (resp.container_mode != ROCE_CHIP_SRQ_MODE_N);
+ rdev->cfg_info.srq_container_mode =
+ roce5_srq_mode_chip_adapt(resp.container_mode);
+ rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3;
+ rdev->cfg_info.warn_th = 0;
+
+ rdev->cfg_info.fake_en = resp.fake_en;
+ rdev->cfg_info.pf_start_bit = resp.pf_start_bit;
+ rdev->cfg_info.pf_end_bit = resp.pf_end_bit;
+ rdev->cfg_info.page_bit = resp.page_bit;
+
+ rdev->cfg_info.port_num = resp.port_num;
+ rdev->cfg_info.host_num = resp.host_num;
+ rdev->cfg_info.pcie_atomic_en = resp.pcie_atomic_en;
+
+ rdev->hca_vf_mode = 0; // V100 fixed to mac mode
+
+ ROCE_DEV_INFO(
+ rdev,
+ "init, func_id(%d), lb_en(%d), lb_mode(%d), srq_mode(%d), scence_id(%d), pcie_atomic_en(%d).",
+ rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode,
+ rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id,
+ rdev->cfg_info.pcie_atomic_en);
+ return 0;
+}
+
+int roce5_del_func_res(struct roce5_device *rdev)
+{
+ roce_set_func_state_cmd_s del_func_res_info;
+ u16 out_size = (u16)sizeof(del_func_res_info);
+ int ret;
+
+ if ((rdev == NULL) || (rdev->hwdev == NULL)) {
+ ROCE_ERR("rdev/Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&del_func_res_info, sizeof(del_func_res_info), 0,
+ sizeof(del_func_res_info));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset del_func_res_info.(ret:%d)", ret);
+ return (-ENOMEM);
+ }
+ del_func_res_info.func_id = rdev->glb_func_id;
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_FUNC_RES,
+ &del_func_res_info,
+ sizeof(del_func_res_info),
+ &del_func_res_info, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (del_func_res_info.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set func state, err(%d), status(0x%x), out size(0x%x)",
+ ret, del_func_res_info.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ return 0;
+}
+
+int roce5_set_bw_ctrl_state(struct roce5_device *rdev, u8 cmd,
+ struct roce5_bw_ctrl_inbuf *inbuf)
+{
+ int ret = 0;
+ roce_cc_cfg_bw_ctrl_cmd_s set_bw_ctrl;
+ u16 out_size = (u16)sizeof(set_bw_ctrl);
+
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&set_bw_ctrl, sizeof(set_bw_ctrl), 0,
+ sizeof(set_bw_ctrl));
+ if (ret != 0) {
+ ROCE_ERR("Memset failed when set_bw_ctrl cmd");
+ return (-ENOMEM);
+ }
+
+ set_bw_ctrl.func_id = rdev->glb_func_id;
+ set_bw_ctrl.cmd = cmd;
+ set_bw_ctrl.cir = inbuf->ctrl_param.cir;
+ set_bw_ctrl.pir = inbuf->ctrl_param.pir;
+ set_bw_ctrl.cnp = inbuf->ctrl_param.cnp;
+ ROCE_INFO("func_id:%u, cmd:%u, cir:%u, pir:%u, cnp:%u",
+ rdev->glb_func_id, set_bw_ctrl.cmd, set_bw_ctrl.cir,
+ set_bw_ctrl.pir, set_bw_ctrl.cnp);
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_SET_BW_CTRL,
+ &set_bw_ctrl, sizeof(set_bw_ctrl),
+ &set_bw_ctrl, &out_size);
+ if ((ret != 0) || (out_size == 0) || (set_bw_ctrl.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set set_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)",
+ ret, set_bw_ctrl.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ return 0;
+}
+
+int roce5_query_bw_ctrl_state(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_param *bw_ctrl_param)
+{
+ int ret = 0;
+ roce_cc_cfg_bw_ctrl_cmd_s query_bw_ctrl;
+ u16 out_size = (u16)sizeof(query_bw_ctrl);
+
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&query_bw_ctrl, sizeof(query_bw_ctrl), 0,
+ sizeof(query_bw_ctrl));
+ if (ret != 0) {
+ ROCE_ERR("Memset failed when set_bw_ctrl cmd");
+ return (-ENOMEM);
+ }
+
+ query_bw_ctrl.func_id = rdev->glb_func_id;
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_QUERY_BW_CTRL,
+ &query_bw_ctrl, sizeof(query_bw_ctrl),
+ &query_bw_ctrl, &out_size);
+ if ((ret != 0) || (out_size == 0) || (query_bw_ctrl.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set query_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)",
+ ret, query_bw_ctrl.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ ret = memcpy_s((void *)bw_ctrl_param,
+ sizeof(struct roce5_bw_ctrl_param),
+ (void *)((u8 *)(&query_bw_ctrl) + ROCE_CMD_DEFAULT_SIZE),
+ sizeof(struct roce5_bw_ctrl_param));
+ if (ret != 0) {
+ ROCE_ERR("memory copy failed. ret%d", ret);
+ return -ENOMEM;
+ }
+
+ ROCE_INFO("query_bw_ctrl_state:func_id:%u, cir:%u, pir:%u, cnp:%u",
+ rdev->glb_func_id, bw_ctrl_param->cir, bw_ctrl_param->pir,
+ bw_ctrl_param->cnp);
+
+ return 0;
+}
+
+#ifdef ROCE_CHIP_TEST
+int roce5_set_vnic_waterline_state(struct roce5_device *rdev,
+ struct roce5_vnic_waterline_ctrl_buf *inbuf,
+ struct roce5_vnic_waterline_ctrl_buf *outbuf)
+{
+ int ret = 0;
+ roce_cc_cfg_vnic_waterline_cmd_s set_vnic_waterline;
+ u16 out_size = (u16)sizeof(set_vnic_waterline);
+
+ (void)outbuf;
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&set_vnic_waterline, sizeof(set_vnic_waterline), 0,
+ sizeof(set_vnic_waterline));
+ if (ret != 0) {
+ ROCE_ERR("Memset failed when set_vnic_waterline cmd");
+ return (-ENOMEM);
+ }
+
+ set_vnic_waterline.func_id = rdev->glb_func_id;
+ set_vnic_waterline.ip_ver = 0;
+ set_vnic_waterline.param.red_waterline =
+ inbuf->waterline_param.red_waterline;
+ set_vnic_waterline.param.yellow_waterline =
+ inbuf->waterline_param.yellow_waterline;
+ set_vnic_waterline.param.ip_info.ipv4.val =
+ inbuf->waterline_param.ip_val;
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev,
+ ROCE_MPU_CMD_CC_SET_VNIC_WATERLINE,
+ &set_vnic_waterline,
+ sizeof(set_vnic_waterline),
+ &set_vnic_waterline, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (set_vnic_waterline.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set set_vnic_waterline cmd, err(%d), status(0x%x), out size(0x%x)",
+ ret, set_vnic_waterline.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ return 0;
+}
+
+int roce5_get_vnic_waterline_state(struct roce5_device *rdev,
+ struct roce5_vnic_waterline_ctrl_buf *inbuf,
+ struct roce5_vnic_waterline_ctrl_buf *outbuf)
+{
+ int ret = 0;
+ roce_cc_cfg_vnic_waterline_cmd_s get_vnic_waterline;
+ u16 out_size = (u16)sizeof(get_vnic_waterline);
+
+ if (rdev->hwdev == NULL) {
+ ROCE_ERR("Hwdev is null");
+ return (-EINVAL);
+ }
+
+ ret = memset_s(&get_vnic_waterline, sizeof(get_vnic_waterline), 0,
+ sizeof(get_vnic_waterline));
+ if (ret != 0) {
+ ROCE_ERR("Memset failed when get_vnic_waterline cmd");
+ return (-ENOMEM);
+ }
+
+ get_vnic_waterline.func_id = rdev->glb_func_id;
+ get_vnic_waterline.ip_ver = 0;
+ get_vnic_waterline.param.ip_info.ipv4.val =
+ inbuf->waterline_param.ip_val;
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev,
+ ROCE_MPU_CMD_CC_GET_VNIC_WATERLINE,
+ &get_vnic_waterline,
+ sizeof(get_vnic_waterline),
+ &get_vnic_waterline, &out_size);
+ if ((ret != 0) || (out_size == 0) ||
+ (get_vnic_waterline.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set get_vnic_waterline cmd, err(%d), status(0x%x), out size(0x%x)",
+ ret, get_vnic_waterline.head.status, out_size);
+ return (-EINVAL);
+ }
+
+ outbuf->waterline_param.red_waterline =
+ get_vnic_waterline.param.red_waterline;
+ outbuf->waterline_param.yellow_waterline =
+ get_vnic_waterline.param.yellow_waterline;
+
+ return 0;
+}
+#endif /* ROCE_CHIP_TEST */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h
new file mode 100644
index 000000000..5fd787a67
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h
@@ -0,0 +1,82 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_CMD_H
+#define ROCE_CMD_H
+
+#include "comm_defs.h"
+#include "roce.h"
+#include "roce_dfx.h"
+#include "roce_srq.h"
+
+#include "roce_mpu_cmd_defs.h"
+#include "roce_cfg_format.h"
+
+#define ROCE_VLAN_ID_MASK 0x7FFF
+#define ROCE_VLAN_N_VID 4096
+
+#define ROCE_CMD_DEFAULT_SIZE 12
+
+#define ERR_EXIST 6
+#define ERR_FULL 12
+#define ERR_NOT_FOUND 13
+
+#define roce5_msg_to_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, out_size) \
+ hinic5_msg_to_mgmt_sync(hwdev, HINIC5_MOD_ROCE, cmd, buf_in, in_size, \
+ buf_out, out_size, 0, HINIC5_CHANNEL_ROCE)
+#define roce5_msg_to_ovs_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, \
+ out_size) \
+ hinic5_msg_to_mgmt_sync(hwdev, HINIC5_MOD_OVS, cmd, buf_in, in_size, \
+ buf_out, out_size, 90000, HINIC5_CHANNEL_OVS)
+
+int roce5_add_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr,
+ size_t mac_addr_len, u32 vlan_id, u16 func_id);
+int roce5_del_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr,
+ size_t mac_addr_len, u32 vlan_id, u16 func_id);
+typedef int (*roce5_modify_mac_tbl)(struct roce5_device *rdev,
+ u8 mac_addr[ETH_ALEN], size_t eth_alen,
+ u32 vlan_id, u16 func_id);
+void roce5_add_ipsu_tbl_mac_entry(struct roce5_device *rdev,
+ u8 mac_addr[ETH_ALEN], u32 vlan_id,
+ u16 func_id, u8 er_id, u16 src_func_id);
+void roce5_del_ipsu_tbl_mac_entry(struct roce5_device *rdev,
+ u8 mac_addr[ETH_ALEN], size_t eth_alen,
+ u32 vlan_id, u8 er_id, u16 src_func_id);
+int roce5_add_ipsu_tbl_ip_entry(struct roce5_device *rdev,
+ struct roce_cmd_mpu_addr_info *addr_info);
+int roce5_del_ipsu_tbl_ip_entry(struct roce5_device *rdev,
+ struct roce_cmd_mpu_addr_info *addr_info);
+int roce5_modify_ipsu_tbl_ip_entry(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry, u8 type);
+void roce5_add_ipsu_tcam_tbl_entry(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry,
+ u32 gid_index);
+void roce5_del_ipsu_tcam_tbl_entry(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry);
+typedef void (*roce5_modify_ipsu_tbl_mac)(void *hwdev, u8 mac_addr[ETH_ALEN],
+ u32 vlan_id, u16 func_id, u8 er_id);
+
+int roce5_set_sysfs_cfg_param(void *hwdev, u16 func_id, u32 *param,
+ u32 param_len, u16 type);
+int roce5_set_bw_ctrl_state(struct roce5_device *rdev, u8 cmd,
+ struct roce5_bw_ctrl_inbuf *inbuf);
+int roce5_query_bw_ctrl_state(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_param *bw_ctrl_param);
+#ifdef ROCE_CHIP_TEST
+int roce5_set_vnic_waterline_state(struct roce5_device *rdev,
+ struct roce5_vnic_waterline_ctrl_buf *inbuf,
+ struct roce5_vnic_waterline_ctrl_buf *outbuf);
+int roce5_get_vnic_waterline_state(struct roce5_device *rdev,
+ struct roce5_vnic_waterline_ctrl_buf *inbuf,
+ struct roce5_vnic_waterline_ctrl_buf *outbuf);
+#endif /* ROCE_CHIP_TEST */
+int roce5_set_func_tbl_func_state(void *hwdev, u8 func_state, u16 func_id);
+int roce5_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value);
+int roce5_set_func_tbl_cpu_endian(struct roce5_device *rdev);
+int roce5_get_feature(struct roce5_device *rdev,
+ roce_get_chip_ability_s *roce_feature);
+int roce5_init_cfg_info(struct roce5_device *rdev);
+int roce5_del_func_res(struct roce5_device *rdev);
+
+#endif /* ROCE_CMD_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c
new file mode 100644
index 000000000..11bd63d04
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c
@@ -0,0 +1,86 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/kernel.h>
+
+#include "roce.h"
+#include "roce_os_compat.h"
+
+int roce5_db_map_user(struct roce5_ucontext *context, unsigned long virt,
+ struct roce5_db *db, u8 shadow_db)
+{
+ int ret = 0;
+ struct roce5_db_page *db_page = NULL;
+
+ if ((context == NULL) || (db == NULL)) {
+ return -EINVAL;
+ }
+ if (shadow_db != 0) {
+ return ret;
+ }
+ mutex_lock(&context->db_page_mutex);
+
+ list_for_each_entry(db_page, &context->db_page_list, list) {
+ if (db_page->user_virt == (virt & PAGE_MASK)) {
+ goto found;
+ }
+ }
+
+ db_page = (struct roce5_db_page *)kmalloc(sizeof(*db_page), GFP_KERNEL);
+ if (db_page == NULL) {
+ ret = -ENOMEM;
+ ROCE_ERR("Failed to alloc DB page");
+ goto out;
+ }
+
+ db_page->user_virt = (virt & PAGE_MASK);
+ db_page->refcnt = 0;
+ db_page->umem = roce5_umem_get(context->ibucontext.device,
+ &context->ibucontext, virt & PAGE_MASK,
+ PAGE_SIZE, 0, 0);
+ if (IS_ERR(db_page->umem)) {
+ ret = (int)PTR_ERR(db_page->umem);
+ ROCE_ERR("Failed to get ib_umem ret:%d", ret);
+ kfree(db_page);
+ goto out;
+ }
+
+ list_add(&db_page->list, &context->db_page_list);
+
+found:
+#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE)
+ db->dma = sg_dma_address(db_page->umem->sgt_append.sgt.sgl) +
+ (virt & ~PAGE_MASK);
+#else
+ db->dma = sg_dma_address(db_page->umem->sg_head.sgl) +
+ (virt & ~PAGE_MASK);
+#endif
+ db->user_page = db_page;
+ ++db_page->refcnt;
+
+out:
+ mutex_unlock(&context->db_page_mutex);
+
+ return ret;
+}
+
+void roce5_db_unmap_user(struct roce5_ucontext *context, struct roce5_db *db,
+ u8 shadow_db)
+{
+ if ((context == NULL) || (db == NULL)) {
+ return;
+ }
+ if (shadow_db != 0) {
+ return;
+ }
+ mutex_lock(&context->db_page_mutex);
+
+ if ((--db->user_page->refcnt) == 0) {
+ list_del(&db->user_page->list);
+ ib_umem_release(db->user_page->umem);
+ kfree(db->user_page);
+ }
+
+ mutex_unlock(&context->db_page_mutex);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h
new file mode 100644
index 000000000..7232e9edc
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h
@@ -0,0 +1,30 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_DB_H
+#define ROCE_DB_H
+
+#include <rdma/ib_umem.h>
+#include <linux/types.h>
+#include <linux/list.h>
+
+struct roce5_db_page {
+ struct list_head list;
+ struct ib_umem *umem;
+ unsigned long user_virt;
+ int refcnt;
+};
+
+struct roce5_db {
+ __be32 *db_record;
+ dma_addr_t dma;
+ struct roce5_db_page *user_page;
+};
+
+static inline void roce5_write64(u32 val[2], void __iomem *dest)
+{
+ __raw_writeq(*(u64 *)(void *)val, dest);
+}
+
+#endif //ROCE_DB_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c b/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c
new file mode 100644
index 000000000..7d589afe1
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c
@@ -0,0 +1,1701 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <net/ipv6.h>
+#include <net/addrconf.h>
+#include <net/bonding.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_user_verbs.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/errno.h>
+#include <linux/inetdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/security.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/netdevice.h>
+#include <linux/sizes.h>
+
+#include "hinic5_crm.h"
+#include "hinic5_hw.h"
+#include "hinic5_mt.h"
+#include "hinic5_srv_nic.h"
+#include "hinic5_rdma.h"
+#include "hinic5_id_tbl.h"
+
+#include "roce_dfx.h"
+#include "roce_mr.h"
+#include "roce_main_extension.h"
+#include "roce_uld_inner.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_netlink.h"
+#include "roce_cmd.h"
+#include "roce_qp.h"
+#include "roce_infra.h"
+#include "rdma_comp_mw_mr.h"
+#include "roce_xrc.h"
+#include "roce_com.h"
+#include "roce_kernel_compat_gen.h"
+#include "roce_netdev.h"
+
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+
+#ifdef ENABLE_HYPER_ROCE
+#include "hyper_roce_tool.h"
+#endif
+
+#include "securec.h"
+
+MODULE_AUTHOR(HIROCE5_DRV_AUTHOR);
+MODULE_DESCRIPTION(HIROCE5_DRV_DESC);
+MODULE_VERSION(HIROCE5_DRV_VERSION);
+MODULE_LICENSE("GPL");
+
+static int g_loop_times = 50000;
+module_param(g_loop_times, int, S_IRUGO);
+MODULE_PARM_DESC(g_loop_times,
+ "default: 50000. The number must be greater than 0.");
+
+static u8 g_destory_flow_timeout = 10;
+module_param(g_destory_flow_timeout, byte, S_IRUGO);
+MODULE_PARM_DESC(g_destory_flow_timeout, "default: 10");
+
+static bool g_local_en = false; // IP mode because IP in this place need switch control whether advancein this place switch
+module_param(g_local_en, bool, S_IRUGO);
+MODULE_PARM_DESC(g_local_en, "default false");
+
+static bool g_full_qp_spec = true; // calculate standard card scenario, supports single cq create full specification qp
+module_param(g_full_qp_spec, bool, S_IRUGO);
+MODULE_PARM_DESC(
+ g_full_qp_spec,
+ "Supports creating a maximum number of QPs through a single CQ. default true.");
+
+static bool g_huge_page_en = 0;
+module_param(g_huge_page_en, bool, S_IRUGO);
+MODULE_PARM_DESC(
+ g_huge_page_en,
+ "default: 0. value: 0 or 1, set huge page en, and users need to reserve huge pages in advance.");
+
+#define ROCE_BOND_SLAVE_BIT_MASK 0xf
+
+/* ****************************************************************************
+ Prototype : get_cpu_endian
+ Description : Acquire CPU's enianness
+ Return Value : 0: little endian; 1: big endian
+
+ 1.Date : 2016/6/23
+ Modification : Created function
+
+**************************************************************************** */
+static int get_cpu_endian(void)
+{
+ int cpu_mode = 0;
+ union {
+ unsigned int i;
+ unsigned char s[4];
+ } c;
+ c.i = 0x12345678;
+
+ if (c.s[0] == 0x12) {
+ ROCE_INFO("CPU is be");
+ cpu_mode = 1;
+ } else {
+ ROCE_INFO("CPU is le");
+ cpu_mode = 0;
+ }
+
+ return cpu_mode;
+}
+
+static int roce5_alloc_hw_resource(struct roce5_device *rdev)
+{
+ int ret;
+
+ ret = roce5_rdma_init_rsvd_lkey(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to init rsvd lkey, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ /* Clear gid table before register for a new IB device */
+ ret = roce5_reset_gid_table(rdev, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to reset gid_table, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_gid_reset;
+ }
+
+ return 0;
+
+err_gid_reset:
+ roce5_rdma_free_rsvd_lkey(rdev);
+
+ return ret;
+}
+
+static void roce5_dealloc_hw_resource(struct roce5_device *rdev)
+{
+ roce5_rdma_free_rsvd_lkey(rdev);
+}
+
+/* 1825 v100 esl sub did */
+#define HI_1825_V100_SUB_DEV_ID_2X100 0x1825
+#define HI_1825_V100_SUB_DEV_ID_COMPUTE_2X200 0x40B1
+#define HI_1825_V100_SUB_DEV_ID_COMPUTE_DY_YH 0x30C4
+#define HI_1825_V100_SUB_DEV_ID_COMPUTE_UBX 0x30C8
+
+static int roce5_init_ver_attr_pci(struct hinic5_lld_dev *lld_dev,
+ struct roce5_device *rdev)
+{
+ struct pci_dev *pdev = container_of(lld_dev->dev, struct pci_dev, dev);
+
+ if ((pdev->device == HINIC5_DEV_ID_25V1_PF) ||
+ (pdev->device == HINIC5_DEV_ID_25V1_VF)) {
+ rdev->device_type = ROCE_DEV_TYPE_HI1825_V100;
+ return 0;
+ }
+
+ switch (pdev->subsystem_device) {
+ case HI_1825_V100_SUB_DEV_ID_2X100:
+ case HI_1825_V100_SUB_DEV_ID_COMPUTE_2X200:
+ case HI_1825_V100_SUB_DEV_ID_COMPUTE_DY_YH:
+ case HI_1825_V100_SUB_DEV_ID_COMPUTE_UBX:
+ rdev->device_type = ROCE_DEV_TYPE_HI1825_V100;
+ break;
+ default:
+ ROCE_DEV_ERR(rdev, "Unknown subsystem id(%u).",
+ pdev->subsystem_device);
+ rdev->device_type = ROCE_DEV_TYPE_UNKNOWN;
+ return -ENXIO;
+ }
+
+ return 0;
+}
+
+static int roce5_init_ver_attr(struct hinic5_lld_dev *lld_dev,
+ struct roce5_device *rdev)
+{
+ int ret;
+
+ switch (lld_dev->dev_type) {
+ case HINIC5_DEVICE_T_PCI:
+ ret = roce5_init_ver_attr_pci(lld_dev, rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to init PCI version attributes, ret(%d).",
+ ret);
+ return ret;
+ }
+ break;
+
+ case HINIC5_DEVICE_T_UB:
+ /*
+ * The 'struct ub_dev' contains a 'module' member that can be used to distinguish the version of the card
+ * under the UBC. However, this member is only available in the most recent version of 'struct ub_dev', and
+ * the current development environment does not include the updated structure definition, making it
+ * inaccessible at present. Here, the fixed version(Hi1825V100) is returned.
+ */
+ rdev->device_type = ROCE_DEV_TYPE_HI1825_V100;
+ break;
+
+ default:
+ ROCE_DEV_ERR(rdev, "Unknown low level device type(%u).",
+ lld_dev->dev_type);
+ rdev->device_type = ROCE_DEV_TYPE_UNKNOWN;
+ return -ENXIO;
+ }
+
+ return 0;
+}
+
+static int roce5_get_perf_cq_step(struct roce5_device *rdev)
+{
+ int perf_step = 0;
+
+ if (ROCE_IS_DEV_1825V100(rdev)) {
+ perf_step = ROCE_1825V100_STEP;
+ } else {
+ ROCE_DEV_INFO(
+ rdev,
+ "not support high performance cq, device type(%u), func_id(%u).",
+ rdev->device_type, rdev->glb_func_id);
+ }
+ return perf_step;
+}
+
+static int roce5_get_perf_cq_cnt(struct roce5_device *rdev)
+{
+ if (rdev->cqn_ctrl.rsvd_cq_step == 0) {
+ return 0;
+ }
+ /* When you insmod the driver, CQ0 is consumed by default. */
+ return (int)((rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs -
+ 1) /
+ rdev->cqn_ctrl.rsvd_cq_step) +
+ 1;
+}
+
+void roce5_init_perf_cq_attr(struct roce5_device *rdev)
+{
+ /* Calculate the step size and number of high-performance CQs */
+ rdev->cqn_ctrl.rsvd_cq_step = roce5_get_perf_cq_step(rdev);
+ rdev->cqn_ctrl.high_perf_cq_cnt = roce5_get_perf_cq_cnt(rdev);
+#ifdef ROCE_VBS_EN
+ // VBS RoCE scenario disable high performance cq
+ rdev->cqn_ctrl.high_perf_cq_cnt = 0;
+#endif
+ rdev->cqn_ctrl.alloc_cq_cnt = 0;
+}
+
+static struct roce5_device *
+roce5_rdev_alloc(struct hinic5_lld_dev *lld_dev, void **uld_dev,
+ const struct rdma_service_cap *rdma_cap)
+{
+ struct roce5_device *rdev = NULL;
+ int ret;
+
+ rdev = (struct roce5_device *)roce5_rdev_alloc_ext();
+ if (rdev == NULL) {
+ ROCE_ERR("Failed to alloc rdev");
+ return NULL;
+ }
+
+ *uld_dev = (void *)rdev;
+ rdev->lld_dev = lld_dev;
+ rdev->hwdev = lld_dev->hwdev;
+ rdev->dev = lld_dev->dev;
+ dma_set_max_seg_size(rdev->dev, SZ_2G);
+ ret = roce5_init_ver_attr(lld_dev, rdev);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to init version attribute of the device, ret(%d)",
+ ret);
+ goto alloc_failed;
+ }
+
+ mutex_init(&rdev->qp_cnt.cur_qps_mutex);
+ mutex_init(&rdev->cqn_ctrl.cq_ctrl_mutex);
+
+ rdev->hwdev_hdl = ((struct hinic5_hwdev *)(rdev->hwdev))->dev_hdl;
+ if (memcpy_s((void *)&rdev->rdma_cap, sizeof(*rdma_cap),
+ (void *)rdma_cap, sizeof(*rdma_cap)) != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy rdma_cap.");
+ goto alloc_failed;
+ }
+
+ rdev->ndev = hinic5_get_netdev_by_lld(rdev->lld_dev);
+ if (rdev->ndev == NULL) {
+ ROCE_DEV_ERR(rdev, "roce add failed, netdev is null.");
+ goto alloc_failed;
+ }
+
+ roce5_init_perf_cq_attr(rdev);
+ rdev->rdev_magic = ROCE_DEVICE_MAGIC;
+ return rdev;
+
+alloc_failed:
+ ib_dealloc_device(&rdev->ib_dev);
+ return NULL;
+}
+
+static int roce5_board_info_get(struct roce5_device *rdev)
+{
+ int ret = 0;
+
+ ret = hinic5_get_board_info(rdev->hwdev, &rdev->board_info,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get board info");
+ return ret;
+ }
+
+ ROCE_DEV_INFO(
+ rdev,
+ "Get board info success, board_type:0x%x, port_num:0x%x, pf_num:0x%x, vf_total_num:0x%x, "
+ "work_mode:0x%x, service_mode:0x%x, port_speed:0x%x.",
+ rdev->board_info.board_type, rdev->board_info.port_num,
+ rdev->board_info.pf_num, rdev->board_info.vf_total_num,
+ rdev->board_info.work_mode, rdev->board_info.service_mode,
+ rdev->board_info.port_speed);
+
+ ret = roce5_board_cfg_check(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to check board cfg info");
+ return ret;
+ }
+
+ return 0;
+}
+
+static void roce5_fix_ibdev_name(struct roce5_device *rdev)
+{
+ int ret;
+
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ ret = strcpy_s(rdev->ib_dev.name, IB_DEVICE_NAME_MAX,
+ "hrn5_bond_%d");
+ if (ret != EOK) {
+ ROCE_DEV_ERR(rdev, "Strcpy_s error");
+ }
+ return;
+ }
+#endif
+ ret = strcpy_s(rdev->ib_dev.name, IB_DEVICE_NAME_MAX, "hrn5_%d");
+ if (ret != EOK) {
+ ROCE_DEV_ERR(rdev, "Strcpy_s error");
+ }
+}
+
+static int roce5_register_template(struct roce5_device *rdev)
+{
+ service_register_template_s svc_template;
+ int ret;
+
+ ret = memset_s(&svc_template, sizeof(svc_template), 0,
+ sizeof(svc_template));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memset svc_template.(ret:%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ svc_template.service_type = SERVICE_T_ROCE;
+ svc_template.scq_ctx_size = rdev->rdma_cap.cqc_entry_sz;
+ svc_template.srq_ctx_size =
+ rdev->rdma_cap.dev_rdma_cap.roce_own_cap.srqc_entry_sz;
+ svc_template.service_handle = rdev;
+ svc_template.embedded_cq_ceq_callback = NULL;
+ svc_template.no_cq_ceq_callback = NULL;
+
+ svc_template.shared_cq_ceq_callback = roce5_cq_completion;
+ svc_template.aeq_level_callback = roce5_async_event_level;
+ svc_template.aeq_callback = roce5_async_event;
+
+ ret = cqm5_service_register(rdev->hwdev, &svc_template);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to register cqm_service, func(%d).",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ return 0;
+}
+
+/* Alloc CEQs and alloc CEQN */
+static int roce5_alloc_ceq(struct roce5_device *rdev)
+{
+ int ret;
+
+ ret = hinic5_alloc_ceqs(rdev->hwdev, SERVICE_T_ROCE,
+ rdev->ib_dev.num_comp_vectors, rdev->ceqn,
+ &rdev->ceq_num);
+ if (ret < 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqes, num_comp_vectors(%d), func_id(%d), ret(%d)",
+ rdev->ib_dev.num_comp_vectors, rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void roce5_free_ceq(struct roce5_device *rdev)
+{
+ int i;
+
+ for (i = 0; i < rdev->ceq_num; ++i) {
+ hinic5_free_ceq(rdev->hwdev, SERVICE_T_ROCE, rdev->ceqn[i]);
+ }
+}
+
+static void roce5_init_hw_info(struct roce5_device *rdev)
+{
+ rdev->hw_info.config_num_ports = (int)rdev->rdma_cap.num_ports;
+ rdev->hw_info.ep_id = hinic5_ep_id(rdev->hwdev);
+ rdev->hw_info.phy_port = 1;
+ rdev->hw_info.is_vf = roce5_func_is_vf(rdev->hwdev);
+ rdev->hw_info.cpu_endian = (u8)get_cpu_endian();
+}
+
+static int roce5_init_dev_info(struct roce5_device *rdev)
+{
+ int ret;
+
+ ret = roce5_set_func_tbl_cpu_endian(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set func_tbl cpu_endian, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_init;
+ }
+
+ /* Set function table to ENABLE */
+ ret = roce5_set_func_tbl_func_state(rdev->hwdev, ROCE_FUNC_ENABLE,
+ rdev->glb_func_id);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set func_tbl, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_init;
+ }
+
+ return 0;
+
+err_init:
+ return ret;
+}
+#ifdef __ROCE_DFX__
+static void device_list_lock_init(struct roce5_device_res *device_res)
+{
+ mutex_init(&device_res->mr_list.node_lock);
+ INIT_LIST_HEAD(&device_res->mr_list.node);
+}
+
+static void device_list_lock_deinit(struct roce5_device_res *device_res)
+{
+ struct roce5_mr *mr = NULL;
+ struct roce5_mr *tmp_mr = NULL;
+
+ mutex_lock(&device_res->mr_list.node_lock);
+ list_for_each_entry_safe(mr, tmp_mr, &device_res->mr_list.node, node) {
+ list_del(&(mr->node));
+ }
+ mutex_unlock(&device_res->mr_list.node_lock);
+
+ mutex_destroy(&(device_res->mr_list.node_lock));
+}
+
+static int roce5_dfx_device_init(struct roce5_device *rdev)
+{
+ rdev->device_res = kzalloc(sizeof(struct roce5_device_res), GFP_KERNEL);
+ if (rdev->device_res == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc device resource.");
+ return -ENOMEM;
+ }
+
+ device_list_lock_init(rdev->device_res);
+
+ return 0;
+}
+
+static void roce5_dfx_device_deinit(struct roce5_device *rdev)
+{
+ if (rdev->device_res != NULL) {
+ device_list_lock_deinit(rdev->device_res);
+ kfree(rdev->device_res);
+ rdev->device_res = NULL;
+ }
+}
+#endif // __ROCE_DFX__
+
+static void roce5_deinit_dev_info(struct roce5_device *rdev)
+{
+ (void)roce5_set_func_tbl_func_state(rdev->hwdev, ROCE_FUNC_DISABLE,
+ rdev->glb_func_id);
+}
+
+static int roce5_ib_register_device(struct roce5_device *rdev)
+{
+#if defined(IB_REG_DEV_HAVE_PORT_CALLBACK)
+ roce5_fix_ibdev_name(rdev);
+ return ib_register_device(&rdev->ib_dev, NULL);
+#elif defined(IB_REG_DEV_HAVE_NAME) && defined(IB_REG_DEV_HAVE_DEVICE)
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ return ib_register_device(&rdev->ib_dev, "hrn5_bond_%d",
+ rdev->dev);
+ }
+#endif
+
+ return ib_register_device(&rdev->ib_dev, "hrn5_%d", rdev->dev);
+#endif
+}
+
+static int roce5_dev_register(struct roce5_device *rdev, char *uld_dev_name)
+{
+ int ret;
+
+ ret = roce5_ib_register_device(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to reg ibdev, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_reg_dev;
+ }
+
+ ret = memcpy_s(uld_dev_name, ROCE_ULD_DEV_NAME_LEN, rdev->ib_dev.name,
+ ROCE_ULD_DEV_NAME_LEN);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy uld_dev_name.(ret:%d)",
+ ret);
+ goto err_memcpy_uld;
+ }
+
+ return 0;
+
+err_memcpy_uld:
+ ib_unregister_device(&rdev->ib_dev);
+err_reg_dev:
+ return ret;
+}
+
+static void roce5_dev_unregister(struct roce5_device *rdev)
+{
+ ib_unregister_device(&rdev->ib_dev);
+}
+
+static int roce5_handle_register_event(struct roce5_device *rdev)
+{
+ int ret;
+
+ ret = roce5_add_real_device_mac(rdev, rdev->ndev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to add real device mac, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ if (netif_running(rdev->ndev) &&
+ test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) {
+ roce5_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ACTIVE);
+ }
+
+ return 0;
+}
+
+static int roce5_init_dev(struct roce5_device *rdev, char *uld_dev_name)
+{
+ int ret;
+
+ ret = roce5_init_dev_info(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init dev info, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_init_dev_info;
+ }
+
+ ret = roce5_alloc_hw_resource(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc hw res, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_alloc_hw_res;
+ }
+
+ ret = roce5_dev_register(rdev, uld_dev_name);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init register, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_init_register;
+ }
+
+ ret = roce5_g_rdev_add(rdev);
+ if (ret != 0) {
+ ROCE_ERR("Failed to add rdev to g_rdev");
+ goto err_add_rdev;
+ }
+
+ ret = roce5_get_feature(rdev, &rdev->cfg_info.feature_cap);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get feature, ret=%d", ret);
+ goto err_get_feature_or_init_sysfs;
+ }
+
+ ret = roce5_init_sysfs(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init sysfs, func_id(%d) ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_get_feature_or_init_sysfs;
+ }
+ rdev->ib_active = true;
+
+ ret = roce5_handle_register_event(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to handle register event, func_id(%d) ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_register_event;
+ }
+
+ ROCE_DEV_INFO(rdev, "Add init all ok, func_id(%d), dev_name(%s)",
+ rdev->glb_func_id, rdev->ib_dev.name);
+
+ return 0;
+
+err_register_event:
+ roce5_remove_sysfs(rdev);
+err_get_feature_or_init_sysfs:
+ roce5_g_rdev_remove(rdev);
+err_add_rdev:
+ roce5_dev_unregister(rdev);
+err_init_register:
+ roce5_dealloc_hw_resource(rdev);
+err_alloc_hw_res:
+ roce5_deinit_dev_info(rdev);
+err_init_dev_info:
+ return ret;
+}
+
+static void roce5_rdev_qp_lb_init(struct roce5_device *rdev)
+{
+ int i = 0;
+ mutex_init(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex);
+ for (; i < ROCE_LB_QPN_MAX; i++) {
+ rdev->qp_lb_scqn_ctrl.qp[i] = 0;
+ }
+}
+
+static int roce5_add_rdev_init(struct roce5_device *rdev)
+{
+ int ret = 0;
+
+ ret = roce5_board_info_get(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get board info");
+ return ret;
+ }
+
+#ifdef ROCE_BONDING_EN
+ ret = roce5_bond_attach(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to attach bond device.");
+ return ret;
+ }
+
+ atomic_set(&rdev->next_port, 0);
+#endif
+
+ rdev->gid_dev = (struct net_device **)kzalloc(
+ rdev->rdma_cap.max_gid_per_port * sizeof(struct net_device *),
+ GFP_KERNEL);
+ if (rdev->gid_dev == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc gid dev memory");
+ return -ENOMEM;
+ }
+
+ ret = roce5_rdma_init_resource(rdev->hwdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init rdma resources.");
+ goto err_init_resource;
+ }
+
+ rdev->glb_func_id = hinic5_global_func_id(rdev->hwdev);
+ roce5_rdev_qp_lb_init(rdev);
+
+ roce5_init_hw_info(rdev);
+
+ ret = roce5_init_cfg_info(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get roce cfg, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_init_cfg;
+ }
+
+ ROCE_DEV_INFO(rdev, "Initializing.");
+ return 0;
+
+err_init_cfg:
+ roce5_rdma_cleanup_resource(rdev->hwdev);
+
+err_init_resource:
+ if (rdev->gid_dev != NULL) {
+ kfree(rdev->gid_dev);
+ }
+ return ret;
+}
+
+static void roce5_add_rdev_unit(struct roce5_device *rdev)
+{
+ roce5_rdma_cleanup_resource(rdev->hwdev);
+ if (rdev->gid_dev != NULL) {
+ kfree(rdev->gid_dev);
+ }
+}
+
+static __be64 rdma_gen_node_guid(u8 *dev_mac)
+{
+ u8 guid[8];
+ u8 mac_addr[6];
+ __be64 node_guid = 0;
+ int ret;
+
+ if (dev_mac == NULL) {
+ ROCE_ERR("Dev_mac is null");
+ return RDMA_INVALID_GUID;
+ }
+
+ ret = memcpy_s((void *)&mac_addr[0], sizeof(mac_addr), (void *)dev_mac,
+ sizeof(mac_addr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy mac_addr.(ret:%d)", ret);
+ return ENOMEM;
+ }
+
+ guid[0] = mac_addr[0] ^
+ DEV_ADDR_FIRST_BYTE_VAL_MASK; // 0 is mac & guid array idx
+ guid[1] = mac_addr[1]; // 1 is mac & guid array idx
+ guid[2] = mac_addr[2]; // 2 is mac & guid array idx
+ guid[3] = 0xff; // 3 is guid array idx
+ guid[4] = 0xfe; // 4 is guid array idx
+ guid[5] = mac_addr[3]; // 5 is guid array idx, 3 is mac array idx
+ guid[6] = mac_addr[4]; // 6 is guid array idx, 4 is mac array idx
+ guid[7] = mac_addr[5]; // 7 is guid array idx, 5 is mac array idx
+
+ /* node_guid is calculated by guid. */
+ node_guid =
+ ((u64)guid[0] << 56) | // 0 is guid array idx, 56 is guid offset
+ ((u64)guid[1] << 48) | // 1 is guid array idx, 48 is guid offset
+ ((u64)guid[2] << 40) | // 2 is guid array idx, 40 is guid offset
+ ((u64)guid[3] << 32) | // 3 is guid array idx, 32 is guid offset
+ ((u64)guid[4] << 24) | // 4 is guid array idx, 24 is guid offset
+ ((u64)guid[5] << 16) | // 5 is guid array idx, 16 is guid offset
+ ((u64)guid[6] << 8) | // 6 is guid array idx, 8 is guid offset
+ (u64)guid[7]; // 7 is guid array idx
+
+ return (__be64)cpu_to_be64(node_guid);
+}
+
+static __be64 roce5_rdma_init_guid(void *hwdev, struct net_device *netdev)
+{
+ struct rdma_comp_priv *comp_priv = NULL;
+
+ if ((hwdev == NULL) || (netdev == NULL)) {
+ ROCE_ERR("Hwdev or netdev is null");
+ return ~0ULL;
+ }
+
+ comp_priv = get_rdma_comp_priv(hwdev);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return ~0ULL;
+ }
+
+ comp_priv->rdma_comp_res.node_guid =
+ rdma_gen_node_guid((u8 *)netdev->dev_addr);
+
+ return comp_priv->rdma_comp_res.node_guid;
+}
+
+#if defined(HAVE_IB_DEVICE_OPS)
+static const struct ib_device_ops dev_ops = {
+ .owner = THIS_MODULE,
+ .driver_id = RDMA_DRIVER_HRN,
+ .uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION,
+
+ .modify_qp = roce5_modify_qp,
+ .query_qp = roce5_query_qp,
+ .destroy_qp = roce5_destroy_qp,
+ .post_send = roce5_post_send,
+ .post_recv = roce5_post_recv,
+ .create_cq = roce5_create_cq,
+ .modify_cq = roce5_modify_cq,
+ .resize_cq = roce5_resize_cq,
+ .destroy_cq = roce5_destroy_cq,
+ .poll_cq = roce5_poll_cq,
+ .req_notify_cq = roce5_arm_cq,
+ .create_srq = roce5_create_srq,
+ .modify_srq = roce5_modify_srq,
+ .query_srq = roce5_query_srq,
+ .destroy_srq = roce5_destroy_srq,
+ .post_srq_recv = roce5_post_srq_recv,
+ .map_mr_sg = roce5_map_kernel_frmr_sg,
+ .get_dma_mr = roce5_get_dma_mr,
+ .reg_user_mr = roce5_reg_user_mr,
+ .dereg_mr = roce5_dereg_mr,
+ .alloc_mr = roce5_alloc_mr,
+ .alloc_mw = roce5_alloc_mw,
+ .dealloc_mw = roce5_dealloc_mw,
+ .query_device = roce5_query_device,
+ .add_gid = roce5_ib_add_gid,
+ .del_gid = roce5_ib_del_gid,
+ .modify_device = roce5_modify_device,
+ .alloc_ucontext = roce5_alloc_ucontext,
+ .dealloc_ucontext = roce5_dealloc_ucontext,
+ .mmap = roce5_mmap,
+ .alloc_pd = roce5_alloc_pd,
+ .dealloc_pd = roce5_dealloc_pd,
+ .create_ah = roce5_create_ah,
+ .query_ah = roce5_query_ah,
+ .destroy_ah = roce5_destroy_ah,
+ .alloc_xrcd = roce5_alloc_xrcd,
+ .dealloc_xrcd = roce5_dealloc_xrcd,
+
+#if defined(ROCE_CREATE_QP_INT)
+ .create_qp = roce5_create_qp,
+#elif defined(ROCE_CREATE_QP_IB_QP)
+ .create_qp = roce5_create_qp_compat,
+#endif
+
+#if defined(HAVE_ALL_ROCE_FUNCS_U32_PORT)
+ .query_port = roce5_query_port,
+#elif defined(HAVE_ALL_ROCE_FUNCS_U8_PORT)
+ .query_port = roce5_query_port_compat,
+#endif
+
+#if defined(GET_LINK_LAYER_HAVE_U32_PORT)
+ .get_link_layer = roce5_port_link_layer,
+#elif defined(GET_LINK_LAYER_HAVE_U8_PORT)
+ .get_link_layer = roce5_port_link_layer_compat,
+#endif
+
+#if defined(QUERY_GID_HAVE_U32_PORT)
+ .query_gid = roce5_query_gid,
+#elif defined(QUERY_GID_HAVE_U8_PORT)
+ .query_gid = roce5_query_gid_compat,
+#endif
+
+#if defined(QUERY_PKEY_HAVE_U32_PORT)
+ .query_pkey = roce5_query_pkey,
+#elif defined(QUERY_PKEY_HAVE_U8_PORT)
+ .query_pkey = roce5_query_pkey_compat,
+#endif
+
+#if defined(MODIFY_PORT_HAVE_U32_PORT)
+ .modify_port = roce5_modify_port,
+#elif defined(MODIFY_PORT_HAVE_U8_PORT)
+ .modify_port = roce5_modify_port_compat,
+#endif
+
+#if defined(GET_PORT_IMMUTABLE_HAVE_U32_PORT)
+ .get_port_immutable = roce5_port_immutable,
+#elif defined(GET_PORT_IMMUTABLE_HAVE_U8_PORT)
+ .get_port_immutable = roce5_port_immutable_compat,
+#endif
+
+#if defined(GET_NETDEV_HAVE_U32_PORT)
+ .get_netdev = roce5_ib_get_netdev,
+#elif defined(GET_NETDEV_HAVE_U8_PORT)
+ .get_netdev = roce5_ib_get_netdev_compat,
+#endif
+
+#if defined(IB_DEVICE_OPS_HAVE_CREATE_USER_AH)
+ .create_user_ah = roce5_create_ah,
+#endif
+
+ INIT_RDMA_OBJ_SIZE(ib_ah, roce5_ah, ibah),
+ INIT_RDMA_OBJ_SIZE(ib_cq, roce5_cq, ibcq),
+ INIT_RDMA_OBJ_SIZE(ib_pd, roce5_pd, ibpd),
+ INIT_RDMA_OBJ_SIZE(ib_srq, roce5_srq, ibsrq),
+ INIT_RDMA_OBJ_SIZE(ib_ucontext, roce5_ucontext, ibucontext),
+ INIT_RDMA_OBJ_SIZE(ib_xrcd, roce5_xrcd, ibxrcd),
+ INIT_RDMA_OBJ_SIZE(ib_mw, roce5_mw, ibmw),
+#if defined(IB_DEVICE_OPS_HAVE_IB_QP_OBJ)
+ INIT_RDMA_OBJ_SIZE(ib_qp, roce5_qp, ibqp),
+#endif
+};
+
+static void roce5_add_init(struct roce5_device *rdev)
+{
+ struct ib_device *ib_dev = &rdev->ib_dev;
+
+ ib_dev->local_dma_lkey = rdev->rdma_cap.reserved_lkey;
+ ib_dev->phys_port_cnt = (u8)rdev->rdma_cap.num_ports;
+ ib_dev->num_comp_vectors =
+ (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ?
+ (int)rdev->rdma_cap.num_comp_vectors :
+ MAX_CEQ_NEED;
+ ib_dev->node_type = RDMA_NODE_IB_CA;
+ ib_dev->node_guid = roce5_rdma_init_guid(rdev->hwdev, rdev->ndev);
+ ib_dev->dma_device = rdev->dev;
+ ib_dev->dev.parent = ib_dev->dma_device;
+ (void)strcpy_s(ib_dev->node_desc, IB_DEVICE_NODE_DESC_MAX, "hrn5");
+
+ rdev->ib_dev.uverbs_cmd_mask = ROCE_UVERBS_CMD_BASE_MASK;
+#if defined(IB_DEV_HAVE_UVERBS_EX_CMD_MASK)
+ rdev->ib_dev.uverbs_ex_cmd_mask = ROCE_UVERBS_EXT_CMD_MASK;
+#endif
+
+ ib_set_device_ops(ib_dev, &dev_ops);
+ roce5_init_dev_ext_handlers(rdev);
+}
+
+#else
+static void roce5_init_dev_qp_handlers(struct roce5_device *rdev)
+{
+ rdev->ib_dev.create_qp = roce5_create_qp_compat;
+ rdev->ib_dev.modify_qp = roce5_modify_qp;
+ rdev->ib_dev.query_qp = roce5_query_qp;
+ rdev->ib_dev.destroy_qp = roce5_destroy_qp_ofa;
+ rdev->ib_dev.post_send = roce5_post_send;
+ rdev->ib_dev.post_recv = roce5_post_recv;
+}
+
+static void roce5_init_dev_cq_handlers(struct roce5_device *rdev)
+{
+ rdev->ib_dev.create_cq = roce5_create_cq_ofa;
+ rdev->ib_dev.modify_cq = roce5_modify_cq;
+ rdev->ib_dev.resize_cq = roce5_resize_cq;
+ rdev->ib_dev.destroy_cq = roce5_destroy_cq_ofa;
+ rdev->ib_dev.poll_cq = roce5_poll_cq;
+ rdev->ib_dev.req_notify_cq = roce5_arm_cq;
+}
+
+static void roce5_init_dev_srq_handlers(struct roce5_device *rdev)
+{
+ rdev->ib_dev.create_srq = roce5_create_srq_ofa;
+ rdev->ib_dev.modify_srq = roce5_modify_srq;
+ rdev->ib_dev.query_srq = roce5_query_srq;
+ rdev->ib_dev.destroy_srq = roce5_destroy_srq_ofa;
+ rdev->ib_dev.post_srq_recv = roce5_post_srq_recv;
+}
+
+static void roce5_init_dev_mr_handlers(struct roce5_device *rdev)
+{
+ rdev->ib_dev.map_mr_sg = roce5_map_kernel_frmr_sg;
+
+ rdev->ib_dev.get_dma_mr = roce5_get_dma_mr;
+ rdev->ib_dev.reg_user_mr = roce5_reg_user_mr;
+ rdev->ib_dev.dereg_mr = roce5_dereg_mr_ofa;
+ rdev->ib_dev.alloc_mr = roce5_alloc_mr;
+
+ rdev->ib_dev.alloc_mw = roce5_alloc_mw_ofa;
+ rdev->ib_dev.dealloc_mw = roce5_dealloc_mw_ofa;
+}
+
+static void roce5_init_dev_handlers(struct roce5_device *rdev)
+{
+ rdev->ib_dev.query_device = roce5_query_device;
+ rdev->ib_dev.query_port = roce5_query_port_compat;
+ rdev->ib_dev.get_link_layer = roce5_port_link_layer_compat;
+ rdev->ib_dev.query_gid = roce5_query_gid_compat;
+ rdev->ib_dev.add_gid = roce5_ib_add_gid;
+ rdev->ib_dev.del_gid = roce5_ib_del_gid;
+ rdev->ib_dev.query_pkey = roce5_query_pkey_compat;
+ rdev->ib_dev.modify_device = roce5_modify_device;
+ rdev->ib_dev.modify_port = roce5_modify_port_compat;
+
+ rdev->ib_dev.alloc_ucontext = roce5_alloc_ucontext_ofa;
+ rdev->ib_dev.dealloc_ucontext = roce5_dealloc_ucontext_ofa;
+ rdev->ib_dev.mmap = roce5_mmap;
+
+ rdev->ib_dev.alloc_pd = roce5_alloc_pd_ofa;
+ rdev->ib_dev.dealloc_pd = roce5_dealloc_pd_ofa;
+
+ rdev->ib_dev.create_ah = roce5_create_ah_ofa;
+ rdev->ib_dev.query_ah = roce5_query_ah;
+ rdev->ib_dev.destroy_ah = roce5_destroy_ah_ofa;
+
+ roce5_init_dev_qp_handlers(rdev);
+ roce5_init_dev_cq_handlers(rdev);
+ roce5_init_dev_srq_handlers(rdev);
+ roce5_init_dev_mr_handlers(rdev);
+
+ rdev->ib_dev.alloc_xrcd = roce5_alloc_xrcd_ofa;
+ rdev->ib_dev.dealloc_xrcd = roce5_dealloc_xrcd_ofa;
+
+ rdev->ib_dev.get_port_immutable = roce5_port_immutable_compat;
+ rdev->ib_dev.get_netdev = roce5_ib_get_netdev_compat;
+}
+
+static void roce5_add_init(struct roce5_device *rdev)
+{
+ int ret;
+
+ rdev->ib_dev.owner = THIS_MODULE;
+ rdev->ib_dev.node_type = RDMA_NODE_IB_CA;
+ rdev->ib_dev.local_dma_lkey = rdev->rdma_cap.reserved_lkey;
+ rdev->ib_dev.phys_port_cnt = (u8)rdev->rdma_cap.num_ports;
+ rdev->ib_dev.num_comp_vectors =
+ (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ?
+ (int)rdev->rdma_cap.num_comp_vectors :
+ MAX_CEQ_NEED;
+ rdev->ib_dev.dma_device = rdev->dev;
+ rdev->ib_dev.dev.parent = rdev->ib_dev.dma_device;
+ rdev->ib_dev.uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION;
+ rdev->ib_dev.node_guid = roce5_rdma_init_guid(rdev->hwdev, rdev->ndev);
+ ret = memcpy_s((void *)rdev->ib_dev.node_desc, ROCE_NODE_DESC_LEN,
+ "hrn5", ROCE_NODE_DESC_LEN);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy node_desc.(ret:%d)", ret);
+ return;
+ }
+
+ rdev->ib_dev.uverbs_cmd_mask = roce5_get_uverbs_cmd_mask();
+ rdev->ib_dev.uverbs_ex_cmd_mask = ROCE_UVERBS_EXT_CMD_MASK;
+
+ roce5_init_dev_handlers(rdev);
+ roce5_init_dev_ext_handlers(rdev);
+ return;
+}
+#endif
+
+static void roce5_mod_param_parse(struct roce5_device *rdev)
+{
+ if (g_loop_times <= 0) {
+ ROCE_DEV_WARN(
+ rdev,
+ "g_loop_times has been amended to 1, because original value(%d) is smaller than "
+ "or equal to 0.",
+ g_loop_times);
+ g_loop_times = 1;
+ }
+ rdev->try_times = g_loop_times;
+ rdev->destory_flow_timeout = g_destory_flow_timeout;
+ rdev->local_en = g_local_en;
+ rdev->full_qp_spec = g_full_qp_spec;
+ rdev->huge_page_en = g_huge_page_en;
+}
+
+static void roce5_init_mac_vlan(struct roce5_device *rdev)
+{
+ INIT_LIST_HEAD(&rdev->mac_vlan_list_head);
+ mutex_init(&rdev->mac_vlan_mutex);
+}
+
+static int roce5_rdev_init(struct roce5_device *rdev)
+{
+ int ret;
+
+ ret = hinic5_stateful_init(rdev->hwdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init stateful resource");
+ return ret;
+ }
+
+ roce5_init_mac_vlan(rdev);
+
+ ret = roce5_add_rdev_init(rdev);
+ if (ret != 0) {
+ goto err;
+ }
+
+ spin_lock_init(&rdev->node_desc_lock);
+ rwlock_init(&rdev->rwlock_func_tbl_bond);
+ mutex_init(&rdev->cap_mask_mutex);
+ spin_lock_init(&rdev->reset_flow_resource_lock);
+ INIT_LIST_HEAD(&rdev->qp_list);
+ roce5_mod_param_parse(rdev);
+
+ roce5_fix_ibdev_name(rdev);
+
+ return 0;
+err:
+ roce5_clean_vlan_device_mac(rdev);
+ hinic5_stateful_deinit(rdev->hwdev);
+ return ret;
+}
+
+static void roce5_rdev_unit(struct roce5_device *rdev)
+{
+ /* FLR by MPU when hotplug, don't need deinit anymore */
+ hinic5_stateful_deinit(rdev->hwdev);
+
+ roce5_add_rdev_unit(rdev);
+
+ mutex_destroy(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex);
+}
+
+static int roce5_add_do_init(struct roce5_device *rdev, char *uld_dev_name)
+{
+ int ret;
+
+ ret = roce5_rdev_init(rdev);
+ if (ret != 0) {
+ goto err_init_rdev;
+ }
+
+ ret = roce5_register_template(rdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to register cqm_service, func_id(%d) ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_cqm_register;
+ }
+
+ ret = hinic5_alloc_db_addr(rdev->hwdev, &rdev->kernel_db_map, NULL);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc db or dwqe, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_alloc_db;
+ }
+#ifdef __ROCE_DFX__
+ ret = roce5_dfx_device_init(rdev);
+ if (ret != 0) {
+ ROCE_ERR("Failed to init dfx device");
+ goto err_init_dfx;
+ }
+#endif
+ roce5_add_init(rdev);
+
+ ret = roce5_alloc_ceq(rdev);
+ if (ret < 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc ceqs, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ goto err_alloc_ceq;
+ }
+
+ ret = roce5_init_dev(rdev, uld_dev_name);
+ if (ret != 0) {
+ goto err_init_dev;
+ }
+
+ return 0;
+
+err_init_dev:
+ roce5_free_ceq(rdev);
+
+err_alloc_ceq:
+#ifdef __ROCE_DFX__
+ roce5_dfx_device_deinit(rdev);
+
+err_init_dfx:
+#endif
+ hinic5_free_db_addr(rdev->hwdev, rdev->kernel_db_map, NULL);
+
+err_alloc_db:
+ cqm5_service_unregister(rdev->hwdev, SERVICE_T_ROCE);
+
+err_cqm_register:
+ roce5_rdev_unit(rdev);
+
+err_init_rdev:
+ return ret;
+}
+
+static int roce5_add_check(const struct hinic5_lld_dev *lld_dev)
+{
+ int ret = 0;
+
+ if (lld_dev == NULL || lld_dev->hwdev == NULL) {
+ ROCE_ERR("Lld_dev is null");
+ return (-EINVAL);
+ }
+
+ ret = roce5_add_check_ext(lld_dev);
+ return ret;
+}
+
+static int roce5_stateful_init(struct hinic5_lld_dev *lld_dev)
+{
+ int ret = 0;
+ struct hinic5_lld_dev *ppf_lld_dev = NULL;
+
+ if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) {
+ return 0;
+ }
+
+ ppf_lld_dev = hinic5_get_ppf_lld_dev(lld_dev);
+ if (ppf_lld_dev == NULL) {
+ ROCE_ERR("Failed to get ppf lld dev, func id(%u).",
+ hinic5_global_func_id(lld_dev->hwdev));
+ return -EINVAL;
+ }
+
+ ret = hinic5_stateful_init(ppf_lld_dev->hwdev);
+ hinic5_lld_dev_put(ppf_lld_dev);
+
+ return ret;
+}
+
+static void roce5_stateful_deinit(struct hinic5_lld_dev *lld_dev)
+{
+ struct hinic5_lld_dev *ppf_lld_dev = NULL;
+
+ if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) {
+ return;
+ }
+
+ ppf_lld_dev = hinic5_get_ppf_lld_dev(lld_dev);
+ if (ppf_lld_dev == NULL) {
+ ROCE_ERR("Failed to get ppf lld dev, func id(%u).",
+ hinic5_global_func_id(lld_dev->hwdev));
+ return;
+ }
+
+ hinic5_stateful_deinit(ppf_lld_dev->hwdev);
+
+ hinic5_lld_dev_put(ppf_lld_dev);
+}
+
+static void roce5_do_remove(struct roce5_device *rdev, u16 glb_func_id,
+ const char *dev_name)
+{
+#ifdef __ROCE_DFX__
+ roce5_dfx_device_deinit(rdev);
+#endif
+
+ roce5_g_rdev_remove(rdev);
+
+ roce5_remove_sysfs(rdev);
+
+ ib_unregister_device(&rdev->ib_dev);
+
+ roce5_dealloc_hw_resource(rdev);
+
+ roce5_deinit_dev_info(rdev);
+
+#ifdef ROCE_BONDING_EN
+ roce5_set_bond_ipsurx_en(true);
+#endif
+
+ roce5_clean_vlan_device_mac(rdev);
+ roce5_clean_real_device_mac(rdev);
+
+ roce5_free_ceq(rdev);
+
+ hinic5_free_db_addr(rdev->hwdev, rdev->kernel_db_map, NULL);
+
+ cqm5_service_unregister(rdev->hwdev, SERVICE_T_ROCE);
+
+ roce5_del_func_res(rdev);
+ ROCE_INFO("Function level resource clear ok, func_id(%d), name(%s)",
+ glb_func_id, dev_name);
+
+ roce5_rdev_unit(rdev);
+}
+
+static void roce5_remove(struct hinic5_lld_dev *lld_dev, void *uld_dev)
+{
+ struct roce5_device *rdev = NULL;
+ char *device_name = NULL;
+ u16 glb_func_id;
+ roce_user_handle_t hdl = { 0 };
+
+ if ((uld_dev == NULL) || (lld_dev == NULL)) {
+ ROCE_ERR("input param is null");
+ return;
+ }
+
+ if (hinic5_support_roce(lld_dev->hwdev, NULL) == false) {
+ ROCE_WARN("Not support RoCE.");
+ return;
+ }
+
+ rdev = (struct roce5_device *)uld_dev;
+ rdev->ib_active = false;
+ device_name = rdev->ib_dev.name;
+ glb_func_id = rdev->glb_func_id;
+
+ ROCE_DEV_INFO(rdev, "Remove start, func_id(%d), name(%s)",
+ rdev->glb_func_id, device_name);
+
+ hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM;
+ hdl.rdev = rdev;
+ roce5_uld_rdev_remove((roce_uld_handle)&hdl);
+
+ roce5_do_remove(rdev, glb_func_id, device_name);
+
+#ifdef ROCE_BONDING_EN
+ roce5_bond_detach(rdev);
+#endif
+
+ ib_dealloc_device(&rdev->ib_dev);
+
+ roce5_stateful_deinit(lld_dev);
+
+ ROCE_INFO("Remove end, func_id(%d), name(%s)", glb_func_id,
+ device_name);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_add
+ Description : roce5_add
+ Input : struct hinic_lld_dev *lld_dev
+ void **uld_dev
+ char *uld_dev_name
+ Output : None
+
+ 1.Date : 2015/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static int roce5_add(struct hinic5_lld_dev *lld_dev, void **uld_dev,
+ char *uld_dev_name)
+{
+ struct roce5_device *rdev = NULL;
+ struct rdma_service_cap rdma_cap;
+ int ret = 0;
+ roce_user_handle_t hdl = { 0 };
+
+ ret = roce5_add_check(lld_dev);
+ if (ret != 0) {
+ return ret;
+ }
+
+ /* return 0(false) if the hwdev don't support ROCE, make sure it probe success */
+ if (hinic5_support_roce(lld_dev->hwdev, &rdma_cap) == false) {
+ ROCE_WARN("Not support RoCE, func: %d",
+ hinic5_global_func_id(lld_dev->hwdev));
+ // this pf no has enable RoCE, but is not affect RoCE driver load, no need return exception value
+ ret = 0;
+ goto err_check;
+ }
+
+ ret = roce5_stateful_init(lld_dev);
+ if (ret != 0) {
+ ROCE_ERR("Failed to add ref to ppf.");
+ goto err_check;
+ }
+
+ rdev = roce5_rdev_alloc(lld_dev, uld_dev, &rdma_cap);
+ if (rdev == NULL) {
+ ROCE_ERR("Failed to alloc rdev, func: %d",
+ hinic5_global_func_id(lld_dev->hwdev));
+ ret = -EINVAL;
+ goto err_alloc;
+ }
+
+ ret = roce5_add_do_init(rdev, uld_dev_name);
+ if (ret != 0) {
+ ROCE_ERR("Failed to init rdev.");
+ goto err_do_init;
+ }
+
+ hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM;
+ hdl.rdev = rdev;
+ ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rdev->shadow, &rdev->shadow_vfid,
+ &rdev->shadow_vf_num);
+ ret = roce5_uld_rdev_add((roce_uld_handle)&hdl);
+ if (ret != 0) {
+ ROCE_ERR("Failed to add uld rdev.");
+ roce5_do_remove(rdev, rdev->glb_func_id, uld_dev_name);
+ goto err_do_init;
+ }
+
+ ROCE_DEV_INFO(rdev, "Device(0x%p) init success.", rdev);
+ return 0;
+
+err_do_init:
+ ib_dealloc_device(&rdev->ib_dev);
+err_alloc:
+ roce5_stateful_deinit(lld_dev);
+err_check:
+ *uld_dev = NULL;
+ return ret;
+}
+
+static int roce5_set_nic_event(struct roce5_device *rdev,
+ const struct hinic5_event_info *event)
+{
+ switch (event->type) {
+ case EVENT_NIC_LINK_UP:
+ if (test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) != 0) {
+ return -1;
+ }
+ ROCE_DEV_INFO(rdev, "Report NIC LINK_UP, func_id(%d)",
+ rdev->glb_func_id);
+ return IB_EVENT_PORT_ACTIVE;
+
+ case EVENT_NIC_LINK_DOWN:
+ if (test_and_clear_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) {
+ return -1;
+ }
+ ROCE_DEV_INFO(rdev, "RoCE report NIC LINK_DOWN, func_id(%d)",
+ rdev->glb_func_id);
+ return IB_EVENT_PORT_ERR;
+
+ case EVENT_NIC_DCB_STATE_CHANGE:
+ ROCE_DEV_INFO(
+ rdev,
+ "DCB state change no longer requires RoCE involvement, func_id(%d)",
+ rdev->glb_func_id);
+ return -1;
+
+ default:
+ ROCE_DEV_ERR(rdev,
+ "Nic event unsupported, func_id(%d), event(%d)",
+ rdev->glb_func_id, event->type);
+ return -1;
+ }
+}
+
+static int roce5_set_ib_event(struct roce5_device *rdev,
+ const struct hinic5_event_info *event)
+{
+ switch (event->service) {
+ case EVENT_SRV_NIC:
+ return roce5_set_nic_event(rdev, event);
+
+ case EVENT_SRV_COMM:
+ return roce5_set_comm_event(rdev, event);
+
+ default:
+ ROCE_DEV_ERR(
+ rdev,
+ "func_id(%d) event from svc(%d) is not supported, event(%d)",
+ rdev->glb_func_id, event->service, event->type);
+ return -1;
+ }
+}
+
+void roce5_event(struct hinic5_lld_dev *lld_dev, void *uld_dev,
+ struct hinic5_event_info *event)
+{
+ struct ib_event ibevent;
+ struct roce5_device *rdev = NULL;
+ int type, ret = 0;
+
+ if ((lld_dev == NULL) || (uld_dev == NULL)) {
+ ROCE_ERR("Input params is null");
+ goto out;
+ }
+ roce5_lock_rdev();
+
+ ret = roce5_get_rdev_by_uld(lld_dev, uld_dev, &rdev, event);
+ if ((ret != 0) || (rdev == NULL)) {
+ ROCE_ERR("find rdev failed");
+ goto err_unlock;
+ }
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ roce5_handle_bonded_port_state_event(rdev);
+ goto err_unlock;
+ }
+#endif
+ type = roce5_set_ib_event(rdev, event);
+ if (type == -1) {
+ goto err_unlock;
+ }
+
+ ibevent.event = (enum ib_event_type)type;
+ ibevent.device = &rdev->ib_dev;
+ ibevent.element.port_num = ROCE_DEFAULT_PORT_NUM;
+
+ if (rdev->ib_active) {
+ ROCE_DEV_INFO(rdev, "report event: %d.", type);
+ ib_dispatch_event(&ibevent);
+ }
+
+err_unlock:
+ roce5_unlock_rdev();
+
+out:
+ return;
+}
+
+int roce5_adm_dfx_get_dev_type(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ const struct roce5_dfx_common_inbuf *inbuf =
+ (struct roce5_dfx_common_inbuf *)buf_in;
+ union roce5_dfx_common_outbuf *outbuf =
+ (union roce5_dfx_common_outbuf *)buf_out;
+
+ struct roce5_device *find_rdev =
+ roce5_get_rdev_by_name(inbuf->dev_name);
+
+ if (in_size < sizeof(struct roce5_dfx_common_inbuf)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size,
+ (int)sizeof(struct roce5_dfx_common_inbuf));
+ return -ENOMEM;
+ }
+ if (find_rdev == NULL) {
+ ROCE_ERR("Failed to find rdev by device name(%s).",
+ inbuf->dev_name);
+ return -EINVAL;
+ }
+
+ outbuf->dev_type = find_rdev->device_type;
+ *out_size = (u32)sizeof(union roce5_dfx_common_outbuf);
+ return 0;
+}
+
+static roce5_adm_cmd_proc_t g_roce5_adm_cmd_proc[] = {
+#ifdef __ROCE_DFX__
+ { ROCE_CMD_GET_QPC_FROM_CACHE, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_QPC_FROM_HOST, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_CQC_FROM_CACHE, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_CQC_FROM_HOST, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_SRQC_FROM_CACHE, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_SRQC_FROM_HOST, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_MPT_FROM_CACHE, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_MPT_FROM_HOST, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_GID_FROM_CACHE, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_QPC_CQC_PI_CI, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_QP_COUNT, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_DEV_ALGO, roce5_adm_dfx_query },
+ { ROCE_CMD_GET_DEV_TYPE, roce5_adm_dfx_get_dev_type },
+#ifdef ENABLE_DEBUG
+ { ROCE_CMD_DFX_ATTACK, roce5_dfx_attack_func },
+#endif
+#ifdef ROCE_CHIP_TEST
+ { ROCE_CMD_SET_VNIC_WATERLINE, roce5_adm_dfx_vnic_wl_ctrl },
+ { ROCE_CMD_GET_VNIC_WATERLINE, roce5_adm_dfx_vnic_wl_ctrl },
+#endif /* ROCE_CHIP_TEST */
+ { ROCE_CMD_ROCE_SET, roce5_adm_dfx_roce_set },
+ { ROCE_CMD_ROCE_SET_DEBUG, roce5_adm_dfx_roce_set_debug },
+#endif /* __ROCE_DFX__ */
+ { ROCE_CMD_ENABLE_BW_CTRL, roce5_adm_dfx_bw_ctrl },
+ { ROCE_CMD_DISABLE_BW_CTRL, roce5_adm_dfx_bw_ctrl },
+ { ROCE_CMD_CHANGE_BW_CTRL_PARAM, roce5_adm_dfx_bw_ctrl },
+ { ROCE_CMD_QUERY_BW_CTRL_PARAM, roce5_adm_dfx_bw_ctrl },
+ { ROCE_CMD_SET_LDCP_PARAM, roce5_adm_dfx_set_ldcp_param },
+ { ROCE_CMD_PORT_TRAFFIC, roce5_adm_dfx_port_traffic },
+ { ROCE_CMD_ULD_IOCTL_EXTEND,
+ (roce5_adm_func_t)roce5_adm_dfx_uld_extend },
+#ifdef ENABLE_HYPER_ROCE
+ { ROCE_CMD_HYPER, roce5_hyper_tool_dispatch },
+#endif
+};
+
+static int roce5_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size,
+ void *buf_out, u32 *out_size)
+{
+ struct roce5_device *rdev = (struct roce5_device *)uld_dev;
+ const roce5_adm_generic_cmd_proc_t *generic_array =
+ (const roce5_adm_generic_cmd_proc_t *)g_roce5_adm_cmd_proc;
+ int elem_count =
+ sizeof(g_roce5_adm_cmd_proc) / sizeof(g_roce5_adm_cmd_proc[0]);
+ roce5_adm_func_t roce5_adm_func = NULL;
+
+ /* check whether is root user */
+ if (!capable(CAP_SYS_ADMIN)) {
+ ROCE_ERR("Invalid permission");
+ return -EPERM; /* if not is root, return error */
+ }
+
+ if (cmd == GET_DRV_VERSION) {
+ return roce5_get_drv_version(buf_out, out_size);
+ }
+
+ if (uld_dev == NULL || buf_in == NULL || buf_out == NULL ||
+ out_size == NULL) {
+ ROCE_ERR("Input params is null");
+ return -EINVAL;
+ }
+
+ if (hinic5_support_roce(rdev->hwdev, NULL) == false) {
+ ROCE_ERR("Not support RoCE.");
+ return -EINVAL;
+ }
+
+ if (cmd == GET_DRV_VERSION) {
+ return roce5_get_drv_version(buf_out, out_size);
+ }
+
+ roce5_adm_func = (roce5_adm_func_t)roce5_adm_generic_find_cmd_func(
+ generic_array, elem_count, cmd);
+ if (roce5_adm_func == NULL) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d)", cmd);
+ return -EINVAL;
+ }
+ if (cmd == ROCE_CMD_ULD_IOCTL_EXTEND) {
+ roce_user_handle_t hdl = { 0 };
+ hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM;
+ hdl.rdev = rdev;
+ return roce5_adm_dfx_uld_extend(&hdl, buf_in, in_size, buf_out,
+ out_size);
+ } else {
+ return roce5_adm_func(rdev, buf_in, in_size, buf_out, out_size);
+ }
+}
+
+struct hinic5_uld_info roce5_info = {
+ .probe = roce5_add,
+ .remove = roce5_remove,
+ .suspend = NULL,
+ .resume = NULL,
+ .event = roce5_event,
+ .ioctl = roce5_ioctl,
+};
+
+struct hinic5_uld_info *roce5_info_get(void)
+{
+ return &roce5_info;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_service_init
+ Description :
+ Input : void
+ Output : None
+ Return Value :
+ Calls :
+ Called By :
+
+ History :
+ 1.Date : 2015/5/27
+ Author :
+ Modification : Created function
+
+**************************************************************************** */
+static int __init roce5_service_init(void)
+{
+ int ret;
+
+ roce5_service_init_pre();
+
+ ret = hinic5_register_uld(SERVICE_T_ROCE, roce5_info_get());
+ if (ret != 0) {
+ ROCE_ERR("Failed to register uld. ret(%d)", ret);
+ return ret;
+ }
+
+ ret = roce5_netlink_init();
+ if (ret != 0) {
+ ROCE_ERR("Failed to init netlink, ret(%d)", ret);
+ hinic5_unregister_uld(SERVICE_T_ROCE);
+ return ret;
+ }
+
+#ifdef ROCE_BONDING_EN
+ ret = roce5_bond_init();
+ if (ret != 0) {
+ ROCE_ERR("Failed to init RoCE Bond. ret(%d).", ret);
+ return ret;
+ }
+#endif
+
+ roce5_uld_service_init();
+
+ ret = roce5_register_netdev_notifier();
+ if (ret != 0) {
+ ROCE_ERR("Failed to register netdev notifier. ret(%d)", ret);
+ return ret;
+ }
+
+ ROCE_INFO("Register roce service successful");
+ return 0;
+}
+
+/* ****************************************************************************
+ Prototype : roce5_service_exit
+ Description : roce service disable
+ Input : void
+ Output : None
+ Return Value :
+ Calls :
+ Called By :
+
+ History :
+ 1.Date : 2015/5/27
+ Author :
+ Modification : Created function
+
+**************************************************************************** */
+static void __exit roce5_service_exit(void)
+{
+ roce5_unregister_netdev_notifier();
+
+ roce5_uld_service_exit();
+
+#ifdef ROCE_BONDING_EN
+ roce5_bond_exit();
+#endif
+
+ roce5_netlink_deinit();
+
+ hinic5_unregister_uld(SERVICE_T_ROCE);
+
+ ROCE_INFO("Unregister roce service successful");
+}
+
+module_init(roce5_service_init);
+module_exit(roce5_service_exit);
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c
new file mode 100644
index 000000000..99d3b3afa
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c
@@ -0,0 +1,1082 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "roce_netdev.h"
+#include "roce_main_extension.h"
+#include "roce_ctx_format.h"
+#include "roce_gid.h"
+#include "roce_cmd.h"
+#include "roce_mpu_cmd.h"
+#include "roce_restore.h"
+#include "rdma_comp.h"
+#include "roce_os_compat.h"
+#include "roce5_gid_type.h"
+
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+#include "hinic5_srv_nic.h"
+
+static struct notifier_block g_roce5_netdev_notifier;
+
+/* ****************************************************************************
+ Prototype : roce5_fill_gid_type
+ Description : fill gid type
+ Input : struct rdma_gid_entry *gid_entry
+ Output : None
+
+ 1.Date : 2016/5/27
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_fill_gid_type(struct rdma_gid_entry *gid_entry,
+ enum roce5_gid_type new_gid_type)
+{
+ static u32 index_roce5_gid_mapping[32][4] = {
+ { 0x120E1436, 3452, 224529784, 0x180E252A },
+ { 0x120E004A, 3472, 224660876, 0x6C0E403E },
+ { 0x120E0042, 3464, 224660836, 0x840E4036 },
+ { 0x120E0042, 3464, 224398688, 0xC00E0036 },
+ { 0x1212143A, 3456, 224791932, 0x1812252E },
+ { 0x1212004E, 3476, 224923024, 0x6C124042 },
+ { 0x12120046, 3468, 224922984, 0x8412403A },
+ { 0x12120046, 3468, 224660836, 0xC012003A },
+ { 0x1216143E, 3460, 225054080, 0x18162532 },
+ { 0x12160052, 3480, 225185172, 0x6C164046 },
+ { 0x1216004A, 3472, 225185132, 0x8416403E },
+ { 0x1216004A, 3472, 224922984, 0xC016003E },
+ { 0x10301458, 3484, 226626968, 0x1830254C },
+ { 0x1030006C, 3504, 226758060, 0x6C304060 },
+ { 0x10300064, 3496, 226758020, 0x84304058 },
+ { 0x10300064, 3496, 226495872, 0xC0300058 },
+ { 0x10401468, 838864300, 227675560, 0x1840255C },
+ { 0x1040007C, 1174408640, 227806652, 0x6C404070 },
+ { 0x10400074, 1040190904, 227806612, 0x84404068 },
+ { 0x10400074, 1040190904, 227544464, 0xC0400068 },
+ { 0x1044146C, 905973168, 227937708, 0x18442560 },
+ { 0x10440080, 1241517508, 228068800, 0x6C444074 },
+ { 0x10440078, 1107299772, 228068760, 0x8444406C },
+ { 0x10440078, 1107299772, 227806612, 0xC044006C },
+ { 0x10481470, 973082036, 228199856, 0x18482564 },
+ { 0x10480084, 1308626376, 228330948, 0x6C484078 },
+ { 0x1048007C, 1174408640, 228330908, 0x84484070 },
+ { 0x1048007C, 1174408640, 228068760, 0xC0480070 },
+ { 0x1262148A, 1040190928, 230034892, 0x1862257E },
+ { 0x1262009E, 1375735268, 230165984, 0x6C624092 },
+ { 0x12620096, 1241517532, 230165944, 0x8462408A },
+ { 0x12620096, 1241517532, 229903796, 0xC062008A },
+ };
+
+ u8 i = 0;
+ gid_entry->dw6_h.bs.gid_type = new_gid_type;
+ gid_entry->dw6_h.bs.gid_update = (new_gid_type == ROCE_IPv4_ROCEv2_GID);
+
+ i = ROCE_GID_MAP_TBL_IDX_GET(gid_entry->dw6_h.bs.tag, (u8)new_gid_type);
+ gid_entry->hdr_len_value = index_roce5_gid_mapping[i][0];
+ if (new_gid_type == ROCE_IPv4_ROCEv2_GID) {
+ *((u32 *)(void *)gid_entry + 1) =
+ cpu_to_be32(index_roce5_gid_mapping[i][1]);
+ *((u32 *)(void *)gid_entry + ROCE_GID_MAP_TBL_IDX2) =
+ cpu_to_be32(
+ index_roce5_gid_mapping[i]
+ [ROCE_GID_MAP_TBL_IDX2]);
+ }
+}
+
+static void roce5_fill_gid_smac(struct net_device *net_dev,
+ struct rdma_gid_entry *gid_entry)
+{
+ int ret;
+ ret = memcpy_s((void *)gid_entry->smac, sizeof(gid_entry->smac),
+ (void *)net_dev->dev_addr, sizeof(gid_entry->smac));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy smac.(ret:%d)", ret);
+ }
+}
+
+/*
+ 2024-04-05 add rdev parameters, because is in roce driver load, via roce5_ib_add_gid to next call, roce5_fill_gid_entry_ext
+ via net_dev get to rdev is empty, function in advance return, cannot execute expected of tunnel assignment logical
+*/
+static void roce5_fill_gid_vlan(const struct roce5_device *rdev,
+ struct net_device *net_dev,
+ struct rdma_gid_entry *gid_entry)
+{
+ if (rdma_vlan_dev_vlan_id(net_dev) != 0xffff) {
+ gid_entry->dw4.bs.cvlan = rdma_vlan_dev_vlan_id(net_dev) &
+ 0xfff;
+ gid_entry->dw6_h.bs.tag = ROCE_GID_VLAN_VALID;
+ }
+
+ roce5_fill_gid_entry_ext(rdev, gid_entry);
+}
+
+/* ****************************************************************************
+ Prototype : roce5_dispatch_event
+ Description : roce5_dispatch_event
+ Input : struct roce5_device *rdev
+ int port_num
+ enum ib_event_type type
+ Output : None
+
+ 1.Date : 2016/3/29
+ Modification : Created function
+
+**************************************************************************** */
+static void roce5_dispatch_event(struct roce5_device *rdev, int port_num,
+ enum ib_event_type type)
+{
+ struct ib_event event;
+ int ret;
+
+ if (rdev == NULL) {
+ ROCE_ERR("Rdev is null");
+ return;
+ }
+
+ ret = memset_s(&event, sizeof(event), 0, sizeof(event));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memset event.(ret:%d)", ret);
+ }
+
+ event.device = &rdev->ib_dev;
+ event.element.port_num = (u8)port_num;
+ event.event = type;
+
+ ROCE_DEV_INFO(rdev, "driver dispatch event: %d.", type);
+ ib_dispatch_event(&event);
+}
+
+static int roce5_del_old_gid_entry(struct roce5_device *rdev)
+{
+ struct rdma_gid_entry old_gid_entry;
+ int ret = 0;
+ int index = 0;
+
+ ret = memset_s(&old_gid_entry, sizeof(old_gid_entry), 0,
+ sizeof(old_gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ ret = roce5_get_gid(rdev, 1, index, &old_gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &old_gid_entry,
+ ROCE_MPU_CMD_DEL_IPSU_IP);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to del ipsu ip entry, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ roce5_del_ipsu_tcam_tbl_entry(rdev, &old_gid_entry);
+ }
+ return ret;
+}
+
+static int roce5_netdev_event_ipv4_dev_scan(struct roce5_device *rdev,
+ struct net_device *event_netdev,
+ struct rdma_gid_entry *gid_entry)
+{
+ struct in_device *in_dev = NULL;
+ int ret = 0;
+ int index = 0;
+
+ ret = roce5_del_old_gid_entry(rdev);
+ if (ret != 0) {
+ ROCE_ERR("Failed to del gid_entry.(ret:%d)", ret);
+ return ret;
+ }
+
+ in_dev = in_dev_get(event_netdev);
+ if (in_dev) {
+ for_ifa(in_dev)
+ {
+ ipv6_addr_set_v4mapped(
+ ifa->ifa_address,
+ (struct in6_addr *)gid_entry->raw);
+
+ roce5_fill_gid_vlan(rdev, event_netdev, gid_entry);
+ roce5_fill_gid_smac(event_netdev, gid_entry);
+
+ /* RoCE V2 */
+ if (rdma_protocol_roce_udp_encap(
+ &rdev->ib_dev, ROCE_DEFAULT_PORT_NUM)) {
+ roce5_fill_gid_type(gid_entry,
+ ROCE_IPv4_ROCEv2_GID);
+ ret = roce5_update_gid_mac(rdev, 0, &index,
+ gid_entry);
+ }
+
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to update type(%u) gid(IPv4) when process netdev's event, func_id(%d)",
+ gid_entry->dw6_h.bs.gid_type,
+ rdev->glb_func_id);
+ in_dev_put(in_dev);
+ return ret;
+ }
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ ret = roce5_modify_ipsu_tbl_ip_entry(
+ rdev, gid_entry,
+ ROCE_MPU_CMD_ADD_IPSU_IP);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to add ipsu ip entry, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ roce5_add_ipsu_tcam_tbl_entry(rdev, gid_entry,
+ index);
+ }
+ roce5_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM,
+ IB_EVENT_GID_CHANGE);
+ }
+ endfor_ifa(in_dev);
+ in_dev_put(in_dev);
+ }
+
+ return 0;
+}
+
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+struct sin6_list {
+ struct list_head list;
+ struct sockaddr_in6 sin6;
+};
+
+static int roce5_gid_fill(struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry,
+ struct net_device *event_netdev,
+ struct sin6_list *sin6_iter)
+{
+ int ret = 0;
+ int index;
+
+ ret = roce5_del_old_gid_entry(rdev);
+ if (ret != 0) {
+ ROCE_ERR("Failed to del gid_entry.(ret:%d)", ret);
+ return ret;
+ }
+
+ if (cpu_to_be64(gid_entry->global.subnet_prefix) ==
+ ROCE_DEFAULT_GID_SUBNET_PREFIX) {
+ return ret;
+ }
+
+ roce5_fill_gid_vlan(rdev, event_netdev, gid_entry);
+ roce5_fill_gid_smac(event_netdev, gid_entry);
+
+ /* RoCE V2 */
+ if (rdma_protocol_roce_udp_encap(&rdev->ib_dev,
+ ROCE_DEFAULT_PORT_NUM)) {
+ roce5_fill_gid_type(gid_entry, ROCE_IPv6_ROCEv2_GID);
+ ret = roce5_update_gid_mac(rdev, 0, &index, gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to update V2 gid(IPv6), func_id(%d)",
+ rdev->glb_func_id);
+ list_del(&sin6_iter->list);
+ kfree(sin6_iter);
+ return ret;
+ }
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ ret = roce5_modify_ipsu_tbl_ip_entry(
+ rdev, gid_entry, ROCE_MPU_CMD_ADD_IPSU_IP);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to add ipsu ip entry, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ roce5_add_ipsu_tcam_tbl_entry(rdev, gid_entry, index);
+ }
+ }
+
+ roce5_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, IB_EVENT_GID_CHANGE);
+
+ return ret;
+}
+
+static int roce5_netdev_event_ipv6_dev_scan(struct roce5_device *rdev,
+ struct net_device *event_netdev,
+ struct rdma_gid_entry *gid_entry)
+{
+ struct sin6_list *sin6_temp = NULL;
+ struct sin6_list *sin6_iter = NULL;
+ struct inet6_dev *in6_dev = NULL;
+ struct inet6_ifaddr *ifp = NULL;
+ struct sin6_list *entry = NULL;
+ int ret;
+
+ LIST_HEAD(sin6_init_list);
+
+ in6_dev = in6_dev_get(event_netdev);
+ if (in6_dev == NULL) {
+ return 0;
+ }
+
+ read_lock_bh(&in6_dev->lock);
+ list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
+ entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
+ if (entry == NULL) {
+ continue;
+ }
+
+ entry->sin6.sin6_family = AF_INET6;
+ entry->sin6.sin6_addr = ifp->addr;
+ list_add_tail(&entry->list, &sin6_init_list);
+ }
+
+ read_unlock_bh(&in6_dev->lock);
+ in6_dev_put(in6_dev);
+
+ list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) {
+ if (memcpy_s((void *)gid_entry->raw, sizeof(union ib_gid),
+ (void *)&sin6_iter->sin6.sin6_addr,
+ sizeof(union ib_gid)) != 0) {
+ ROCE_WARN("Failed to do memcpy_s");
+ }
+
+ ret = roce5_gid_fill(rdev, gid_entry, event_netdev, sin6_iter);
+ if (ret != 0) {
+ goto err_free;
+ }
+
+ list_del(&sin6_iter->list);
+ kfree(sin6_iter);
+ }
+
+ return 0;
+
+err_free:
+ list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) {
+ list_del(&sin6_iter->list);
+ kfree(sin6_iter);
+ }
+ return ret;
+}
+#endif
+
+static int roce5_netdev_event_ip_dev_scan(struct roce5_device *rdev,
+ struct net_device *event_netdev)
+{
+ struct rdma_gid_entry gid_entry;
+ int ret;
+
+ ret = memset_s(&gid_entry, sizeof(gid_entry), 0, sizeof(gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ ret = roce5_netdev_event_ipv4_dev_scan(rdev, event_netdev, &gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to scan IPv4, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+ ret = roce5_netdev_event_ipv6_dev_scan(rdev, event_netdev, &gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to scan IPv6, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+#endif
+
+ return 0;
+}
+
+static struct roce5_vlan_dev_list *
+roce5_find_vlan_device_list(const struct list_head *mac_list,
+ const struct net_device *netdev)
+{
+ struct roce5_vlan_dev_list *vlan_dev_list = NULL;
+
+ list_for_each_entry(vlan_dev_list, mac_list, list) {
+ if (vlan_dev_list->net_dev == netdev) {
+ return vlan_dev_list;
+ }
+ }
+
+ return NULL;
+}
+
+static int roce5_add_vlan_device(struct roce5_device *rdev,
+ struct net_device *netdev)
+{
+ struct roce5_vlan_dev_list *new_list = NULL;
+ int ret;
+
+ new_list = (struct roce5_vlan_dev_list *)kzalloc(sizeof(*new_list),
+ GFP_KERNEL);
+ if (new_list == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Malloc mac_vlan list_head is NULL, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+
+ new_list->ref_cnt = 1;
+ new_list->net_dev = netdev;
+ new_list->vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID(
+ ((u32)rdma_vlan_dev_vlan_id(netdev)));
+ ret = memcpy_s(new_list->mac, ROCE_MAC_ADDR_LEN, netdev->dev_addr,
+ ROCE_MAC_ADDR_LEN);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy mac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ INIT_LIST_HEAD(&new_list->list);
+ list_add_tail(&new_list->list, &rdev->mac_vlan_list_head);
+
+ return 0;
+}
+
+static void roce5_del_vlan_device(struct roce5_device *rdev,
+ struct roce5_vlan_dev_list *old_list)
+{
+ list_del(&old_list->list);
+ kfree(old_list);
+}
+
+int roce5_add_vlan_device_mac(struct roce5_device *rdev,
+ struct net_device *netdev)
+{
+ int ret = 0;
+ u32 vlan_id = 0;
+
+ vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID(
+ ((u32)rdma_vlan_dev_vlan_id(netdev)));
+ ROCE_DEV_INFO(
+ rdev,
+ "enter roce5_add_vlan_device_mac, vlan_id(0x%x), func_id(%d).",
+ vlan_id, rdev->glb_func_id);
+
+ ret = roce5_add_mac_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr,
+ ETH_ALEN, vlan_id, rdev->glb_func_id);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set vlan device mac, vlan_id(0x%x), func_id(%d)",
+ vlan_id, rdev->glb_func_id);
+ return ret;
+ }
+
+ roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, vlan_id,
+ rdev->glb_func_id,
+ hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ ret = roce5_add_bond_vlan_slave_mac(
+ rdev, (u8 *)netdev->dev_addr, (u16)vlan_id);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set bond vlan slave mac, vlan_id(0x%x), func_id(%d)",
+ vlan_id, rdev->glb_func_id);
+ goto err_add_bond_vlan_slave_mac;
+ }
+ }
+#endif
+
+ return ret;
+
+#ifdef ROCE_BONDING_EN
+err_add_bond_vlan_slave_mac:
+ roce5_del_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, ETH_ALEN,
+ vlan_id, hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+
+ (void)hinic5_del_mac(rdev->hwdev, (u8 *)netdev->dev_addr, (u16)vlan_id,
+ rdev->glb_func_id, HINIC5_CHANNEL_ROCE);
+
+ return ret;
+#endif
+}
+
+void roce5_del_vlan_device_mac(struct roce5_device *rdev,
+ struct roce5_vlan_dev_list *old_list)
+{
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ roce5_del_bond_vlan_slave_mac(rdev, old_list->mac,
+ (u16)old_list->vlan_id);
+ }
+#endif
+
+ roce5_del_ipsu_tbl_mac_entry(rdev, old_list->mac, ETH_ALEN,
+ old_list->vlan_id,
+ hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+
+ (void)roce5_del_mac_tbl_mac_entry(rdev, old_list->mac, ETH_ALEN,
+ old_list->vlan_id, rdev->glb_func_id);
+}
+
+static int roce5_update_vlan_device_mac(struct roce5_device *rdev,
+ struct net_device *netdev)
+{
+ struct roce5_vlan_dev_list *old_list = NULL;
+ int ret;
+
+ mutex_lock(&rdev->mac_vlan_mutex);
+ old_list =
+ roce5_find_vlan_device_list(&rdev->mac_vlan_list_head, netdev);
+ if (old_list == NULL) {
+ mutex_unlock(&rdev->mac_vlan_mutex);
+ return 0;
+ }
+
+ if (ROCE_MEMCMP(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN) !=
+ 0) {
+ roce5_del_vlan_device_mac(rdev, old_list);
+
+ ret = roce5_add_vlan_device_mac(rdev, netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to add mac_vlan, func_id(%d)",
+ rdev->glb_func_id);
+ mutex_unlock(&rdev->mac_vlan_mutex);
+ return ret;
+ }
+
+ ret = memcpy_s(old_list->mac, ROCE_MAC_ADDR_LEN,
+ netdev->dev_addr, ROCE_MAC_ADDR_LEN);
+ if (ret != 0) {
+ roce5_del_vlan_device_mac(rdev, old_list);
+ mutex_unlock(&rdev->mac_vlan_mutex);
+ return ret;
+ }
+ }
+
+ mutex_unlock(&rdev->mac_vlan_mutex);
+
+ return 0;
+}
+
+int roce5_add_real_device_mac(struct roce5_device *rdev,
+ struct net_device *netdev)
+{
+ int ret;
+
+ roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, 0,
+ rdev->glb_func_id,
+ hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ ret = roce5_add_bond_real_slave_mac(rdev,
+ (u8 *)netdev->dev_addr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to add bond read device ipsu mac, func_id(%d)",
+ rdev->glb_func_id);
+ roce5_del_ipsu_tbl_mac_entry(
+ rdev, (u8 *)netdev->dev_addr, ETH_ALEN, 0,
+ hinic5_er_id(rdev->hwdev), rdev->glb_func_id);
+ return ret;
+ }
+ }
+#endif
+
+ ret = memcpy_s(rdev->mac, sizeof(rdev->mac), netdev->dev_addr,
+ sizeof(rdev->mac));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy mac.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+void roce5_del_real_device_mac(struct roce5_device *rdev)
+{
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ roce5_del_bond_real_slave_mac(rdev);
+ }
+#endif
+
+ roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0,
+ hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+}
+
+static int roce5_update_real_device_mac(struct roce5_device *rdev,
+ struct net_device *netdev)
+{
+ int ret;
+
+ roce5_del_real_device_mac(rdev);
+
+ ret = roce5_add_real_device_mac(rdev, netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to add real device ipsu mac, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ return 0;
+}
+
+void roce5_clean_vlan_device_mac(struct roce5_device *rdev)
+{
+ struct roce5_vlan_dev_list *pos = NULL;
+ struct roce5_vlan_dev_list *tmp = NULL;
+
+ mutex_lock(&rdev->mac_vlan_mutex);
+ list_for_each_entry_safe(pos, tmp, &rdev->mac_vlan_list_head, list) {
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ (void)roce5_del_bond_vlan_slave_mac(rdev, pos->mac,
+ (u16)pos->vlan_id);
+ }
+#endif
+ roce5_del_ipsu_tbl_mac_entry(rdev, pos->mac, ETH_ALEN,
+ pos->vlan_id,
+ hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+
+ (void)roce5_del_mac_tbl_mac_entry(rdev, pos->mac, ETH_ALEN,
+ pos->vlan_id,
+ rdev->glb_func_id);
+
+ list_del(&pos->list);
+ kfree(pos);
+ }
+ mutex_unlock(&rdev->mac_vlan_mutex);
+}
+
+void roce5_clean_real_device_mac(struct roce5_device *rdev)
+{
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ (void)roce5_del_bond_real_slave_mac(rdev);
+ }
+#endif
+
+ roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0,
+ hinic5_er_id(rdev->hwdev),
+ rdev->glb_func_id);
+}
+
+void roce5_ifconfig_up_down_event_report(struct roce5_device *rdev,
+ u8 net_event)
+{
+ struct ib_event event = {};
+
+ event.device = &rdev->ib_dev;
+ event.event = net_event;
+ event.element.port_num = ROCE_DEFAULT_PORT_NUM;
+
+ ib_dispatch_event(&event);
+}
+
+static int roce5_netdev_event_raw_dev(unsigned long event,
+ struct roce5_device *rdev,
+ struct net_device *event_netdev)
+{
+ int ret;
+
+ if (event == NETDEV_CHANGEADDR) {
+ ret = roce5_update_real_device_mac(rdev, event_netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to update real device mac, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = roce5_netdev_event_ip_dev_scan(rdev, event_netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to scan ip_dev, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+ }
+
+#ifdef ROCE_BONDING_EN
+ if (roce5_bond_is_active(rdev)) {
+ if ((event == NETDEV_DOWN) || (event == NETDEV_UP)) {
+ roce5_handle_bonded_port_state_event(rdev);
+ }
+ return 0;
+ }
+#endif
+ if (event == NETDEV_DOWN) {
+ if (test_and_clear_bit(ROCE5_PORT_EVENT, &rdev->status) != 0) {
+ roce5_ifconfig_up_down_event_report(rdev,
+ IB_EVENT_PORT_ERR);
+ }
+ }
+
+ if (event == NETDEV_UP) {
+ if (test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) {
+ roce5_ifconfig_up_down_event_report(
+ rdev, IB_EVENT_PORT_ACTIVE);
+ }
+ }
+
+ return 0;
+}
+
+static int roce5_netdev_event_vlan_dev(unsigned long event,
+ struct roce5_device *rdev,
+ struct net_device *event_netdev)
+{
+ int ret;
+
+ if (event == NETDEV_CHANGEADDR) {
+ ret = roce5_update_vlan_device_mac(rdev, event_netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to update vlan device mac, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = roce5_netdev_event_ip_dev_scan(rdev, event_netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to scan vlan device ip list, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static int roce5_netdev_event(struct notifier_block *nb, unsigned long event,
+ void *ptr)
+{
+ struct net_device *event_netdev = NULL;
+ struct roce5_device *rdev = NULL;
+ struct hinic5_lld_dev *lld_dev = NULL;
+ struct net_device *real_dev = NULL;
+ int ret;
+
+ if (nb == NULL || ptr == NULL) {
+ goto err_out;
+ }
+
+ event_netdev = netdev_notifier_info_to_dev(ptr);
+ if (event_netdev->type != ARPHRD_ETHER) {
+ goto err_out;
+ }
+
+ real_dev = (rdma_vlan_dev_real_dev(event_netdev) != 0) ?
+ rdma_vlan_dev_real_dev(event_netdev) :
+ event_netdev;
+
+ lld_dev = hinic5_get_lld_dev_by_netdev(real_dev);
+ if (lld_dev == NULL) {
+ goto err_out;
+ }
+
+ /*
+ * hinic5_get_uld_dev internal will call uld5_dev_hold add device reference count
+ * via uld5_dev_put free reference count
+ */
+ rdev = (struct roce5_device *)hinic5_get_uld_dev(lld_dev,
+ SERVICE_T_ROCE);
+
+ /*
+ * passed in of rdev is empty value when,roce5_bond_event_cfg_rdev will found lld corresponding to bond device rdev, call uld5_dev_hold add device reference count
+ * passed in of rdev non-null value when, reserved rdev original value, not will call uld5_dev_hold add device reference count
+ * via uld5_dev_put free reference count
+ */
+ ret = roce5_bond_event_cfg_rdev(lld_dev, (void *)rdev, &rdev, true);
+ if (ret != 0 || rdev == NULL) {
+ goto err_out;
+ }
+
+ /* not need extra of roce5_is_port_of_net_dev check */
+ if (real_dev == event_netdev) {
+ ret = roce5_netdev_event_raw_dev(event, rdev, event_netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to deal with raw device, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_put;
+ }
+ } else {
+ ret = roce5_netdev_event_vlan_dev(event, rdev, event_netdev);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to deal with vlan device, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_put;
+ }
+ }
+
+err_put:
+ uld5_dev_put(rdev->lld_dev, SERVICE_T_ROCE);
+err_out:
+ return NOTIFY_DONE;
+}
+
+int roce5_register_netdev_notifier(void)
+{
+ int ret;
+
+ g_roce5_netdev_notifier.notifier_call = roce5_netdev_event;
+ ret = register_netdevice_notifier(&g_roce5_netdev_notifier);
+ if (ret != 0) {
+ ROCE_ERR("Failed to register netdev notifier, ret(%d)", ret);
+ g_roce5_netdev_notifier.notifier_call = NULL;
+ }
+
+ return ret;
+}
+
+void roce5_unregister_netdev_notifier(void)
+{
+ if (g_roce5_netdev_notifier.notifier_call != NULL) {
+ unregister_netdevice_notifier(&g_roce5_netdev_notifier);
+ g_roce5_netdev_notifier.notifier_call = NULL;
+ }
+}
+
+static int roce5_set_gid_entry(const struct ib_gid_attr *attr,
+ struct rdma_gid_entry *gid_entry)
+{
+ int ret = 0;
+ enum rdma_network_type network_type;
+ static enum roce5_gid_type gid_type_map[] = {
+ // map rdma_network_type to roce5_gid_type
+ [RDMA_NETWORK_IB] = ROCE_ROCEv1_GID,
+ [RDMA_NETWORK_ROCE_V1] = ROCE_ROCEv1_GID,
+ [RDMA_NETWORK_IPV4] = ROCE_IPv4_ROCEv2_GID,
+ [RDMA_NETWORK_IPV6] = ROCE_IPv6_ROCEv2_GID,
+ };
+
+ ret = memset_s(gid_entry, sizeof(*gid_entry), 0, sizeof(*gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ network_type = rdma_gid_attr_network_type(attr);
+ ret = memcpy_s((void *)gid_entry->raw, sizeof(union ib_gid),
+ (void *)(&attr->gid), sizeof(union ib_gid));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy gid_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ if (network_type == RDMA_NETWORK_ROCE_V1 ||
+ network_type == RDMA_NETWORK_IB) {
+ ROCE_ERR("network type(%d) is unsupported.", network_type);
+ return -EINVAL;
+ }
+
+ roce5_fill_gid_vlan(to_roce5_dev(attr->device), attr->ndev, gid_entry);
+ roce5_fill_gid_type(gid_entry, gid_type_map[network_type]);
+ roce5_fill_gid_smac(attr->ndev, gid_entry);
+
+ return 0;
+}
+
+/* add vlan_mac list or ref_cnt + 1
+ * add mac_vlan when new list add
+ */
+static int roce5_notify_vlan(struct roce5_device *rdev,
+ const struct ib_gid_attr *attr)
+{
+ int ret;
+ struct roce5_vlan_dev_list *old_list = NULL;
+
+ mutex_lock(&rdev->mac_vlan_mutex);
+ ROCE_DEV_INFO(rdev, "Add vlan: func_id(%d), netdev(%s).",
+ rdev->glb_func_id, attr->ndev->name);
+ old_list = roce5_find_vlan_device_list(&rdev->mac_vlan_list_head,
+ attr->ndev);
+ if (old_list != NULL) {
+ old_list->ref_cnt++;
+ ret = 0;
+ goto out;
+ }
+
+ ret = roce5_add_vlan_device(rdev, attr->ndev);
+ if (ret != 0) {
+ goto out;
+ }
+
+ ret = roce5_add_vlan_device_mac(rdev, attr->ndev);
+ if (ret != 0) {
+ old_list = roce5_find_vlan_device_list(
+ &rdev->mac_vlan_list_head, attr->ndev);
+ if (old_list) {
+ roce5_del_vlan_device(rdev, old_list);
+ }
+ }
+
+out:
+ mutex_unlock(&rdev->mac_vlan_mutex);
+
+ return ret;
+}
+
+int roce5_ib_add_gid_common(const struct ib_gid_attr *attr)
+{
+ int ret;
+ struct rdma_gid_entry gid_entry;
+ struct roce5_device *rdev = NULL;
+ unsigned int index;
+
+ // the input gid pointer is not needed to check on Euler OS
+ if (attr == NULL) {
+ ROCE_ERR("Input pointer is NULL");
+ return (-EINVAL);
+ }
+
+ rdev = to_roce5_dev(attr->device);
+ index = attr->index;
+
+ ret = roce5_set_gid_entry(attr, &gid_entry);
+ if (ret != 0) {
+ ROCE_ERR("Failed to set gid entry.");
+ return ret;
+ }
+
+ ret = roce5_update_gid(rdev, 0, index, &gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to update gid. ret(%d),func_id(%d)",
+ ret, rdev->glb_func_id);
+ return ret;
+ }
+
+ rdev->gid_dev[index] = attr->ndev;
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &gid_entry,
+ ROCE_MPU_CMD_ADD_IPSU_IP);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to add ipsu ip entry, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ roce5_add_ipsu_tcam_tbl_entry(rdev, &gid_entry, index);
+ }
+
+ if (rdma_vlan_dev_real_dev(attr->ndev)) {
+ ret = roce5_notify_vlan(rdev, attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to nofify vlan, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+int roce5_ib_add_gid(const struct ib_gid_attr *attr,
+ __always_unused void **context)
+{
+ return roce5_ib_add_gid_common(attr);
+}
+
+int roce5_ib_del_gid(const struct ib_gid_attr *attr,
+ __always_unused void **context)
+{
+ struct roce5_vlan_dev_list *old_list = NULL;
+ struct roce5_device *rdev = NULL;
+ struct rdma_gid_entry gid_entry;
+ u32 index = 0;
+ int ret;
+
+ if ((attr == NULL) || (attr->device == NULL)) {
+ ROCE_ERR("Attr or attr->device is null");
+ return (-EINVAL);
+ }
+
+ rdev = to_roce5_dev(attr->device);
+ index = attr->index;
+ if (index >= rdev->rdma_cap.max_gid_per_port) {
+ ROCE_DEV_ERR(rdev, "Invalid gid index(%u), func_id(%d)", index,
+ rdev->glb_func_id);
+ return (-EINVAL);
+ }
+
+ /* del vlan_mac list or ref_cnt - 1 */
+ /* del mac_vlan when ref_cnt = 0 */
+ if (rdev->ndev != rdev->gid_dev[index]) {
+ mutex_lock(&rdev->mac_vlan_mutex);
+ old_list = roce5_find_vlan_device_list(
+ &rdev->mac_vlan_list_head, rdev->gid_dev[index]);
+ if (old_list) {
+ old_list->ref_cnt--;
+ if (old_list->ref_cnt == 0) {
+ roce5_del_vlan_device_mac(rdev, old_list);
+
+ roce5_del_vlan_device(rdev, old_list);
+ }
+ }
+ mutex_unlock(&rdev->mac_vlan_mutex);
+ }
+
+ ret = memset_s(&gid_entry, sizeof(gid_entry), 0, sizeof(gid_entry));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ ret = roce5_get_gid(rdev, 1, index, &gid_entry);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d), index(%u)",
+ rdev->glb_func_id, index);
+ return ret;
+ }
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &gid_entry,
+ ROCE_MPU_CMD_DEL_IPSU_IP);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to del ipsu ip entry, func_id(%d)",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ roce5_del_ipsu_tcam_tbl_entry(rdev, &gid_entry);
+ }
+
+ /*
+ * delete gid no longer send cmd to ucode which write 0s to target entry,
+ * preventing PPE from building malformed packets. */
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h
new file mode 100644
index 000000000..f77f75dbc
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h
@@ -0,0 +1,70 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_NETDEV_H
+#define ROCE_NETDEV_H
+
+#include <net/ipv6.h>
+#include <net/bonding.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/inetdevice.h>
+#include <linux/if_vlan.h>
+
+#include <rdma/ib_addr.h>
+
+#include "roce.h"
+#include "roce_compat.h"
+#include "roce_mix.h"
+#include "roce_cmd.h"
+
+#ifdef ROCE_BONDING_EN
+#include "hinic5_srv_nic.h"
+#endif
+
+#include "securec.h"
+
+#define ROCE_GID_SET_VLAN_32BIT_VLAID(vlan_id) \
+ ((vlan_id) | (((u32)1) << 31)) /* the 31 bit is vlan eable bit */
+#define ROCE_MAC_ADDR_LEN 6
+
+/* ipv4 header len. unit:4B */
+#define IPV4_HDR_LEN 5
+
+#define IP_ECN_PERMIT 0x2 //ip header tclass bit[0:2]
+
+#define ROCE_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL
+
+#define ROCE_GID_MAP_TBL_IDX_GET(tag, new_gid_type) \
+ ((u8)(((tag) << 2) | (new_gid_type)))
+
+#define ROCE_GID_MAP_TBL_IDX2 2
+
+enum roce5_gid_vlan_type { ROCE_GID_VLAN_INVALID = 0, ROCE_GID_VLAN_VALID };
+
+enum roce5_gid_tunnel_type {
+ ROCE_GID_TUNNEL_INVALID = 0,
+ ROCE_GID_TUNNEL_VALID
+};
+
+struct roce5_vlan_dev_list {
+ u8 mac[ETH_ALEN];
+ u32 vlan_id;
+ u32 ref_cnt;
+ struct net_device *net_dev;
+ struct list_head list;
+};
+
+#if !defined(INETDEV_HAVE_FOR_IFA)
+#define for_ifa(in_dev) \
+ { \
+ struct in_ifaddr *ifa; \
+ for (ifa = (in_dev)->ifa_list; ifa; ifa = ifa->ifa_next)
+#define endfor_ifa(in_dev) }
+#endif
+
+int roce5_add_real_device_mac(struct roce5_device *rdev,
+ struct net_device *netdev);
+
+#endif /* ROCE_NETDEV_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c
new file mode 100644
index 000000000..6a16c7f29
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c
@@ -0,0 +1,413 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved.
+
+
+#include <linux/genetlink.h>
+#include <linux/netlink.h>
+#include <linux/rtnetlink.h>
+#include <linux/skbuff.h>
+#include <linux/socket.h>
+#include <net/genetlink.h>
+#include <net/net_namespace.h>
+#include <net/netlink.h>
+
+#include "roce.h"
+#include "hinic5_crm.h"
+#include "hinic5_lld.h"
+#include "roce_drv_msg_intf.h"
+#include "roce_infra.h"
+#include "roce_kernel_compat_gen.h"
+#include "roce_netlink_process.h"
+#include "securec.h"
+#include "udk_usrnl.h"
+#ifdef ENABLE_HYPER_ROCE
+#include "hyper_roce_netlink.h"
+#endif
+#include "roce_netlink.h"
+
+static int roce5_netlink_cmd_dispatch(struct sk_buff *skb,
+ struct genl_info *nl_info);
+
+static DEFINE_RWLOCK(g_cmd_process_rwlock); /* lock for cmd_cbs */
+static roce5_netlink_cmd_excute_callback cmd_cbs[USRNL_ROCE_CMD_NUM];
+
+static int roce5_netlink_register_cb(enum roce_usrnl_cmd cmd,
+ roce5_netlink_cmd_excute_callback cb)
+{
+ int ret = 0;
+
+ if (cmd >= USRNL_ROCE_CMD_NUM) {
+ ROCE_ERR("Invalid command(%u)", cmd);
+ return -EINVAL;
+ }
+
+ write_lock(&g_cmd_process_rwlock);
+
+ if (cmd_cbs[cmd] != NULL) {
+ ROCE_ERR(
+ "Repeated command setting, the callback of the command(%u) already exists.",
+ cmd);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ cmd_cbs[cmd] = cb;
+
+out:
+ write_unlock(&g_cmd_process_rwlock);
+ return ret;
+}
+
+static void roce5_netlink_unregister_cb(enum roce_usrnl_cmd cmd)
+{
+ if (cmd >= USRNL_ROCE_CMD_NUM) {
+ ROCE_ERR("Invalid command(%u)", cmd);
+ return;
+ }
+
+ write_lock(&g_cmd_process_rwlock);
+ cmd_cbs[cmd] = NULL;
+ write_unlock(&g_cmd_process_rwlock);
+}
+
+int roce5_netlink_register_cbs(roce5_netlink_cmd_process cmd_procs[],
+ u8 cmd_num)
+{
+ int ret, i;
+
+ for (i = 0; i < cmd_num; ++i) {
+ ret = roce5_netlink_register_cb(cmd_procs[i].cmd,
+ cmd_procs[i].callback);
+ if (ret != 0) {
+ ROCE_ERR("Failed to set callback in netlink, ret(%d)",
+ ret);
+ goto err_out;
+ }
+ }
+
+ return 0;
+
+err_out:
+ for (--i; i >= 0; --i) {
+ roce5_netlink_unregister_cb(cmd_procs[i].cmd);
+ }
+ return ret;
+}
+
+static roce5_netlink_cmd_process g_cmd_process[] = {
+#ifdef ROCE_BONDING_EN
+ { USRNL_ROCE_CMD_QUERY_QP_TX_PORT, roce5_genl_query_qp_tx_port },
+ { USRNL_ROCE_CMD_SET_QP_TX_PORT, roce5_genl_set_qp_tx_port },
+ { USRNL_ROCE_CMD_QUERY_QP_RX_PORT, roce5_genl_query_qp_rx_port },
+ { USRNL_ROCE_CMD_QUERY_BOND_PORT_INFO,
+ roce5_genl_query_bond_port_info },
+ { USRNL_ROCE_CMD_SET_QP_UDP_SRC_PORT, roce5_genl_set_qp_udp_src_port },
+ { USRNL_ROCE_CMD_QUERY_QP_UDP_SRC_PORT,
+ roce5_genl_query_qp_udp_src_port },
+#endif
+ { USRNL_ROCE_CMD_QUERY_NEXT_WQE_IDX, roce5_genl_query_next_wqe_idx },
+ { USRNL_ROCE_CMD_QUERY_QPC, roce5_genl_query_qpc },
+ { USRNL_ROCE_CMD_QUERY_CQC, roce5_genl_query_cqc },
+ { USRNL_ROCE_CMD_QP_HASH_VALUE_PROCESS,
+ roce5_genl_qp_hash_value_process },
+ { USRNL_ROCE_CMD_ULD,
+ roce5_netlink_uld_process }, /* usr kernel of uld interface */
+};
+
+static int roce5_netlink_init_cbs(void)
+{
+ int ret;
+
+ ret = roce5_netlink_register_cbs(
+ g_cmd_process,
+ sizeof(g_cmd_process) / sizeof(roce5_netlink_cmd_process));
+ if (ret != 0) {
+ ROCE_ERR("Failed to register netlink callbacks, ret(%d).", ret);
+ return ret;
+ }
+
+#ifdef ENABLE_HYPER_ROCE
+ ret = roce5_hyper_netlink_init();
+ if (ret != 0) {
+ ROCE_ERR("Failed to init hyper roce in netlink, ret(%d).", ret);
+ return ret;
+ }
+#endif
+
+ return 0;
+}
+
+#if defined(NLA_POLICY_HAVE_MIN_MAX)
+static const struct nla_policy roce5_policy[USRNL_GENL_ATTR_MAX + 1] = {
+ [USRNL_GENL_ATTR] = { .type = NLA_BINARY,
+ .len = CMD_BUF_LEN_MAX,
+ .min = 0,
+ .max = CMD_BUF_LEN_MAX },
+};
+#else
+static const struct nla_policy roce5_policy[USRNL_GENL_ATTR_MAX + 1] = {
+ [USRNL_GENL_ATTR] = { .type = NLA_BINARY, .len = CMD_BUF_LEN_MAX },
+};
+#endif
+
+static const struct genl_ops roce5_genl_ops[] = { {
+ .cmd = USRNL_DEV_DRV_CMD_EXECUTE,
+#ifdef HAVE_GENL_OPS_FIELD_VALIDATE
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+#endif
+ .flags = GENL_ADMIN_PERM,
+ .policy = roce5_policy,
+ .doit = roce5_netlink_cmd_dispatch,
+} };
+
+static struct genl_family roce5_genl_family = {
+ .hdrsize = 0,
+ .name = ROCE_GENL_FAMILY_NAME,
+ .version = 1,
+ .maxattr = USRNL_GENL_ATTR_MAX,
+ .netnsok = true,
+ .parallel_ops = true,
+ .ops = roce5_genl_ops,
+ .n_ops = ARRAY_SIZE(roce5_genl_ops),
+};
+
+static void roce5_drv_msg_hdr_push(struct sk_buff *skb,
+ usrnl_drv_msg_hdr_s *in_msg_hdr,
+ uint16_t out_msg_data_len)
+{
+ usrnl_drv_msg_hdr_s *out_msg_hdr = NULL;
+
+ out_msg_hdr = skb_push(skb, ROCE_DRV_MSG_HDRLEN);
+ out_msg_hdr->cmd = in_msg_hdr->cmd;
+ out_msg_hdr->msg_len = ROCE_DRV_MSG_HDRLEN + out_msg_data_len;
+ out_msg_hdr->module = in_msg_hdr->module;
+ out_msg_hdr->src_func_idx = in_msg_hdr->src_func_idx;
+ out_msg_hdr->err_code = 0;
+}
+
+static void roce5_nlahdr_push(struct sk_buff *skb, uint16_t type)
+{
+ struct nlattr *nlahdr = NULL;
+ uint16_t attrlen = (uint16_t)skb->len;
+ uint32_t padlen = (uint32_t)nla_padlen(attrlen);
+
+ nlahdr = (struct nlattr *)skb_push(skb, NLA_HDRLEN);
+ nlahdr->nla_len = (uint16_t)nla_attr_size(attrlen);
+ nlahdr->nla_type = type;
+
+ if ((skb->end - skb->tail) >= padlen) {
+ (void)skb_put(skb, padlen);
+ }
+}
+
+static struct nlmsghdr *nlmsg_push(struct sk_buff *skb, uint32_t portid,
+ uint32_t seq, uint16_t type, uint32_t len,
+ uint16_t flags)
+{
+ struct nlmsghdr *nlh = NULL;
+ uint32_t size = (uint32_t)nlmsg_msg_size(len);
+
+ nlh = (struct nlmsghdr *)skb_push(skb, NLMSG_ALIGN(size));
+ if (nlh != NULL) {
+ nlh->nlmsg_type = type;
+ nlh->nlmsg_len = skb->len;
+ nlh->nlmsg_flags = flags;
+ nlh->nlmsg_pid = portid;
+ nlh->nlmsg_seq = seq;
+ if (!__builtin_constant_p(size) ||
+ NLMSG_ALIGN(size) - size != 0) {
+ (void)memset_s((char *)nlmsg_data(nlh) + len,
+ NLMSG_ALIGN(size) - size, 0,
+ NLMSG_ALIGN(size) - size);
+ }
+ }
+
+ return nlh;
+}
+
+static inline struct nlmsghdr *roce5_nlmsg_push(struct sk_buff *skb,
+ uint32_t portid, uint32_t seq,
+ uint16_t type, uint32_t payload,
+ uint16_t flags)
+{
+ if (unlikely(skb_headroom(skb) <
+ (unsigned int)nlmsg_total_size(payload))) {
+ return NULL;
+ }
+
+ return nlmsg_push(skb, portid, seq, type, payload, flags);
+}
+
+static void *roce5_genlmsg_push(struct sk_buff *skb, uint32_t portid,
+ uint32_t seq, const struct genl_family *family,
+ uint16_t flags, uint8_t cmd)
+{
+ struct nlmsghdr *nlh = NULL;
+ struct genlmsghdr *hdr = NULL;
+
+ nlh = roce5_nlmsg_push(skb, portid, seq, (int)family->id,
+ (int)(GENL_HDRLEN + family->hdrsize), flags);
+ if (nlh == NULL) {
+ return NULL;
+ }
+
+ hdr = nlmsg_data(nlh);
+ hdr->cmd = cmd;
+ hdr->version = (uint8_t)family->version;
+ hdr->reserved = 0;
+
+ return (void *)nlh;
+}
+
+static inline roce5_netlink_cmd_excute_callback
+roce5_netlink_get_callback(uint16_t cmd)
+{
+ roce5_netlink_cmd_excute_callback cb = NULL;
+
+ if (cmd >= USRNL_ROCE_CMD_NUM) {
+ ROCE_ERR("Invalid command(%u).", cmd);
+ return NULL;
+ }
+
+ read_lock(&g_cmd_process_rwlock);
+ cb = cmd_cbs[cmd];
+ read_unlock(&g_cmd_process_rwlock);
+
+ return cb;
+}
+
+static int32_t
+roce5_netlink_buf_handler(struct nlattr *cmd_attr,
+ roce5_netlink_cmd_excute_callback callback,
+ struct roce5_device *rdev, struct genl_info *nl_info,
+ struct sk_buff *reply)
+{
+ int32_t ret = 0;
+ uint16_t buf_in_len = 0;
+ uint16_t max_buf_out_len = 0;
+ uint16_t buf_out_len = 0;
+ usrnl_drv_msg_hdr_s *in_msg_hdr = NULL;
+
+ in_msg_hdr = (usrnl_drv_msg_hdr_s *)nla_data(cmd_attr);
+ if (in_msg_hdr->msg_len < ROCE_DRV_MSG_HDRLEN) {
+ ROCE_DEV_ERR(rdev, "Invalid parameter: msg_len(%d).",
+ in_msg_hdr->msg_len);
+ return -EINVAL;
+ }
+
+ buf_in_len = in_msg_hdr->msg_len - ROCE_DRV_MSG_HDRLEN;
+ max_buf_out_len =
+ NLMSG_DEFAULT_SIZE -
+ (NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN + ROCE_DRV_MSG_HDRLEN);
+
+ if (buf_in_len == 0) {
+ ret = callback(rdev, NULL, 0, reply->data, max_buf_out_len,
+ &buf_out_len);
+ } else {
+ ret = callback(rdev, nla_data(cmd_attr) + ROCE_DRV_MSG_HDRLEN,
+ buf_in_len, reply->data, max_buf_out_len,
+ &buf_out_len);
+ }
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Do netlink %u cmd failed, errcode:%d",
+ nl_info->genlhdr->cmd, ret);
+ nlmsg_free(reply);
+ return -EIO;
+ }
+
+ if (buf_out_len != 0) {
+ skb_put(reply, buf_out_len);
+ }
+
+ roce5_drv_msg_hdr_push(reply, in_msg_hdr, buf_out_len);
+
+ return 0;
+}
+
+static int roce5_netlink_cmd_dispatch(struct sk_buff *skb,
+ struct genl_info *nl_info)
+{
+ int32_t ret = 0;
+ struct sk_buff *reply = NULL;
+ struct nlattr *cmd_attr = NULL;
+ roce5_drv_msg_hdr_s *in_msg_hdr = NULL;
+ roce5_netlink_cmd_excute_callback callback = NULL;
+ struct roce5_device *rdev = NULL;
+
+ cmd_attr = nl_info->attrs[USRNL_GENL_ATTR];
+ if (cmd_attr == NULL) {
+ ROCE_ERR("Do netlink cmd from portid:%u failed for NULL attr",
+ nl_info->snd_portid);
+ return -EINVAL;
+ } else if (nla_len(cmd_attr) < ROCE_DRV_MSG_HDRLEN) {
+ ROCE_ERR("Invalid parameter: nla_len(%d).", nla_len(cmd_attr));
+ return -EINVAL;
+ }
+
+ in_msg_hdr = (roce5_drv_msg_hdr_s *)nla_data(cmd_attr);
+
+ rdev = roce5_get_rdev_by_card_func_id(
+ in_msg_hdr->card_id, in_msg_hdr->common_hdr.src_func_idx);
+ if (rdev == NULL) {
+ ROCE_ERR("Invalid param: src_func_idx: %u, card_id: %llu",
+ in_msg_hdr->common_hdr.src_func_idx,
+ in_msg_hdr->card_id);
+ return -EINVAL;
+ }
+
+ callback = roce5_netlink_get_callback(in_msg_hdr->common_hdr.cmd);
+ if (callback == NULL) {
+ ROCE_DEV_ERR(rdev, "Unable to handle cmd: %u",
+ in_msg_hdr->common_hdr.cmd);
+ return -EINVAL;
+ }
+
+ reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (reply == NULL) {
+ ROCE_DEV_ERR(rdev, "Alloc reply buf for netlink cmd failed");
+ return -ENOMEM;
+ }
+
+ skb_reserve(reply, NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN +
+ ROCE_DRV_MSG_HDRLEN);
+
+ ret = roce5_netlink_buf_handler(cmd_attr, callback, rdev, nl_info,
+ reply);
+ if (ret != 0) {
+ return ret;
+ }
+
+ roce5_nlahdr_push(reply, USRNL_GENL_ATTR);
+ roce5_genlmsg_push(reply, nl_info->snd_portid, nl_info->snd_seq,
+ &roce5_genl_family, 0, nl_info->genlhdr->cmd);
+
+ genlmsg_reply(reply, nl_info);
+
+ return 0;
+}
+
+int32_t roce5_netlink_init(void)
+{
+ int ret;
+
+ ret = roce5_netlink_init_cbs();
+ if (ret != 0) {
+ ROCE_ERR("Failed to init netlink callbacks, ret(%d).", ret);
+ return ret;
+ }
+
+ ret = genl_register_family(&roce5_genl_family);
+ if (ret != 0) {
+ ROCE_ERR("Failed to register netlink, ret(%d).", ret);
+ return ret;
+ }
+
+ ROCE_INFO("netlink init successfully.");
+ return 0;
+}
+
+void roce5_netlink_deinit(void)
+{
+ genl_unregister_family(&roce5_genl_family);
+ ROCE_INFO("netlink deinit successfully.");
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h
new file mode 100644
index 000000000..9c4cfff02
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h
@@ -0,0 +1,53 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved.
+
+
+#ifndef ROCE5_NETLINK_H
+#define ROCE5_NETLINK_H
+
+#include <linux/types.h>
+#include "roce_drv_msg_intf.h"
+#include "roce.h"
+
+#define ROCE_GENL_FAMILY_NAME "roce5_genl"
+#define ROCE_DRV_MSG_HDRLEN (sizeof(roce5_drv_msg_hdr_s))
+#define CMD_BUF_LEN_MAX 2048
+
+typedef int32_t (*roce5_netlink_cmd_excute_callback)(struct roce5_device *rdev,
+ void *in, uint16_t in_len,
+ void *out,
+ uint16_t out_len,
+ uint16_t *out_size);
+
+enum roce_usrnl_cmd {
+ USRNL_ROCE_CMD_QUERY_QP_TX_PORT,
+ USRNL_ROCE_CMD_SET_QP_TX_PORT,
+ USRNL_ROCE_CMD_QUERY_QP_RX_PORT,
+ USRNL_ROCE_CMD_QUERY_BOND_PORT_INFO,
+ USRNL_ROCE_CMD_SET_QP_UDP_SRC_PORT,
+ USRNL_ROCE_CMD_QUERY_QP_UDP_SRC_PORT,
+ USRNL_ROCE_CMD_QUERY_NEXT_WQE_IDX,
+ USRNL_ROCE_CMD_QUERY_QPC,
+ USRNL_ROCE_CMD_QUERY_CQC,
+ USRNL_ROCE_CMD_QP_HASH_VALUE_PROCESS,
+ USRNL_ROCE_CMD_ULD, /* uld use */
+
+ /* Command number for HyperRoCE */
+ USRNL_ROCE_HYPER_ROCE_START = 16,
+ USRNL_ROCE_CMD_HYPER_QP_SET = USRNL_ROCE_HYPER_ROCE_START,
+ USRNL_ROCE_CMD_MODIFY_QP_EXTEND,
+ USRNL_ROCE_HYPER_ROCE_END = 31,
+ USRNL_ROCE_CMD_NUM
+};
+
+typedef struct {
+ enum roce_usrnl_cmd cmd;
+ roce5_netlink_cmd_excute_callback callback;
+} roce5_netlink_cmd_process;
+
+int32_t roce5_netlink_init(void);
+void roce5_netlink_deinit(void);
+int roce5_netlink_register_cbs(roce5_netlink_cmd_process cmd_procs[],
+ u8 cmd_num);
+
+#endif /* ROCE5_NETLINK_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c
new file mode 100644
index 000000000..8b7445ab0
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c
@@ -0,0 +1,832 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include <linux/types.h>
+#include <linux/semaphore.h>
+
+#include "hinic5_cqm.h"
+#include "hinic5_mt.h"
+#include "roce_cqm_cmd.h"
+#include "roce_cmd.h"
+#include "roce_com.h"
+#include "roce_dfx.h"
+#include "roce_main_extension.h"
+#include "roce_mix.h"
+#include "roce_npu_cmd.h"
+#include "roce_npu_cmd_defs.h"
+#include "roce_qp.h"
+#include "roce_restore.h"
+#include "roce_srq.h"
+#include "roce_uld_inner.h"
+#include "roce_uld_kernel_api.h"
+#include "roce_netlink_process.h"
+#ifdef ROCE_BONDING_EN
+#include "roce_bond.h"
+#endif
+
+static int32_t roce5_set_hash_value(struct roce5_device *rdev, uint32_t qpn,
+ uint32_t hash_type, uint32_t new_hash_value)
+{
+ int32_t ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ void *modify_hash_value = NULL;
+ size_t buf_size = 0;
+
+ if (ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) != 0) {
+ buf_size = sizeof(roce_uni_cmd_modify_hash_s);
+ }
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf,
+ (uint16_t)buf_size, NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+ modify_hash_value = cqm_cmd_inbuf->buf;
+
+ if (ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) != 0) {
+ roce_uni_cmd_modify_hash_s *cmd =
+ (roce_uni_cmd_modify_hash_s *)modify_hash_value;
+ cmd->com.index = cpu_to_be32(qpn);
+ cmd->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ cmd->info.type = cpu_to_be32(hash_type);
+ cmd->info.hash_value = cpu_to_be32(new_hash_value);
+ }
+
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_HASH_VALUE_QP,
+ cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d), qpn(%u), type(%u), hash_value(%u), ret=%d",
+ rdev->glb_func_id, qpn, hash_type, new_hash_value, ret);
+ ret = -1;
+ }
+
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+static int32_t roce5_get_hash_value(struct roce5_device *rdev, uint32_t qpn,
+ uint32_t hash_type, uint32_t *hash_value)
+{
+ int32_t ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_get_hash_s *get_hash_value = NULL;
+ roce_verbs_qp_hash_info_s *get_hash_value_out = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (uint16_t)sizeof(roce_uni_cmd_get_hash_s), &cqm_cmd_outbuf,
+ (uint16_t)sizeof(roce_verbs_qp_hash_info_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ get_hash_value = (roce_uni_cmd_get_hash_s *)cqm_cmd_inbuf->buf;
+ get_hash_value->com.index = cpu_to_be32(qpn);
+ get_hash_value->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ get_hash_value->info.type = cpu_to_be32(hash_type);
+
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_HASH_VALUE_QP,
+ cqm_cmd_inbuf, cqm_cmd_outbuf,
+ ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send ROCE_CMD_GET_HASH_VALUE_QP command, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -1;
+ goto out;
+ }
+
+ get_hash_value_out = (roce_verbs_qp_hash_info_s *)cqm_cmd_outbuf->buf;
+ *hash_value = be32_to_cpu(get_hash_value_out->hash_value);
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+ return ret;
+}
+
+#ifdef ROCE_BONDING_EN
+int32_t roce5_genl_query_qp_tx_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size)
+{
+ int32_t ret = 0;
+ uint32_t slave_cnt = 0;
+ struct roce5_qp *rqp = NULL;
+ uint32_t func_tbl_value = 0;
+ struct roce5_qp_in *msg_in = NULL;
+ struct roce5_port_out *msg_out = NULL;
+
+ if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) ||
+ out == NULL || out_len == 0 ||
+ out_len < sizeof(struct roce5_port_out) || out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d",
+ !!in, in_len, !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_in *)in;
+ *out_size = sizeof(struct roce5_port_out);
+ msg_out = (struct roce5_port_out *)out;
+
+ if (!roce5_bond_is_active(rdev)) {
+ msg_out->port = 0;
+ return 0;
+ }
+
+ rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp");
+ return -EINVAL;
+ }
+
+ read_lock(&rdev->rwlock_func_tbl_bond);
+ func_tbl_value = rdev->func_tbl_bond;
+ slave_cnt = func_tbl_value & 0xfU;
+ if (slave_cnt != 0) {
+ msg_out->port = (func_tbl_value >>
+ (ROCE_BOND_FWD_ID_TBL_ALL_BITS -
+ ((rqp->tx_hash_value % slave_cnt + 1) *
+ ROCE_BOND_FWD_ID_TBL_PER_BITS))) &
+ ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK;
+ }
+ read_unlock(&rdev->rwlock_func_tbl_bond);
+
+ if (slave_cnt == 0) {
+ ROCE_ERR("slave_cnt is 0");
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+err_out:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int32_t roce5_set_new_tx_hash_value(struct roce5_device *rdev,
+ struct roce5_qp *rqp,
+ struct roce5_qp_port_in *msg_in)
+{
+ int32_t ret = 0;
+ uint32_t func_tbl_value = 0;
+ uint32_t slave_cnt = 0;
+ uint32_t target_hash_port = 0;
+ uint32_t old_hash_port = 0;
+ uint32_t new_tx_hash_value;
+
+ read_lock(&rdev->rwlock_func_tbl_bond);
+ func_tbl_value = rdev->func_tbl_bond;
+ read_unlock(&rdev->rwlock_func_tbl_bond);
+
+ slave_cnt = func_tbl_value & 0xfU;
+ if (slave_cnt == 0) {
+ ROCE_DEV_ERR(rdev, "slave_cnt is 0.");
+ return -EINVAL;
+ }
+ target_hash_port = msg_in->port & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK;
+ old_hash_port =
+ (func_tbl_value >> (ROCE_BOND_FWD_ID_TBL_ALL_BITS -
+ ((rqp->tx_hash_value % slave_cnt + 1) *
+ ROCE_BOND_FWD_ID_TBL_PER_BITS))) &
+ ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK;
+
+ if (target_hash_port == old_hash_port) {
+ return 0;
+ }
+
+ new_tx_hash_value =
+ rqp->tx_hash_value +
+ (((slave_cnt - old_hash_port) + target_hash_port) % slave_cnt);
+ ret = roce5_set_hash_value(rdev, rqp->qpn, ROCE_VERBS_QP_HASH_TYPE_BOND,
+ new_tx_hash_value);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify QP(0x%06x) Bond hash value, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ return ret;
+ }
+
+ rqp->tx_hash_value = new_tx_hash_value;
+
+ return 0;
+}
+
+int32_t roce5_genl_set_qp_tx_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out, uint16_t out_len,
+ uint16_t *out_size)
+{
+ int32_t ret = 0;
+ struct roce5_qp_port_in *msg_in = NULL;
+ struct roce5_qp *rqp = NULL;
+
+ if (in == NULL || in_len == 0 ||
+ in_len < sizeof(struct roce5_qp_port_in) || out_size == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid param, in:%d, in_len:%u, out_size:%d",
+ !!in, in_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_port_in *)in;
+
+ if (!roce5_bond_is_active(rdev)) {
+ return 0;
+ }
+
+ /* check qp exist */
+ rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp");
+ return -EINVAL;
+ }
+
+ ret = roce5_set_new_tx_hash_value(rdev, rqp, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set new tx hash value, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_out;
+ }
+
+err_out:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int roce5_uni_cmd_get_qp_rx_port(struct roce5_device *rdev, uint32_t qpn,
+ uint32_t *rx_port)
+{
+ int32_t ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_get_port_info_s *get_rx_port_inbuf = NULL;
+ roce_uni_cmd_get_port_info_outbuf_s *get_rx_port_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (uint16_t)sizeof(roce_uni_cmd_get_port_info_s), &cqm_cmd_outbuf,
+ (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ get_rx_port_inbuf = (roce_uni_cmd_get_port_info_s *)cqm_cmd_inbuf->buf;
+ get_rx_port_inbuf->com.index = cpu_to_be32(qpn);
+ get_rx_port_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK));
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_RX_PORT_QP,
+ cqm_cmd_inbuf, cqm_cmd_outbuf,
+ ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send ROCE_CMD_GET_RX_PORT_QP command, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -1;
+ goto out;
+ }
+ get_rx_port_outbuf =
+ (roce_uni_cmd_get_port_info_outbuf_s *)cqm_cmd_outbuf->buf;
+ *rx_port = be32_to_cpu(get_rx_port_outbuf->value.rx_port);
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+ return ret;
+}
+
+static int32_t roce5_get_qp_rx_port(struct roce5_device *rdev, uint32_t qpn,
+ uint32_t *rx_port)
+{
+ if (ROCE5_SUPPORT_GET_QP_RX_PORT_MSG_V2(rdev) != 0) {
+ return roce5_uni_cmd_get_qp_rx_port(rdev, qpn, rx_port);
+ }
+ return -EINVAL;
+}
+
+int32_t roce5_genl_query_qp_rx_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size)
+{
+ int32_t ret = 0;
+ uint32_t rx_port = 0;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_qp_in *msg_in = NULL;
+ struct roce5_port_out *msg_out = NULL;
+
+ if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) ||
+ out == NULL || out_len == 0 ||
+ out_len < sizeof(struct roce5_port_out) || out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d",
+ !!in, in_len, !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_in *)in;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp");
+ return -EINVAL;
+ }
+
+ if (rqp->qp_type != IB_QPT_RC) {
+ ROCE_DEV_ERR(rdev, "not support qp type(%d), only support RC.",
+ rqp->qp_type);
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ *out_size = sizeof(struct roce5_port_out);
+ msg_out = (struct roce5_port_out *)out;
+
+ if (!roce5_bond_is_active(rdev)) {
+ msg_out->port = 0;
+ ret = 0;
+ goto err_out;
+ }
+
+ ret = roce5_get_qp_rx_port(rdev, msg_in->qpn, &rx_port);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get QP(0x%06x) rx port",
+ msg_in->qpn);
+ goto err_out;
+ }
+
+ msg_out->port = rx_port;
+
+err_out:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+int32_t roce5_genl_query_bond_port_info(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size)
+{
+ uint32_t func_tbl_value = 0;
+ struct roce5_bond_device *bond_dev = NULL;
+ struct roce5_bond_port_info_out *msg_out = NULL;
+ uint32_t alive_port_num;
+ uint32_t i;
+
+ if (out == NULL || out_len == 0 ||
+ out_len < sizeof(struct roce5_bond_port_info_out) ||
+ out_size == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid param, out:%d, out_len:%u, out_size:%d",
+ !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ *out_size = sizeof(struct roce5_bond_port_info_out);
+ msg_out = (struct roce5_bond_port_info_out *)out;
+
+ if (!roce5_bond_is_active(rdev)) {
+ return 0;
+ }
+
+ bond_dev = rdev->bond_dev;
+
+ /* Read func_tbl_bond and derive alive port info under lock to prevent data race */
+ read_lock(&rdev->rwlock_func_tbl_bond);
+ func_tbl_value = rdev->func_tbl_bond;
+ alive_port_num = func_tbl_value & 0xfU;
+ for (i = 0; i < alive_port_num; i++) {
+ msg_out->alive_port[i] =
+ (func_tbl_value >>
+ (ROCE_BOND_FWD_ID_TBL_ALL_BITS -
+ ((i + 1) * ROCE_BOND_FWD_ID_TBL_PER_BITS))) &
+ ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK;
+ }
+ read_unlock(&rdev->rwlock_func_tbl_bond);
+ msg_out->alive_port_num =
+ (int32_t)alive_port_num; /* netlink protocol field */
+
+ /* Read slave info under slave_lock (separate from rwlock to avoid sleeping in atomic context) */
+ msg_out->original_port_num =
+ (int32_t)bond_dev->slave_cnt; /* netlink protocol field */
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ msg_out->original_port[i] = bond_dev->slaves[i].func_id;
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+
+ return 0;
+}
+
+static int32_t roce5_uni_cmd_modify_udp_src_port(struct roce5_device *rdev,
+ uint32_t qpn,
+ uint32_t new_udp_src_port)
+{
+ int32_t ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_set_udp_src_port_s *modify_udp_src_port = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (uint16_t)sizeof(roce_uni_cmd_set_udp_src_port_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ modify_udp_src_port =
+ (roce_uni_cmd_set_udp_src_port_s *)cqm_cmd_inbuf->buf;
+ modify_udp_src_port->com.index = cpu_to_be32(qpn);
+ modify_udp_src_port->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ modify_udp_src_port->info.udp_src_port = cpu_to_be32(new_udp_src_port);
+
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_UDP_SRC_PORT_QP,
+ cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -1;
+ }
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+static int32_t roce5_modify_udp_src_port(struct roce5_device *rdev,
+ uint32_t qpn,
+ uint32_t new_udp_src_port)
+{
+ if (ROCE5_SUPPORT_MODIFY_UDP_SRC_PORT_MSG_V2(rdev) != 0) {
+ return roce5_uni_cmd_modify_udp_src_port(rdev, qpn,
+ new_udp_src_port);
+ }
+ return -EINVAL;
+}
+
+int32_t roce5_genl_set_qp_udp_src_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size)
+{
+ int32_t ret = 0;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_qp_udp_src_port_in *msg_in = NULL;
+
+ if (in == NULL || in_len == 0 ||
+ in_len < sizeof(struct roce5_qp_udp_src_port_in) ||
+ out_size == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid param, in:%d, in_len:%u, out_size:%d",
+ !!in, in_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_udp_src_port_in *)in;
+
+ /* check qp exist */
+ rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp");
+ return -EINVAL;
+ }
+
+ ret = roce5_modify_udp_src_port(rdev, rqp->qpn, msg_in->udp_src_port);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to modify QP(0x%06x) hash value, func_id(%d)",
+ rqp->qpn, rdev->glb_func_id);
+ ret = -EIO;
+ goto err_out;
+ }
+
+err_out:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int32_t roce5_uni_cmd_get_qp_udp_src_port(struct roce5_device *rdev,
+ uint32_t qpn,
+ uint32_t *udp_src_port)
+{
+ int32_t ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_get_port_info_s *get_udp_src_port = NULL;
+ roce_uni_cmd_get_port_info_outbuf_s *get_udp_src_port_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s),
+ &cqm_cmd_outbuf,
+ (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ get_udp_src_port = (roce_uni_cmd_get_port_info_s *)cqm_cmd_inbuf->buf;
+ get_udp_src_port->com.index = cpu_to_be32(qpn);
+ get_udp_src_port->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK));
+
+ ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_UDP_SRC_PORT_QP,
+ cqm_cmd_inbuf, cqm_cmd_outbuf,
+ ROCE_CMD_TIME_CLASS_A);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send ROCE_CMD_GET_UDP_SRC_PORT_QP command, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -1;
+ goto out;
+ }
+
+ get_udp_src_port_outbuf =
+ (roce_uni_cmd_get_port_info_outbuf_s *)cqm_cmd_outbuf->buf;
+ *udp_src_port =
+ be32_to_cpu(get_udp_src_port_outbuf->value.udp_src_port);
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+
+ return ret;
+}
+
+static int32_t roce5_get_qp_udp_src_port(struct roce5_device *rdev,
+ uint32_t qpn, uint32_t *udp_src_port)
+{
+ if (ROCE5_SUPPORT_GET_QP_UDP_SRC_PORT_MSG_V2(rdev) != 0) {
+ return roce5_uni_cmd_get_qp_udp_src_port(rdev, qpn,
+ udp_src_port);
+ }
+ return -EINVAL;
+}
+
+int32_t roce5_genl_query_qp_udp_src_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size)
+{
+ int32_t ret = 0;
+ uint32_t udp_src_port = 0;
+ struct roce5_qp_in *msg_in = NULL;
+ struct roce5_port_out *msg_out = NULL;
+
+ if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) ||
+ out == NULL || out_len == 0 ||
+ out_len < sizeof(struct roce5_port_out) || out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d",
+ !!in, in_len, !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_in *)in;
+
+ ret = roce5_get_qp_udp_src_port(rdev, msg_in->qpn, &udp_src_port);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to get QP(0x%06x) udp src port",
+ msg_in->qpn);
+ return ret;
+ }
+
+ *out_size = sizeof(struct roce5_port_out);
+ msg_out = (struct roce5_port_out *)out;
+
+ msg_out->port = udp_src_port;
+
+ return 0;
+}
+#endif
+
+int32_t roce5_genl_query_next_wqe_idx(struct roce5_device *rdev, void *in,
+ u16 in_len, void *out, u16 out_len,
+ u16 *out_size)
+{
+ int32_t ret = 0;
+ int32_t next_wqe_idx = 0;
+ uint32_t container_mode = 0;
+ uint32_t container_size = 0;
+ struct roce5_next_wqe_idx_in *msg_in = NULL;
+ struct roce5_next_wqe_idx_out *msg_out = NULL;
+ roce_verbs_srq_attr_s *srq_attr = NULL;
+
+ if (in == NULL || in_len == 0 ||
+ in_len < sizeof(struct roce5_next_wqe_idx_in) || out == NULL ||
+ out_len == 0 || out_len < sizeof(struct roce5_next_wqe_idx_out) ||
+ out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d",
+ !!in, in_len, !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ srq_attr = kmalloc(sizeof(roce_verbs_srq_attr_s), GFP_KERNEL);
+ if (srq_attr == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to allocate memory for dfx_query_outbuf");
+ return -ENOMEM;
+ }
+
+ msg_in = (struct roce5_next_wqe_idx_in *)in;
+ msg_out = (struct roce5_next_wqe_idx_out *)out;
+ ret = roce5_dfx_cmd_query_srq(rdev, msg_in->srqn, srq_attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to do roce5_dfx_cmd_query_srq, ret(%d)",
+ ret);
+ ret = -EINVAL;
+ goto free_outbuf;
+ }
+
+ srq_attr->dw2.value = be32_to_cpu(srq_attr->dw2.value);
+ srq_attr->dw3.value = be32_to_cpu(srq_attr->dw3.value);
+ srq_attr->cont.dw2.value = be32_to_cpu(srq_attr->cont.dw2.value);
+ if (srq_attr->dw3.bs.container == 0) {
+ next_wqe_idx = (int32_t)(srq_attr->dw2.bs.next_wqe_idx);
+ } else {
+ container_mode = MAX_SUPPORT_CONTAINER_MODE -
+ srq_attr->cont.dw2.bs.cont_size;
+ container_size = roce5_get_container_sz(container_mode);
+ next_wqe_idx = (int32_t)(container_size *
+ srq_attr->cont.dw2.bs.head_idx);
+ }
+
+ msg_out->next_wqe_idx = next_wqe_idx;
+ *out_size = sizeof(struct roce5_next_wqe_idx_out);
+ ret = 0;
+
+free_outbuf:
+ kfree(srq_attr);
+ return ret;
+}
+
+int32_t roce5_genl_query_qpc(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out, uint16_t out_len,
+ uint16_t *out_size)
+{
+ int32_t ret = 0;
+ struct roce5_qp_in *msg_in = NULL;
+ roce_verbs_qp_attr_s *verbs_qp_attr = NULL;
+
+ if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) ||
+ out == NULL || out_len == 0 ||
+ out_len < sizeof(roce_verbs_qp_attr_s) || out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d",
+ !!in, in_len, !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_in *)in;
+ verbs_qp_attr = (roce_verbs_qp_attr_s *)out;
+
+ ret = roce5_qp_query(rdev, msg_in->qpn, (u32 *)(void *)verbs_qp_attr,
+ sizeof(roce_verbs_qp_attr_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query qpc, ret(%0x)", ret);
+ return ret;
+ }
+
+ *out_size = sizeof(roce_verbs_qp_attr_s);
+ return 0;
+}
+
+int32_t roce5_genl_query_cqc(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out, uint16_t out_len,
+ uint16_t *out_size)
+{
+ int32_t ret = 0;
+ struct roce5_cq_in *msg_in = NULL;
+ roce_verbs_cq_attr_s *verbs_cq_attr = NULL;
+
+ if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_cq_in) ||
+ out == NULL || out_len == 0 ||
+ out_len < sizeof(roce_verbs_cq_attr_s) || out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d",
+ !!in, in_len, !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_cq_in *)in;
+ verbs_cq_attr = (roce_verbs_cq_attr_s *)out;
+
+ ret = roce5_dfx_query_cq(rdev, msg_in->cqn,
+ (u32 *)(void *)verbs_cq_attr,
+ sizeof(roce_verbs_cq_attr_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query cqc, ret(%0x)", ret);
+ return ret;
+ }
+
+ *out_size = sizeof(roce_verbs_cq_attr_s);
+ return 0;
+}
+
+int32_t roce5_genl_qp_hash_value_process(struct roce5_device *rdev, void *in,
+ u16 in_len, void *out, u16 out_len,
+ u16 *out_size)
+{
+ uint32_t hash_value = 0;
+ struct roce5_qp_hash_value_in *msg_in = NULL;
+ struct roce5_qp_hash_value_out *msg_out = NULL;
+ int32_t ret;
+
+ if (in == NULL || in_len == 0 ||
+ in_len < sizeof(struct roce5_qp_hash_value_in)) {
+ ROCE_DEV_ERR(rdev, "Invalid input param, in:%d, in_len:%u",
+ !!in, in_len);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_qp_hash_value_in *)in;
+
+ if (msg_in->type == ROCE_QP_HASH_VALUE_GET) {
+ if (out == NULL || out_len == 0 ||
+ out_len < sizeof(struct roce5_qp_hash_value_out) ||
+ out_size == NULL) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Invalid output param, out:%d, out_len:%u, out_size:%d",
+ !!out, out_len, !!out_size);
+ return -EINVAL;
+ }
+ ret = roce5_get_hash_value(rdev, msg_in->qpn,
+ ROCE_VERBS_QP_HASH_TYPE_UBC,
+ &hash_value);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to get QP(0x%06x) hash value, func_id(%d)",
+ msg_in->qpn, rdev->glb_func_id);
+ return ret;
+ }
+
+ *out_size = sizeof(struct roce5_qp_hash_value_out);
+ msg_out = (struct roce5_qp_hash_value_out *)out;
+ msg_out->hash_value = hash_value;
+ } else {
+ ret = roce5_set_hash_value(rdev, msg_in->qpn,
+ ROCE_VERBS_QP_HASH_TYPE_UBC,
+ msg_in->hash_value);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set QP(0x%06x) hash value, func_id(%d)",
+ msg_in->qpn, rdev->glb_func_id);
+ }
+ }
+
+ return ret;
+}
+
+int32_t roce5_netlink_uld_process(struct roce5_device *rdev, void *buf_in,
+ u16 buf_in_len, void *buf_out,
+ u16 buf_out_len, u16 *buf_out_size)
+{
+ int32_t ret = 0;
+ roce_user_handle_t handle;
+
+ handle.rdev = rdev;
+ handle.magic = ROCE_ULD_HANDLE_MAGIC_NUM;
+
+ ret = roce5_uld_user2kernel_msg_handle((void *)&handle, buf_in,
+ buf_in_len, buf_out, buf_out_len,
+ buf_out_size);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "ULD process err, func_id(%d)",
+ rdev->glb_func_id);
+ }
+ return ret;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h
new file mode 100644
index 000000000..d09bcf733
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h
@@ -0,0 +1,133 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_NETLINK_PROCESS_H
+#define ROCE_NETLINK_PROCESS_H
+
+#include "roce.h"
+#include "roce_compat.h"
+
+typedef enum {
+ ROCE_QP_HASH_VALUE_SET,
+ ROCE_QP_HASH_VALUE_GET,
+} ROCE_QP_HASH_VALUE_PROC_TYPE;
+
+#ifdef ROCE_BONDING_EN
+/* and above next data structure is adaptation V100 use */
+struct roce5_get_rx_port {
+ __be32 rx_port;
+};
+
+struct roce5_get_udp_src_port {
+ __be32 udp_src_port;
+};
+
+typedef struct tag_roce_get_qp_rx_port {
+ roce_verbs_cmd_com_s com;
+} roce_get_qp_rx_port_s;
+
+typedef struct tag_roce_modify_hash_value {
+ roce_verbs_cmd_com_s com;
+ u32 hash_value;
+} roce_modify_hash_value_s;
+
+typedef struct tag_roce_modify_udp_src_port {
+ roce_verbs_cmd_com_s com;
+ u32 udp_src_port;
+} roce_modify_udp_src_port_s;
+
+typedef struct roce_get_qp_udp_src_port {
+ roce_verbs_cmd_com_s com;
+} roce_get_qp_udp_src_port_s;
+
+struct roce5_qp_port_in {
+ uint32_t qpn;
+ uint32_t port;
+};
+
+struct roce5_port_out {
+ uint32_t port;
+};
+
+struct roce5_bond_port_info_out {
+ int32_t original_port_num;
+ uint8_t original_port[8];
+ int32_t rsvd1;
+
+ int32_t alive_port_num;
+ uint8_t alive_port[8];
+ int32_t rsvd2;
+};
+
+struct roce5_qp_udp_src_port_in {
+ uint32_t qpn;
+ uint32_t udp_src_port;
+};
+
+#endif
+
+struct roce5_qp_in {
+ uint32_t qpn;
+};
+
+struct roce5_cq_in {
+ uint32_t cqn;
+};
+
+struct roce5_next_wqe_idx_in {
+ uint32_t srqn;
+};
+
+struct roce5_next_wqe_idx_out {
+ int32_t next_wqe_idx;
+};
+
+struct roce5_qp_hash_value_in {
+ uint32_t type;
+ uint32_t qpn;
+ uint32_t hash_value;
+};
+
+struct roce5_qp_hash_value_out {
+ uint32_t hash_value;
+};
+
+#ifdef ROCE_BONDING_EN
+
+int32_t roce5_genl_query_qp_tx_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size);
+int32_t roce5_genl_set_qp_tx_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out, uint16_t out_len,
+ uint16_t *out_size);
+int32_t roce5_genl_query_qp_rx_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size);
+int32_t roce5_genl_query_bond_port_info(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size);
+int32_t roce5_genl_set_qp_udp_src_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size);
+int32_t roce5_genl_query_qp_udp_src_port(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size);
+
+#endif
+
+int32_t roce5_genl_query_next_wqe_idx(struct roce5_device *rdev, void *in,
+ u16 in_len, void *out, u16 out_len,
+ u16 *out_size);
+int32_t roce5_genl_query_qpc(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out, uint16_t out_len,
+ uint16_t *out_size);
+int32_t roce5_genl_query_cqc(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out, uint16_t out_len,
+ uint16_t *out_size);
+int32_t roce5_genl_qp_hash_value_process(struct roce5_device *rdev, void *in,
+ u16 in_len, void *out, u16 out_len,
+ u16 *out_size);
+int32_t roce5_netlink_uld_process(struct roce5_device *rdev, void *buf_in,
+ u16 buf_in_len, void *buf_out,
+ u16 buf_out_len, u16 *buf_out_size);
+#endif /* ROCE_NETLINK_PROCESS_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c
new file mode 100644
index 000000000..69549fbe3
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c
@@ -0,0 +1,74 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "securec.h"
+
+#include "roce_cqm_cmd.h"
+
+void roce5_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf,
+ cqm_cmd_buf_s *cqm_cmd_outbuf)
+{
+ if (cqm_cmd_outbuf != NULL) {
+ cqm5_cmd_free(ex_handle, cqm_cmd_outbuf);
+ }
+
+ if (cqm_cmd_inbuf != NULL) {
+ cqm5_cmd_free(ex_handle, cqm_cmd_inbuf);
+ }
+}
+
+int roce5_cqm_cmd_zalloc_inoutbuf(void *ex_handle,
+ cqm_cmd_buf_s **cqm_cmd_inbuf, u16 inbuf_size,
+ cqm_cmd_buf_s **cqm_cmd_outbuf,
+ u16 outbuf_size)
+{
+ int ret;
+
+ if (cqm_cmd_inbuf != NULL) {
+ *cqm_cmd_inbuf = cqm5_cmd_alloc(ex_handle);
+
+ if (*cqm_cmd_inbuf == NULL) {
+ ret = -ENOMEM;
+ goto inbuf_err;
+ }
+ (*cqm_cmd_inbuf)->size = inbuf_size;
+
+ ret = memset_s((*cqm_cmd_inbuf)->buf, inbuf_size, 0,
+ inbuf_size);
+ if (ret != 0) {
+ ret = -EINVAL;
+ goto inbuf_err;
+ }
+ }
+
+ if (cqm_cmd_outbuf != NULL) {
+ *cqm_cmd_outbuf = cqm5_cmd_alloc(ex_handle);
+
+ if (*cqm_cmd_outbuf == NULL) {
+ ret = -ENOMEM;
+ goto outbuf_err;
+ }
+ (*cqm_cmd_outbuf)->size = outbuf_size;
+
+ ret = memset_s((*cqm_cmd_outbuf)->buf, outbuf_size, 0,
+ outbuf_size);
+ if (ret != 0) {
+ ret = -EINVAL;
+ goto outbuf_err;
+ }
+ }
+
+ return 0;
+outbuf_err:
+ if (cqm_cmd_outbuf != NULL) {
+ roce5_cqm5_cmd_free_inoutbuf(ex_handle, NULL, *cqm_cmd_outbuf);
+ *cqm_cmd_outbuf = NULL;
+ }
+inbuf_err:
+ if (cqm_cmd_inbuf != NULL) {
+ roce5_cqm5_cmd_free_inoutbuf(ex_handle, *cqm_cmd_inbuf, NULL);
+ *cqm_cmd_inbuf = NULL;
+ }
+ return ret;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h
new file mode 100644
index 000000000..0130fa320
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h
@@ -0,0 +1,21 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_CQM_CMD_H
+#define ROCE_CQM_CMD_H
+#ifdef ROCE_KNL_DT
+#include <linux/compiler_types.h>
+#endif
+#include "hinic5_cqm.h"
+
+#include "roce.h"
+
+void roce5_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf,
+ cqm_cmd_buf_s *cqm_cmd_outbuf);
+int roce5_cqm_cmd_zalloc_inoutbuf(void *ex_handle,
+ cqm_cmd_buf_s **cqm_cmd_inbuf, u16 inbuf_size,
+ cqm_cmd_buf_s **cqm_cmd_outbuf,
+ u16 outbuf_size);
+
+#endif //ROCE_CQM_CMD_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c
new file mode 100644
index 000000000..3000c6b97
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c
@@ -0,0 +1,240 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+#include "roce.h"
+#include "roce_sysfs.h"
+#include "roce_cc_format.h"
+
+#include "roce_cmd.h"
+#include "roce_multipath_sysfs.h"
+
+#define MAX_STRING_FORMAT_LEN 64
+#define to_roce5_ecn_multipath_ctx(_kobj) \
+ container_of(_kobj, struct roce5_ecn_multipath_ctx, ecn_multipath_root)
+
+static void roce_ecn_multipath_sysfs_release(struct kobject *kobj)
+{
+}
+
+static ssize_t roce5_show_slow_path_psn_threshold(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx =
+ to_roce5_ecn_multipath_ctx(kobj);
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n",
+ (u32)ecn_multipath_ctx->slow_path_psn_threshold);
+}
+
+static ssize_t roce5_store_slow_path_psn_threshold(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf,
+ size_t count)
+{
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx =
+ to_roce5_ecn_multipath_ctx(kobj);
+ u32 old_slow_path_psn_threshold = 0;
+ int slow_path_psn_threshold = 0;
+ int ret;
+ char remaining;
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ ret = sscanf_s(buf, "%d %c", &slow_path_psn_threshold, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, slow_path_psn_threshold should be unsigned int\n");
+ return -EIO;
+ }
+
+ if (slow_path_psn_threshold >
+ ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX) {
+ ROCE_ERR(
+ "Invalid input, slow_path_psn_threshold(%d) over max slow_path_psn_threshold(%u), slow_path_psn_threshold",
+ slow_path_psn_threshold,
+ (u32)ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX);
+ return -EIO;
+ }
+
+ if (ecn_multipath_ctx->slow_path_psn_threshold !=
+ (u32)slow_path_psn_threshold) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_slow_path_psn_threshold =
+ ecn_multipath_ctx->slow_path_psn_threshold;
+ ecn_multipath_ctx->slow_path_psn_threshold =
+ (u32)slow_path_psn_threshold;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Fail to update slow_path_psn_threshold(%d), ret(%d).",
+ slow_path_psn_threshold, ret);
+ ecn_multipath_ctx->slow_path_psn_threshold =
+ old_slow_path_psn_threshold;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+static ssize_t roce5_show_rtt_req_event_mode(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx =
+ to_roce5_ecn_multipath_ctx(kobj);
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_multipath_ctx->rtt_req_event_mode);
+}
+
+static ssize_t roce5_store_rtt_req_event_mode(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ u32 old_rtt_req_event_mode = 0;
+ u32 rtt_req_event_mode = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx =
+ to_roce5_ecn_multipath_ctx(kobj);
+ char remaining;
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ ret = sscanf_s(buf, "%u %c", &rtt_req_event_mode, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, rtt_req_event_mode should be unsigned int ret(%d), rtt_req_event_mode(%u)\n",
+ ret, rtt_req_event_mode);
+ return -EIO;
+ }
+
+ if ((rtt_req_event_mode & ROCE_MULTIPATH_RTT_EN_MASK) != 0) {
+ ROCE_ERR(
+ "Invalid input, rtt_req_event_mode (%u)only 0 and 1 supported\n",
+ rtt_req_event_mode);
+ return -EIO;
+ }
+
+ if (ecn_multipath_ctx->rtt_req_event_mode != (u32)rtt_req_event_mode) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_rtt_req_event_mode = ecn_multipath_ctx->rtt_req_event_mode;
+ ecn_multipath_ctx->rtt_req_event_mode = (u32)rtt_req_event_mode;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Fail to update rtt_req_event_mode(%d), ret(%d).",
+ rtt_req_event_mode, ret);
+ ecn_multipath_ctx->rtt_req_event_mode =
+ old_rtt_req_event_mode;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+ROCE_ATTR_RW(slow_path_psn_threshold, roce5_show_slow_path_psn_threshold,
+ roce5_store_slow_path_psn_threshold);
+ROCE_ATTR_RW(rtt_req_event_mode, roce5_show_rtt_req_event_mode,
+ roce5_store_rtt_req_event_mode);
+
+static struct attribute *ecn_multipath_ctx_attrs[] = {
+ ROCE_ATTR_PTR(rtt_req_event_mode),
+ ROCE_ATTR_PTR(slow_path_psn_threshold),
+ NULL,
+};
+
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+static const struct attribute_group ecn_multipath_ctx_attr_grp = {
+ .attrs = ecn_multipath_ctx_attrs,
+};
+
+static const struct attribute_group *ecn_multipath_ctx_groups[] = {
+ &ecn_multipath_ctx_attr_grp,
+ NULL,
+};
+#endif
+
+static struct kobj_type roce_ecn_multipath_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = roce_ecn_multipath_sysfs_release,
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+ .default_groups = ecn_multipath_ctx_groups,
+#else
+ .default_attrs = ecn_multipath_ctx_attrs,
+#endif
+};
+
+int roce5_init_ecn_multipath_sysfs(
+ struct roce5_device *rdev, struct kobject *kobj,
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx)
+{
+ int ret = 0;
+
+ if (rdev->device_type != ROCE_DEV_TYPE_HI1825_V100) {
+ return 0;
+ }
+ if (ecn_multipath_ctx == NULL) {
+ pr_err_ratelimited(
+ "[ROCE, ERR] %s: ecn_multipath_ctx is null\n",
+ __func__);
+ return -EINVAL;
+ }
+
+ ecn_multipath_ctx->rtt_req_event_mode =
+ ROCE_MULTIPATH_DEFAULT_RTT_EVENT_MODE;
+ ecn_multipath_ctx->slow_path_psn_threshold =
+ ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD;
+
+ ret = kobject_init_and_add(&ecn_multipath_ctx->ecn_multipath_root,
+ &roce_ecn_multipath_ktype, kobj,
+ "roce5_multipath");
+ if (ret != 0) {
+ ROCE_ERR("Failed to add kobject roce5_ultipath.(ret:%d)\n",
+ ret);
+ kobject_put(&ecn_multipath_ctx->ecn_multipath_root);
+ return ret;
+ }
+
+ return 0;
+}
+
+void roce5_remove_ecn_multipath_sysfs(
+ struct roce5_device *rdev, struct kobject *kobj,
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx)
+{
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ if (ecn_multipath_ctx == NULL) {
+ ROCE_ERR("ecn_multipath_ctx is null\n");
+ return;
+ }
+
+ kobject_put(&ecn_multipath_ctx->ecn_multipath_root);
+ }
+
+ return;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h
new file mode 100644
index 000000000..2a26de8e2
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h
@@ -0,0 +1,30 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifndef ROCE_MULTIPATH_SYSFS_H
+#define ROCE_MULTIPATH_SYSFS_H
+
+#include <linux/types.h>
+#include <linux/stat.h>
+#include <linux/kobject.h>
+
+struct roce5_device;
+
+#define ROCE_MULTIPATH_DEFAULT_RTT_EVENT_MODE 0
+#define ROCE_MULTIPATH_RTT_EN_MASK 0xfffffffe
+#define ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD 1
+#define ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX 255
+
+struct roce5_ecn_multipath_ctx {
+ struct kobject ecn_multipath_root;
+ u32 slow_path_psn_threshold;
+ u32 rtt_req_event_mode;
+};
+int roce5_init_ecn_multipath_sysfs(
+ struct roce5_device *rdev, struct kobject *kobj,
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx);
+void roce5_remove_ecn_multipath_sysfs(
+ struct roce5_device *rdev, struct kobject *kobj,
+ struct roce5_ecn_multipath_ctx *ecn_multipath_ctx);
+#endif /* ROCE_MULTIPATH_SYSFS_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c
new file mode 100644
index 000000000..869a97571
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c
@@ -0,0 +1,2157 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include <linux/types.h>
+#include <linux/stat.h>
+
+#include "roce.h"
+#include "ccp_algo_format.h"
+#include "hinic5_hw.h"
+#include "roce_cc_format.h"
+#include "roce_cmd.h"
+#include "roce_com.h"
+#include "roce_dfx.h"
+#include "roce_infra.h"
+#include "roce_kernel_compat_gen.h"
+#include "roce_mix.h"
+#include "roce_mpu_cmd.h"
+#include "roce_multipath_sysfs.h"
+#include "roce_npu_cmd_srq_defs.h"
+#include "roce_qp.h"
+#include "roce_ucc_sysfs.h"
+#include "securec.h"
+#include "roce_sysfs.h"
+
+#define MAX_STRING_FORMAT_LEN 64
+
+/* generic macro template: generate show/store function, used for rp_ctx parameters (with range check) */
+#define ROCE5_PARAM_SHOW_STORE_RP(_name, _member, _min_macro, _max_macro) \
+ static ssize_t roce5_show_##_name( \
+ struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
+ { \
+ struct roce5_ecn_rp_ctx *ecn_rp_ctx = \
+ to_roce5_ecn_rp_ctx(kobj); \
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \
+ (int)ecn_rp_ctx->_member); \
+ } \
+ static ssize_t roce5_store_##_name(struct kobject *kobj, \
+ struct kobj_attribute *attr, \
+ const char *buf, size_t count) \
+ { \
+ int ret, _val; \
+ u32 _old; \
+ struct roce5_ecn_ctx *ecn_ctx = \
+ to_roce5_ecn_ctx(kobj->parent); \
+ struct roce5_ecn_rp_ctx *ecn_rp_ctx = \
+ to_roce5_ecn_rp_ctx(kobj); \
+ struct roce5_device *rdev = \
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx); \
+ ret = sscanf_s(buf, "%d", &_val); \
+ if (ret != 1) \
+ return -EIO; \
+ if (ROCE5_PARAM_CHECK_BOUND( \
+ _val, \
+ g_roce_cc_ecn_param[rdev->cc_param_version] \
+ ._min_macro, \
+ g_roce_cc_ecn_param[rdev->cc_param_version] \
+ ._max_macro)) \
+ return -EIO; \
+ if (ecn_rp_ctx->_member != (u32)_val) { \
+ mutex_lock(&ecn_ctx->ecn_mutex); \
+ _old = ecn_rp_ctx->_member; \
+ ecn_rp_ctx->_member = (u32)_val; \
+ ret = roce5_update_ecn_param(ecn_ctx); \
+ if (ret != 0) { \
+ ecn_rp_ctx->_member = _old; \
+ mutex_unlock(&ecn_ctx->ecn_mutex); \
+ return -EIO; \
+ } \
+ mutex_unlock(&ecn_ctx->ecn_mutex); \
+ } \
+ return (ssize_t)count; \
+ }
+
+/* generic macro template: generate show/store function, used for np_ctx parameters (with range check) */
+#define ROCE5_PARAM_SHOW_STORE_NP(_name, _member, _min_macro, _max_macro) \
+ static ssize_t roce5_show_##_name( \
+ struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
+ { \
+ struct roce5_ecn_np_ctx *ecn_np_ctx = \
+ to_roce5_ecn_np_ctx(kobj); \
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \
+ (int)ecn_np_ctx->_member); \
+ } \
+ static ssize_t roce5_store_##_name(struct kobject *kobj, \
+ struct kobj_attribute *attr, \
+ const char *buf, size_t count) \
+ { \
+ int ret, _val; \
+ u32 _old; \
+ struct roce5_ecn_ctx *ecn_ctx = \
+ to_roce5_ecn_ctx(kobj->parent); \
+ struct roce5_ecn_np_ctx *ecn_np_ctx = \
+ to_roce5_ecn_np_ctx(kobj); \
+ struct roce5_device *rdev = \
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx); \
+ ret = sscanf_s(buf, "%d", &_val); \
+ if (ret != 1) \
+ return -EIO; \
+ if (ROCE5_PARAM_CHECK_BOUND( \
+ _val, \
+ g_roce_cc_ecn_param[rdev->cc_param_version] \
+ ._min_macro, \
+ g_roce_cc_ecn_param[rdev->cc_param_version] \
+ ._max_macro)) \
+ return -EIO; \
+ if (ecn_np_ctx->_member != (u32)_val) { \
+ mutex_lock(&ecn_ctx->ecn_mutex); \
+ _old = ecn_np_ctx->_member; \
+ ecn_np_ctx->_member = (u32)_val; \
+ ret = roce5_update_ecn_param(ecn_ctx); \
+ if (ret != 0) { \
+ ecn_np_ctx->_member = _old; \
+ mutex_unlock(&ecn_ctx->ecn_mutex); \
+ return -EIO; \
+ } \
+ mutex_unlock(&ecn_ctx->ecn_mutex); \
+ } \
+ return (ssize_t)count; \
+ }
+
+/* generic macro template: generate show/store function, used for ldcp_ctx parameters (with range check) */
+#define ROCE5_PARAM_SHOW_STORE_LDCP(_name, _member, _min_macro, _max_macro) \
+ static ssize_t roce5_show_##_name( \
+ struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
+ { \
+ struct roce5_ecn_ldcp_ctx *ecn_ldcp_ctx = \
+ to_roce5_ecn_ldcp_ctx(kobj); \
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \
+ (int)ecn_ldcp_ctx->_member); \
+ } \
+ static ssize_t roce5_store_##_name(struct kobject *kobj, \
+ struct kobj_attribute *attr, \
+ const char *buf, size_t count) \
+ { \
+ int ret, _val; \
+ u32 _old; \
+ struct roce5_ecn_ctx *ecn_ctx = \
+ to_roce5_ecn_ctx(kobj->parent); \
+ struct roce5_ecn_ldcp_ctx *ecn_ldcp_ctx = \
+ to_roce5_ecn_ldcp_ctx(kobj); \
+ struct roce5_device *rdev = \
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx); \
+ ret = sscanf_s(buf, "%d", &_val); \
+ if (ret != 1) { \
+ ROCE_ERR("Failed to get " #_member ", ret(%d).", ret); \
+ return -EIO; \
+ } \
+ if (ROCE5_PARAM_CHECK_BOUND( \
+ _val, \
+ g_roce_cc_ecn_param[rdev->cc_param_version] \
+ ._min_macro, \
+ g_roce_cc_ecn_param[rdev->cc_param_version] \
+ ._max_macro)) { \
+ ROCE_ERR(#_member "(%u) param is out bound.", _val); \
+ return -EIO; \
+ } \
+ if (ecn_ldcp_ctx->_member != (u32)_val) { \
+ mutex_lock(&ecn_ctx->ecn_mutex); \
+ _old = ecn_ldcp_ctx->_member; \
+ ecn_ldcp_ctx->_member = (u32)_val; \
+ ret = roce5_update_ecn_param(ecn_ctx); \
+ if (ret != 0) { \
+ ecn_ldcp_ctx->_member = _old; \
+ mutex_unlock(&ecn_ctx->ecn_mutex); \
+ return -EIO; \
+ } \
+ mutex_unlock(&ecn_ctx->ecn_mutex); \
+ } \
+ return (ssize_t)count; \
+ }
+
+const struct roce_cc_ecn_param g_roce_cc_ecn_param[] = {
+ { 0 }, // invalid
+ { .default_ip_enable_en = 1,
+ .default_bypass = 0,
+ .default_min_cnp_period = 16,
+ .default_alpha_dec_period = 160,
+ .default_rate_dec_period = 32,
+ .default_rate_inc_period = 480,
+ .default_factor_gita = 7,
+ .default_initial_alpha = 1023,
+ .default_rate_inc_ai = 2,
+ .default_rate_inc_hai = 8,
+ .default_rate_first_set = 1024,
+ .default_rate_target_clamp = 1,
+ .default_cnp_prio_enable = 0,
+ .default_cnp_prio = 0,
+ .default_token_period = 16,
+ .default_min_rate = 1,
+ .default_ecn_cnp_enable = 1,
+ .default_flow_ctrl_unit = 0,
+ .default_rtt_rsp_prio = 0,
+ .default_rtt_rsp_prio_enable = 0,
+ .default_rtt_version = 0,
+ .max_rtt_version = 0,
+ .min_bypass = 0,
+ .max_bypass = 0,
+ .min_rate_inc_hai = 1,
+ .max_rate_inc_hai = 255,
+ .min_rate_inc_ai = 1,
+ .max_rate_inc_ai = 63,
+ .min_initial_alpha = 1,
+ .max_initial_alpha = 1023,
+ .min_rate_inc_period = 1,
+ .max_rate_inc_period = 1023,
+ .min_alpha_dec_period = 1,
+ .max_alpha_dec_period = 1023,
+ .min_token_period = 4,
+ .max_token_period = 255,
+ .min_flow_min_rate = 1,
+ .max_flow_min_rate = 63,
+ .min_rate_dec_period = 1,
+ .max_rate_dec_period = 255,
+ .min_min_cnp_period = 1,
+ .max_min_cnp_period = 255,
+ .min_factor_gita = 1,
+ .max_factor_gita = 15,
+ .min_rate_first_set = 128,
+ .max_rate_first_set = 8191,
+ .default_ldcp_wnd_min = 5,
+ .default_ldcp_wnd_default = 1024,
+ .default_ldcp_alpha = 2,
+ .default_ldcp_beta = 12,
+ .default_ldcp_gamma = 9,
+ .default_ldcp_eta = 3,
+ .min_ldcp_wnd_min = 1,
+ .max_ldcp_wnd_min = 0x8,
+ .min_ldcp_wnd_default = 1024,
+ .max_ldcp_wnd_default = 32768,
+ .min_ldcp_alpha = 0,
+ .max_ldcp_alpha = 0xf,
+ .min_ldcp_beta = 0,
+ .max_ldcp_beta = 0xf,
+ .min_ldcp_gamma = 0,
+ .max_ldcp_gamma = 0xf,
+ .min_ldcp_eta = 0,
+ .max_ldcp_eta = 0xf }, // v1
+ { .default_ip_enable_en = 1,
+ .default_bypass = 0,
+ .default_min_cnp_period = 16,
+ .default_alpha_dec_period = 160,
+ .default_rate_dec_period = 48,
+ .default_rate_inc_period = 480,
+ .default_factor_gita = 7,
+ .default_initial_alpha = 1023,
+ .default_rate_inc_ai = 2,
+ .default_rate_inc_hai = 12,
+ .default_rate_first_set = 1024,
+ .default_rate_target_clamp = 1,
+ .default_cnp_prio_enable = 0,
+ .default_cnp_prio = 0,
+ .default_token_period = 16,
+ .default_min_rate = 1,
+ .default_ecn_cnp_enable = 1,
+ .default_flow_ctrl_unit = 0,
+ .default_rtt_rsp_prio = 0,
+ .default_rtt_rsp_prio_enable = 0,
+ .default_rtt_version = 0,
+ .max_rtt_version = 0,
+ .min_bypass = 0,
+ .max_bypass = 2,
+ .min_rate_inc_hai = 1,
+ .max_rate_inc_hai = 255,
+ .min_rate_inc_ai = 1,
+ .max_rate_inc_ai = 255,
+ .min_initial_alpha = 1,
+ .max_initial_alpha = 1023,
+ .min_rate_inc_period = 1,
+ .max_rate_inc_period = 1023,
+ .min_alpha_dec_period = 1,
+ .max_alpha_dec_period = 1023,
+ .min_token_period = 4,
+ .max_token_period = 255,
+ .min_flow_min_rate = 1,
+ .max_flow_min_rate = 4095,
+ .min_rate_dec_period = 1,
+ .max_rate_dec_period = 255,
+ .min_min_cnp_period = 1,
+ .max_min_cnp_period = 255,
+ .min_factor_gita = 1,
+ .max_factor_gita = 15,
+ .min_rate_first_set = 128,
+ .max_rate_first_set = 50000,
+ .default_ldcp_wnd_min = 5,
+ .default_ldcp_wnd_default = 1024,
+ .default_ldcp_alpha = 2,
+ .default_ldcp_beta = 9,
+ .default_ldcp_gamma = 6,
+ .default_ldcp_eta = 6,
+ .min_ldcp_wnd_min = 1,
+ .max_ldcp_wnd_min = 0x8,
+ .min_ldcp_wnd_default = 1024,
+ .max_ldcp_wnd_default = 32768,
+ .min_ldcp_alpha = 0,
+ .max_ldcp_alpha = 0xf,
+ .min_ldcp_beta = 0,
+ .max_ldcp_beta = 0xf,
+ .min_ldcp_gamma = 0,
+ .max_ldcp_gamma = 0xf,
+ .min_ldcp_eta = 0,
+ .max_ldcp_eta = 0xf }, // v2
+};
+
+static int roce5_update_ver_check(struct roce5_device *rdev, u32 cc_algo)
+{
+ u32 algo_bitmap = 0;
+ /* driver intercept not support algorithm. 1823V200 only supports LDCP algorithm, 1825V100 current only supports DCQCN, IPQCN, user custom A & B algorithm */
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ algo_bitmap = BIT(ROCE_CC_DISABLE);
+ if (ROCE5_SUPPORT_CC_GET_DCQCN_ENABLE(rdev) != 0) {
+ algo_bitmap |= BIT(ROCE_CC_DCQCN_ALGO);
+ }
+ if (ROCE5_SUPPORT_CC_GET_LDCP_ENABLE(rdev) != 0) {
+ algo_bitmap |= BIT(ROCE_CC_LDCP_ALGO);
+ }
+ if (ROCE5_SUPPORT_CC_GET_IPQCN_ENABLE(rdev) != 0) {
+ algo_bitmap |= BIT(ROCE_CC_IPQCN_ALGO);
+ }
+ if (ROCE5_SUPPORT_CC_GET_USER_ALGO_ENABLE(rdev) != 0) {
+ algo_bitmap |= BIT(ROCE_CC_USER_A_ALGO);
+ algo_bitmap |= BIT(ROCE_CC_USER_B_ALGO);
+ }
+ }
+
+ if (ROCE_CC_ALGO_TEST_BIT(algo_bitmap, cc_algo) == 0) {
+ ROCE_DEV_ERR(rdev, "func_id(0x%x), cc algo(0x%x) not support!",
+ rdev->glb_func_id, cc_algo);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int roce5_update_cfg_cc_param(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx)
+{
+ roce_cc_param_s cc_param;
+ int ret;
+
+ ret = roce5_update_ver_check(rdev, ecn_ctx->cc_algo);
+ if (ret != 0) {
+ return ret;
+ }
+
+ (void)memset_s(&cc_param, sizeof(cc_param), 0, sizeof(cc_param));
+
+ cc_param.dw0.bs.bypass = ecn_ctx->rp_ctx.bypass;
+ cc_param.dw0.bs.rtt_rsp_prio = ecn_ctx->np_ctx.rtt_rsp_prio;
+ cc_param.dw0.bs.rtt_rsp_prio_enable =
+ ecn_ctx->np_ctx.rtt_rsp_prio_enable;
+ cc_param.dw0.bs.rtt_version = ecn_ctx->np_ctx.rtt_version;
+ cc_param.dw0.bs.ecn_cnp_en = ecn_ctx->np_ctx.ecn_cnp_en;
+ cc_param.dw0.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period;
+ cc_param.dw0.bs.cnp_prio_enable = ecn_ctx->np_ctx.cnp_prio_enable;
+ cc_param.dw0.bs.slow_path_psn_threshold =
+ ecn_ctx->multipath_ctx.slow_path_psn_threshold;
+ cc_param.dw0.bs.rtt_req_event_mode =
+ ecn_ctx->multipath_ctx.rtt_req_event_mode;
+ cc_param.dw1.bs.cnp_prio = ecn_ctx->np_ctx.cnp_prio;
+ cc_param.dw1.bs.cnp_cos =
+ roce5_get_db_cos_from_vlan_pri(rdev, cc_param.dw1.bs.cnp_prio);
+ cc_param.dw1.bs.cc_appid = ecn_ctx->cc_algo;
+
+ if (cc_param.dw1.bs.cc_appid == ROCE_CC_LDCP_ALGO) {
+ cc_param.dw0.bs.algo_mode = CCP_ALGO_MODE_CWND;
+ } else if (cc_param.dw1.bs.cc_appid == ROCE_CC_DCQCN_ALGO ||
+ cc_param.dw1.bs.cc_appid == ROCE_CC_IPQCN_ALGO) {
+ cc_param.dw0.bs.algo_mode = CCP_ALGO_MODE_RATE;
+ } else {
+ cc_param.dw0.bs.algo_mode = ecn_ctx->ucc_ctx.algo_mode;
+ }
+
+ if (cc_param.dw0.bs.algo_mode == CCP_ALGO_MODE_CWND) {
+ cc_param.dw0.bs.flow_ctrl_unit = FLOW_CTRL_UNIT_QP;
+ } else {
+ cc_param.dw0.bs.flow_ctrl_unit = ecn_ctx->np_ctx.flow_ctrl_unit;
+ }
+
+ ROCE_DEV_INFO(rdev, "set cc_appid(%u), func_id(%u).",
+ cc_param.dw1.bs.cc_appid, rdev->glb_func_id);
+
+ ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id,
+ (u32 *)&cc_param,
+ sizeof(roce_cc_param_s),
+ (u16)ROCE_MPU_CMD_CC_CFG_CC_PARAM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set cc param, func_id(%d) ret(%d)",
+ rdev->glb_func_id, ret);
+ }
+
+ return ret;
+}
+
+static int roce5_update_dcqcn_param(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx)
+{
+ ccp_dcqcn_para_s dcqcn_param;
+ int ret;
+
+ if (ecn_ctx->cc_algo != ROCE_CC_DCQCN_ALGO) {
+ ROCE_DEV_INFO(
+ rdev,
+ "func_id(%d) cc_algo(%u) is not dcqcn, ignore update.",
+ rdev->glb_func_id, ecn_ctx->cc_algo);
+ return 0;
+ }
+
+ (void)memset_s(&dcqcn_param, sizeof(dcqcn_param), 0,
+ sizeof(dcqcn_param));
+
+ dcqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period;
+ dcqcn_param.dw0.bs.min_rate = ecn_ctx->rp_ctx.min_rate;
+ dcqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period;
+ dcqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period;
+ dcqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai;
+ dcqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai;
+ dcqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period;
+ dcqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period;
+ dcqcn_param.dw3.bs.gita_shift = ecn_ctx->rp_ctx.factor_gita;
+ dcqcn_param.dw3.bs.rate_target_clamp =
+ ecn_ctx->rp_ctx.rate_target_clamp;
+ dcqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha;
+ // adjust_en
+ dcqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set;
+
+ ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id,
+ (u32 *)(void *)&dcqcn_param,
+ sizeof(ccp_dcqcn_para_s),
+ (u16)ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set dcqcn param, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ }
+
+ return ret;
+}
+
+static int roce5_update_ldcp_param(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx)
+{
+ roce_ldcp_param_s ldcp_param;
+ int ret;
+
+ if (ecn_ctx->cc_algo != ROCE_CC_LDCP_ALGO) {
+ ROCE_DEV_INFO(
+ rdev,
+ "func_id(%d) cc_algo(%u) is not ldcp, ignore update.",
+ rdev->glb_func_id, ecn_ctx->cc_algo);
+ return 0;
+ }
+
+ ROCE_DEV_INFO(
+ rdev,
+ "update ldcp param, func_id(%d), alpha(%u), beta(%u), gamma(%u), eta(%u) wnd_min(%u) wnd_default(%u)",
+ rdev->glb_func_id, ecn_ctx->ldcp_ctx.alpha,
+ ecn_ctx->ldcp_ctx.beta, ecn_ctx->ldcp_ctx.gamma,
+ ecn_ctx->ldcp_ctx.eta, ecn_ctx->ldcp_ctx.wnd_min,
+ ecn_ctx->ldcp_ctx.wnd_default);
+
+ (void)memset_s(&ldcp_param, sizeof(ldcp_param), 0, sizeof(ldcp_param));
+
+ ldcp_param.dw0.bs.alpha = ecn_ctx->ldcp_ctx.alpha;
+ ldcp_param.dw0.bs.beta = ecn_ctx->ldcp_ctx.beta;
+ ldcp_param.dw0.bs.gamma = ecn_ctx->ldcp_ctx.gamma;
+ ldcp_param.dw0.bs.eta = ecn_ctx->ldcp_ctx.eta;
+ ldcp_param.dw0.bs.wnd_min = ecn_ctx->ldcp_ctx.wnd_min;
+ ldcp_param.dw1.bs.wnd_default = ecn_ctx->ldcp_ctx.wnd_default;
+ ldcp_param.dw0.bs.set_flag = 1;
+
+ ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id,
+ (u32 *)(void *)&ldcp_param,
+ sizeof(roce_ldcp_param_s),
+ (u16)ROCE_MPU_CMD_CC_CFG_LDCP_PARAM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set ldcp param, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ }
+
+ return ret;
+}
+
+static int roce5_update_ipqcn_param(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx)
+{
+ roce_ipqcn_param_s ipqcn_param;
+ int ret;
+
+ if (ecn_ctx->cc_algo != ROCE_CC_IPQCN_ALGO) {
+ ROCE_DEV_INFO(
+ rdev,
+ "func_id(%d) cc_algo(%u) is not ipqcn, ignore update.",
+ rdev->glb_func_id, ecn_ctx->cc_algo);
+ return 0;
+ }
+
+ (void)memset_s(&ipqcn_param, sizeof(ipqcn_param), 0,
+ sizeof(ipqcn_param));
+
+ ipqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period;
+ ipqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate;
+ ipqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period;
+ ipqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period;
+ ipqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai;
+ ipqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai;
+ ipqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period;
+ ipqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period;
+ ipqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita;
+ ipqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp;
+ ipqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha;
+ ipqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set;
+
+ ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id,
+ (u32 *)(void *)&ipqcn_param,
+ sizeof(roce_ipqcn_param_s),
+ (u16)ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to set ipqcn param, func_id(%d) ret(%d)",
+ rdev->glb_func_id, ret);
+ }
+
+ return ret;
+}
+
+typedef int (*roce5_update_param_t)(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx);
+
+static roce5_update_param_t roce5_update_param_funcs[] = {
+ roce5_update_cfg_cc_param, roce5_update_dcqcn_param,
+ roce5_update_ipqcn_param, roce5_update_ldcp_param,
+ roce5_update_ucc_param,
+};
+
+int roce5_update_ecn_param(const struct roce5_ecn_ctx *ecn_ctx)
+{
+ unsigned int i;
+ int ret = 0;
+ struct roce5_device *rdev =
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx);
+
+ /* current only supports LDCP algorithm, its remaining algorithm can only modify its algorithm parameters, cannot switch */
+ for (i = 0; i < (sizeof(roce5_update_param_funcs) /
+ sizeof(roce5_update_param_t));
+ i++) {
+ ret = roce5_update_param_funcs[i](rdev, ecn_ctx);
+ if (ret != 0) {
+ ret = -EINVAL;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+static ssize_t roce5_show_cc_algo(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ switch ((int)ecn_ctx->cc_algo) {
+ case ROCE_CC_DISABLE:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "disable");
+ case ROCE_CC_DCQCN_ALGO:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "dcqcn");
+ case ROCE_CC_LDCP_ALGO:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "ldcp");
+ case ROCE_CC_IPQCN_ALGO:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n",
+ "hc3_1/ipqcn");
+ case ROCE_CC_USER_A_ALGO:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n",
+ "user_cc_6");
+ case ROCE_CC_USER_B_ALGO:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n",
+ "user_cc_7");
+ default:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n",
+ "error_type");
+ }
+}
+
+static ssize_t roce5_store_cc_algo(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf,
+ size_t count)
+{
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+ u32 old_cc_algo = 0;
+ int cc_algo = 0;
+ const char *ptr = NULL;
+ int ret;
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ /* dcqcn -> 0x0, ldcp -> 0x2, hc3_1 -> 0x80 */
+ if (strncmp(buf, "disable", strlen("disable")) == 0) {
+ cc_algo = ROCE_CC_DISABLE;
+ } else if (strncmp(buf, "dcqcn", strlen("dcqcn")) == 0) {
+ cc_algo = ROCE_CC_DCQCN_ALGO;
+ } else if ((strncmp(buf, "ipqcn", strlen("ipqcn")) == 0) ||
+ (strncmp(buf, "hc3_1", strlen("hc3_1")) == 0)) {
+ cc_algo = ROCE_CC_IPQCN_ALGO;
+ } else if (strncmp(buf, "ldcp", strlen("ldcp")) == 0) {
+ cc_algo = ROCE_CC_LDCP_ALGO;
+ } else if (strncmp(buf, "user_cc_", strlen("user_cc_")) == 0) {
+ if (strlen(buf) > ROCE_USER_CC_ID_MAX_LEN) {
+ ROCE_ERR(
+ "Invalid buffer length, expected <= %d, got %zu, Buffer: '%s'",
+ ROCE_USER_CC_ID_MAX_LEN, strlen(buf), buf);
+ return -EINVAL;
+ }
+ ptr = buf + strlen("user_cc_");
+ cc_algo = (unsigned char)(*ptr) - CHAR_0;
+ if (cc_algo != ROCE_CC_USER_A_ALGO &&
+ cc_algo != ROCE_CC_USER_B_ALGO) {
+ ROCE_ERR("Invalid cc_algo(%d),buf(%s)", cc_algo, buf);
+ return -EINVAL;
+ }
+ } else {
+ ROCE_ERR("Invalid cc_algo(%d),buf(%s)", cc_algo, buf);
+ return -EIO;
+ }
+
+ if (ecn_ctx->cc_algo != (u32)cc_algo) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_cc_algo = ecn_ctx->cc_algo;
+ ecn_ctx->cc_algo = (u32)cc_algo;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Failed to update cc_algo(%d), ret(%d).",
+ cc_algo, ret);
+ ecn_ctx->cc_algo = old_cc_algo;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+static ssize_t roce5_show_ecn_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_prio_enable_ctx *prio_enable_ctx =
+ container_of(attr, struct roce5_prio_enable_ctx, enable);
+
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)prio_enable_ctx->prio_en);
+}
+
+static ssize_t roce5_store_ecn_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ int prio_en = 0;
+ u32 old_prio_en = 0;
+ struct roce5_ecn_rp_ctx *rp_ctx = NULL;
+ struct roce5_ecn_np_ctx *np_ctx = NULL;
+ struct roce5_ecn_ctx *ecn_ctx = NULL;
+ struct roce5_prio_enable_ctx *prio_enable_ctx =
+ container_of(attr, struct roce5_prio_enable_ctx, enable);
+ struct roce5_ecn_enable_ctx *ecn_enable_ctx =
+ (struct roce5_ecn_enable_ctx *)prio_enable_ctx->ecn_enable_ctx;
+
+ if (ecn_enable_ctx->np_rp == ROCE_DCQCN_NP) {
+ np_ctx = container_of(ecn_enable_ctx, struct roce5_ecn_np_ctx,
+ enable_ctx);
+ ecn_ctx = container_of(np_ctx, struct roce5_ecn_ctx, np_ctx);
+ } else {
+ rp_ctx = container_of(ecn_enable_ctx, struct roce5_ecn_rp_ctx,
+ enable_ctx);
+ ecn_ctx = container_of(rp_ctx, struct roce5_ecn_ctx, rp_ctx);
+ }
+
+ ret = sscanf_s(buf, "%d", &prio_en);
+ if (ret != 1) {
+ return -EIO;
+ }
+
+ if (((u32)prio_en & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (prio_enable_ctx->prio_en != (u32)prio_en) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_prio_en = prio_enable_ctx->prio_en;
+ prio_enable_ctx->prio_en = (u32)prio_en;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update prio_en(%d), ret(%d).",
+ prio_en, ret);
+ prio_enable_ctx->prio_en = old_prio_en;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+static ssize_t roce5_show_ecn_ip_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ip_prio_enable_ctx *ip_prio_enable_ctx =
+ container_of(attr, struct roce5_ip_prio_enable_ctx, ip_enable);
+
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ip_prio_enable_ctx->prio_en);
+}
+
+static ssize_t roce5_store_ecn_ip_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ int prio_en = 0;
+ u32 old_prio_en = 0;
+ struct roce5_ip_prio_enable_ctx *ip_prio_enable_ctx =
+ container_of(attr, struct roce5_ip_prio_enable_ctx, ip_enable);
+ struct roce5_ecn_ip_enable_ctx *ip_enable_ctx =
+ (struct roce5_ecn_ip_enable_ctx *)
+ ip_prio_enable_ctx->ecn_ip_enable_ctx;
+ struct roce5_ecn_ctx *ecn_ctx = container_of(
+ ip_enable_ctx, struct roce5_ecn_ctx, ip_enable_ctx);
+
+ ret = sscanf_s(buf, "%d", &prio_en);
+ if (ret != 1) {
+ return -EIO;
+ }
+
+ if (((u32)prio_en & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (ip_prio_enable_ctx->prio_en != (u32)prio_en) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_prio_en = ip_prio_enable_ctx->prio_en;
+ ip_prio_enable_ctx->prio_en = (u32)prio_en;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update prio_en(%d), ret(%d).",
+ prio_en, ret);
+ ip_prio_enable_ctx->prio_en = old_prio_en;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+/* rp_ctx parameters use macro generate show/store function */
+ROCE5_PARAM_SHOW_STORE_RP(bypass, bypass, min_bypass, max_bypass)
+ROCE5_PARAM_SHOW_STORE_RP(rate_inc_hai, rate_inc_hai, min_rate_inc_hai,
+ max_rate_inc_hai)
+ROCE5_PARAM_SHOW_STORE_RP(rate_inc_ai, rate_inc_ai, min_rate_inc_ai,
+ max_rate_inc_ai)
+ROCE5_PARAM_SHOW_STORE_RP(initial_alpha, initial_alpha, min_initial_alpha,
+ max_initial_alpha)
+ROCE5_PARAM_SHOW_STORE_RP(factor_gita, factor_gita, min_factor_gita,
+ max_factor_gita)
+ROCE5_PARAM_SHOW_STORE_RP(rate_inc_period, rate_inc_period, min_rate_inc_period,
+ max_rate_inc_period)
+ROCE5_PARAM_SHOW_STORE_RP(rate_dec_period, rate_dec_period, min_rate_dec_period,
+ max_rate_dec_period)
+ROCE5_PARAM_SHOW_STORE_RP(token_period, token_period, min_token_period,
+ max_token_period)
+ROCE5_PARAM_SHOW_STORE_RP(min_rate, min_rate, min_flow_min_rate,
+ max_flow_min_rate)
+ROCE5_PARAM_SHOW_STORE_RP(alpha_dec_period, alpha_dec_period,
+ min_alpha_dec_period, max_alpha_dec_period)
+ROCE5_PARAM_SHOW_STORE_RP(rate_first_set, rate_first_set, min_rate_first_set,
+ max_rate_first_set)
+
+/* rate_target_clamp special handle: use ROCE_ENABLE_CHECK_MASK but non- range check */
+static ssize_t roce5_show_rate_target_clamp(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ecn_rp_ctx *ecn_rp_ctx = to_roce5_ecn_rp_ctx(kobj);
+
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_rp_ctx->rate_target_clamp);
+}
+
+static ssize_t roce5_store_rate_target_clamp(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ int rate_target_clamp = 0;
+ u32 old_rate_target_clamp = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_rp_ctx *ecn_rp_ctx = to_roce5_ecn_rp_ctx(kobj);
+
+ ret = sscanf_s(buf, "%d", &rate_target_clamp);
+ if (ret != 1) {
+ return -EIO;
+ }
+
+ if (((u32)rate_target_clamp & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (ecn_rp_ctx->rate_target_clamp != (u32)rate_target_clamp) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_rate_target_clamp = ecn_rp_ctx->rate_target_clamp;
+ ecn_rp_ctx->rate_target_clamp = (u32)rate_target_clamp;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Fail to update rate_target_clamp(%d), ret(%d).",
+ rate_target_clamp, ret);
+ ecn_rp_ctx->rate_target_clamp = old_rate_target_clamp;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+/* np_ctx parameters use macro generate show/store function */
+ROCE5_PARAM_SHOW_STORE_NP(min_cnp_period, min_cnp_period, min_min_cnp_period,
+ max_min_cnp_period)
+
+/* flow_ctrl_unit special handle: use ROCE_ENABLE_CHECK_MASK check */
+static ssize_t roce5_show_flow_ctrl_unit(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_np_ctx->flow_ctrl_unit);
+}
+static ssize_t roce5_store_flow_ctrl_unit(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret, _val;
+ u32 _old;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ char remaining;
+
+ ret = sscanf_s(buf, "%d %c", &_val, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, flow_ctrl_unit should be unsigned int, ret(%d), flow_ctrl_unit(%d)",
+ ret, _val);
+ return -EIO;
+ }
+
+ if (((u32)_val & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->flow_ctrl_unit != (u32)_val) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ _old = ecn_np_ctx->flow_ctrl_unit;
+ ecn_np_ctx->flow_ctrl_unit = (u32)_val;
+ ret = roce5_update_ecn_param(ecn_ctx);
+
+ if (ret != 0) {
+ ecn_np_ctx->flow_ctrl_unit = _old;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+ return (ssize_t)count;
+}
+
+/* rtt_version special handle: only check maximum value */
+static ssize_t roce5_show_rtt_version(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_np_ctx->rtt_version);
+}
+static ssize_t roce5_store_rtt_version(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ u32 _val, _old;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ struct roce5_device *rdev =
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx);
+ char remaining;
+
+ ret = sscanf_s(buf, "%u %c", &_val, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, rtt_version should be unsigned int, ret(%d), rtt_version(%u)",
+ ret, _val);
+ return -EIO;
+ }
+
+ if (_val >
+ g_roce_cc_ecn_param[rdev->cc_param_version].max_rtt_version) {
+ ROCE_ERR("Invalid input, rtt_version(%u) over max rtt version",
+ _val);
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->rtt_version != _val) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ _old = ecn_np_ctx->rtt_version;
+ ecn_np_ctx->rtt_version = _val;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ecn_np_ctx->rtt_version = _old;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+ return (ssize_t)count;
+}
+
+/* rtt_rsp_prio special handle: use PRI_ARRAY_LEN - 1 check */
+static ssize_t roce5_show_rtt_rsp_prio(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n",
+ ecn_np_ctx->rtt_rsp_prio);
+}
+
+static ssize_t roce5_store_rtt_rsp_prio(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ u32 rtt_rsp_prio = 0;
+ u32 old_rtt_rsp_prio = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+
+ char remaining;
+ ret = sscanf_s(buf, "%u %c", &rtt_rsp_prio, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, rtt_rsp_prio should be unsigned int, ret(%d), rtt_rsp_prio(%u)\n",
+ ret, rtt_rsp_prio);
+ return -EIO;
+ }
+
+ if (rtt_rsp_prio > (PRI_ARRAY_LEN - 1)) {
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->rtt_rsp_prio != (u32)rtt_rsp_prio) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_rtt_rsp_prio = ecn_np_ctx->rtt_rsp_prio;
+ ecn_np_ctx->rtt_rsp_prio = (u32)rtt_rsp_prio;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update rtt_rsp_prio(%d), ret(%d).",
+ rtt_rsp_prio, ret);
+ ecn_np_ctx->rtt_rsp_prio = old_rtt_rsp_prio;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+/* rtt_rsp_prio_enable special handle: use ROCE_ENABLE_CHECK_MASK check */
+static ssize_t roce5_show_rtt_rsp_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n",
+ ecn_np_ctx->rtt_rsp_prio_enable);
+}
+
+static ssize_t roce5_store_rtt_rsp_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ u32 rtt_rsp_prio_enable = 0;
+ u32 old_rtt_rsp_prio_enable = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+
+ char remaining;
+ ret = sscanf_s(buf, "%u %c", &rtt_rsp_prio_enable, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, rtt_rsp_prio_enable should be unsigned int, ret(%d), rtt_rsp_prio_enable(%u)",
+ ret, rtt_rsp_prio_enable);
+ return -EIO;
+ }
+
+ if ((rtt_rsp_prio_enable & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->rtt_rsp_prio_enable != (u32)rtt_rsp_prio_enable) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_rtt_rsp_prio_enable = ecn_np_ctx->rtt_rsp_prio_enable;
+ ecn_np_ctx->rtt_rsp_prio_enable = (u32)rtt_rsp_prio_enable;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Fail to update rtt_rsp_prio_enable(%d), ret(%d).",
+ rtt_rsp_prio_enable, ret);
+ ecn_np_ctx->rtt_rsp_prio_enable =
+ old_rtt_rsp_prio_enable;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+/* ecn_cnp_en special handle: use ROCE_ENABLE_CHECK_MASK check */
+static ssize_t roce5_show_ecn_cnp_en(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_np_ctx->ecn_cnp_en);
+}
+
+static ssize_t roce5_store_ecn_cnp_en(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ u32 ecn_cnp_en = 0;
+ u32 old_ecn_cnp_en = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+
+ char remaining;
+ ret = sscanf_s(buf, "%u %c", &ecn_cnp_en, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR(
+ "Invalid input, ecn_cnp_en should be unsigned int, ret(%d), ecn_cnp_en(%u)",
+ ret, ecn_cnp_en);
+ return -EIO;
+ }
+
+ if ((ecn_cnp_en & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->ecn_cnp_en != (u32)ecn_cnp_en) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_ecn_cnp_en = ecn_np_ctx->ecn_cnp_en;
+ ecn_np_ctx->ecn_cnp_en = (u32)ecn_cnp_en;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update ecn_cnp_en(%d), ret(%d).",
+ ecn_cnp_en, ret);
+ ecn_np_ctx->ecn_cnp_en = old_ecn_cnp_en;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+/* cnp_prio_enable special handle: use ROCE_ENABLE_CHECK_MASK check */
+static ssize_t roce5_show_cnp_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_np_ctx->cnp_prio_enable);
+}
+
+static ssize_t roce5_store_cnp_prio_enable(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ int cnp_prio_enable = 0;
+ u32 old_cnp_prio_enable = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+
+ ret = sscanf_s(buf, "%d", &cnp_prio_enable);
+ if (ret != 1) {
+ return -EIO;
+ }
+
+ if (((u32)cnp_prio_enable & ROCE_ENABLE_CHECK_MASK) != 0) {
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->cnp_prio_enable != (u32)cnp_prio_enable) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_cnp_prio_enable = ecn_np_ctx->cnp_prio_enable;
+ ecn_np_ctx->cnp_prio_enable = (u32)cnp_prio_enable;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update cnp_prio_enable(%d), ret(%d).",
+ cnp_prio_enable, ret);
+ ecn_np_ctx->cnp_prio_enable = old_cnp_prio_enable;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+/* cnp_prio special handle: use PRI_ARRAY_LEN - 1 check */
+static ssize_t roce5_show_cnp_prio(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_np_ctx->cnp_prio);
+}
+
+static ssize_t roce5_store_cnp_prio(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret = 0;
+ int cnp_prio = 0;
+ u32 old_cnp_prio = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj);
+
+ ret = sscanf_s(buf, "%d", &cnp_prio);
+ if (ret != 1) {
+ return -EIO;
+ }
+
+ if (((u32)cnp_prio) > (PRI_ARRAY_LEN - 1)) {
+ return -EIO;
+ }
+
+ if (ecn_np_ctx->cnp_prio != (u32)cnp_prio) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_cnp_prio = ecn_np_ctx->cnp_prio;
+ ecn_np_ctx->cnp_prio = (u32)cnp_prio;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update cnp_prio(%d), ret(%d).",
+ cnp_prio, ret);
+ ecn_np_ctx->cnp_prio = old_cnp_prio;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+static int
+roce5_init_ecn_enable_sysfs(struct kobject *kobj,
+ struct roce5_ecn_enable_ctx *ecn_enable_ctx)
+{
+ int ret = 0;
+ int i = 0;
+ int j = 0;
+ const char *prio_arr[8] = { "0", "1", "2", "3", "4", "5", "6", "7" };
+ struct roce5_prio_enable_ctx *prio_enable = NULL;
+
+ ecn_enable_ctx->enable_root = kobject_create_and_add("enable", kobj);
+ if (ecn_enable_ctx->enable_root == NULL) {
+ ROCE_ERR("Failed to create kobject for ecn enable.(ret:%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < PRI_ARRAY_LEN; i++) {
+ prio_enable = &ecn_enable_ctx->prio_enable[i];
+ prio_enable->ecn_enable_ctx = (void *)ecn_enable_ctx;
+ prio_enable->prio = (u32)i;
+ prio_enable->prio_en = 1;
+ sysfs_attr_init(&prio_enable->enable.attr);
+ prio_enable->enable.attr.name = prio_arr[i];
+ prio_enable->enable.attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP);
+ prio_enable->enable.show = roce5_show_ecn_prio_enable;
+ prio_enable->enable.store = roce5_store_ecn_prio_enable;
+ ret = sysfs_create_file(ecn_enable_ctx->enable_root,
+ &prio_enable->enable.attr);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to create file for ecn enable.(ret:%d)",
+ ret);
+ goto err_create_file;
+ }
+ }
+
+ return 0;
+
+err_create_file:
+ for (j = i - 1; j >= 0; j--) {
+ sysfs_remove_file(ecn_enable_ctx->enable_root,
+ &ecn_enable_ctx->prio_enable[i].enable.attr);
+ }
+
+ kobject_put(ecn_enable_ctx->enable_root);
+
+ return ret;
+}
+
+static void
+roce5_remove_ecn_enable_sysfs(struct kobject *kobj,
+ struct roce5_ecn_enable_ctx *ecn_enable_ctx)
+{
+ int i;
+
+ for (i = 0; i < PRI_ARRAY_LEN; i++) {
+ sysfs_remove_file(ecn_enable_ctx->enable_root,
+ &ecn_enable_ctx->prio_enable[i].enable.attr);
+ }
+
+ kobject_put(ecn_enable_ctx->enable_root);
+}
+
+static int
+roce5_init_ecn_ip_enable_sysfs(struct kobject *kobj,
+ struct roce5_ecn_ip_enable_ctx *ip_enable_ctx)
+{
+ int ret = 0;
+ int i = 0;
+ int j = 0;
+ const char *prio_arr[8] = { "0", "1", "2", "3", "4", "5", "6", "7" };
+ struct roce5_ip_prio_enable_ctx *ip_prio_enable = NULL;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+ struct roce5_device *rdev =
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx);
+ ip_enable_ctx->ip_enable_root =
+ kobject_create_and_add("ip_enable", kobj);
+ if (ip_enable_ctx->ip_enable_root == NULL) {
+ ROCE_ERR("Failed to create kobject for ecn ip enable.(ret:%d)",
+ ret);
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < PRI_ARRAY_LEN; i++) {
+ ip_prio_enable = &ip_enable_ctx->ip_prio_enable[i];
+ ip_prio_enable->ecn_ip_enable_ctx = (void *)ip_enable_ctx;
+ ip_prio_enable->prio = (u32)i;
+ ip_prio_enable->prio_en =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_ip_enable_en;
+ sysfs_attr_init(&ip_prio_enable->ip_enable.attr);
+ ip_prio_enable->ip_enable.attr.name = prio_arr[i];
+ ip_prio_enable->ip_enable.attr.mode =
+ (S_IWUSR | S_IRUSR | S_IRGRP);
+ ip_prio_enable->ip_enable.show = roce5_show_ecn_ip_prio_enable;
+ ip_prio_enable->ip_enable.store =
+ roce5_store_ecn_ip_prio_enable;
+ ret = sysfs_create_file(ip_enable_ctx->ip_enable_root,
+ &ip_prio_enable->ip_enable.attr);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to create file for ecn ip enable.(ret:%d)",
+ ret);
+ goto err_create_file;
+ }
+ }
+
+ return 0;
+
+err_create_file:
+ for (j = i - 1; j >= 0; j--) {
+ sysfs_remove_file(
+ ip_enable_ctx->ip_enable_root,
+ &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr);
+ }
+
+ kobject_put(ip_enable_ctx->ip_enable_root);
+
+ return ret;
+}
+
+static void
+roce5_remove_ecn_ip_enable_sysfs(struct kobject *kobj,
+ struct roce5_ecn_ip_enable_ctx *ip_enable_ctx)
+{
+ int i;
+
+ for (i = 0; i < PRI_ARRAY_LEN; i++) {
+ sysfs_remove_file(
+ ip_enable_ctx->ip_enable_root,
+ &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr);
+ }
+
+ kobject_put(ip_enable_ctx->ip_enable_root);
+}
+
+ROCE_ATTR_RW(bypass, roce5_show_bypass, roce5_store_bypass);
+ROCE_ATTR_RW(alpha_dec_period, roce5_show_alpha_dec_period,
+ roce5_store_alpha_dec_period);
+ROCE_ATTR_RW(rate_dec_period, roce5_show_rate_dec_period,
+ roce5_store_rate_dec_period);
+ROCE_ATTR_RW(rate_inc_period, roce5_show_rate_inc_period,
+ roce5_store_rate_inc_period);
+ROCE_ATTR_RW(factor_gita, roce5_show_factor_gita, roce5_store_factor_gita);
+ROCE_ATTR_RW(initial_alpha, roce5_show_initial_alpha,
+ roce5_store_initial_alpha);
+ROCE_ATTR_RW(rate_inc_ai, roce5_show_rate_inc_ai, roce5_store_rate_inc_ai);
+ROCE_ATTR_RW(rate_inc_hai, roce5_show_rate_inc_hai, roce5_store_rate_inc_hai);
+ROCE_ATTR_RW(rate_first_set, roce5_show_rate_first_set,
+ roce5_store_rate_first_set);
+ROCE_ATTR_RW(rate_target_clamp, roce5_show_rate_target_clamp,
+ roce5_store_rate_target_clamp);
+ROCE_ATTR_RW(token_period, roce5_show_token_period, roce5_store_token_period);
+ROCE_ATTR_RW(min_rate, roce5_show_min_rate, roce5_store_min_rate);
+
+static struct attribute *ecn_rp_ctx_attrs[] = {
+ ROCE_ATTR_PTR(bypass),
+ ROCE_ATTR_PTR(alpha_dec_period),
+ ROCE_ATTR_PTR(rate_dec_period),
+ ROCE_ATTR_PTR(rate_inc_period),
+ ROCE_ATTR_PTR(factor_gita),
+ ROCE_ATTR_PTR(initial_alpha),
+ ROCE_ATTR_PTR(rate_inc_ai),
+ ROCE_ATTR_PTR(rate_inc_hai),
+ ROCE_ATTR_PTR(rate_first_set),
+ ROCE_ATTR_PTR(rate_target_clamp),
+ ROCE_ATTR_PTR(token_period),
+ ROCE_ATTR_PTR(min_rate),
+ NULL,
+};
+
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+static const struct attribute_group ecn_rp_ctx_attr_grp = {
+ .attrs = ecn_rp_ctx_attrs,
+};
+
+static const struct attribute_group *ecn_rp_ctx_attr_grps[] = {
+ &ecn_rp_ctx_attr_grp,
+ NULL,
+};
+#endif
+
+static void roce_ecn_rp_sysfs_release(struct kobject *kobj)
+{
+}
+
+static struct kobj_type roce_ecn_rp_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = roce_ecn_rp_sysfs_release,
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+ .default_groups = ecn_rp_ctx_attr_grps,
+#else
+ .default_attrs = ecn_rp_ctx_attrs,
+#endif
+};
+
+static int roce5_init_ecn_rp_sysfs(struct kobject *kobj,
+ struct roce5_ecn_rp_ctx *ecn_rp_ctx)
+{
+ int ret = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+ struct roce5_device *rdev =
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx);
+
+ ecn_rp_ctx->bypass =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_bypass;
+ ecn_rp_ctx->alpha_dec_period =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_alpha_dec_period;
+ ecn_rp_ctx->rate_dec_period =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_rate_dec_period;
+ ecn_rp_ctx->rate_inc_period =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_rate_inc_period;
+ ecn_rp_ctx->factor_gita =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_factor_gita;
+ ecn_rp_ctx->initial_alpha = g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_initial_alpha;
+ ecn_rp_ctx->rate_inc_ai =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_rate_inc_ai;
+ ecn_rp_ctx->rate_inc_hai =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_rate_inc_hai;
+ ecn_rp_ctx->rate_first_set = g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_rate_first_set;
+ ecn_rp_ctx->rate_target_clamp =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_rate_target_clamp;
+ ecn_rp_ctx->token_period =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_token_period;
+ ecn_rp_ctx->min_rate =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_min_rate;
+ ret = kobject_init_and_add(&ecn_rp_ctx->ecn_rp_root, &roce_ecn_rp_ktype,
+ kobj, "roce5_rp");
+ if (ret != 0) {
+ ROCE_ERR("Failed to add kobject roce5_rp.(ret:%d)", ret);
+ kobject_put(&ecn_rp_ctx->ecn_rp_root);
+ return ret;
+ }
+
+ ecn_rp_ctx->enable_ctx.np_rp = ROCE_DCQCN_RP;
+ ret = roce5_init_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root,
+ &ecn_rp_ctx->enable_ctx);
+ if (ret != 0) {
+ goto err_init_prio_enable_sysfs;
+ }
+
+ return 0;
+
+err_init_prio_enable_sysfs:
+ kobject_put(&ecn_rp_ctx->ecn_rp_root);
+
+ return ret;
+}
+
+static void roce5_remove_ecn_rp_sysfs(struct kobject *kobj,
+ struct roce5_ecn_rp_ctx *ecn_rp_ctx)
+{
+ roce5_remove_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root,
+ &ecn_rp_ctx->enable_ctx);
+
+ kobject_put(&ecn_rp_ctx->ecn_rp_root);
+}
+
+ROCE_ATTR_RW(rtt_rsp_prio, roce5_show_rtt_rsp_prio, roce5_store_rtt_rsp_prio);
+ROCE_ATTR_RW(rtt_rsp_prio_enable, roce5_show_rtt_rsp_prio_enable,
+ roce5_store_rtt_rsp_prio_enable);
+ROCE_ATTR_RW(rtt_version, roce5_show_rtt_version, roce5_store_rtt_version);
+ROCE_ATTR_RW(ecn_cnp_en, roce5_show_ecn_cnp_en, roce5_store_ecn_cnp_en);
+ROCE_ATTR_RW(flow_ctrl_unit, roce5_show_flow_ctrl_unit,
+ roce5_store_flow_ctrl_unit);
+ROCE_ATTR_RW(min_cnp_period, roce5_show_min_cnp_period,
+ roce5_store_min_cnp_period);
+ROCE_ATTR_RW(cnp_prio_enable, roce5_show_cnp_prio_enable,
+ roce5_store_cnp_prio_enable);
+ROCE_ATTR_RW(cnp_prio, roce5_show_cnp_prio, roce5_store_cnp_prio);
+
+static struct attribute *ecn_np_ctx_attrs[] = {
+ ROCE_ATTR_PTR(min_cnp_period),
+ ROCE_ATTR_PTR(cnp_prio_enable),
+ ROCE_ATTR_PTR(cnp_prio),
+ ROCE_ATTR_PTR(rtt_rsp_prio),
+ ROCE_ATTR_PTR(rtt_rsp_prio_enable),
+ ROCE_ATTR_PTR(rtt_version),
+ ROCE_ATTR_PTR(ecn_cnp_en),
+ ROCE_ATTR_PTR(flow_ctrl_unit),
+ NULL,
+};
+
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+static const struct attribute_group ecn_np_ctx_attr_grp = {
+ .attrs = ecn_np_ctx_attrs,
+};
+
+static const struct attribute_group *ecn_np_ctx_attr_grps[] = {
+ &ecn_np_ctx_attr_grp,
+ NULL,
+};
+#endif
+
+static void roce_ecn_np_sysfs_release(struct kobject *kobj)
+{
+}
+
+static struct kobj_type roce_ecn_np_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = roce_ecn_np_sysfs_release,
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+ .default_groups = ecn_np_ctx_attr_grps,
+#else
+ .default_attrs = ecn_np_ctx_attrs,
+#endif
+};
+
+static int roce5_init_ecn_np_sysfs(struct kobject *kobj,
+ struct roce5_ecn_np_ctx *ecn_np_ctx)
+{
+ int ret = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+ struct roce5_device *rdev =
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx);
+
+ ecn_np_ctx->min_cnp_period = g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_min_cnp_period;
+ ecn_np_ctx->cnp_prio_enable =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_cnp_prio_enable;
+ ecn_np_ctx->cnp_prio =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_cnp_prio;
+ ecn_np_ctx->ecn_cnp_en = g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_ecn_cnp_enable;
+ ecn_np_ctx->flow_ctrl_unit = g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_flow_ctrl_unit;
+ ecn_np_ctx->rtt_rsp_prio =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_rtt_rsp_prio;
+ ecn_np_ctx->rtt_rsp_prio_enable =
+ g_roce_cc_ecn_param[rdev->cc_param_version]
+ .default_rtt_rsp_prio_enable;
+ ecn_np_ctx->rtt_version =
+ g_roce_cc_ecn_param[rdev->cc_param_version].default_rtt_version;
+ ret = kobject_init_and_add(&ecn_np_ctx->ecn_np_root, &roce_ecn_np_ktype,
+ kobj, "roce5_np");
+ if (ret != 0) {
+ ROCE_ERR("Failed to add kobject roce5_np.(ret:%d)", ret);
+ kobject_put(&ecn_np_ctx->ecn_np_root);
+ return ret;
+ }
+
+ ecn_np_ctx->enable_ctx.np_rp = ROCE_DCQCN_NP;
+ ret = roce5_init_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root,
+ &ecn_np_ctx->enable_ctx);
+ if (ret != 0) {
+ goto err_init_prio_enable_sysfs;
+ }
+
+ return 0;
+
+err_init_prio_enable_sysfs:
+ kobject_put(&ecn_np_ctx->ecn_np_root);
+ return ret;
+}
+
+/* ldcp_ctx parameters use macro generate show/store function */
+ROCE5_PARAM_SHOW_STORE_LDCP(wnd_min, wnd_min, min_ldcp_wnd_min,
+ max_ldcp_wnd_min);
+ROCE5_PARAM_SHOW_STORE_LDCP(wnd_default, wnd_default, min_ldcp_wnd_default,
+ max_ldcp_wnd_default);
+ROCE5_PARAM_SHOW_STORE_LDCP(alpha, alpha, min_ldcp_alpha, max_ldcp_alpha);
+ROCE5_PARAM_SHOW_STORE_LDCP(beta, beta, min_ldcp_beta, max_ldcp_beta);
+ROCE5_PARAM_SHOW_STORE_LDCP(gamma, gamma, min_ldcp_gamma, max_ldcp_gamma);
+ROCE5_PARAM_SHOW_STORE_LDCP(eta, eta, min_ldcp_eta, max_ldcp_eta);
+
+ROCE_ATTR_RW(wnd_min, roce5_show_wnd_min, roce5_store_wnd_min);
+ROCE_ATTR_RW(wnd_default, roce5_show_wnd_default, roce5_store_wnd_default);
+ROCE_ATTR_RW(alpha, roce5_show_alpha, roce5_store_alpha);
+ROCE_ATTR_RW(beta, roce5_show_beta, roce5_store_beta);
+ROCE_ATTR_RW(gamma, roce5_show_gamma, roce5_store_gamma);
+ROCE_ATTR_RW(eta, roce5_show_eta, roce5_store_eta);
+
+static struct attribute *ecn_ldcp_ctx_attrs[] = {
+ ROCE_ATTR_PTR(wnd_min),
+ ROCE_ATTR_PTR(wnd_default),
+ ROCE_ATTR_PTR(alpha),
+ ROCE_ATTR_PTR(beta),
+ ROCE_ATTR_PTR(gamma),
+ ROCE_ATTR_PTR(eta),
+ NULL,
+};
+
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+static const struct attribute_group ecn_ldcp_ctx_attr_grp = {
+ .attrs = ecn_ldcp_ctx_attrs,
+};
+
+static const struct attribute_group *ecn_ldcp_ctx_attr_grps[] = {
+ &ecn_ldcp_ctx_attr_grp,
+ NULL,
+};
+#endif
+
+static void roce_ecn_ldcp_sysfs_release(struct kobject *kobj)
+{
+}
+
+static struct kobj_type roce_ecn_ldcp_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = roce_ecn_ldcp_sysfs_release,
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+ .default_groups = ecn_ldcp_ctx_attr_grps,
+#else
+ .default_attrs = ecn_ldcp_ctx_attrs,
+#endif
+};
+
+static int roce5_init_ecn_ldcp_sysfs(struct kobject *kobj,
+ roce5_ecn_ldcp_ctx_t *ecn_ldcp_ctx)
+{
+ int ret;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+ struct roce5_device *rdev =
+ container_of(ecn_ctx, struct roce5_device, ecn_ctx);
+
+ ecn_ldcp_ctx->wnd_min =
+ g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_wnd_min;
+ ecn_ldcp_ctx->wnd_default =
+ g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_wnd_default;
+ ecn_ldcp_ctx->alpha =
+ g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_alpha;
+ ecn_ldcp_ctx->beta =
+ g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_beta;
+ ecn_ldcp_ctx->gamma =
+ g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_gamma;
+ ecn_ldcp_ctx->eta =
+ g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_eta;
+ ret = kobject_init_and_add(&ecn_ldcp_ctx->ecn_ldcp_root,
+ &roce_ecn_ldcp_ktype, kobj, "roce5_ldcp");
+ if (ret != 0) {
+ ROCE_ERR("Failed to add kobject roce5_ldcp.(ret:%d)", ret);
+ kobject_put(&ecn_ldcp_ctx->ecn_ldcp_root);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void roce5_remove_ecn_ldcp_sysfs(roce5_ecn_ldcp_ctx_t *ecn_ldcp_ctx)
+{
+ kobject_put(&ecn_ldcp_ctx->ecn_ldcp_root);
+}
+
+static void roce5_remove_ecn_np_sysfs(struct kobject *kobj,
+ struct roce5_ecn_np_ctx *ecn_np_ctx)
+{
+ roce5_remove_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root,
+ &ecn_np_ctx->enable_ctx);
+
+ kobject_put(&ecn_np_ctx->ecn_np_root);
+}
+
+static ssize_t roce5_show_ecn_ver(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n",
+ (int)ecn_ctx->ecn_ver);
+}
+
+static ssize_t roce5_store_ecn_ver(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf,
+ size_t count)
+{
+ int ret = 0;
+ int ecn_ver = 0;
+ u32 old_ecn_ver = 0;
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj);
+
+ ret = sscanf_s(buf, "%d", &ecn_ver);
+ if (ret != 1) {
+ return -EIO;
+ }
+
+ if (ROCE5_PARAM_CHECK_BOUND(ecn_ver, ECN_VER_DCQCN, ECN_VER_PATHQCN)) {
+ return -EIO;
+ }
+
+ if (ecn_ctx->ecn_ver != (u32)ecn_ver) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_ecn_ver = ecn_ctx->ecn_ver;
+ ecn_ctx->ecn_ver = (u32)ecn_ver;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Fail to update ecn_ver(%d), ret(%d).",
+ ecn_ver, ret);
+ ecn_ctx->ecn_ver = old_ecn_ver;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+ROCE_ATTR_RW(ecn_ver, roce5_show_ecn_ver, roce5_store_ecn_ver);
+ROCE_ATTR_RW(cc_algo, roce5_show_cc_algo, roce5_store_cc_algo);
+
+static struct attribute *ecn_ctx_attrs[] = {
+ ROCE_ATTR_PTR(ecn_ver),
+ ROCE_ATTR_PTR(cc_algo),
+ NULL,
+};
+
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+static const struct attribute_group ecn_ctx_attr_grp = {
+ .attrs = ecn_ctx_attrs,
+};
+
+static const struct attribute_group *ecn_ctx_attr_grps[] = {
+ &ecn_ctx_attr_grp,
+ NULL,
+};
+#endif
+
+static void roce_ecn_sysfs_release(struct kobject *kobj)
+{
+}
+
+static struct kobj_type roce_ecn_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = roce_ecn_sysfs_release,
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+ .default_groups = ecn_ctx_attr_grps,
+#else
+ .default_attrs = ecn_ctx_attrs,
+#endif
+};
+
+static void roce5_init_cc_algo(struct roce5_device *rdev,
+ struct roce5_ecn_ctx *ecn_ctx)
+{
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ ecn_ctx->cc_algo = ROCE_CC_DCQCN_ALGO;
+ } else {
+ ecn_ctx->cc_algo = ROCE_CC_DISABLE;
+ }
+}
+
+static void roce5_init_cc_param_version(struct roce5_device *rdev)
+{
+ rdev->cc_param_version =
+ ROCE5_SUPPORT_CC_PARAM_V1(rdev) != 0 ? ROCE_CC_PARAM_V1 :
+ ROCE5_SUPPORT_CC_PARAM_V2(rdev) != 0 ? ROCE_CC_PARAM_V2 :
+ ROCE_CC_PARAM_INVALID;
+}
+
+static int roce5_init_ecn_sysfs(struct roce5_device *rdev,
+ struct roce5_ecn_ctx *ecn_ctx)
+{
+ int ret = 0;
+ struct net_device *ndev = rdev->ndev;
+
+ ret = memset_s(ecn_ctx, sizeof(*ecn_ctx), 0, sizeof(*ecn_ctx));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset ecn_ctx.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ mutex_init(&ecn_ctx->ecn_mutex);
+
+ ecn_ctx->ecn_ver = ECN_VER_DCQCN;
+ roce5_init_cc_algo(rdev, ecn_ctx);
+ roce5_init_cc_param_version(rdev);
+
+ ret = kobject_init_and_add(&ecn_ctx->ecn_root, &roce_ecn_ktype,
+ &ndev->dev.kobj, "ecn");
+ if (ret != 0) {
+ ROCE_ERR("Failed to add kobject ecn.(ret:%d)", ret);
+ kobject_put(&ecn_ctx->ecn_root);
+ return ret;
+ }
+
+ ret = roce5_init_ecn_ldcp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ldcp_ctx);
+ if (ret != 0) {
+ goto err_init_ldcp_sysfs;
+ }
+
+ ret = roce5_init_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx);
+ if (ret != 0) {
+ goto err_init_np_sysfs;
+ }
+
+ ret = roce5_init_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx);
+ if (ret != 0) {
+ goto err_init_rp_sysfs;
+ }
+
+ ret = roce5_init_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root,
+ &ecn_ctx->ip_enable_ctx);
+ if (ret != 0) {
+ goto err_init_ip_enable_sysfs;
+ }
+
+ ret = roce5_init_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx);
+ if (ret != 0) {
+ goto err_init_ecn_ucc_sysfs;
+ }
+
+ ret = roce5_init_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root,
+ &ecn_ctx->multipath_ctx);
+ if (ret != 0) {
+ goto err_init_multipath_sysfs;
+ }
+
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ goto err_update;
+ }
+
+ return 0;
+
+err_update:
+ roce5_remove_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root,
+ &ecn_ctx->multipath_ctx);
+err_init_multipath_sysfs:
+ roce5_remove_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx);
+err_init_ecn_ucc_sysfs:
+ roce5_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root,
+ &ecn_ctx->ip_enable_ctx);
+err_init_ip_enable_sysfs:
+ roce5_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx);
+err_init_rp_sysfs:
+ roce5_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx);
+err_init_np_sysfs:
+ roce5_remove_ecn_ldcp_sysfs(&ecn_ctx->ldcp_ctx);
+err_init_ldcp_sysfs:
+ kobject_put(&ecn_ctx->ecn_root);
+ return ret;
+}
+
+static void roce5_prase_qpc(roce_verbs_qp_attr_s *qp_attr)
+{
+ qp_attr->com_info.rsvd = be32_to_cpu(qp_attr->com_info.rsvd);
+ qp_attr->ext_seg.dfx_info.dw0.value =
+ be32_to_cpu(qp_attr->ext_seg.dfx_info.dw0.value);
+ qp_attr->ext_seg.dfx_info.dw2.value =
+ be32_to_cpu(qp_attr->ext_seg.dfx_info.dw2.value);
+ qp_attr->ext_seg.dfx_info.dw4.value =
+ be32_to_cpu(qp_attr->ext_seg.dfx_info.dw4.value);
+}
+
+static ssize_t roce5_store_dfx_qpc(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf,
+ size_t count)
+{
+ int ret;
+ u32 qpn;
+ struct roce5_dfx_qpc_ctx *qpc_ctx = NULL;
+ struct roce5_dfx_ctx *dfx_ctx = NULL;
+ struct roce5_device *rdev = NULL;
+ roce_verbs_qp_attr_s qp_attr;
+ struct rdma_service_cap *rdma_cap = NULL;
+
+ if (kobj == NULL || attr == NULL || buf == NULL || count == 0) {
+ ROCE_ERR("Invalid input para.");
+ return -EINVAL;
+ }
+
+ qpc_ctx = container_of(attr, struct roce5_dfx_qpc_ctx, kattr);
+ dfx_ctx = container_of(qpc_ctx, struct roce5_dfx_ctx, qpc_ctx);
+ rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx);
+
+ (void)memset_s(&qp_attr, sizeof(roce_verbs_qp_attr_s), 0,
+ sizeof(roce_verbs_qp_attr_s));
+
+ ret = (int)sscanf_s(buf, "%u", &qpn);
+ if (ret != 1) {
+ ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret);
+ return -EIO;
+ }
+
+ rdma_cap = &rdev->rdma_cap;
+ if ((qpn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_qps) ||
+ (qpn < ROCE_MIN_QPN)) {
+ ROCE_DEV_ERR(rdev, "Invalid qpn(%u), max_qps(%u)", qpn,
+ rdma_cap->dev_rdma_cap.roce_own_cap.max_qps);
+ return -EIO;
+ }
+
+ ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)&qp_attr,
+ sizeof(roce_verbs_qp_attr_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query qp_attr, ret(%d)", ret);
+ return (ssize_t)ret;
+ }
+
+ roce5_prase_qpc(&qp_attr);
+
+ roce5_dfx_print("qpn(0x%x): sqc pi(0x%x), sqc ci(0x%x).", qpn,
+ qp_attr.ext_seg.dfx_info.dw0.bs.sq_pi,
+ qp_attr.ext_seg.dfx_info.dw0.bs.sq_ci);
+ roce5_dfx_print(
+ "sqac ci(0x%x), sqac wqe_prefetch_ci(0x%x), rqc pi(0x%x), rqc ci(0x%x).",
+ qp_attr.ext_seg.dfx_info.dw2.bs.sqa_ci,
+ qp_attr.ext_seg.dfx_info.dw2.bs.sqa_wqe_prefetch_ci,
+ qp_attr.ext_seg.dfx_info.dw4.bs.rq_pi,
+ qp_attr.ext_seg.dfx_info.dw4.bs.rq_ci);
+
+ return (ssize_t)count;
+}
+
+static ssize_t roce5_store_dfx_cqc(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf,
+ size_t count)
+{
+ int ret;
+ u32 cqn;
+ struct roce5_dfx_cqc_ctx *cqc_ctx = NULL;
+ struct roce5_dfx_ctx *dfx_ctx = NULL;
+ struct roce5_device *rdev = NULL;
+ roce_verbs_cq_attr_s cq_attr;
+ roce5_cq_query_outbuf_s cq_query;
+ struct rdma_service_cap *rdma_cap = NULL;
+
+ if (kobj == NULL || attr == NULL || buf == NULL || count == 0) {
+ ROCE_ERR("Invalid input para.");
+ return -EINVAL;
+ }
+
+ cqc_ctx = container_of(attr, struct roce5_dfx_cqc_ctx, kattr);
+ dfx_ctx = container_of(cqc_ctx, struct roce5_dfx_ctx, cqc_ctx);
+ rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx);
+
+ (void)memset_s(&cq_query, sizeof(roce5_cq_query_outbuf_s), 0,
+ sizeof(roce5_cq_query_outbuf_s));
+
+ ret = (int)sscanf_s(buf, "%u", &cqn);
+ if (ret != 1) {
+ ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret);
+ return -EIO;
+ }
+
+ rdma_cap = &rdev->rdma_cap;
+ if (cqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs) {
+ ROCE_DEV_ERR(rdev, "Invalid cqn(%u), max_cqs(%u)", cqn,
+ rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs);
+ return -EIO;
+ }
+
+ ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)&cq_query,
+ sizeof(roce5_cq_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query cqc, ret(%d)", ret);
+ return (ssize_t)ret;
+ }
+
+ if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) {
+ (void)memcpy_s(&cq_attr, sizeof(roce_verbs_cq_attr_s),
+ &cq_query, sizeof(roce_verbs_cq_attr_s));
+ cq_attr.dfx_info.dw0.value =
+ be32_to_cpu(cq_attr.dfx_info.dw0.value);
+ cq_attr.dfx_info.dw1.value =
+ be32_to_cpu(cq_attr.dfx_info.dw1.value);
+ roce5_dfx_print("cqn(0x%x): pi(0x%x), ci(0x%x).", cqn,
+ cq_attr.dfx_info.dw1.bs.pi,
+ cq_attr.dfx_info.dw0.bs.ci);
+ } else {
+ ROCE_DEV_ERR(rdev,
+ "Unsupport rdev device_type, device_type(%d)",
+ rdev->device_type);
+ }
+
+ return (ssize_t)count;
+}
+
+static ssize_t roce5_store_dfx_srqc(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ u32 srqn;
+ struct roce5_dfx_srqc_ctx *srqc_ctx = NULL;
+ struct roce5_dfx_ctx *dfx_ctx = NULL;
+ struct roce5_device *rdev = NULL;
+ roce_verbs_srq_attr_s srq_attr;
+ struct rdma_service_cap *rdma_cap = NULL;
+
+ if (kobj == NULL || attr == NULL || buf == NULL || count == 0) {
+ ROCE_ERR("Invalid input para.");
+ return -EINVAL;
+ }
+
+ srqc_ctx = container_of(attr, struct roce5_dfx_srqc_ctx, kattr);
+ dfx_ctx = container_of(srqc_ctx, struct roce5_dfx_ctx, srqc_ctx);
+ rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx);
+
+ (void)memset_s(&srq_attr, sizeof(roce_verbs_srq_attr_s), 0,
+ sizeof(roce_verbs_srq_attr_s));
+
+ ret = (int)sscanf_s(buf, "%u", &srqn);
+ if (ret != 1) {
+ ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret);
+ return -EIO;
+ }
+
+ rdma_cap = &rdev->rdma_cap;
+ if (srqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs) {
+ ROCE_DEV_ERR(rdev, "Invalid srqn(%u), max_srqs(%u)", srqn,
+ rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs);
+ return -EIO;
+ }
+
+ ret = roce5_dfx_cmd_query_srq(rdev, srqn, &srq_attr);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to query srq_attr, ret(%d)", ret);
+ return (ssize_t)ret;
+ }
+
+ srq_attr.dfx_info.dw0.value = be32_to_cpu(srq_attr.dfx_info.dw0.value);
+
+ roce5_dfx_print("srqn(0x%x): pcnt(0x%x), ccnt(0x%x).", srqn,
+ srq_attr.dfx_info.dw0.bs.pcnt,
+ srq_attr.dfx_info.dw0.bs.ccnt);
+
+ return (ssize_t)count;
+}
+
+static int roce5_init_dfx_sub_ctx_sysfs(struct kobject *kobj,
+ struct roce5_dfx_ctx *dfx_ctx)
+{
+ int ret;
+ struct kobj_attribute *qpc_kattr = NULL;
+ struct kobj_attribute *cqc_kattr = NULL;
+ struct kobj_attribute *srqc_kattr = NULL;
+
+ if (kobj == NULL || dfx_ctx == NULL) {
+ ROCE_ERR("Invalid kobj or dfx_ctx.");
+ return -EINVAL;
+ }
+
+ qpc_kattr = &dfx_ctx->qpc_ctx.kattr;
+ cqc_kattr = &dfx_ctx->cqc_ctx.kattr;
+ srqc_kattr = &dfx_ctx->srqc_ctx.kattr;
+
+ sysfs_attr_init(&qpc_kattr->attr);
+ qpc_kattr->attr.name = "qpc";
+ qpc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP);
+ qpc_kattr->store = roce5_store_dfx_qpc;
+ ret = sysfs_create_file(kobj, &qpc_kattr->attr);
+ if (ret != 0) {
+ ROCE_ERR("QPC failed to do sysfs_create_file, ret(%d)", ret);
+ return ret;
+ }
+
+ sysfs_attr_init(&cqc_kattr->attr);
+ cqc_kattr->attr.name = "cqc";
+ cqc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP);
+ cqc_kattr->store = roce5_store_dfx_cqc;
+ ret = sysfs_create_file(kobj, &cqc_kattr->attr);
+ if (ret != 0) {
+ ROCE_ERR("CQC failed to do sysfs_create_file, ret(%d)", ret);
+ goto err_create_cqc_sysfs;
+ }
+
+ sysfs_attr_init(&srqc_kattr->attr);
+ srqc_kattr->attr.name = "srqc";
+ srqc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP);
+ srqc_kattr->store = roce5_store_dfx_srqc;
+ ret = sysfs_create_file(kobj, &srqc_kattr->attr);
+ if (ret != 0) {
+ ROCE_ERR("SRQC failed to do sysfs_create_file, ret(%d)", ret);
+ goto err_create_srqc_sysfs;
+ }
+
+ return 0;
+
+err_create_srqc_sysfs:
+ sysfs_remove_file(kobj, &cqc_kattr->attr);
+
+err_create_cqc_sysfs:
+ sysfs_remove_file(kobj, &qpc_kattr->attr);
+
+ return ret;
+}
+
+static int roce5_init_dfx_sysfs(struct net_device *ndev,
+ struct roce5_dfx_ctx *dfx_ctx)
+{
+ int ret;
+
+ (void)memset_s(dfx_ctx, sizeof(*dfx_ctx), 0, sizeof(*dfx_ctx));
+
+ dfx_ctx->dfx_root =
+ kobject_create_and_add("roce5_dfx", &ndev->dev.kobj);
+ if (dfx_ctx->dfx_root == NULL) {
+ ROCE_ERR("Failed to do kobject_create_and_add.");
+ return -ENOMEM;
+ }
+
+ ret = roce5_init_dfx_sub_ctx_sysfs(dfx_ctx->dfx_root, dfx_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Failed to init dfx sub ctx sysfs, ret(%d)", ret);
+ kobject_put(dfx_ctx->dfx_root);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void roce5_remove_ecn_sysfs(struct roce5_device *rdev)
+{
+ struct roce5_ecn_ctx *ecn_ctx = &rdev->ecn_ctx;
+ roce5_remove_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root,
+ &ecn_ctx->multipath_ctx);
+
+ roce5_remove_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx);
+
+ roce5_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root,
+ &ecn_ctx->ip_enable_ctx);
+
+ roce5_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx);
+
+ roce5_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx);
+
+ roce5_remove_ecn_ldcp_sysfs(&ecn_ctx->ldcp_ctx);
+
+ kobject_put(&ecn_ctx->ecn_root);
+}
+
+int roce5_init_sysfs(struct roce5_device *rdev)
+{
+ int ret;
+
+ ret = roce5_init_ecn_sysfs(rdev, &rdev->ecn_ctx);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init ecn sysfs, ret(%d)", ret);
+ return ret;
+ }
+
+ ret = roce5_init_dfx_sysfs(rdev->ndev, &rdev->dfx_ctx);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to init dfx sysfs, ret(%d)", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void roce5_remove_dfx_sub_sysfs(struct roce5_dfx_ctx *dfx_ctx)
+{
+ if (dfx_ctx->dfx_root != NULL) {
+ sysfs_remove_file(dfx_ctx->dfx_root,
+ &dfx_ctx->srqc_ctx.kattr.attr);
+
+ sysfs_remove_file(dfx_ctx->dfx_root,
+ &dfx_ctx->cqc_ctx.kattr.attr);
+
+ sysfs_remove_file(dfx_ctx->dfx_root,
+ &dfx_ctx->qpc_ctx.kattr.attr);
+
+ kobject_put(dfx_ctx->dfx_root);
+
+ dfx_ctx->dfx_root = NULL;
+ }
+}
+
+void roce5_remove_sysfs(struct roce5_device *rdev)
+{
+ roce5_remove_dfx_sub_sysfs(&rdev->dfx_ctx);
+ roce5_remove_ecn_sysfs(rdev);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h
new file mode 100644
index 000000000..814f29ddc
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h
@@ -0,0 +1,241 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_SYSFS_H
+#define ROCE_SYSFS_H
+
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+#include <linux/netdevice.h>
+#include <linux/kobject.h>
+#include "roce_ucc_sysfs.h"
+#include "roce_multipath_sysfs.h"
+
+#define ROCE_MIN_QPN 2
+#define BYTE_TO_BIT 8
+
+#define PRI_ARRAY_LEN 8
+
+#define ROCE_MAX_PORT_CNT 8
+#define ROCE_MAX_PF_NUM 32
+
+#define ROCE_ENABLE_CHECK_MASK 0xfffffffe
+
+/* ECN VER */
+#define ECN_VER_DCQCN 0
+#define ECN_VER_PATHQCN 1
+
+#define FLOW_CTRL_UNIT_QP 0
+#define FLOW_CTRL_UNIT_IP 1
+
+enum { ROCE_DCQCN_NP = 0, ROCE_DCQCN_RP = 1 };
+
+enum {
+ ROCE_CC_PARAM_INVALID = 0,
+ ROCE_CC_PARAM_V1 = 1,
+ ROCE_CC_PARAM_V2 = 2,
+};
+
+#define HRN0_K_MAGIC_NUM_3_SYSFS 3
+#define HRN0_K_MAGIC_NUM_7_SYSFS 7
+#define HRN0_K_MAGIC_NUM_8_SYSFS 8
+#define HRN0_K_MAGIC_NUM_M_SYSFS 1000000
+
+#define to_roce5_ecn_ctx(_kobj) \
+ container_of(_kobj, struct roce5_ecn_ctx, ecn_root)
+#define to_roce5_ecn_np_ctx(_kobj) \
+ container_of(_kobj, struct roce5_ecn_np_ctx, ecn_np_root)
+#define to_roce5_ecn_rp_ctx(_kobj) \
+ container_of(_kobj, struct roce5_ecn_rp_ctx, ecn_rp_root)
+#define to_roce5_ecn_ldcp_ctx(_kobj) \
+ container_of(_kobj, struct roce5_ecn_ldcp_ctx, ecn_ldcp_root)
+
+#define ROCE5_PARAM_CHECK_BOUND(val, min, max) \
+ ((val) < (min)) || ((val) > (max))
+
+#define INIT_ROCE_KOBJ_ATTR(_name, _mode, _show, _store) \
+ { \
+ .attr = { .name = __stringify(_name), .mode = _mode }, \
+ .show = (_show), .store = (_store), \
+ }
+
+#define ROCE_ATTR_RW(_name, _show, _store) \
+ static struct kobj_attribute roce_attr_##_name = INIT_ROCE_KOBJ_ATTR( \
+ _name, (S_IWUSR | S_IRUSR | S_IRGRP), (_show), (_store))
+
+#define ROCE_ATTR_PTR(_name) (&roce_attr_##_name.attr)
+
+#define ROCE_CC_ALGO_TEST_BIT(bitmap, cc_algo) (!!(bitmap & (1U << cc_algo)))
+
+struct roce5_prio_enable_ctx {
+ struct kobj_attribute enable;
+ void *ecn_enable_ctx;
+ u32 prio_en;
+ u32 prio;
+};
+
+struct roce5_ip_prio_enable_ctx {
+ struct kobj_attribute ip_enable;
+ void *ecn_ip_enable_ctx;
+ u32 prio_en;
+ u32 prio;
+};
+
+struct roce5_ecn_ip_enable_ctx {
+ struct kobject *ip_enable_root;
+ struct roce5_ip_prio_enable_ctx ip_prio_enable[PRI_ARRAY_LEN];
+};
+
+struct roce5_ecn_enable_ctx {
+ struct kobject *enable_root;
+ struct roce5_prio_enable_ctx prio_enable[PRI_ARRAY_LEN];
+ u32 np_rp;
+};
+
+struct roce5_dfx_qpc_ctx {
+ struct kobj_attribute kattr;
+};
+
+struct roce5_dfx_cqc_ctx {
+ struct kobj_attribute kattr;
+};
+
+struct roce5_dfx_srqc_ctx {
+ struct kobj_attribute kattr;
+};
+
+struct roce5_dfx_ctx {
+ struct kobject *dfx_root;
+ struct roce5_dfx_qpc_ctx qpc_ctx;
+ struct roce5_dfx_cqc_ctx cqc_ctx;
+ struct roce5_dfx_srqc_ctx srqc_ctx;
+};
+
+struct roce5_ecn_rp_ctx {
+ struct kobject ecn_rp_root;
+ u32 alpha_dec_period;
+ u32 rate_dec_period;
+ u32 rate_inc_period;
+ u32 factor_gita;
+ u32 initial_alpha;
+ u32 rate_inc_ai;
+ u32 rate_inc_hai;
+ u32 rate_first_set;
+ u32 rate_target_clamp;
+ u32 token_period;
+ u32 min_rate;
+ u32 bypass;
+ struct roce5_ecn_enable_ctx enable_ctx;
+};
+
+struct roce5_ecn_np_ctx {
+ struct kobject ecn_np_root;
+ u32 min_cnp_period;
+ u32 cnp_prio_enable;
+ u32 cnp_prio;
+ u32 rtt_rsp_prio;
+ u32 rtt_rsp_prio_enable;
+ u32 rtt_version;
+ u32 ecn_cnp_en;
+ u32 flow_ctrl_unit;
+ struct roce5_ecn_enable_ctx enable_ctx;
+};
+
+typedef struct roce5_ecn_ldcp_ctx {
+ struct kobject ecn_ldcp_root;
+ u32 wnd_min;
+ u32 wnd_default; /* initial default window size, unit: Byte */
+ u32 alpha;
+ u32 beta;
+ u32 gamma;
+ u32 eta;
+} roce5_ecn_ldcp_ctx_t;
+
+struct roce5_ecn_ctx {
+ struct kobject ecn_root;
+ struct mutex ecn_mutex;
+ u32 ecn_ver;
+ u32 cc_algo;
+ roce5_ecn_ldcp_ctx_t ldcp_ctx;
+ struct roce5_ecn_np_ctx np_ctx;
+ struct roce5_ecn_rp_ctx rp_ctx;
+ struct roce5_ecn_ip_enable_ctx ip_enable_ctx;
+ struct roce5_ecn_ucc_ctx ucc_ctx;
+ struct roce5_ecn_multipath_ctx multipath_ctx;
+};
+
+struct roce_cc_ecn_param {
+ u32 default_ip_enable_en;
+ u32 default_bypass;
+ u32 default_min_cnp_period;
+ u32 default_alpha_dec_period;
+ u32 default_rate_dec_period;
+ u32 default_rate_inc_period;
+ u32 default_factor_gita;
+ u32 default_initial_alpha;
+ u32 default_rate_inc_ai;
+ u32 default_rate_inc_hai;
+ u32 default_rate_first_set;
+ u32 default_rate_target_clamp;
+ u32 default_cnp_prio_enable;
+ u32 default_cnp_prio;
+ u32 default_token_period;
+ u32 default_min_rate;
+ u32 default_ecn_cnp_enable;
+ u32 default_flow_ctrl_unit;
+ u32 default_rtt_rsp_prio;
+ u32 default_rtt_rsp_prio_enable;
+ u32 default_rtt_version;
+
+ u32 max_rtt_version;
+ u32 min_bypass;
+ u32 max_bypass;
+ u32 min_rate_inc_hai;
+ u32 max_rate_inc_hai;
+ u32 min_rate_inc_ai;
+ u32 max_rate_inc_ai;
+ u32 min_initial_alpha;
+ u32 max_initial_alpha;
+ u32 min_rate_inc_period;
+ u32 max_rate_inc_period;
+ u32 min_alpha_dec_period;
+ u32 max_alpha_dec_period;
+ u32 min_token_period;
+ u32 max_token_period;
+ u32 min_flow_min_rate;
+ u32 max_flow_min_rate;
+ u32 min_rate_dec_period;
+ u32 max_rate_dec_period;
+ u32 min_min_cnp_period;
+ u32 max_min_cnp_period;
+ u32 min_factor_gita;
+ u32 max_factor_gita;
+ u32 min_rate_first_set;
+ u32 max_rate_first_set;
+
+ u32 default_ldcp_wnd_min;
+ u32 default_ldcp_wnd_default;
+ u32 default_ldcp_alpha;
+ u32 default_ldcp_beta;
+ u32 default_ldcp_gamma;
+ u32 default_ldcp_eta;
+
+ u32 min_ldcp_wnd_min;
+ u32 max_ldcp_wnd_min;
+ u32 min_ldcp_wnd_default;
+ u32 max_ldcp_wnd_default;
+ u32 min_ldcp_alpha;
+ u32 max_ldcp_alpha;
+ u32 min_ldcp_beta;
+ u32 max_ldcp_beta;
+ u32 min_ldcp_gamma;
+ u32 max_ldcp_gamma;
+ u32 min_ldcp_eta;
+ u32 max_ldcp_eta;
+};
+
+int roce5_update_ecn_param(const struct roce5_ecn_ctx *ecn_ctx);
+
+#endif //ROCE_SYSFS_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c
new file mode 100644
index 000000000..e163f895b
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c
@@ -0,0 +1,472 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+
+#include "roce_sysfs.h"
+#include "roce_cc_format.h"
+#include "roce_mpu_cmd.h"
+#include "roce_cmd.h"
+#include "ccp_algo_format.h"
+#include "roce_kernel_compat_gen.h"
+
+#define MAX_STRING_FORMAT_LEN 64
+#define to_roce5_ecn_ucc_ctx(_kobj) \
+ container_of(_kobj, struct roce5_ecn_ucc_ctx, ecn_ucc_root)
+
+static char g_param_buffer[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN];
+static char g_param_show_tmp[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN];
+static char g_param_show_format[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN];
+
+static int roce5_ucc_store_input_check(struct kobject *kobj, const char *buf,
+ u8 param_index)
+{
+ if (kobj == NULL || buf == NULL) {
+ ROCE_ERR("kobj or buf is null\n");
+ return -EINVAL;
+ }
+
+ if (param_index >= ROCE_UCC_MAX_PARAM_NUM) {
+ ROCE_ERR("invalid index:%hhu.\n", param_index);
+ return -EINVAL;
+ }
+
+ if (strlen(buf) > (ROCE_UCC_PARAM_BUFF_LEN - 1)) {
+ ROCE_ERR("input buffer is beyond 1022.\n");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static int roce5_init_ucc_global_buffer(u8 param_index, const char *buf)
+{
+ int ret;
+ ret = memset_s(g_param_show_tmp[param_index],
+ sizeof(g_param_show_tmp[param_index]), 0,
+ sizeof(g_param_show_tmp[param_index]));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset buffer.(ret:%d)\n", ret);
+ return -EIO;
+ }
+
+ ret = strncpy_s(g_param_buffer[param_index],
+ sizeof(g_param_buffer[param_index]), buf,
+ sizeof(g_param_buffer[param_index]) - 1);
+ if (ret != 0) {
+ ROCE_ERR("Failed to strncpy buffer.(ret:%d)\n", ret);
+ return -EIO;
+ }
+
+ g_param_buffer[param_index][ROCE_UCC_PARAM_BUFF_LEN - 1] = '\0';
+
+ return 0;
+}
+
+static int check_key_unique(char (*pkey)[ROCE_UCC_MAX_KEY_SIZE], u8 key_num)
+{
+ u8 key_index = 0;
+
+ if (key_num >= ROCE_UCC_MAX_KEY_SIZE) {
+ ROCE_ERR("invalid key value:%hhu.\n", key_num);
+ return -EINVAL;
+ }
+
+ for (key_index = 0; key_index < key_num; key_index++) {
+ if (strcmp(pkey[key_index], pkey[key_num]) == 0) {
+ ROCE_ERR("key:%s is not unique.\n", pkey[key_num]);
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
+static int parse_each_line(char *param_line,
+ char (*pkey)[ROCE_UCC_MAX_KEY_SIZE], u8 key_num,
+ u32 *param_value, u8 *param_bits, u16 *sum_bits)
+{
+ u64 temp_value = 0;
+ u64 temp_bits = 0;
+ u64 temp_bit_sum = *sum_bits;
+ char extra = 0;
+ char value_str[ROCE_UCC_PARAM_VALUE_LEN] = { 0 };
+ char bits_str[ROCE_UCC_PARAM_BITS_LEN] = { 0 };
+
+ if (key_num >= ROCE_UCC_MAX_KEY_SIZE) {
+ ROCE_ERR("invalid key value:%hhu.\n", key_num);
+ return -EINVAL;
+ }
+
+ /* input of each line per per key:value:length perform parse, obtain value and length size, and check key whether repeat */
+ if (sscanf_s(param_line, " %31[^: ] : %10[0-9] : %2[0-9] %c",
+ pkey[key_num], (unsigned)sizeof(pkey[key_num]), value_str,
+ (unsigned)sizeof(value_str), bits_str,
+ (unsigned)sizeof(bits_str), &extra,
+ 1) == ROCE_UCC_LINE_PARAM_TH) {
+ if (kstrtoull(value_str, NUM_10, &temp_value) < 0) {
+ ROCE_ERR("invalid input value:%s.\n", value_str);
+ return -EINVAL;
+ }
+
+ if (kstrtoull(bits_str, NUM_10, &temp_bits) < 0) {
+ ROCE_ERR("invalid input bits:%s.\n", bits_str);
+ return -EINVAL;
+ }
+
+ if ((temp_bits == 0) || (temp_bits > TYPE_BITS(u32)) ||
+ (temp_value > BITS_VALUE(temp_bits))) {
+ ROCE_ERR("bits:%llu or value:%llu is invalid.\n",
+ temp_bits, temp_value);
+ return -EINVAL;
+ }
+
+ temp_bit_sum += temp_bits;
+ if (temp_bit_sum > TYPE_BITS(u32)) {
+ ROCE_ERR("the length of bits:%llu beyond 32.\n",
+ temp_bit_sum);
+ return -EINVAL;
+ }
+
+ if (check_key_unique(pkey, key_num) != 0) {
+ return -EINVAL;
+ }
+
+ *param_value = (u32)temp_value;
+ *param_bits = (u8)temp_bits;
+ *sum_bits = (u16)temp_bit_sum;
+ return 0;
+ }
+
+ ROCE_ERR("param format is wrong, please enter key:value:length param, "
+ "value less than 4294967296, length less than 33.\n");
+ return -EINVAL;
+}
+
+static int roce5_parse_ucc_param(u32 *ucc_param, char *param_buffer,
+ char *show_tmpkey)
+{
+ u8 key_num = 0;
+ u32 temp_param = 0;
+ /* param_value\param_bits used to store each line parse of values size and so occupy bit width result */
+ u32 param_value = 0;
+ u8 param_bits = 0;
+ u16 sum_bits = 0;
+ /* param_line\ppbuffer used for input parameters partinterval parse */
+ char *param_line = NULL;
+ char **ppbuffer = ¶m_buffer;
+
+ char(*pkey)[ROCE_UCC_MAX_KEY_SIZE] = NULL;
+ char arr_tempkey[ROCE_UCC_MAX_KEY_NUM][ROCE_UCC_MAX_KEY_SIZE] = { 0 };
+ char append_buff[ROCE_UCC_LINE_MAX_LEN] = { 0 };
+ pkey = arr_tempkey;
+
+ while ((param_line = strsep(ppbuffer, "\n")) != NULL) {
+ if (*param_line == '\0') {
+ break;
+ }
+ /* per per key:value:length parse input of each one line */
+ if (parse_each_line(param_line, pkey, key_num, ¶m_value,
+ ¶m_bits, &sum_bits) != 0) {
+ return -EIO;
+ }
+
+ if (snprintf_s(append_buff, ROCE_UCC_LINE_MAX_LEN,
+ ROCE_UCC_LINE_MAX_LEN - 1, "%s:%u:%hhu\n",
+ pkey[key_num], param_value, param_bits) < 0) {
+ ROCE_ERR("snprintf_s err!\n");
+ return -EIO;
+ }
+
+ if (strncat_s(show_tmpkey, ROCE_UCC_PARAM_BUFF_LEN, append_buff,
+ ROCE_UCC_PARAM_BUFF_LEN - strlen(show_tmpkey) -
+ 1) != 0) {
+ ROCE_ERR("strncat_s err!\n");
+ return -EIO;
+ }
+ key_num++;
+ temp_param = key_num > 1 ? ((temp_param << param_bits) +
+ param_value) :
+ param_value;
+ }
+
+ if (sum_bits < TYPE_BITS(u32)) {
+ temp_param = temp_param << (TYPE_BITS(u32) - sum_bits);
+ }
+
+ if (ucc_param != NULL) {
+ *ucc_param = temp_param;
+ }
+ return 0;
+}
+
+static int roce5_store_ucc_param_inner(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, u8 param_index)
+{
+ int ret = 0;
+ u32 ucc_param = 0;
+ u32 old_ucc_param = 0;
+ struct roce5_ecn_ctx *ecn_ctx = NULL;
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = NULL;
+
+ if (roce5_ucc_store_input_check(kobj, buf, param_index) != 0) {
+ return -EINVAL;
+ }
+
+ ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj);
+
+ if (roce5_init_ucc_global_buffer(param_index, buf) != 0) {
+ return -EIO;
+ }
+
+ if (roce5_parse_ucc_param(&ucc_param, g_param_buffer[param_index],
+ g_param_show_tmp[param_index]) != 0) {
+ return -EIO;
+ }
+
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ if (strncpy_s(g_param_show_format[param_index],
+ sizeof(g_param_show_format[param_index]),
+ g_param_show_tmp[param_index],
+ sizeof(g_param_show_format[param_index]) - 1) != 0) {
+ ROCE_ERR("Failed to strncpy buffer.\n");
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+
+ if ((ecn_ucc_ctx != NULL) &&
+ (ecn_ucc_ctx->param[param_index] != ucc_param)) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_ucc_param = ecn_ucc_ctx->param[param_index];
+ ecn_ucc_ctx->param[param_index] = ucc_param;
+ ret = (ssize_t)roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ecn_ucc_ctx->param[param_index] = old_ucc_param;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return 0;
+}
+
+ssize_t roce5_show_ucc_param(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ struct roce5_ucc_param_ctx *ucc_param =
+ (struct roce5_ucc_param_ctx *)to_roce5_ucc_param_ctx(attr);
+ return sprintf_s(buf, sizeof(g_param_show_format[ucc_param->param_id]),
+ "%s", g_param_show_format[ucc_param->param_id]);
+}
+
+ssize_t roce5_store_ucc_param(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct roce5_ucc_param_ctx *ucc_param =
+ (struct roce5_ucc_param_ctx *)to_roce5_ucc_param_ctx(attr);
+ return (roce5_store_ucc_param_inner(kobj, attr, buf,
+ ucc_param->param_id) == 0) ?
+ (ssize_t)count :
+ -EIO;
+}
+
+static ssize_t roce5_show_algo_mode(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj);
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ switch ((int)ecn_ucc_ctx->algo_mode) {
+ case CCP_ALGO_MODE_CWND:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n",
+ (u32)CCP_ALGO_MODE_CWND);
+ case CCP_ALGO_MODE_RATE:
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n",
+ (u32)CCP_ALGO_MODE_RATE);
+ default:
+ break;
+ }
+ return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "error_type");
+}
+
+static ssize_t roce5_store_algo_mode(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent);
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj);
+ u32 old_algo_mode = 0;
+ int algo_mode = 0;
+ int ret;
+ char remaining;
+
+ if ((attr == NULL) || (buf == NULL)) {
+ ROCE_ERR("attr or buf is null");
+ return -EINVAL;
+ }
+
+ ret = sscanf_s(buf, "%d %c", &algo_mode, &remaining, 1);
+ if (ret != 1) {
+ ROCE_ERR("Invalid input, algo_mode should be unsigned int\n");
+ return -EIO;
+ }
+
+ if (algo_mode != CCP_ALGO_MODE_CWND &&
+ algo_mode != CCP_ALGO_MODE_RATE) {
+ ROCE_ERR("Invalid algo_mode(%d),buf(%s)\n", algo_mode, buf);
+ return -EIO;
+ }
+
+ if (ecn_ucc_ctx->algo_mode != (u32)algo_mode) {
+ mutex_lock(&ecn_ctx->ecn_mutex);
+ old_algo_mode = ecn_ucc_ctx->algo_mode;
+ ecn_ucc_ctx->algo_mode = (u32)algo_mode;
+ ret = roce5_update_ecn_param(ecn_ctx);
+ if (ret != 0) {
+ ROCE_ERR("Failed to update algo_mode(%d), ret(%d).",
+ algo_mode, ret);
+ ecn_ucc_ctx->algo_mode = old_algo_mode;
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ return -EIO;
+ }
+ mutex_unlock(&ecn_ctx->ecn_mutex);
+ }
+
+ return (ssize_t)count;
+}
+
+ROCE_UCC_PARAM_RW(param0, 0);
+ROCE_UCC_PARAM_RW(param1, 1);
+ROCE_UCC_PARAM_RW(param2, 2);
+ROCE_UCC_PARAM_RW(param3, 3);
+ROCE_UCC_PARAM_RW(param4, 4);
+ROCE_UCC_PARAM_RW(param5, 5);
+ROCE_UCC_PARAM_RW(param6, 6);
+ROCE_UCC_PARAM_RW(param7, 7);
+ROCE_UCC_PARAM_RW(param8, 8);
+ROCE_UCC_PARAM_RW(param9, 9);
+ROCE_UCC_PARAM_RW(param10, 10);
+ROCE_UCC_PARAM_RW(param11, 11);
+ROCE_UCC_PARAM_RW(param12, 12);
+ROCE_UCC_PARAM_RW(param13, 13);
+ROCE_UCC_PARAM_RW(param14, 14);
+ROCE_UCC_PARAM_RW(param15, 15);
+ROCE_ATTR_RW(algo_mode, roce5_show_algo_mode, roce5_store_algo_mode);
+
+static struct attribute *ecn_ucc_ctx_attrs[] = {
+ ROCE_UCC_PARAM_ATTR_PTR(param0), ROCE_UCC_PARAM_ATTR_PTR(param1),
+ ROCE_UCC_PARAM_ATTR_PTR(param2), ROCE_UCC_PARAM_ATTR_PTR(param3),
+ ROCE_UCC_PARAM_ATTR_PTR(param4), ROCE_UCC_PARAM_ATTR_PTR(param5),
+ ROCE_UCC_PARAM_ATTR_PTR(param6), ROCE_UCC_PARAM_ATTR_PTR(param7),
+ ROCE_UCC_PARAM_ATTR_PTR(param8), ROCE_UCC_PARAM_ATTR_PTR(param9),
+ ROCE_UCC_PARAM_ATTR_PTR(param10), ROCE_UCC_PARAM_ATTR_PTR(param11),
+ ROCE_UCC_PARAM_ATTR_PTR(param12), ROCE_UCC_PARAM_ATTR_PTR(param13),
+ ROCE_UCC_PARAM_ATTR_PTR(param14), ROCE_UCC_PARAM_ATTR_PTR(param15),
+ ROCE_ATTR_PTR(algo_mode), NULL,
+};
+
+static void roce_ecn_ucc_sysfs_release(struct kobject *kobj)
+{
+}
+
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+static const struct attribute_group ecn_ucc_ctx_attr_grp = {
+ .attrs = ecn_ucc_ctx_attrs,
+};
+
+static const struct attribute_group *ecn_ucc_ctx_groups[] = {
+ &ecn_ucc_ctx_attr_grp,
+ NULL,
+};
+#endif
+
+static struct kobj_type roce_ecn_ucc_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = roce_ecn_ucc_sysfs_release,
+#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP)
+ .default_groups = ecn_ucc_ctx_groups,
+#else
+ .default_attrs = ecn_ucc_ctx_attrs,
+#endif
+};
+
+int roce5_update_ucc_param(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx)
+{
+ roce_ucc_param_s ucc_param;
+ int ret;
+ int param_idx = 0;
+
+ if (ecn_ctx->cc_algo != ROCE_CC_USER_A_ALGO &&
+ ecn_ctx->cc_algo != ROCE_CC_USER_B_ALGO) {
+ ROCE_DEV_INFO(
+ rdev,
+ "func_id(%d) cc_algo(%u) is not user algo, ignore update.",
+ rdev->glb_func_id, ecn_ctx->cc_algo);
+ return 0;
+ }
+ (void)memset_s(&ucc_param, sizeof(ucc_param), 0, sizeof(ucc_param));
+
+ for (; param_idx < ROCE_UCC_MAX_PARAM_NUM; param_idx++) {
+ ucc_param.param[param_idx] = ecn_ctx->ucc_ctx.param[param_idx];
+ }
+
+ ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id,
+ (u32 *)(void *)&ucc_param,
+ sizeof(roce_ucc_param_s),
+ (u16)ROCE_MPU_CMD_CC_CFG_UCC_PARAM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set user algo param, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ }
+
+ return ret;
+}
+
+int roce5_init_ecn_ucc_sysfs(struct kobject *kobj,
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx)
+{
+ int ret = 0;
+ int param_idx = 0;
+
+ if (ecn_ucc_ctx == NULL) {
+ pr_err_ratelimited("[ROCE, ERR] %s: ecn_ucc_ctx is null\n",
+ __func__);
+ return -EINVAL;
+ }
+
+ ecn_ucc_ctx->algo_mode = CCP_ALGO_MODE_CWND;
+ for (; param_idx < ROCE_UCC_MAX_PARAM_NUM; param_idx++) {
+ ecn_ucc_ctx->param[param_idx] = ROCE_DEFAULT_UCC_PARAM;
+ }
+
+ ret = kobject_init_and_add(&ecn_ucc_ctx->ecn_ucc_root,
+ &roce_ecn_ucc_ktype, kobj, "roce5_ucc");
+ if (ret != 0) {
+ ROCE_ERR("Failed to add kobject roce5_ucc.(ret:%d)\n", ret);
+ kobject_put(&ecn_ucc_ctx->ecn_ucc_root);
+ return ret;
+ }
+
+ return 0;
+}
+
+void roce5_remove_ecn_ucc_sysfs(struct kobject *kobj,
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx)
+{
+ if (ecn_ucc_ctx == NULL) {
+ ROCE_ERR("ecn_ucc_ctx is null\n");
+ return;
+ }
+
+ kobject_put(&ecn_ucc_ctx->ecn_ucc_root);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h
new file mode 100644
index 000000000..bbbe92b30
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h
@@ -0,0 +1,73 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
+
+
+#ifndef ROCE_UCC_SYSFS_H
+#define ROCE_UCC_SYSFS_H
+
+#include <linux/types.h>
+#include <linux/stat.h>
+#include <linux/kobject.h>
+
+#define ROCE_UCC_MAX_PARAM_NUM 16
+#define ROCE_UCC_PARAM_BUFF_LEN 1024
+#define ROCE_DEFAULT_UCC_PARAM 0
+#define ROCE_UCC_MAX_KEY_SIZE 32
+#define ROCE_UCC_MAX_KEY_NUM 32
+#define ROCE_UCC_LINE_MAX_LEN 64
+#define ROCE_UCC_PARAM_VALUE_LEN 11
+#define ROCE_UCC_PARAM_BITS_LEN 3
+#define ROCE_UCC_LINE_PARAM_TH 3
+
+#define CHAR_0 '0'
+#define CHAR_9 '9'
+#define NUM_10 10
+#define ROCE_USER_CC_ID_LABLE_LEN 8
+#define ROCE_USER_CC_ID_MAX_LEN (ROCE_USER_CC_ID_LABLE_LEN + 2)
+
+#define TYPE_BITS(type) (sizeof(type) * 8)
+#define BITS_VALUE(bits) (((u64)1 << (bits)) - 1)
+
+/* forward-declaration */
+struct roce5_device;
+struct roce5_ecn_ctx;
+
+typedef struct roce_ucc_param {
+ u32 param[ROCE_UCC_MAX_PARAM_NUM];
+} roce_ucc_param_s;
+
+struct roce5_ecn_ucc_ctx {
+ struct kobject ecn_ucc_root;
+ u32 algo_mode;
+ u32 param[ROCE_UCC_MAX_PARAM_NUM];
+};
+
+struct roce5_ucc_param_ctx {
+ struct kobj_attribute param;
+ u32 param_id;
+};
+
+ssize_t roce5_show_ucc_param(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf);
+ssize_t roce5_store_ucc_param(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count);
+#define ROCE_UCC_PARAM_RW(_name, _idx) \
+ static struct roce5_ucc_param_ctx roce_ucc_attr_##_name = { \
+ .param = INIT_ROCE_KOBJ_ATTR( \
+ _name, (S_IWUSR | S_IRUSR | S_IRGRP), \
+ roce5_show_ucc_param, roce5_store_ucc_param), \
+ .param_id = (_idx), \
+ }
+
+#define ROCE_UCC_PARAM_ATTR_PTR(_name) (&roce_ucc_attr_##_name.param.attr)
+#define to_roce5_ucc_param_ctx(_kattr) \
+ container_of(_kattr, struct roce5_ucc_param_ctx, param)
+
+int roce5_update_ucc_param(struct roce5_device *rdev,
+ const struct roce5_ecn_ctx *ecn_ctx);
+int roce5_init_ecn_ucc_sysfs(struct kobject *kobj,
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx);
+void roce5_remove_ecn_ucc_sysfs(struct kobject *kobj,
+ struct roce5_ecn_ucc_ctx *ecn_ucc_ctx);
+
+#endif /* ROCE_UCC_SYSFS_H */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c
new file mode 100644
index 000000000..145904486
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c
@@ -0,0 +1,13 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved.
+
+
+#include "roce_main_extension.h"
+#include "roce_mr_chip_ext.h"
+
+void roce5_init_dev_ext_handlers(struct roce5_device *rdev)
+{
+ rdev->ib_dev.check_mr_status = roce5_check_mr_status;
+ rdev->ib_dev.reg_user_fast_mr = roce5_reg_user_fast_mr;
+ rdev->ib_dev.map_frmr_sg = roce5_map_user_frmr_sg;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h
new file mode 100644
index 000000000..a5f0847f8
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h
@@ -0,0 +1,17 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_MAIN_CHIP_EXT_C
+#define ROCE_MAIN_CHIP_EXT_C
+
+#define ROCE_UVERBS_CMD_EXT_MASK \
+ ((1ULL << IB_USER_VERBS_CMD_CREATE_DCT) | \
+ (1ULL << IB_USER_VERBS_CMD_QUERY_DCT) | \
+ (1ULL << IB_USER_VERBS_CMD_ARM_DCT) | \
+ (1ULL << IB_USER_VERBS_CMD_DESTROY_DCT) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_MR) | \
+ (1ULL << IB_USER_VERBS_CMD_REG_FAST_MR) | \
+ (1ULL << IB_USER_VERBS_CMD_DEREG_FAST_MR) | \
+ (1ULL << IB_USER_VERBS_CMD_MAP_FRMR_SG))
+
+#endif /* ROCE_MAIN_CHIP_EXT_C */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c
new file mode 100644
index 000000000..c460ca8a5
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c
@@ -0,0 +1,348 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#include "roce_mr.h"
+#include "roce_infra.h"
+#include "roce_mr_extension.h"
+#include "roce_main_extension.h"
+#include "roce_restore.h"
+#include "hmm_common.h"
+#include "roce_user.h"
+
+int roce5_check_alloc_mr_type(enum ib_mr_type mr_type)
+{
+ int ret = 0;
+
+ if (mr_type != IB_MR_TYPE_MEM_REG && mr_type != IB_MR_INDIRECT_REG &&
+ mr_type != IB_MR_TYPE_FAST_MR) {
+ ret = -EINVAL;
+ ROCE_ERR("mr_type is invalid. mr_type:%u", mr_type);
+ }
+
+ return ret;
+}
+
+enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type)
+{
+ switch (ib_mr_type) {
+ case IB_MR_TYPE_MEM_REG:
+ return RDMA_DMA_MR;
+
+ case IB_MR_INDIRECT_REG:
+ return RDMA_INDIRECT_MR;
+
+ case IB_MR_TYPE_FAST_MR:
+ return RDMA_FRMR;
+
+ default:
+ return RDMA_DMA_MR;
+ }
+}
+
+static int roce5_alloc_usr_frmr(struct ib_pd *ibpd, struct ib_udata *udata,
+ struct roce5_reg_frmr_cmd *ucmd,
+ struct roce5_device **rdev,
+ struct roce5_mr **rmr)
+{
+ struct ib_umem **fumem = NULL;
+ if (ibpd == NULL) {
+ ROCE_ERR("Ibpd is null");
+ return -EINVAL;
+ }
+
+ if (udata == NULL) {
+ ROCE_ERR("Udata is null");
+ return -EINVAL;
+ }
+
+ *rdev = to_roce5_dev(ibpd->device);
+
+ if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd))) {
+ ROCE_DEV_ERR((*rdev),
+ "Failed to copy_from_udata while creating frmr");
+ return -EFAULT;
+ }
+
+ *rmr = (struct roce5_mr *)kzalloc(sizeof(struct roce5_mr), GFP_KERNEL);
+ if (*rmr == NULL) {
+ ROCE_DEV_ERR((*rdev), "Failed to alloc memory for roce mr");
+ return -ENOMEM;
+ }
+
+ fumem = (struct ib_umem **)kzalloc(
+ sizeof(struct ib_umem *) * ROCE5_FRMR_UMEM_NUM, GFP_KERNEL);
+ if (fumem == NULL) {
+ ROCE_DEV_ERR((*rdev), "Failed to alloc memory for frmr umem");
+ kfree(*rmr);
+ return -ENOMEM;
+ }
+ (*rmr)->fumem = fumem;
+ return 0;
+}
+
+static int roce5_get_frmr_umem(struct roce5_mr *rmr, struct ib_pd *ibpd,
+ struct roce5_reg_frmr_cmd *ucmd,
+ struct roce5_device *rdev)
+{
+ rmr->umem = ib_umem_get(ibpd->uobject->context, ucmd->pbl_vaddr,
+ PAGE_SIZE_4K, 0, 0);
+ if (rmr->umem == NULL) {
+ ROCE_DEV_ERR(rdev, "Rmr->umem is null");
+ return -ENOMEM;
+ }
+
+ if (IS_ERR(rmr->umem)) {
+ ROCE_DEV_ERR(rdev, "Failed to get ib umem, func_id(%d)",
+ rdev->glb_func_id);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static int roce5_init_frmr_attr(u64 *pbl_paddr, struct roce5_mr *rmr,
+ struct roce5_device *rdev, struct ib_pd *ibpd)
+{
+ int ret;
+ u32 mpt_xid_mode;
+ struct roce5_pd *rpd = NULL;
+
+ *pbl_paddr = sg_dma_address(rmr->umem->sg_head.sgl);
+ if (*pbl_paddr == 0ULL) {
+ ROCE_DEV_ERR(rdev, "Invalid pbl_paddr");
+ return -EINVAL;
+ }
+
+ rmr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE;
+ mpt_xid_mode = roce5_get_mpt_xid_mode(rdev);
+ ret = roce5_alloc_tpt(rdev, &rmr->rdmamr, ROCE_FRMR_MAX_PAGES,
+ PAGE_SHIFT, mpt_xid_mode);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, ret(%d)", ret);
+ return ret;
+ }
+
+ rpd = to_roce5_pd(ibpd);
+ roce5_set_rdma_mr(&rmr->rdmamr, RDMA_FRMR, rpd->pdn, 0ULL, 0ULL,
+ ROCE5_MR_DEFAULT_ACCESS);
+
+ return 0;
+}
+
+struct ib_mr *roce5_reg_user_fast_mr(struct ib_pd *ibpd, struct ib_udata *udata)
+{
+ int ret;
+ struct roce5_device *rdev = NULL;
+ struct roce5_mr *rmr = NULL;
+ struct roce5_reg_frmr_cmd ucmd;
+ u64 pbl_paddr;
+
+ ret = roce5_alloc_usr_frmr(ibpd, udata, &ucmd, &rdev, &rmr);
+ if (ret != 0) {
+ ROCE_ERR("Failed to pre check, ret(%d)", ret);
+ goto err_out;
+ }
+
+ ret = roce5_get_frmr_umem(rmr, ibpd, &ucmd, rdev);
+ if (ret != 0) {
+ goto err_ib_umem_get;
+ }
+
+ /* Use valloc to allocate a page of memory, just need to get the first physical address */
+ ret = roce5_init_frmr_attr(&pbl_paddr, rmr, rdev, ibpd);
+ if (ret != 0) {
+ goto err_pbl_addr;
+ }
+
+ ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(rmr->rdmamr),
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to enable mpt of frmr, ret(%d)",
+ ret);
+ goto err_enable_mpt;
+ }
+
+ /* User mode also needs to send out pbl_paddr */
+ if (ib_copy_to_udata(udata, &pbl_paddr, sizeof(u64)) != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to copy data to user space, func_id(%d)",
+ rdev->glb_func_id);
+ ret = -EFAULT;
+ goto err_enable_mpt;
+ }
+
+ rmr->ibmr.lkey = rmr->rdmamr.key;
+ rmr->ibmr.rkey = rmr->rdmamr.key;
+
+ return &rmr->ibmr;
+
+err_enable_mpt:
+ roce5_free_tpt(rdev, &rmr->rdmamr);
+
+err_pbl_addr:
+ ib_umem_release(rmr->umem);
+
+err_ib_umem_get:
+ kfree(rmr->fumem, sizeof(struct ib_umem *) * ROCE5_FRMR_UMEM_NUM);
+ kfree(rmr);
+
+err_out:
+ return (struct ib_mr *)ERR_PTR((long)ret);
+}
+
+int roce5_frmr_check_user_param(struct ib_mr *ibmr, unsigned int page_num,
+ struct ib_udata *udata)
+{
+ if (ibmr == NULL) {
+ ROCE_ERR("Ibmr is null");
+ return -EINVAL;
+ }
+
+ if (page_num == 0 || page_num > ROCE5_FRMR_UMEM_NUM) {
+ ROCE_ERR("page_num(%d) is invalid", page_num);
+ return -EINVAL;
+ }
+
+ if (udata == NULL) {
+ ROCE_ERR("udata is NULL");
+ return -EINVAL;
+ }
+ return 0;
+}
+
+void roce5_frmr_free_addr_list(unsigned long long *va_list,
+ unsigned long long *pa_list)
+{
+ vfree(va_list);
+ vfree(pa_list);
+}
+
+int roce5_frmr_alloc_addr_list(struct roce5_device *rdev,
+ unsigned long long **va_list,
+ unsigned long long **pa_list, u32 len)
+{
+ int ret;
+ unsigned long long *val = NULL;
+ unsigned long long *pal = NULL;
+
+ val = (unsigned long long *)vmalloc(len);
+ if (val == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to vmalloc va_list");
+ return -EAGAIN;
+ }
+ pal = (unsigned long long *)vmalloc(len);
+ if (pal == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to vmalloc pa_list");
+ vfree(val);
+ return -EAGAIN;
+ }
+ ret = memset_s(val, len, 0, len);
+ if (ret) {
+ ROCE_DEV_ERR(rdev, "Failed to memset va_list.(ret:%d)", ret);
+ ret = -ENOMEM;
+ goto err_free;
+ }
+ ret = memset_s(pal, len, 0, len);
+ if (ret) {
+ ROCE_DEV_ERR(rdev, "Failed to memset pa_list.(ret:%d)", ret);
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ *va_list = val;
+ *pa_list = pal;
+ return 0;
+
+err_free:
+ roce5_frmr_free_addr_list(val, pal);
+ return ret;
+}
+
+static int process_va_list_and_allocate_umem(
+ struct roce5_mr *rmr, struct roce5_device *rdev, struct ib_mr *ibmr,
+ unsigned long long *va_list, unsigned long long *pa_list, u32 page_num)
+{
+ u32 i;
+ int ret;
+ for (i = 0; i < page_num; i++) {
+ ROCE_DEV_INFO(rdev, "i:%d, va(%llu)", i, va_list[i]);
+ rmr->fumem[i] = ib_umem_get(ibmr->pd->uobject->context,
+ va_list[i], FRMR_PAGE_SIZE, 0, 0);
+ if (IS_ERR(rmr->fumem[i])) {
+ ret = (int)PTR_ERR(rmr->fumem[i]);
+ roce5_free_frmr_umem(rmr->fumem, i - 1);
+ ROCE_DEV_ERR(rdev,
+ "Alloc umem failed, func_id(%d) err:%d",
+ rdev->glb_func_id, ret);
+ return ret;
+ }
+
+ /* use valloc allocate a page of memory, only needs need get the a physical address */
+ pa_list[i] = sg_dma_address(rmr->fumem[i]->sg_head.sgl);
+ ROCE_DEV_INFO(rdev, "PA_LIST:%llu", pa_list[i]);
+ if (pa_list[i] == 0ULL) {
+ ret = -EINVAL;
+ roce5_free_frmr_umem(rmr->fumem, i);
+ ROCE_DEV_ERR(rdev, "Invalid physicaL addr index(%d)",
+ i);
+ return ret;
+ }
+ }
+ return 0;
+}
+
+int roce5_map_user_frmr_sg(struct ib_mr *ibmr, unsigned int page_num,
+ struct ib_udata *udata)
+{
+ struct roce5_mr *rmr = NULL;
+ struct roce5_device *rdev = NULL;
+ int ret;
+ unsigned long long *va_list = NULL;
+ unsigned long long *pa_list = NULL;
+ u32 len;
+
+ ret = roce5_frmr_check_user_param(ibmr, page_num, udata);
+ if (ret) {
+ return ret;
+ }
+
+ len = page_num * sizeof(unsigned long long);
+ rmr = to_roce5_mr(ibmr);
+ rdev = to_roce5_dev(ibmr->device);
+ ret = roce5_frmr_alloc_addr_list(rdev, &va_list, &pa_list, len);
+ if (ret) {
+ return ret;
+ }
+
+ ROCE_DEV_INFO(rdev, "func_id(%d), page_num(%u)", rdev->glb_func_id,
+ page_num);
+
+ ret = ib_copy_from_udata((void *)va_list, udata, len);
+ if (ret) {
+ ROCE_DEV_ERR(rdev, "Failed to copy udata, ret(%d)", ret);
+ goto err_free;
+ }
+
+ ret = process_va_list_and_allocate_umem(rmr, rdev, ibmr, va_list,
+ pa_list, page_num);
+ if (ret != 0) {
+ goto err_free;
+ }
+
+ /* 讲 pa information pass to user mode */
+ if (ib_copy_to_udata(udata, pa_list, len)) {
+ ret = -EFAULT;
+ ROCE_DEV_ERR(rdev, "Failed to copy udata, func_id(%d)",
+ rdev->glb_func_id);
+ goto err_free;
+ }
+
+ rmr->fumem_nr = page_num;
+ roce5_frmr_free_addr_list(va_list, pa_list);
+ return 0;
+
+err_free:
+ rmr->fumem_nr = 0;
+ roce5_frmr_free_addr_list(va_list, pa_list);
+ return ret;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h
new file mode 100644
index 000000000..c206ca7f6
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h
@@ -0,0 +1,24 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_MR_CHIP_EXT_H
+#define ROCE_MR_CHIP_EXT_H
+
+#include <linux/types.h>
+#include <linux/io-mapping.h>
+#include <linux/list.h>
+#include <linux/cdev.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_umem.h>
+
+int roce5_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
+ struct ib_mr_status *mr_status);
+struct ib_mr *roce5_reg_user_fast_mr(struct ib_pd *ibpd,
+ struct ib_udata *udata);
+int roce5_map_user_frmr_sg(struct ib_mr *ibmr, unsigned int page_num,
+ struct ib_udata *udata);
+
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c
new file mode 100644
index 000000000..903da9898
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c
@@ -0,0 +1,44 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
+
+
+#include "roce_qp_extension.h"
+
+void roce5_set_qp_dif_attr(struct roce5_qp *rqp,
+ const struct ib_qp_init_attr *init_attr,
+ const struct roce5_device *rdev)
+{
+ if (((unsigned int)init_attr->create_flags &
+ IB_QP_CREATE_SIGNATURE_EN) != 0) {
+ rqp->signature_en = true;
+ ROCE_DEV_INFO(rdev,
+ "func(%d) qp(%u) roce5_create_qp signature_en.",
+ rdev->glb_func_id, rqp->qpn);
+ }
+
+ if (((unsigned int)init_attr->create_flags &
+ IB_QP_CREATE_SIGNATURE_DOUBLE_SGL) != 0) {
+ rqp->double_sgl_mode = 1;
+ ROCE_DEV_INFO(
+ rdev,
+ "func(%d) qp(%u) roce5_create_qp double_sgl_mode.",
+ rdev->glb_func_id, rqp->qpn);
+ }
+
+ if (((unsigned int)init_attr->create_flags &
+ IB_QP_CREATE_SIGNATURE_EXT_MTU) != 0) {
+ rqp->ext_mtu = 1;
+ ROCE_DEV_INFO(rdev, "func(%d) qp(%u) roce5_create_qp ext_mtu.",
+ rdev->glb_func_id, rqp->qpn);
+ }
+
+ if (((unsigned int)init_attr->create_flags &
+ IB_QP_CREATE_SIGNATURE_EXT_MTU_MODE) != 0) {
+ rqp->ext_mtu_mode = 1;
+ ROCE_DEV_INFO(rdev,
+ "func(%d) qp(%u) roce5_create_qp ext_mtu_mode.",
+ rdev->glb_func_id, rqp->qpn);
+ }
+
+ return;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c
new file mode 100644
index 000000000..a6323afd0
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c
@@ -0,0 +1,26 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved.
+
+
+#include "roce_srq_extension.h"
+
+void roce5_srq_container_init(struct ib_srq_init_attr *init_attr,
+ struct roce5_srq *rsrq, struct roce5_device *rdev)
+{
+ u8 srq_cont_en = 0;
+ srq_cont_en = init_attr->container_flag;
+
+ rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC);
+ rsrq->container_flag = srq_cont_en ? init_attr->container_flag :
+ rdev->cfg_info.srq_container_en;
+ rsrq->container_mode =
+ srq_cont_en ?
+ init_attr->container_mode :
+ (rsrq->xrc_en ? rdev->cfg_info.xrc_srq_container_mode :
+ rdev->cfg_info.srq_container_mode);
+ rsrq->container_warn_th = init_attr->attr.srq_limit;
+ rsrq->rqe_cnt_th = srq_cont_en ? init_attr->container_warn_th :
+ rdev->cfg_info.warn_th;
+
+ rsrq->container_size = roce5_get_container_sz(rsrq->container_mode);
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h
new file mode 100644
index 000000000..b956ce82c
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h
@@ -0,0 +1,13 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef HYPER_ROCE_COMMON
+#define HYPER_ROCE_COMMON
+
+enum roce5_hyper_mode {
+ ROCE_HYPER_MODE_DISABLE,
+ ROCE_HYPER_MODE_UDP,
+ ROCE_HYPER_MODE_AR,
+};
+
+#endif /* HYPER_ROCE_COMMON */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c
new file mode 100644
index 000000000..ca38186cc
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c
@@ -0,0 +1,511 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include <linux/types.h>
+#include "hinic5_rdma.h"
+#include "rdma_comp.h"
+#include "roce_hyper_npu_cmd.h"
+#include "roce_npu_cmd.h"
+#include "roce_hyper_npu_cmd.h"
+#include "comm_defs.h"
+
+#include "roce.h"
+#include "comm_defs.h"
+#include "roce_qp.h"
+#include "roce_qp_ext.h"
+#include "roce_netlink.h"
+#include "roce_cqm_cmd.h"
+#include "hyper_roce_common.h"
+#include "hyper_roce_qp.h"
+#include "hyper_roce_netlink.h"
+
+#define HYPER_ROCE_RDMARC_ENTRY_SIZE_BYTES 32
+
+static int roce5_send_cmd_set_qp_hyper_mode(struct roce5_device *rdev,
+ struct roce5_hyper_set_in *msg_in)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_enable_hyper_roce_s *input = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_enable_hyper_roce_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ input = (roce_uni_cmd_enable_hyper_roce_s *)cqm_cmd_inbuf->buf;
+ input->com.index = cpu_to_be32(msg_in->qpn);
+ input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ input->com.opt = cpu_to_be32(RoCE_CMD_ENABLE_HYPER_ROCE_QP);
+
+ input->cmd_info.dw0.bs.hyper_mode = msg_in->mode;
+ input->cmd_info.dw0.bs.srp_en = msg_in->sack_en;
+ input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value);
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto out;
+ }
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+static int
+roce5_send_cmd_modify_qp_extend(struct roce5_device *rdev,
+ struct roce5_hyper_extend_config_in *msg_in)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_enable_hyper_roce_s *input = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_enable_hyper_roce_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ input = (roce_uni_cmd_enable_hyper_roce_s *)cqm_cmd_inbuf->buf;
+ input->com.index = cpu_to_be32(msg_in->qpn);
+ input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ input->com.opt = cpu_to_be32(RoCE_CMD_ENABLE_HYPER_ROCE_QP);
+
+ switch (msg_in->attr_mask) {
+ case IBV_QP_ATTR_EXTEND_UDP_SRC_PORT:
+ input->cmd_info.dw0.bs.base_src_port = msg_in->udp_src_port;
+ break;
+ case IBV_QP_ATTR_EXTEND_HYROCE_FEATURE:
+ input->cmd_info.dw0.bs.hyper_mode = msg_in->feature.type;
+ input->cmd_info.dw0.bs.srp_en = msg_in->feature.sack_enable;
+ break;
+ case IBV_QP_ATTR_EXTEND_LB_MODE:
+ input->cmd_info.dw0.bs.lb_mode = msg_in->lb_mode;
+ break;
+ case IBV_QP_ATTR_EXTEND_MP_CONFIG:
+ input->cmd_info.dw1.bs.sport_repeat_num =
+ msg_in->mp.flowlet_pkg_num;
+ input->cmd_info.dw1.bs.path_num = msg_in->mp.path_num;
+ input->cmd_info.dw1.bs.sport_interval = msg_in->mp.interval;
+ input->cmd_info.dw1.bs.path_rr_enable =
+ msg_in->mp.path_rr_enable;
+ input->cmd_info.dw1.bs.cc_mode = msg_in->mp.cc_mode;
+ break;
+ case IBV_QP_ATTR_EXTEND_AR_CONFIG:
+ input->cmd_info.dw1.bs.port_rr_enable =
+ msg_in->ar.port_rr_enable;
+ break;
+ case IBV_QP_ATTR_EXTEND_SACK_CONFIG:
+ input->cmd_info.dw2.bs.oor_range_m = msg_in->sack.oor_range;
+ input->cmd_info.dw2.bs.srp_range_n = msg_in->sack.srp_range;
+ break;
+ default:
+ ROCE_DEV_ERR(rdev, "Unsupport attr mask!\n");
+ break;
+ }
+
+ input->cmd_info.dw0.bs.attr_ext_mask = msg_in->attr_mask;
+ input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value);
+ input->cmd_info.dw1.value = cpu_to_be32(input->cmd_info.dw1.value);
+ input->cmd_info.dw2.value = cpu_to_be32(input->cmd_info.dw2.value);
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto out;
+ }
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+static int
+roce5_send_cmd_set_qp_msnc(struct roce5_device *rdev, u32 qpn,
+ struct roce5_qp_hyper_extend *rqp_hyper_ext)
+{
+ int ret = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_init_msnc_s *input = NULL;
+ u64 bitmap_addr = 0;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_init_msnc_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ input = (roce_uni_cmd_init_msnc_s *)cqm_cmd_inbuf->buf;
+ input->com.index = cpu_to_be32(qpn);
+ input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ input->com.opt = cpu_to_be32(RoCE_CMD_INIT_MSNC_QP);
+
+ input->cmd_info.dw0.bs.sack_bitmap_size =
+ HYPER_ROCE_TX_SACK_BITMAP_SIZE;
+ input->cmd_info.dw0.bs.txmsn_entry_size = HYPER_ROCE_MSN_ENTRY_SIZE;
+ input->cmd_info.dw0.bs.txmsn_table_size = HYPER_ROCE_MSN_TABLE_SIZE;
+ input->cmd_info.dw0.bs.txmsn_table_gpa_h =
+ (rqp_hyper_ext->tx_msnc.dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) &
+ 0xffffff;
+ input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value);
+
+ input->cmd_info.txmsn_table_gpa_l =
+ cpu_to_be32((u32)(rqp_hyper_ext->tx_msnc.dma_addr >>
+ ROCE5_RDMARC_GPA_L_SHIFT));
+
+ input->cmd_info.dw2.bs.rxmsn_entry_size0 = HYPER_ROCE_MSN_ENTRY_SIZE;
+ input->cmd_info.dw2.bs.rxmsn_table_size0 = HYPER_ROCE_MSN_TABLE_SIZE;
+ input->cmd_info.dw2.bs.rxmsn_table_gpa0_h =
+ (rqp_hyper_ext->rx_msnc.dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) &
+ 0xffffff;
+ input->cmd_info.dw2.value = cpu_to_be32(input->cmd_info.dw2.value);
+
+ input->cmd_info.rxmsn_table_gpa0_l =
+ cpu_to_be32((u32)(rqp_hyper_ext->rx_msnc.dma_addr >>
+ ROCE5_RDMARC_GPA_L_SHIFT));
+
+ bitmap_addr = (u64)rqp_hyper_ext->rx_msnc.dma_addr +
+ HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE *
+ HYPER_ROCE_RDMARC_ENTRY_SIZE_BYTES;
+
+ input->cmd_info.dw4.bs.sack_bitmap_size =
+ HYPER_ROCE_RX_SACK_BITMAP_SIZE;
+ input->cmd_info.dw4.bs.rxmsn_entry_size1 = HYPER_ROCE_MSN_ENTRY_SIZE;
+ input->cmd_info.dw4.bs.rxmsn_table_size1 = HYPER_ROCE_MSN_TABLE_SIZE;
+ input->cmd_info.dw4.bs.rxmsn_table_gpa1_h =
+ (bitmap_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & 0xffffff;
+ input->cmd_info.dw4.value = cpu_to_be32(input->cmd_info.dw4.value);
+
+ input->cmd_info.rxmsn_table_gpa1_l =
+ cpu_to_be32((u32)(bitmap_addr >> ROCE5_RDMARC_GPA_L_SHIFT));
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RoCE_CMD_INIT_MSNC_QP command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto out;
+ }
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+static int roce5_alloc_msnc(struct roce5_device *rdev,
+ struct roce5_qp_hyper_extend *rqp_hyper_ext)
+{
+ int ret;
+ ret = roce5_rdma_rdmarc_alloc(rdev->hwdev,
+ HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE,
+ &rqp_hyper_ext->tx_msnc);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdmarc, func_id(%u).",
+ rdev->glb_func_id);
+ return ret;
+ }
+
+ ret = roce5_rdma_rdmarc_alloc(
+ rdev->hwdev,
+ HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE +
+ HYPER_ROCE_RXMSN_BITMAP_RDMARC_ALLOC_SIZE,
+ &rqp_hyper_ext->rx_msnc);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdmarc, func_id(%u).",
+ rdev->glb_func_id);
+ roce5_rdma_rdmarc_free(rdev->hwdev, &rqp_hyper_ext->tx_msnc);
+ return ret;
+ }
+
+ rqp_hyper_ext->msnc_en = true;
+
+ return 0;
+}
+
+static int32_t roce5_genl_set_qp_hyper_mode(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len,
+ uint16_t *out_size)
+{
+ int32_t ret = 0;
+ struct roce5_hyper_set_in *msg_in = NULL;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_qp_hyper_extend *rqp_hyper_ext = NULL;
+
+ if (in == NULL || in_len == 0 ||
+ in_len < sizeof(struct roce5_hyper_set_in) || out_size == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid param, in:%d, in_len:%u, out_size:%d",
+ !!in, in_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_hyper_set_in *)in;
+
+ if (msg_in->qpn == ROCE_QP_SMI_QP_NUM ||
+ msg_in->qpn == ROCE_QP_GSI_QP_NUM) {
+ ROCE_DEV_ERR(rdev, "Invalid qpn(%u).", msg_in->qpn);
+ return -EINVAL;
+ }
+
+ rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp.");
+ return -EINVAL;
+ }
+
+ mutex_lock(&rqp->mutex);
+
+ if (rqp->qp_state >= IB_QPS_RTR) {
+ ROCE_DEV_ERR(rdev, "Invalid qp state, current state(%u).",
+ rqp->qp_state);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = roce5_send_cmd_set_qp_hyper_mode(rdev, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret);
+ goto out;
+ }
+
+ rqp_hyper_ext = (struct roce5_qp_hyper_extend *)roce5_qp_get_ext_buf(
+ rqp, ROCE_EXT_HYPER_ROCE);
+ if (rqp_hyper_ext == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to the buf of qp hyper extend, qpn(%u).",
+ rqp->qpn);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ rqp_hyper_ext->mode = msg_in->mode;
+ rqp_hyper_ext->sack_en = msg_in->sack_en;
+
+ /* ROCE_HYPER_MODE_DISABLE indicates not enable out-of-order receive, not need allocate rdmarc resource; enable sack_en selective retransmission, must enable out-of-order receive */
+ if (rqp_hyper_ext->mode != ROCE_HYPER_MODE_DISABLE) {
+ ret = roce5_alloc_msnc(rdev, rqp_hyper_ext);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to alloc msnc, ret(%d).",
+ ret);
+ goto out;
+ }
+
+ ret = roce5_send_cmd_set_qp_msnc(rdev, msg_in->qpn,
+ rqp_hyper_ext);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret);
+ goto out;
+ }
+ }
+out:
+ mutex_unlock(&rqp->mutex);
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int32_t roce5_genl_set_default_multipath_config(
+ struct roce5_device *rdev, struct roce5_hyper_extend_config_in *msg_in)
+{
+ int ret = 0;
+
+ msg_in->lb_mode = HYPER_ROCE_DEFAULT_LB_MODE;
+ msg_in->mp.flowlet_pkg_num = HYPER_ROCE_DEFAULT_MULTI_PKG_NUM;
+ msg_in->mp.path_num = HYPER_ROCE_DEFAULT_MULTI_PATH_NUM;
+ msg_in->mp.interval = HYPER_ROCE_DEFAULT_MULTI_INTERVAL;
+ msg_in->mp.path_rr_enable = HYPER_ROCE_DEFAULT_MULTI_PATHRR_ENABLE;
+ msg_in->mp.cc_mode = HYPER_ROCE_DEFAULT_MULTI_CC_MODE;
+
+ msg_in->attr_mask = IBV_QP_ATTR_EXTEND_LB_MODE;
+ ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret);
+ return ret;
+ }
+
+ msg_in->attr_mask = IBV_QP_ATTR_EXTEND_MP_CONFIG;
+ ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret);
+ return ret;
+ }
+
+ return ret;
+}
+
+static int32_t roce5_modify_hyper_config(
+ struct roce5_device *rdev, struct roce5_hyper_extend_config_in *msg_in,
+ struct roce5_qp_hyper_extend *rqp_hyper_ext, struct roce5_qp *rqp)
+{
+ int ret = 0;
+
+ if (msg_in->attr_mask == IBV_QP_ATTR_EXTEND_HYROCE_FEATURE) {
+ ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret);
+ return ret;
+ }
+
+ rqp_hyper_ext->mode = (u8)msg_in->feature.type;
+ rqp_hyper_ext->version = (u8)msg_in->feature.version;
+ rqp_hyper_ext->sack_en = (u8)msg_in->feature.sack_enable;
+
+ /* ROCE_HYPER_MODE_DISABLE indicates not enable out-of-order receive, not need allocate rdmarc resource; enable sack_en selective retransmission, must enable out-of-order receive */
+ if (rqp_hyper_ext->mode != ROCE_HYPER_MODE_DISABLE) {
+ ret = roce5_alloc_msnc(rdev, rqp_hyper_ext);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to alloc msnc, ret(%d).",
+ ret);
+ return ret;
+ }
+
+ ret = roce5_send_cmd_set_qp_msnc(rdev, msg_in->qpn,
+ rqp_hyper_ext);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to send CMDQ, ret(%d)",
+ ret);
+ return ret;
+ }
+ }
+
+ ret = roce5_genl_set_default_multipath_config(rdev, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to set default mp config, ret(%d).",
+ ret);
+ return ret;
+ }
+ } else {
+ if (rqp_hyper_ext->mode == ROCE_HYPER_MODE_DISABLE) {
+ ROCE_DEV_ERR(rdev,
+ "Hyper_roce is disable, current mode(%u).",
+ rqp_hyper_ext->mode);
+ ret = -EINVAL;
+ return ret;
+ }
+
+ ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret);
+ return ret;
+ }
+ }
+
+ return ret;
+}
+
+static int32_t roce5_genl_modify_qp_extend(struct roce5_device *rdev, void *in,
+ uint16_t in_len, void *out,
+ uint16_t out_len, uint16_t *out_size)
+{
+ int32_t ret = 0;
+ struct roce5_hyper_extend_config_in *msg_in = NULL;
+ struct roce5_qp *rqp = NULL;
+ struct roce5_qp_hyper_extend *rqp_hyper_ext = NULL;
+
+ if (in == NULL || in_len == 0 ||
+ in_len < sizeof(struct roce5_hyper_extend_config_in) ||
+ out_size == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Invalid param, in:%d, in_len:%u, out_size:%d",
+ !!in, in_len, !!out_size);
+ return -EINVAL;
+ }
+
+ msg_in = (struct roce5_hyper_extend_config_in *)in;
+
+ if (msg_in->qpn == ROCE_QP_SMI_QP_NUM ||
+ msg_in->qpn == ROCE_QP_GSI_QP_NUM) {
+ ROCE_DEV_ERR(rdev, "Invalid qpn(%u).", msg_in->qpn);
+ return -EINVAL;
+ }
+
+ rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn);
+ if (rqp == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to look up rqp.");
+ return -EINVAL;
+ }
+
+ mutex_lock(&rqp->mutex);
+
+ if (rqp->qp_state >= IB_QPS_RTR) {
+ ROCE_DEV_ERR(rdev, "Invalid qp state, current state(%u).",
+ rqp->qp_state);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ rqp_hyper_ext = (struct roce5_qp_hyper_extend *)roce5_qp_get_ext_buf(
+ rqp, ROCE_EXT_HYPER_ROCE);
+ if (rqp_hyper_ext == NULL) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to the buf of qp hyper extend, qpn(%u).",
+ rqp->qpn);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = roce5_modify_hyper_config(rdev, msg_in, rqp_hyper_ext, rqp);
+ if (ret) {
+ ROCE_DEV_ERR(rdev, "Failed to modify hyper roce config.");
+ }
+out:
+ mutex_unlock(&rqp->mutex);
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static roce5_netlink_cmd_process g_cmd_process[] = {
+ { USRNL_ROCE_CMD_HYPER_QP_SET, roce5_genl_set_qp_hyper_mode },
+ { USRNL_ROCE_CMD_MODIFY_QP_EXTEND, roce5_genl_modify_qp_extend },
+};
+
+int roce5_hyper_netlink_init(void)
+{
+ int ret;
+
+ ret = roce5_netlink_register_cbs(
+ g_cmd_process,
+ sizeof(g_cmd_process) / sizeof(roce5_netlink_cmd_process));
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to register hyper roce netlink callbacks, ret(%d).",
+ ret);
+ return ret;
+ }
+
+ return 0;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h
new file mode 100644
index 000000000..27de1b070
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef HYPER_ROCE_NETLINK_H
+#define HYPER_ROCE_NETLINK_H
+
+#include <linux/types.h>
+
+#define HYPER_ROCE_DEFAULT_LB_MODE 1
+#define HYPER_ROCE_DEFAULT_MULTI_PKG_NUM 4
+#define HYPER_ROCE_DEFAULT_MULTI_PATH_NUM 4
+#define HYPER_ROCE_DEFAULT_MULTI_INTERVAL 1
+#define HYPER_ROCE_DEFAULT_MULTI_PATHRR_ENABLE 0
+#define HYPER_ROCE_DEFAULT_MULTI_CC_MODE 2
+
+struct roce5_hyper_set_in {
+ uint32_t qpn;
+ uint8_t mode; /* @see enum roce_hyper_mode */
+ uint8_t sack_en;
+ uint8_t rsvd[2];
+};
+
+int roce5_hyper_netlink_init(void);
+
+enum ibv_qp_attr_extend_mask {
+ IBV_QP_ATTR_EXTEND_UDP_SRC_PORT = 1 << 0, /* source UDP port number */
+ IBV_QP_ATTR_EXTEND_HYROCE_FEATURE = 1 << 1, /* high-order RoCE of feature */
+ IBV_QP_ATTR_EXTEND_LB_MODE = 1 << 2, /* load balance mode */
+ IBV_QP_ATTR_EXTEND_MP_CONFIG = 1 << 3, /* Multi-Path multipath mode configuration */
+ IBV_QP_ATTR_EXTEND_AR_CONFIG =
+ 1 << 4, /* Adaptive-Routing multipath mode configuration */
+ IBV_QP_ATTR_EXTEND_SACK_CONFIG =
+ 1 << 5, /* Selective Ack selective retransmission parameters configuration */
+};
+
+/* high-order RoCE feature configuration structure */
+struct ibv_hyroce_feature {
+ uint32_t type; /* high-order RoCE type, value range: enum ibv_hyroce_feature_type */
+ uint32_t version; /* high-order RoCE version, value range: enum ibv_hyroce_feature_version */
+ uint32_t sack_enable; /* selective retransmission switch, 0- disable, 1- enable */
+
+ uint32_t resv[32]; /* reserved field, used for not to extension feature */
+};
+
+/* multipath (Multi-Path) configuration parameters structure */
+struct ibv_mp_config {
+ uint32_t flowlet_pkg_num; /* each sub-flow (flowlet) of packet count, used for flow splitting */
+ uint32_t path_num; /* multipath of path count */
+ uint32_t interval; /* UDP port number increment interval, used for distinguish different path */
+ uint32_t path_rr_enable; /* path whether supports poll (RR) select network port, 0- not support, 1- supports */
+ uint32_t cc_mode;
+
+ uint32_t resv[32]; /* reserved field, used for not to extension */
+};
+
+/* adaptive path by (Adaptive Routing) configuration structure */
+struct ibv_ar_config {
+ uint32_t port_rr_enable; /* whether enable network port side port poll (RR) per-packet function, 0- disable, 1- enable */
+
+ uint32_t resv[32]; /* reserved field, used for not to extension */
+};
+
+/* selective retransmission (SACK - Selective Ack) configuration structure */
+struct ibv_sack_config {
+ uint32_t srp_range; /* TX(send) direction maximum repass packet count */
+ uint32_t oor_range; /* RX(receive) direction out-of-order repass window of packet count */
+
+ uint32_t resv[32]; /* reserved field, used for not to extension */
+};
+
+struct roce5_hyper_extend_config_in {
+ uint32_t qpn;
+ uint32_t udp_src_port; /* source UDP port number */
+ struct ibv_hyroce_feature feature; /* high-order RoCE feature */
+ uint32_t lb_mode; /* load balance mode, value range: enum ibv_lb_mode type */
+ struct ibv_mp_config mp; /* multipath (Multi-Path) configuration */
+ struct ibv_ar_config ar; /* adaptive path by (Adaptive Routing) configuration */
+ struct ibv_sack_config sack; /* selective retransmission (SACK) configuration */
+ enum ibv_qp_attr_extend_mask attr_mask;
+};
+
+#endif /* HYPER_ROCE_NETLINK_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c
new file mode 100644
index 000000000..1cc831f16
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c
@@ -0,0 +1,29 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include "roce_qp.h"
+#include "roce_restore.h"
+#include "hyper_roce_common.h"
+#include "hyper_roce_qp.h"
+
+int roce5_hyper_qp_init(void *parent, void *ext_data)
+{
+ struct roce5_qp_hyper_extend *hyper_qp_ext =
+ (struct roce5_qp_hyper_extend *)ext_data;
+ hyper_qp_ext->mode = ROCE_HYPER_MODE_DISABLE;
+ hyper_qp_ext->sack_en = false;
+ return 0;
+}
+
+void roce5_hyper_qp_cleanup(void *parent, void *ext_data)
+{
+ struct roce5_qp *rqp = (struct roce5_qp *)parent;
+ struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device);
+ struct roce5_qp_hyper_extend *hyper_qp_ext =
+ (struct roce5_qp_hyper_extend *)ext_data;
+
+ if (hyper_qp_ext->msnc_en) {
+ roce5_rdma_rdmarc_free(rdev->hwdev, &(hyper_qp_ext->tx_msnc));
+ roce5_rdma_rdmarc_free(rdev->hwdev, &(hyper_qp_ext->rx_msnc));
+ }
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h
new file mode 100644
index 000000000..52c38f0a4
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h
@@ -0,0 +1,35 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef HYPER_ROCE_QP_H
+#define HYPER_ROCE_QP_H
+
+#include <linux/types.h>
+#include "base_type.h"
+#include "hinic5_rdma.h"
+
+/* Entry_size for TXMSN/RXMSN Table 00: 32B 01: 64B Others: reserved. */
+#define HYPER_ROCE_MSN_ENTRY_SIZE 0u
+/* Entry number for TXMSN/RXMSN Table 00: 256 01: 512 10: 1024 11: 2048 Others: reserved. */
+#define HYPER_ROCE_MSN_TABLE_SIZE 0u
+/* 000: 128bit 001: 256bit 010: 512bit 011: 1024bit 100: 2048bit Others: reserved. */
+#define HYPER_ROCE_TX_SACK_BITMAP_SIZE 0u
+/* 011: 1024bit 100: 2048bit Others: reserved. */
+#define HYPER_ROCE_RX_SACK_BITMAP_SIZE 3u
+#define HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE \
+ (256u << (HYPER_ROCE_MSN_ENTRY_SIZE & 0x3))
+#define HYPER_ROCE_RXMSN_BITMAP_RDMARC_ALLOC_SIZE (4)
+
+struct roce5_qp_hyper_extend {
+ u8 mode; /* @see enum roce_hyper_mode */
+ u8 version;
+ u8 sack_en;
+ struct rdma_rdmarc tx_msnc;
+ struct rdma_rdmarc rx_msnc;
+ bool msnc_en;
+};
+
+int roce5_hyper_qp_init(void *parent, void *ext_data);
+void roce5_hyper_qp_cleanup(void *parent, void *ext_data);
+
+#endif /* HYPER_ROCE_QP_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c
new file mode 100644
index 000000000..fe4e5cf64
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c
@@ -0,0 +1,60 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include "roce.h"
+#include "roce_dfx.h"
+#include "hyper_roce_tool_pktdrop.h"
+#include "hyper_roce_tool.h"
+
+enum roce5_hyper_cmds {
+ ROCE_HYPER_CMD_START = 1,
+ ROCE_HYPER_CMD_START_DROP_QP,
+ ROCE_HYPER_CMD_STOP_DROP_QP,
+ ROCE_HYPER_CMD_MAX
+};
+
+const roce5_adm_func_t hyper_roce5_adm_func[] = {
+ [ROCE_HYPER_CMD_START_DROP_QP] = roce5_hyper_tool_start_drop_qp,
+ [ROCE_HYPER_CMD_STOP_DROP_QP] = roce5_hyper_tool_stop_drop_qp,
+};
+
+int roce5_hyper_tool_dispatch(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ const struct roce5_hyper_cmd_head *head = NULL;
+ const void *data_ptr = NULL;
+ u32 data_len;
+ int ret;
+
+ if (in_size < ROCE_HYPER_CMD_HEAD_SIZE) {
+ ROCE_DEV_ERR(rdev, "Invalid command length(%d).", in_size);
+ return -EINVAL;
+ }
+
+ data_ptr = (void *)((char *)buf_in + ROCE_HYPER_CMD_HEAD_SIZE);
+ data_len = in_size - ROCE_HYPER_CMD_HEAD_SIZE;
+ head = data_ptr;
+
+ if (head->sub_cmd < ROCE_HYPER_CMD_START ||
+ head->sub_cmd >= ROCE_HYPER_CMD_MAX) {
+ ROCE_DEV_ERR(rdev, "Unknown hyper roce command(%d).",
+ head->sub_cmd);
+ return -EINVAL;
+ }
+
+ if (hyper_roce5_adm_func[head->sub_cmd] == NULL) {
+ ROCE_DEV_ERR(rdev, "The callback of cmd(%d) is NULL.",
+ head->sub_cmd);
+ return -EINVAL;
+ }
+
+ ret = hyper_roce5_adm_func[head->sub_cmd](rdev, buf_in, in_size,
+ buf_out, out_size);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to execute the callback of cmd(%d)",
+ head->sub_cmd);
+ return ret;
+ }
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h
new file mode 100644
index 000000000..2438bbb78
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h
@@ -0,0 +1,21 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef HYPER_ROCE_TOOL_H
+#define HYPER_ROCE_TOOL_H
+
+#include "base_type.h"
+#include "roce.h"
+
+#define ROCE_CMD_HYPER 0x220
+
+struct roce5_hyper_cmd_head {
+ u32 sub_cmd;
+};
+
+#define ROCE_HYPER_CMD_HEAD_SIZE sizeof(struct roce5_hyper_cmd_head)
+
+int roce5_hyper_tool_dispatch(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+
+#endif /* HYPER_ROCE_TOOL_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c
new file mode 100644
index 000000000..7d305e069
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c
@@ -0,0 +1,142 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include <linux/types.h>
+#include "hinic5_rdma.h"
+#include "rdma_comp.h"
+#include "roce_npu_cmd.h"
+#include "roce_hyper_npu_cmd.h"
+#include "comm_defs.h"
+
+#include "roce.h"
+#include "roce_qp.h"
+#include "roce_qp_ext.h"
+#include "roce_netlink.h"
+#include "roce_cqm_cmd.h"
+#include "hyper_roce_tool_pktdrop.h"
+
+int roce5_hyper_tool_start_drop_qp(struct roce5_device *rdev,
+ const void *buf_in, u32 in_size,
+ void *buf_out, u32 *out_size)
+{
+ int ret = 0;
+ u32 drop_pkt_ratio = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_modify_qp_drop_percent_s *input = NULL;
+ struct pktdrop_cmd_inbuf *in = NULL;
+
+ if (in_size < sizeof(struct pktdrop_cmd_inbuf)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(struct pktdrop_cmd_inbuf));
+ return -EINVAL;
+ }
+
+ if (*out_size < sizeof(union pktdrop_outbuf)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)",
+ (int)*out_size, (int)sizeof(union pktdrop_outbuf));
+ return -ENOMEM;
+ }
+
+ in = (struct pktdrop_cmd_inbuf *)buf_in;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_modify_qp_drop_percent_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ drop_pkt_ratio = in->data.attr.dw15.bs.drop_pkt_ratio;
+ input = (roce_uni_cmd_modify_qp_drop_percent_s *)cqm_cmd_inbuf->buf;
+ input->com.index = cpu_to_be32(in->qpn);
+ input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ input->com.opt = cpu_to_be32(RoCE_CMD_MODIFY_QP_DROP_PERCENT);
+
+ input->cmd_info.dw0.bs.drop_precent = drop_pkt_ratio;
+ input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value);
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto out;
+ }
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
+
+int roce5_hyper_tool_stop_drop_qp(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ int ret = 0;
+ u32 drop_pkt_ratio = 0;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ roce_uni_cmd_modify_qp_drop_percent_s *input = NULL;
+ struct pktdrop_cmd_inbuf *in = NULL;
+
+ if (in_size < sizeof(struct pktdrop_cmd_inbuf)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(struct pktdrop_cmd_inbuf));
+ return -EINVAL;
+ }
+
+ if (*out_size < sizeof(union pktdrop_outbuf)) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)",
+ (int)*out_size, (int)sizeof(union pktdrop_outbuf));
+ return -ENOMEM;
+ }
+
+ in = (struct pktdrop_cmd_inbuf *)buf_in;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_modify_qp_drop_percent_s), NULL, 0);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)",
+ rdev->glb_func_id, ret);
+ return -ENOMEM;
+ }
+
+ input = (roce_uni_cmd_modify_qp_drop_percent_s *)cqm_cmd_inbuf->buf;
+ input->com.index = cpu_to_be32(in->qpn);
+ input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK);
+ input->com.opt = cpu_to_be32(RoCE_CMD_MODIFY_QP_DROP_PERCENT);
+
+ input->cmd_info.dw0.bs.drop_precent = drop_pkt_ratio;
+ input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value);
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL,
+ ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(
+ rdev,
+ "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)",
+ ret, rdev->glb_func_id);
+ goto out;
+ }
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL);
+ return ret;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h
new file mode 100644
index 000000000..4d3dbd1d5
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h
@@ -0,0 +1,23 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef HYPER_ROCE_TOOL_PKTDROP_H
+#define HYPER_ROCE_TOOL_PKTDROP_H
+#include "hyper_roce_pub_cmd.h"
+#include "roce.h"
+
+struct pktdrop_cmd_inbuf {
+ u32 cmd_type;
+ u32 hyper_roce_sub_cmd;
+ u32 mode;
+ u32 qpn;
+ u32 drop_level;
+ struct pktdrop_l2d_tbl_dfx data;
+};
+
+int roce5_hyper_tool_start_drop_qp(struct roce5_device *rdev,
+ const void *buf_in, u32 in_size,
+ void *buf_out, u32 *out_size);
+int roce5_hyper_tool_stop_drop_qp(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+#endif
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce.h
new file mode 100644
index 000000000..3fbc3343b
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce.h
@@ -0,0 +1,665 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_H
+#define ROCE_H
+
+#include <linux/types.h>
+#include <linux/io-mapping.h>
+#include <linux/list.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_umem.h>
+#include <rdma/uverbs_ioctl.h>
+#include "roce_verbs_compat.h"
+
+#include "hinic5_hw.h"
+#include "hinic5_crm.h"
+#include "hinic5_hw_cfg.h"
+#include "hinic5_lld.h"
+#include "hinic5_cqm.h"
+#include "hinic5_rdma.h"
+#include "nic_cfg_comm.h"
+#include "hmm_common.h"
+
+#include "roce_infra.h"
+#include "roce_db.h"
+#include "roce_sysfs.h"
+
+#include "roce_uld_kernel_api.h"
+#include "roce_npu_cmd_defs.h"
+#include "roce_npu_cmd_ext_defs.h"
+
+#define HIROCE5_DRV_NAME "roce5_drv"
+#define HIROCE5_DRV_AUTHOR "Huawei Technologies CO., Ltd"
+#define HIROCE5_DRV_DESC \
+ "Huawei(R) Intelligent Network Interface Card, RoCE Driver"
+#define HIROCE5_DRV_VERSION GLOBAL_VERSION_STR
+
+#define RDMA_DRIVER_HRN 8395
+#define ROCE_IB_UVERBS_ABI_VERSION 1
+#define ROCE_ULD_DEV_NAME_LEN 16
+
+#define MAX_CEQ_NEED 256
+
+#define DEV_ADDR_FIRST_BYTE_VAL_MASK 2
+
+#define ROCE_NODE_DESC_LEN 5
+
+#define ROCE_SQ_WQEBB_SIZE 64
+
+#define ROCE_GID_LEN 16
+
+#define ROCE_PCI_CFG_REGS_BAR0 0
+#define ROCE_PCI_CFG_REGS_BAR3 3
+
+#define DEFAULT_ROCE_DEV_NODE_PRI 0666
+
+#define ROCE_MAX_PORT_NUM 8
+
+#ifndef BIG_ENDIAN
+#define BIG_ENDIAN 0x4321
+#endif
+
+#ifndef LITTLE_ENDIAN
+#define LITTLE_ENDIAN 0x1234
+#endif
+
+#ifndef BYTE_ORDER
+#define BYTE_ORDER LITTLE_ENDIAN
+#endif
+
+#define MAX_ROCE_DEV (28 * 4)
+
+#define CHECK_1LAYER_IS_NULL(ptr) ((ptr) == NULL)
+#define CHECK_2LAYER_IS_NULL(ptr, l2_member) \
+ (CHECK_1LAYER_IS_NULL(ptr) || (((ptr)->l2_member) == NULL))
+
+static inline bool roce5_func_is_vf(void *hwdev)
+{
+ return hinic5_func_type(hwdev) == TYPE_VF;
+}
+
+static inline bool roce5_func_is_pf(void *hwdev)
+{
+ return hinic5_func_type(hwdev) == TYPE_PF;
+}
+
+enum {
+ ROCE5_1_PORT_NUM = 1, //EVB single die
形mode
+ ROCE5_2_PORT_NUM = 2,
+ ROCE5_4_PORT_NUM = 4
+};
+
+enum {
+ ROCE5_25G_PORT_SPEED = 25,
+ ROCE5_100G_PORT_SPEED = 100,
+ ROCE5_200G_PORT_SPEED = 200,
+ ROCE5_400G_PORT_SPEED =
+ 201, //because MPU bit width not
够衍generate of custom enum, refer to port_speed_mode_e
+ ROCE5_800G_PORT_SPEED = 202
+};
+
+enum {
+ ROCE5_INVALID_HCA = -1,
+ ROCE5_2_100G_HCA = 0,
+ ROCE5_4_25G_HCA = 1,
+ ROCE5_2_25G_HCA = 2,
+ ROCE5_2_200G_HCA = 3,
+ ROCE5_1_100G_HCA = 4,
+ ROCE5_2_400G_HCA = 5,
+ ROCE5_4_200G_HCA = 6
+};
+
+enum ROCE5_100G_BW_PARAM_E {
+ ROCE5_100G_CIR = 46500000,
+ ROCE5_100G_PIR = 52500000,
+ ROCE5_100G_CNP = 100
+};
+
+enum ROCE5_25G_BW_PARAM_E {
+ ROCE5_25G_CIR = 23200000,
+ ROCE5_25G_PIR = 25500000,
+ ROCE5_25G_CNP = 3
+};
+
+enum roce_bitshift_e {
+ BYTES_TO_2B_SHIFT = 1,
+ BYTES_TO_4B_SHIFT,
+ BYTES_TO_8B_SHIFT,
+ BYTES_TO_16B_SHIFT,
+ BYTES_TO_32B_SHIFT
+};
+
+/* BIG/LITTLE ENGIAN switch */
+#ifdef HW_CONVERT_ENDIAN
+#define roce5_convert_be32(val) (val)
+#define roce5_convert_cpu32(val) (val)
+#else
+#define roce5_convert_be32(val) cpu_to_be32(val)
+#define roce5_convert_cpu32(val) be32_to_cpu(val)
+#endif
+
+enum roce5_aeq_type {
+ /* ofed err */
+ OFED_ET_PATH_MIG = 0,
+ OFED_ET_COMM_EST,
+ OFED_ET_SQ_DRAINED,
+ OFED_ET_SRQ_QP_LAST_WQE,
+ OFED_ET_WQ_CATAS_ERR,
+
+ OFED_ET_PATH_MIG_FAILED = 5,
+ OFED_ET_WQ_INVAL_REQ_ERR,
+ OFED_ET_WQ_ACCESS_ERR,
+ OFED_ET_CQ_ERR,
+ OFED_ET_SRQ_LIMIT,
+ OFED_ET_SRQ_CATAS_ERR,
+
+ /* non ofed err */
+ NON_OFED_ET_QPC_LOOKUP_ERR = 11,
+ NON_OFED_ET_OTHER_TYPE_ERR,
+
+ /* non ofed extend err */
+ NON_OFED_ET_DEBUG_ERR = 32,
+
+ /* NOF AA err */
+ OFED_NOF_AA_QP_DISCONNECT = 64,
+ OFED_NOF_AA_MASTER_CHANGE,
+
+ INVAL_ET_ERR
+};
+
+enum {
+ ROCE_CMD_TIME_CLASS_A = 30000, //30s, TODO: use ASIC card after need modify is 3s
+ ROCE_CMD_TIME_CLASS_B = 40000, //40s, TODO: use ASIC card after need modify is 4s
+ ROCE_CMD_TIME_CLASS_C = 50000 //50s, TODO: use ASIC card after need modify is 5s
+};
+
+enum roce5_ctrl_status { ROCE5_PORT_EVENT = BIT(0) };
+
+#define ROCE_DEFAULT_PORT_NUM 1
+
+#define ROCE_UVERBS_CMD_BASE_MASK \
+ ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \
+ (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \
+ (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \
+ (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \
+ (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \
+ (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \
+ (1ULL << IB_USER_VERBS_CMD_REG_MR) | \
+ (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \
+ (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \
+ (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \
+ (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \
+ (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \
+ (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \
+ (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \
+ (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \
+ (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \
+ (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \
+ (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) | \
+ (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW) | \
+ (1ULL << IB_USER_VERBS_CMD_OPEN_XRCD) | \
+ (1ULL << IB_USER_VERBS_CMD_CLOSE_XRCD) | \
+ (1ULL << IB_USER_VERBS_CMD_OPEN_QP))
+
+#define ROCE_UVERBS_EXT_CMD_MASK \
+ ((1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ) | \
+ (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_QP))
+
+enum roce5_load_balance_mode_e {
+ ROCE_LB_MODE_0 = 0,
+ ROCE_LB_MODE_1,
+ ROCE_LB_MODE_2,
+ ROCE_LB_MODE_N,
+};
+
+enum push_ofed_device_status {
+ ROCE_DEV_STATUS_NORMAL = 0,
+ ROCE_DEV_STATUS_CMDQ_TIMEOUT
+};
+
+enum roce5_func_state { ROCE_FUNC_DISABLE = 0, ROCE_FUNC_ENABLE };
+
+enum {
+ ROCE_Cl_TYPE_QPC = 0x0, /* cl_start: 0x0, cl_end: 0x7f, cl_size: 0 */
+ ROCE_CL_TYPE_MPT = 0x1, /* cl_start: 0x150, cl_end: 0x15f, cl_size: 0 */
+ ROCE_CL_TYPE_SQ_WQE =
+ 0x2, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */
+ ROCE_CL_TYPE_RQ_WQE =
+ 0x3, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */
+ ROCE_CL_TYPE_CQC_SRQC =
+ 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */
+ ROCE_CL_TYPE_RDMARC =
+ 0x5, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */
+ ROCE_CL_TYPE_CMTT =
+ 0x6, /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */
+ ROCE_CL_TYPE_DMTT = 0x7 /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */
+};
+
+#define XRC_CONTAINER_FLAG ((u32)(1L << 20))
+
+struct roce5_buf_list {
+ void *buf;
+ dma_addr_t map;
+};
+
+struct roce5_buf {
+ struct roce5_buf_list direct;
+ struct roce5_buf_list *page_list;
+ int nbufs;
+
+ int npages;
+ int page_shift;
+};
+
+struct shadow_hw_db {
+ u32 page_idx;
+ u32 refcnt;
+};
+
+struct roce5_ucontext {
+ struct ib_ucontext ibucontext;
+ void __iomem *db_map;
+ void __iomem *dwqe_map;
+ u64 db_dma_addr;
+ u64 dwqe_dma_addr;
+ struct list_head db_page_list;
+ struct mutex db_page_mutex;
+
+ struct shadow_hw_db shadow_db_ctx;
+};
+
+struct roce5_qp_cnt {
+ struct mutex cur_qps_mutex;
+ u32 alloc_qp_cnt;
+ u32 del_qp_cnt;
+};
+
+struct roce5_dev_hw_info {
+ int config_num_ports; /* Number of ports from configuration file */
+ int hca_type; /* HCA version: 4x25G or 2x100G */
+ u8 phy_port;
+ u8 ep_id; /* EP ID */
+ u8 cpu_endian;
+ u8 rsvd;
+
+ bool is_vf;
+};
+
+struct roce5_dev_cfg_info {
+ u8 scence_id; /* load scenes ID as aa_en */
+ u8 lb_en; /* load balance enable */
+ u8 lb_mode; /* load balance mode */
+ struct {
+ u8 pcie_atomic_en : 1;
+ u8 rsvd : 7;
+ };
+
+ u8 srq_container_en;
+ u8 srq_container_mode;
+ u8 xrc_srq_container_mode;
+ u8 warn_th;
+
+ u8 fake_en;
+ u8 page_bit;
+ u8 pf_start_bit;
+ u8 pf_end_bit;
+
+ u8 port_num; /* cfg data port num */
+ u8 host_num; /* cfg data host num */
+ u8 rsvd1[2];
+ roce_get_chip_ability_s feature_cap;
+};
+#define ROCE_LB_QPN_MAX \
+ 1 //less qp, same a smf on of qp will distribution to same a icq queue, because this decrease low lb of qp count
+struct roce5_qp_lb_scqn_ctrl {
+ struct mutex qp_lb_scqn_ctrl_mutex;
+ //record device in lb of qpn
+ u32 qp[ROCE_LB_QPN_MAX];
+};
+
+struct roce5_cq_ctrl {
+ struct mutex cq_ctrl_mutex;
+ u8 alloc_cq_cnt;
+ u8 high_perf_cq_cnt;
+ u8 rsvd_cq_step;
+ unsigned long cq_bitmap;
+};
+
+enum roce5_device_type { ROCE_DEV_TYPE_UNKNOWN, ROCE_DEV_TYPE_HI1825_V100 };
+
+enum roce5_perf_cq_step { ROCE_1825V100_STEP = 8 };
+
+#define ROCE_IS_DEV_1825V100(rdev) \
+ (((rdev)->device_type) == ROCE_DEV_TYPE_HI1825_V100)
+
+#ifdef __ROCE_DFX__
+struct roce5_list {
+ struct list_head node;
+ struct mutex node_lock;
+};
+struct roce5_device_res {
+ struct roce5_list mr_list;
+};
+#endif //__ROCE_DFX__
+
+//uld service can extension structure space
+#define ROCE_DEVICE_ULD_DEF_LEN 64
+#define ROCE_DEVICE_MAGIC 0xaabacada
+struct roce5_device_uld_def {
+ u8 self_def[ROCE_DEVICE_ULD_DEF_LEN];
+};
+
+struct roce5_device {
+ struct ib_device ib_dev;
+ struct hinic5_lld_dev *lld_dev;
+ struct net_device *ndev;
+ struct device *dev;
+ void *hwdev;
+ void *hwdev_hdl;
+ //struct dev_version_info dev_ver; /*version info */
+ struct hinic5_board_info board_info;
+
+ void __iomem *kernel_db_map;
+ void __iomem *kernel_dwqe_map;
+
+ spinlock_t node_desc_lock;
+ struct mutex cap_mask_mutex;
+ struct rdma_service_cap rdma_cap;
+
+ u8 mac[6]; /* Mac addr. */
+ u16 glb_func_id;
+ unsigned long status;
+ int ceq_num;
+ int ceqn[MAX_CEQ_NEED];
+ int try_times;
+ int destory_flow_timeout;
+ bool ib_active;
+ enum roce5_device_type device_type;
+
+ u8 group_rc_cos;
+ u8 group_ud_cos;
+ u8 group_xrc_cos;
+ u8 mpt_xid_mode;
+ struct roce5_qp_lb_scqn_ctrl qp_lb_scqn_ctrl;
+ struct roce5_dev_hw_info hw_info; /* Hw info read from nic/pcie */
+ struct roce5_dev_cfg_info cfg_info; /* Cfg data info read from MPU */
+
+#ifdef ROCE_BONDING_EN
+ enum ib_port_state port_state;
+ struct roce5_bond_device *bond_dev;
+ atomic_t next_port;
+#endif
+
+ struct mutex mac_vlan_mutex;
+ struct list_head mac_vlan_list_head;
+ struct net_device **gid_dev;
+
+ struct roce5_ecn_ctx ecn_ctx;
+ struct roce5_dfx_ctx dfx_ctx;
+
+ struct roce5_qp_cnt qp_cnt;
+ struct roce5_cq_ctrl cqn_ctrl;
+
+ struct srcu_struct mr_srcu;
+ atomic_t num_prefetch;
+ struct completion comp_prefetch;
+ enum push_ofed_device_status dev_status_to_ofed;
+
+ spinlock_t reset_flow_resource_lock;
+ struct list_head qp_list;
+
+ void *fake_data_buf;
+ dma_addr_t fake_data_page_addr;
+ void *pri_roce_dev
+ [ROCE_MAX_PORT_NUM]; //mailbox, ppf store all pf's roce device, to improve the effiency.
+ int hca_vf_mode; //HCA uses mode to init MAC/IP vf table
+ bool local_en; //STUB local
+ bool full_qp_spec;
+ bool huge_page_en;
+ bool is_vroce;
+ u8 cc_param_version;
+ u8 shadow;
+ u8 rsvd[2];
+ u32 shadow_vfid;
+ u16 shadow_vf_num;
+
+ rwlock_t rwlock_func_tbl_bond;
+ u32 func_tbl_bond;
+
+#ifdef __ROCE_DFX__
+ struct roce5_device_res *device_res;
+ struct roce5_ucontext *ucontext;
+#endif //__ROCE_DFX__
+
+ struct roce5_device_uld_def uld_def;
+ u32 rdev_magic; /* used for uld check, preventstop踩踏 */
+};
+
+int roce5_db_map_user(struct roce5_ucontext *context, unsigned long virt,
+ struct roce5_db *db, u8 shadow_db);
+void roce5_db_unmap_user(struct roce5_ucontext *context, struct roce5_db *db,
+ u8 shadow_db);
+
+int roce5_buf_write_mtt(struct roce5_device *rdev, struct mtt *mtt,
+ struct tag_cqm_buf *buf);
+int roce5_umem_write_mtt(struct roce5_device *rdev, struct mtt *mtt,
+ struct ib_umem *umem, bool use_huge);
+
+int roce5_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
+ struct ib_udata *uhw);
+
+int roce5_query_port_compat(struct ib_device *ibdev, u8 port,
+ struct ib_port_attr *props);
+
+int roce5_query_port(struct ib_device *ibdev, u32 port,
+ struct ib_port_attr *props);
+
+enum rdma_link_layer roce5_port_link_layer_compat(struct ib_device *ibdev,
+ u8 port_num);
+
+enum rdma_link_layer roce5_port_link_layer(struct ib_device *ibdev,
+ u32 port_num);
+
+int roce5_query_gid_compat(struct ib_device *ibdev, u8 port, int index,
+ union ib_gid *gid);
+
+int roce5_query_gid(struct ib_device *ibdev, u32 port, int index,
+ union ib_gid *gid);
+
+int roce5_query_pkey_compat(struct ib_device *ibdev, u8 port, u16 index,
+ u16 *pkey);
+
+int roce5_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
+
+int roce5_modify_device(struct ib_device *ibdev, int mask,
+ struct ib_device_modify *props);
+
+int roce5_modify_port_compat(struct ib_device *ibdev, u8 port, int mask,
+ struct ib_port_modify *props);
+
+int roce5_modify_port(struct ib_device *ibdev, u32 port, int mask,
+ struct ib_port_modify *props);
+
+int roce5_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma);
+
+int roce5_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
+
+int roce5_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
+
+int roce5_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
+
+int roce5_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
+
+int roce5_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
+
+void roce5_dealloc_ucontext(struct ib_ucontext *ibcontext);
+
+int roce5_alloc_ucontext(struct ib_ucontext *ibucontext,
+ struct ib_udata *udata);
+
+int roce5_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int roce5_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata);
+
+int roce5_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata);
+int roce5_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
+
+int roce5_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+
+int roce5_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int roce5_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period);
+
+int roce5_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
+
+int roce5_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
+
+int roce5_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
+
+struct ib_qp *roce5_create_qp_compat(struct ib_pd *ibpd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int roce5_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int roce5_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata);
+
+int roce5_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+
+int roce5_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
+
+int roce5_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int roce5_destroy_ah(struct ib_ah *ibah, u32 flags);
+
+int roce5_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr);
+
+int roce5_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr);
+
+int roce5_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
+
+int roce5_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
+
+struct ib_mr *roce5_get_dma_mr(struct ib_pd *ibpd, int access);
+struct ib_mr *roce5_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+ u32 max_num_sg);
+struct ib_mr *roce5_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
+ u64 virt_addr, int access,
+ struct ib_udata *udata);
+
+int roce5_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+ int sg_nents, unsigned int *sg_offset);
+
+int roce5_dealloc_mw(struct ib_mw *ibmw);
+
+int roce5_port_immutable_compat(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_immutable *immutable);
+int roce5_port_immutable(struct ib_device *ibdev, u32 port_num,
+ struct ib_port_immutable *immutable);
+int roce5_query_device_status(struct ib_device *ibdev, int *dev_status);
+
+int roce5_init_sysfs(struct roce5_device *rdev);
+void roce5_remove_sysfs(struct roce5_device *rdev);
+
+struct net_device *roce5_ib_get_netdev_compat(struct ib_device *ibdev,
+ u8 port_num);
+struct net_device *roce5_ib_get_netdev(struct ib_device *ibdev, u32 port_num);
+
+void roce5_async_event(void *svc_hd, u8 event_type, u8 *val);
+u8 roce5_async_event_level(void *svc_hd, u8 event_type, u8 *val);
+void roce5_ifconfig_up_down_event_report(struct roce5_device *rdev,
+ u8 net_event);
+int roce5_register_netdev_notifier(void);
+void roce5_unregister_netdev_notifier(void);
+void roce5_clean_vlan_device_mac(struct roce5_device *rdev);
+void roce5_clean_real_device_mac(struct roce5_device *rdev);
+
+int roce5_ib_add_gid(const struct ib_gid_attr *attr,
+ __always_unused void **context);
+
+int roce5_ib_del_gid(const struct ib_gid_attr *attr,
+ __always_unused void **context);
+
+/* OFA - OpenFabrics Alliance */
+struct ib_pd *roce5_alloc_pd_ofa(struct ib_device *ibdev,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata);
+
+int roce5_dealloc_pd_ofa(struct ib_pd *ibpd);
+
+struct ib_srq *roce5_create_srq_ofa(struct ib_pd *ibpd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata);
+
+struct ib_ucontext *roce5_alloc_ucontext_ofa(struct ib_device *ibdev,
+ struct ib_udata *udata);
+
+int roce5_dealloc_ucontext_ofa(struct ib_ucontext *ibcontext);
+
+struct ib_xrcd *roce5_alloc_xrcd_ofa(struct ib_device *ibdev,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata);
+
+int roce5_dealloc_xrcd_ofa(struct ib_xrcd *ibxrcd);
+
+int roce5_destroy_srq_ofa(struct ib_srq *ibsrq);
+
+int roce5_post_srq_recv_ofa(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
+ struct ib_recv_wr **bad_wr);
+
+int roce5_post_recv_ofa(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+ struct ib_recv_wr **bad_wr);
+
+struct ib_cq *roce5_create_cq_ofa(struct ib_device *ibdev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *ibcontext,
+ struct ib_udata *udata);
+
+int roce5_destroy_cq_ofa(struct ib_cq *ibcq);
+
+struct ib_ah *roce5_create_ah_ofa(struct ib_pd *pd,
+ struct rdma_ah_attr *ah_attr,
+ struct ib_udata *udata);
+
+int roce5_destroy_ah_ofa(struct ib_ah *ibah);
+
+struct ib_mw *roce5_alloc_mw_ofa(struct ib_pd *ibpd, enum ib_mw_type type,
+ struct ib_udata *udata);
+
+int roce5_dealloc_mw_ofa(struct ib_mw *ibmw);
+
+int roce5_dereg_mr_ofa(struct ib_mr *ibmr);
+
+int roce5_destroy_qp_ofa(struct ib_qp *ibqp);
+
+int roce5_ib_add_gid_ofa(const union ib_gid *gid,
+ const struct ib_gid_attr *attr,
+ __always_unused void **context);
+
+int roce5_g_rdev_add(struct roce5_device *rdev);
+
+void roce5_g_rdev_remove(struct roce5_device *rdev);
+
+struct roce5_device *roce5_get_rdev_by_card_func_id(uint64_t card_id,
+ uint16_t func_id);
+
+struct roce5_device *roce5_get_rdev_by_name(const char *dev_name);
+
+#endif //ROCE_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h
new file mode 100644
index 000000000..da2b4f65a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h
@@ -0,0 +1,87 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_COMPAT_H
+#define ROCE_COMPAT_H
+
+#include <linux/delay.h>
+#include <linux/types.h>
+
+#define ROCE_LIKELY(x) likely(x)
+
+#define ROCE_UNLIKELY(x) unlikely(x)
+
+static inline u32 roce_ilog2(u32 n)
+{
+ return ilog2(n);
+}
+#define ROCE_ILOG2(n) roce_ilog2(n)
+
+static inline u32 roce_roundup_pow_of_two(u32 n)
+{
+ return roundup_pow_of_two(n);
+}
+#define ROCE_ROUNDUP_POW_OF_TWO(n) roce_roundup_pow_of_two(n)
+
+#ifndef _lint
+#define ROCE_MDELAY(n) mdelay(n)
+#define ROCE_UDELAY(n) udelay(n)
+#else
+#define ROCE_MDELAY(n) ((void)n)
+#define ROCE_UDELAY(n) ((void)n)
+#endif
+
+#define US_PERF_DELAY 100
+#define ROCE_DELAY() ROCE_UDELAY(US_PERF_DELAY)
+
+#define ROCE_ALIGN(a, b) ALIGN(a, b)
+#define ROCE_MAX(a, b) max(a, b)
+#define ROCE_MIN(a, b) min(a, b)
+#define ROCE_DIV_ROUND_UP(a, b) DIV_ROUND_UP(a, b)
+#define ROCE_FLS(n) fls(n)
+
+#define ROCE_MEMCMP(a, b, count) memcmp(a, b, count)
+#define ROCE_IO_MAPPING_MAP_WC(mapping, offset) \
+ io_mapping_map_wc(mapping, offset)
+#define ROCE_IOREMAP(phys_addr, size) ioremap(phys_addr, size)
+#define ROCE_IOUNMAP(addr) iounmap(addr)
+#define ROCE_IO_MAPPING_UNMAP(vaddr) io_mapping_unmap(vaddr)
+
+#ifndef wc_wmb
+
+#if defined(__i386__)
+#define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory")
+#elif defined(__x86_64__)
+#define wc_wmb() asm volatile("sfence" ::: "memory")
+#elif defined(__ia64__)
+#define wc_wmb() asm volatile("fwb" ::: "memory")
+#else
+#define wc_wmb() wmb()
+#endif
+
+#endif
+
+typedef struct tag_roce_verbs_cmd_com {
+ union {
+ u32 value;
+
+ struct {
+ u32 version : 8;
+ u32 rsvd : 8;
+ u32 cmd_bitmask : 16; //CMD_TYPE_BITMASK_E
+ } bs;
+ } dw0;
+
+ u32 index; //qpn/cqn/srqn/mpt_index/gid idx
+} roce_verbs_cmd_com_s;
+
+typedef struct tag_roce_xq_mtt_info {
+ u32 mtt_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */
+ u32 mtt_num; /* Number of cmtt, which needs to be assigned by the driver when the is kicked out by queue. */
+ u32 mtt_cache_line_start; /* The driver needs to read the driver from the configuration file. */
+ u32 mtt_cache_line_end; /* The driver needs to read the driver from the configuration file. */
+ u32 mtt_cache_line_size; /* 0:256B,1:512B */
+} roce_xq_mtt_info_s;
+
+#endif //ROCE_COMPAT_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h
new file mode 100644
index 000000000..649a4275b
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h
@@ -0,0 +1,17 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
+
+
+#ifndef ROCE_EVENT_EXTENSION_H
+#define ROCE_EVENT_EXTENSION_H
+
+#include "roce.h"
+#include "roce_event.h"
+
+int to_unkown_event_str_index(uint8_t event_type, const uint8_t *val);
+void roce5_event_report_extend(const struct roce5_device *rdev, u8 *val,
+ int event_str_index);
+int roce5_async_event_handle_extend(u8 event_type, u8 *val,
+ struct roce5_device *rdev);
+
+#endif /* ROCE_EVENT_EXTENSION_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h
new file mode 100644
index 000000000..54cfc7bf6
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
+
+
+#ifndef ROCE_GID_H
+#define ROCE_GID_H
+
+#include <linux/types.h>
+
+struct rdma_gid_entry {
+ union {
+ u8 raw[16];
+ struct {
+ __be64 subnet_prefix;
+ __be64 interface_id;
+ } global;
+ };
+ union {
+ struct {
+ u32 rsvd : 8;
+ u32 cvlan : 12; /* in layer vlan customer vlan */
+ u32 svlan : 12; /* outside layer vlan */
+ } bs;
+ u32 value;
+ } dw4;
+
+ union {
+ u32 hdr_len_value;
+ };
+
+ union {
+ struct {
+ u16 tag : 2; /* 0: no has vlan; 1: one layer vlan; 2: 2 layer vlan; 3: stag */
+ u16 tunnel : 1; // rsvd for ppe, don't use. 'tunnel'
+ u16 gid_type : 2;
+ u16 ppe_rsvd1 : 1;
+ u16 outer_tag : 2; // rsvd for ppe, don't use. 'outer_tag'
+ u16 ppe_rsvd3 : 1; // rsvd for ppe, don't use. 'stag'
+ u16 gid_update : 1;
+ u16 rsvd : 6;
+ } bs;
+ u16 value;
+ } dw6_h;
+
+ u8 smac[6];
+};
+
+#endif // ROCE_GID_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h
new file mode 100644
index 000000000..b055d5c92
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h
@@ -0,0 +1,77 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved.
+
+
+#ifndef ROCE_INFRA_H
+#define ROCE_INFRA_H
+
+#include <linux/kern_levels.h>
+#include <linux/sched.h>
+
+#define ROCE_INFO(format, ...) \
+ printk(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_WARN(format, ...) \
+ printk(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_ERR(format, ...) \
+ printk(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+
+#define ROCE_INFO_LIMIT(format, ...) \
+ printk_ratelimited(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format \
+ "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_WARN_LIMIT(format, ...) \
+ printk_ratelimited(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format \
+ "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_ERR_LIMIT(format, ...) \
+ printk_ratelimited(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+
+#define ROCE_INFO_ONCE(format, ...) \
+ printk_once(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_WARN_ONCE(format, ...) \
+ printk_once(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_ERR_ONCE(format, ...) \
+ printk_once(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+
+/* Importent, ensure dev is not null pointer when using ROCE_DEV_XXX() !!! */
+#define ROCE_DEV_INFO(dev, format, ...) \
+ dev_info((dev)->hwdev_hdl, "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_DEV_WARN(dev, format, ...) \
+ dev_warn((dev)->hwdev_hdl, "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_DEV_ERR(dev, format, ...) \
+ dev_err((dev)->hwdev_hdl, "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+
+#define ROCE_DEV_INFO_LIMIT(dev, format, ...) \
+ dev_info_ratelimited((dev)->hwdev_hdl, \
+ "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_DEV_WARN_LIMIT(dev, format, ...) \
+ dev_warn_ratelimited((dev)->hwdev_hdl, \
+ "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+#define ROCE_DEV_ERR_LIMIT(dev, format, ...) \
+ dev_err_ratelimited((dev)->hwdev_hdl, \
+ "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \
+ current->pid, __func__, __LINE__, ##__VA_ARGS__)
+
+#define ROCE_PRINT_GID(dev, raw, format, ...) \
+ ROCE_DEV_WARN( \
+ (dev), \
+ "gid:%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x==>" format, \
+ *((u16 *)(raw) + IP_INDEX_0), *((u16 *)(raw) + IP_INDEX_1), \
+ *((u16 *)(raw) + IP_INDEX_2), *((u16 *)(raw) + IP_INDEX_3), \
+ *((u16 *)(raw) + IP_INDEX_4), *((u16 *)(raw) + IP_INDEX_5), \
+ *((u16 *)(raw) + IP_INDEX_6), *((u16 *)(raw) + IP_INDEX_7), \
+ ##__VA_ARGS__)
+
+#endif //ROCE_INFRA_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h
new file mode 100644
index 000000000..b1bca23b0
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h
@@ -0,0 +1,6 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_KERNEL_COMPAT_GEN_H
+#define ROCE_KERNEL_COMPAT_GEN_H
+#endif /* ROCE_KERNEL_COMPAT_GEN_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h
new file mode 100644
index 000000000..2244bb236
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h
@@ -0,0 +1,77 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_MAIN_EXTENSION_H
+#define ROCE_MAIN_EXTENSION_H
+
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_hw.h"
+#include "hinic5_crm.h"
+#include "hinic5_lld.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce.h"
+#include "roce_mix.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_qp.h"
+#include "roce_dfx.h"
+#include "roce_cmd.h"
+#include "roce_user.h"
+#include "rdma_comp.h"
+#ifdef ROCE_CHIP_TEST
+#include "roce_main_chip_ext.h"
+#endif
+
+#define ROCE_MAX_WQES_COMPUTE (4 * K_UNIT - 1)
+#define RDMA_MAX_SQ_DESC_SZ_COMPUTE 512
+#define ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE 448
+
+#define ROCE5_ULP_CMD(rdev, bitmask) \
+ (((rdev)->is_vroce) ? (VERBS_CMD_TYPE_VROCE_BITMASK) : (bitmask))
+
+static inline u64 roce5_get_uverbs_cmd_mask(void)
+{
+ u64 mask = 0;
+ mask |= ROCE_UVERBS_CMD_BASE_MASK;
+#ifdef ROCE_CHIP_TEST
+ mask |= ROCE_UVERBS_CMD_EXT_MASK;
+#endif
+ return mask;
+}
+
+struct hinic5_uld_info *roce5_info_get(void);
+
+void roce5_service_init_pre(void);
+
+void roce5_lock_rdev(void);
+
+void roce5_unlock_rdev(void);
+
+int roce5_get_rdev_by_uld(struct hinic5_lld_dev *lld_dev, void *uld_dev,
+ struct roce5_device **rdev,
+ struct hinic5_event_info *event);
+
+void roce5_init_dev_ext_handlers(struct roce5_device *rdev);
+
+int roce5_board_cfg_check(struct roce5_device *rdev);
+
+void roce5_remove_clean_res_ext(struct roce5_device *rdev);
+
+int roce5_set_comm_event(const struct roce5_device *rdev,
+ const struct hinic5_event_info *event);
+
+void *roce5_rdev_alloc_ext(void);
+
+void roce5_fill_gid_entry_ext(const struct roce5_device *rdev,
+ struct rdma_gid_entry *gid_entry);
+
+void roce5_cfg_info_update_ext(struct roce5_device *rdev,
+ roce_get_cfg_info_resp_s *resp);
+
+int roce5_add_check_ext(const struct hinic5_lld_dev *lld_dev);
+
+#endif /* ROCE_MAIN_EXTENSION_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h
new file mode 100644
index 000000000..ff0464a31
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h
@@ -0,0 +1,14 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_MR_EXTEND_H
+#define ROCE_MR_EXTEND_H
+
+#include <rdma/ib_verbs.h>
+#include "roce.h"
+
+int roce5_check_alloc_mr_type(enum ib_mr_type mr_type);
+enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type);
+
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h
new file mode 100644
index 000000000..6b8a3113a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h
@@ -0,0 +1,237 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_OS_COMPAT_H
+#define ROCE_OS_COMPAT_H
+#include <rdma/ib_verbs.h>
+#include "roce.h"
+#include "roce_restore.h"
+#include "roce_verbs_compat.h"
+#include "roce_kernel_compat_gen.h"
+#if defined(UVERBS_IOCTL_HAVE_RDMA_UDATA_TO_DRV_CONTEXT)
+#include <rdma/uverbs_ioctl.h>
+#endif
+
+#define PAGE_SIZE_4KB (1ULL << PAGE_SHIFT_4KB)
+#define PAGE_SIZE_64KB (1ULL << PAGE_SHIFT_64KB)
+#define PAGE_SIZE_2MB (1ULL << PAGE_SHIFT_2MB)
+
+#define BIT_NUM_PER_BYTE (8)
+#define PAGE_SHIFT_4KB (12)
+#define PAGE_SHIFT_64KB (16)
+#define PAGE_SHIFT_2MB (21)
+
+#if !defined(HAVE_IB_UMEM_NUM_DMA_BLOCKS)
+static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem,
+ unsigned long pgsz)
+{
+ return (size_t)((ALIGN(umem->address + umem->length, pgsz) -
+ ALIGN_DOWN(umem->address, pgsz))) /
+ pgsz;
+}
+#endif
+
+#if !defined(HAVE_RDMA_UMEM_FOR_EACH_DMA_BLOCK)
+#define rdma_umem_for_each_dma_block(umem, biter, pgsz) \
+ for (__rdma_umem_block_iter_start(biter, umem, pgsz); \
+ __rdma_block_iter_next(biter);)
+
+static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter,
+ struct ib_umem *umem,
+ unsigned long pgsz)
+{
+ __rdma_block_iter_start(biter, umem->sg_head.sgl, umem->nmap, pgsz);
+}
+static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter)
+{
+ return __rdma_block_iter_next(biter) && biter->__sg_numblocks--;
+}
+#endif
+
+#if defined(IB_UMEM_NEED_IB_UMEM_FIND_BEST_PGSZ)
+static inline unsigned int rdma_find_pg_bit(unsigned long addr,
+ unsigned long pgsz_bitmap)
+{
+ unsigned long align;
+ unsigned long pgsz;
+
+ align = addr & -addr;
+
+ pgsz = pgsz_bitmap & ~(-align << 1);
+ if (!pgsz)
+ return __ffs(pgsz_bitmap);
+
+ return __fls(pgsz);
+}
+
+static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
+ unsigned long pgsz_bitmap,
+ unsigned long virt)
+{
+ struct scatterlist *sg;
+ unsigned int best_pg_bit;
+ unsigned long va, pgoff;
+ dma_addr_t mask;
+ int i;
+
+ if (WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0))))
+ return 0;
+
+ va = virt;
+ mask = roundup_pow_of_two(umem->length);
+ pgoff = umem->address & ~PAGE_MASK;
+
+ for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) {
+ mask |= (sg_dma_address(sg) + pgoff) ^ va;
+ if (i && i != (umem->nmap - 1))
+ mask |= sg_dma_len(sg);
+ va += sg_dma_len(sg) - pgoff;
+ pgoff = 0;
+ }
+ best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap);
+
+ return BIT_ULL(best_pg_bit);
+}
+#endif
+
+static inline u32 roce5_umem_get_page_count(struct ib_umem *umem)
+{
+#if defined(HAVE_IB_UMEM_PAGE_COUNT)
+ return (u32)ib_umem_page_count(umem);
+#elif defined(HAVE_IB_UMEM_NUM_PAGES)
+ return (u32)ib_umem_num_pages(umem);
+#endif
+}
+
+static inline u32 roce5_umem_get_page_shift(struct ib_umem *umem)
+{
+#if defined(UMEM_HAVE_PAGE_SHIFT)
+ return umem->page_shift;
+#else
+ return PAGE_SHIFT;
+#endif
+}
+
+static inline u32 roce5_umem_get_page_size(struct ib_umem *umem)
+{
+#if defined(UMEM_HAVE_PAGE_SHIFT)
+ return BIT((unsigned int)umem->page_shift);
+#else
+ return PAGE_SIZE;
+#endif
+}
+
+static inline u32 roce5_get_page_size(struct roce5_device *rdev, bool use_huge)
+{
+ if (use_huge && rdev->huge_page_en) {
+ return PAGE_SIZE_2MB;
+ }
+ return PAGE_SIZE;
+}
+
+static inline size_t roce5_get_page_size_cap(size_t page_size_cap,
+ size_t length)
+{
+ size_t max_shift;
+
+ if (length == 0) {
+ return 0;
+ }
+
+ max_shift = fls_long(length) - 1;
+ if (max_shift == sizeof(size_t) * BIT_NUM_PER_BYTE - 1)
+ return page_size_cap;
+ else
+ return page_size_cap & (((size_t)1 << (max_shift + 1)) - 1);
+}
+
+static inline u32 roce5_umem_get_buf_page_shift(struct roce5_device *rdev,
+ struct ib_umem *umem,
+ u64 buf_addr)
+{
+ u32 real_page_size_cap, page_shift;
+
+ real_page_size_cap = (u32)roce5_get_page_size_cap(
+ (size_t)rdev->rdma_cap.page_size_cap, umem->length);
+
+ if (rdev->huge_page_en) {
+ page_shift = (u32)order_base_2(ib_umem_find_best_pgsz(
+ umem, real_page_size_cap, buf_addr));
+ } else {
+ page_shift = roce5_umem_get_page_shift(umem);
+ }
+
+ return page_shift;
+}
+
+static inline u32 roce5_umem_get_mtt_npages(struct roce5_device *rdev,
+ struct ib_umem *umem,
+ u32 page_shift)
+{
+ u32 npages;
+
+ if (rdev->huge_page_en) {
+ npages = ib_umem_num_dma_blocks(umem, BIT(page_shift));
+ } else {
+ npages = roce5_umem_get_page_count(umem);
+ }
+
+ return npages;
+}
+
+/* These APIs can be reused by different OSs */
+static inline struct ib_umem *roce5_umem_get(struct ib_device *ibdev,
+ struct ib_ucontext *context,
+ unsigned long addr, size_t size,
+ int access, int dmasync)
+{
+#if (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \
+ !defined(MMU_INTERVAL_NOTIFIER)) || \
+ (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \
+ !defined(IB_UMEM_GET_HAVE_IB_DEVICE))
+ struct uverbs_attr_bundle *attrs;
+#endif
+
+#if (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \
+ !defined(MMU_INTERVAL_NOTIFIER)) || \
+ (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \
+ !defined(IB_UMEM_GET_HAVE_IB_DEVICE))
+ attrs = container_of(&context, struct uverbs_attr_bundle, context);
+#endif
+
+#if defined(IB_UMEM_GET_HAVE_IB_UCONTEXT)
+ return ib_umem_get(context, addr, size, access, dmasync);
+#elif defined(IB_UMEM_GET_HAVE_IB_DEVICE) && defined(MMU_INTERVAL_NOTIFIER)
+ return ib_umem_get(ibdev, addr, size, access);
+#elif defined(IB_UMEM_GET_HAVE_DMASYNC)
+ return ib_umem_get(&attrs->driver_udata, addr, size, access, dmasync);
+#else
+ return ib_umem_get(&attrs->driver_udata, addr, size, access);
+#endif
+}
+
+#if defined(IB_UMEM_GET_PEER)
+static inline struct ib_umem *
+roce5_mr_umem_get_peer(struct roce5_device *rdev, size_t length, int access,
+ u64 addr, struct ib_udata *udata, bool use_huge)
+{
+ u64 page_size = PAGE_SIZE;
+ size_t buf_size = length;
+
+ page_size = roce5_get_page_size(rdev, use_huge);
+ buf_size = (size_t)ROCE_ALIGN(buf_size, page_size);
+
+#if defined(IB_UMEM_GET_PEER_HAVE_IBDEVICE) && defined(MMU_INTERVAL_NOTIFIER)
+ return ib_umem_get_peer(&rdev->ib_dev, addr, buf_size, access,
+ (IB_PEER_MEM_ALLOW | IB_PEER_MEM_INVAL_SUPP));
+#else
+ return ib_umem_get_peer(udata, addr, buf_size, access,
+ (IB_PEER_MEM_ALLOW | IB_PEER_MEM_INVAL_SUPP));
+#endif
+}
+#endif
+
+struct roce5_ucontext *
+roce5_queue_to_ucontext(void *ibq, struct ib_udata *udata, int q_type);
+
+#endif
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h
new file mode 100644
index 000000000..2cbcd5d71
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h
@@ -0,0 +1,61 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_QP_EXTENSION_H
+#define ROCE_QP_EXTENSION_H
+
+#include <rdma/ib_verbs.h>
+
+#include "roce_qp.h"
+#include "roce_qp_exp.h"
+
+int to_roce5_qp_type(enum ib_qp_type qp_type);
+
+bool roce5_check_qp_modify_ok(enum ib_qp_state cur_state,
+ enum ib_qp_state next_state, enum ib_qp_type type,
+ enum ib_qp_attr_mask mask,
+ enum rdma_link_layer ll);
+
+int roce5_is_qp_normal(struct roce5_qp *rqp, struct ib_qp_init_attr *init_attr);
+
+void roce5_set_qp_dif_attr(struct roce5_qp *rqp,
+ const struct ib_qp_init_attr *init_attr,
+ const struct roce5_device *rdev);
+
+#ifdef ROCE_EXTEND
+
+#define ROCE_QP_VBS_FLAG (1U << 22)
+#define ROCE_QP_OSD_FLAG (1U << 23)
+#define QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC 204800 //200K
+
+struct roce5_set_qp_ext_attr_cmd {
+ u32 qpn;
+ u32 attr_mask;
+};
+
+struct roce5_modify_qp_vbs_cmd {
+ u32 qpn;
+ u64 ci_record_addr;
+};
+
+struct roce5_vbs_qp {
+ cqm_qpc_mpt_s *vbs_sqpc_info; /* vbs private data */
+ struct roce5_db db; /* to record ci_record_pa */
+};
+
+long roce5_vbs_create_sqpc(struct roce5_device *rdev, void *buf);
+#endif
+
+void roce5_destroy_pcqc_ext(struct roce5_qp *rqp);
+int roce5_modify_qp_pre_ext(struct roce5_device *rdev, struct roce5_qp *rqp,
+ struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata);
+void roce5_set_attr_ext(struct roce5_qp *rqp, const struct ib_qp_attr *attr,
+ roce_verbs_qp_attr_s *qp_attr, int attr_mask);
+void roce5_transmit_ci_record_addr_ext(struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr);
+void roce5_cos_update_ext(struct roce5_device *rdev, u8 *cos,
+ enum ib_qp_type qp_type);
+
+#endif /* ROCE_QP_EXTENSION_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h
new file mode 100644
index 000000000..c772472a6
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h
@@ -0,0 +1,88 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_RESTORE_H
+#define ROCE_RESTORE_H
+#include <rdma/ib_verbs.h>
+
+#include "roce.h"
+#include "roce_qp.h"
+#include "roce_srq.h"
+#include "roce_cq.h"
+#include "roce_mr.h"
+#include "roce_mix.h"
+#include "roce_pd.h"
+#include "roce_xrc.h"
+#include "hinic5_cqm.h"
+
+static inline struct roce5_device *to_roce5_dev(const struct ib_device *ibdev)
+{
+ return container_of(ibdev, struct roce5_device, ib_dev);
+}
+
+static inline struct roce5_ucontext *
+to_roce5_ucontext(const struct ib_ucontext *ibucontext)
+{
+ return container_of(ibucontext, struct roce5_ucontext, ibucontext);
+}
+
+static inline struct roce5_qp *to_roce5_qp(const struct ib_qp *ibqp)
+{
+ return container_of(ibqp, struct roce5_qp, ibqp);
+}
+
+static inline struct roce5_qp *cqmobj_to_roce_qp(const cqm_object_s *object)
+{
+ cqm_qpc_mpt_s *qpc_info = NULL;
+ qpc_info = container_of(object, cqm_qpc_mpt_s, object);
+ return (struct roce5_qp *)qpc_info->priv;
+}
+
+static inline struct roce5_srq *to_roce5_srq(const struct ib_srq *ibsrq)
+{
+ return container_of(ibsrq, struct roce5_srq, ibsrq);
+}
+
+static inline struct roce5_srq *cqmobj_to_roce5_srq(const cqm_object_s *object)
+{
+ cqm_queue_s *cqm_srq = NULL;
+ cqm_srq = container_of(object, cqm_queue_s, object);
+ return (struct roce5_srq *)cqm_srq->priv;
+}
+
+static inline struct roce5_cq *to_roce5_cq(const struct ib_cq *ibcq)
+{
+ return container_of(ibcq, struct roce5_cq, ibcq);
+}
+
+static inline struct roce5_mr *to_roce5_mr(const struct ib_mr *ibmr)
+{
+ return container_of(ibmr, struct roce5_mr, ibmr);
+}
+
+static inline struct roce5_mw *to_roce5_mw(const struct ib_mw *ibmw)
+{
+ return container_of(ibmw, struct roce5_mw, ibmw);
+}
+
+static inline struct roce5_mr *
+rdma_mr_to_roce5_mr(const struct hmm_rdma *rdmamr)
+{
+ return container_of(rdmamr, struct roce5_mr, rdmamr);
+}
+
+static inline struct roce5_ah *to_roce5_ah(const struct ib_ah *ibah)
+{
+ return container_of(ibah, struct roce5_ah, ibah);
+}
+
+static inline struct roce5_pd *to_roce5_pd(const struct ib_pd *ibpd)
+{
+ return container_of(ibpd, struct roce5_pd, ibpd);
+}
+
+static inline struct roce5_xrcd *to_roce5_xrcd(const struct ib_xrcd *ibxrcd)
+{
+ return container_of(ibxrcd, struct roce5_xrcd, ibxrcd);
+}
+#endif
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h
new file mode 100644
index 000000000..b931f9426
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h
@@ -0,0 +1,38 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_SRQ_EXTENSION_H
+#define ROCE_SRQ_EXTENSION_H
+
+#include <linux/types.h>
+#include "roce_srq.h"
+
+/**
+ * container_mode:
+ * mode: 0 -> container_size: 16
+ * mode: 1 -> container_size: 8
+ * mode: 2 -> container_size: 4
+ * mode: 3 -> container_size: 2
+ */
+enum roce5_srq_mode {
+ ROCE_SRQ_MODE_0 = 0,
+ ROCE_SRQ_MODE_1,
+ ROCE_SRQ_MODE_2,
+ ROCE_SRQ_MODE_3
+};
+
+enum roce5_srq_cont_num_mode {
+ ROCE_SRQ_CONT_NUM_MODE3 = 2,
+ ROCE_SRQ_CONT_NUM_MODE2 = 4,
+ ROCE_SRQ_CONT_NUM_MODE1 = 8,
+ ROCE_SRQ_CONT_NUM_MODE0 = 16
+};
+
+u8 roce5_get_container_sz(u32 container_mode);
+u8 roce5_calculate_cont_th(u32 srq_limit);
+void roce5_srq_container_init(struct ib_srq_init_attr *init_attr,
+ struct roce5_srq *rsrq,
+ struct roce5_device *rdev);
+
+#endif /* ROCE_SRQ_EXTENSION_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h
new file mode 100644
index 000000000..81effef16
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h
@@ -0,0 +1,100 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_USER_H
+#define ROCE_USER_H
+
+#include <linux/types.h>
+
+struct roce5_alloc_ucontext_resp {
+ uint32_t num_qps; /* the maximum of qps */
+ uint32_t num_xsrqs; /* the maximum of xsrqs */
+ uint32_t cqe_size; /* the cqe entry size that device support, the valid value is 32B or 64B; */
+ uint32_t wqebb_size; /* the wqebb entry size that device support, the valid value is 64B or 128B */
+ uint32_t dwqe_size; /* the direct maximum wqe size that device support, 256B */
+ uint32_t max_msg_size; /* the maximum message size that device support, (2G -1)B */
+ uint32_t max_comp_vector; /* the maximum completion vector */
+ uint32_t max_inline_size; /* the maximum of qp inline size that device support */
+ uint32_t dev_caps;
+ uint8_t phy_port;
+ uint8_t storage_aa_en;
+ uint16_t func_id;
+ uint64_t card_id;
+ uint16_t db_offset;
+ uint16_t dwqe_offset;
+ struct {
+ uint16_t pcie_atomic_en : 1;
+ uint16_t rsvd1 : 15;
+ };
+ uint8_t srq_container_en;
+ uint8_t srq_container_mode;
+ uint8_t xrc_srq_container_mode;
+ uint8_t warn_th;
+ uint8_t huge_page_en;
+ uint8_t rsvd;
+ uint8_t max_send_sge; /* the maximum of sq sge */
+ uint8_t max_recv_sge; /* the maximum of rq sge */
+ uint32_t max_sq_wqe_len; /* the maximum sq wqe length that device support */
+ uint64_t roce_user_ext_ability;
+ uint64_t roce_uld_ext_ability;
+};
+
+struct roce5_create_cq_cmd {
+ u64 buf_addr;
+ u64 db_addr;
+};
+
+struct roce5_resize_cq_cmd {
+ u64 buf_addr; /* resize cq's 'buf va */
+ u64 stage; /* describe the resize stage,0 or 1; */
+};
+
+struct roce5_create_srq_cmd {
+ u64 buf_addr;
+ u64 db_addr;
+ u32 rsvd;
+};
+
+struct create_qp_cmd {
+ u64 buf_addr; /* describe the qp buf address that used to store cqe; */
+ u64 db_addr; /* describe the qp sw db address */
+ u32 comp_mask; /* kernel can judge whether handle receive inline through comp_mask */
+ u8 log_sq_bb_count; /* wqebb number = 2 << log_sq_bb_count */
+ u8 log_sq_stride; /* wqebb size = 2 << log_sq_stride */
+ u16 reserved; /* reserved is used to aligned cmd */
+};
+
+struct roce5_modify_srq_cmd {
+ u8 container_flag;
+ u8 container_warn_th;
+ u16 rsvd;
+};
+
+struct roce5_reg_frmr_cmd {
+ u64 pbl_vaddr;
+};
+
+struct roce5_modify_qp_ex_resp_data {
+ u8 is_cos_valid;
+ u8 cos;
+ u8 is_ssn_valid;
+ u8 rsvd0;
+ u32 ssn;
+ u32 rsvd[6];
+#ifdef ROCE_LATENCY_PROF_EN
+ uint8_t latency_prof_en;
+ uint8_t rsvd1[3];
+ uint32_t frequency_reduction_ratio;
+ uint64_t non_ptp_time_diff;
+#endif
+};
+
+struct roce5_create_ah_resp {
+ u8 dmac[ETH_ALEN];
+ u16 vlan_id;
+ u8 cos;
+ u8 rsvd[3];
+};
+
+#endif //ROCE_USER_H
diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h
new file mode 100644
index 000000000..cd87856ea
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h
@@ -0,0 +1,24 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_VERBS_COMPAT_H
+#define ROCE_VERBS_COMPAT_H
+
+#include <rdma/ib_verbs.h>
+#include "roce_kernel_compat_gen.h"
+
+enum { ROCE_Q_TYPE_QP, ROCE_Q_TYPE_SRQ, ROCE_Q_TYPE_CQ };
+
+#if !defined(HAVE_RDMA_AH_INIT_ATTR)
+/* kernel 4.19 not defined but kernel 5.10 defined, reused for ah creating */
+struct rdma_ah_init_attr {
+ struct rdma_ah_attr *ah_attr;
+};
+
+enum ib_wr_opcode_compat {
+ IB_WR_REG_MR_INTEGRITY = IB_WR_REG_SIG_MR,
+};
+
+#endif
+
+#endif /* ROCE_VERBS_COMPAT_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h
new file mode 100644
index 000000000..099b76a1a
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h
@@ -0,0 +1,79 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
+
+
+#ifndef ROCE_BOND_H
+#define ROCE_BOND_H
+
+#include <net/bonding.h>
+#include <linux/netdevice.h>
+
+#include "hinic5_lld.h"
+#include "drv_bond_api.h"
+
+#include "roce.h"
+#include "roce_qp.h"
+#include "roce_cmd.h"
+
+#define ROCE_BOND_MIN_SLAVE 1
+#define ROCE_BOND_MAX_SLAVE BOND_PORT_MAX_NUM
+
+#define ROCE_BOND_RSVD_VLAN_ID 4095
+
+#define ROCE_BOND_FWD_ID_TBL_ALL_BITS 32U
+#define ROCE_BOND_FWD_ID_TBL_PER_BITS 3U
+#define ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK 0x7U
+
+struct roce5_bond_slave {
+ struct net_device *netdev;
+ struct hinic5_lld_dev *lld_dev;
+ struct netdev_lag_lower_state_info netdev_state;
+ u32 update_cnt;
+ u16 func_id;
+ u8 er_id;
+};
+
+struct roce5_bond_transition_res {
+ struct hinic5_lld_dev *ppf_lld;
+};
+
+struct roce5_bond_device {
+ char name[IFNAMSIZ]; /* the name of nic bond device */
+ struct list_head entry;
+ struct hinic5_lld_dev *bond_lld_dev;
+ struct roce5_bond_slave slaves[ROCE_BOND_MAX_SLAVE];
+ struct mutex slave_lock; /* lock for slaves */
+ u32 slave_cnt;
+ atomic_t next_port;
+ struct work_struct detach_work;
+ struct roce5_device *attached_rdev;
+ struct bond_attr attr;
+ struct roce5_bond_transition_res transition_res;
+ u8 need_clean_on_remove;
+};
+
+bool roce5_bond_is_active(struct roce5_device *rdev);
+void roce5_bond_detach(struct roce5_device *rdev);
+struct net_device *roce5_bond_get_netdev(struct roce5_device *rdev);
+
+int roce5_add_bond_real_slave_mac(struct roce5_device *rdev, u8 *mac);
+int roce5_add_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac,
+ u16 vlan_id);
+void roce5_del_bond_real_slave_mac(struct roce5_device *rdev);
+void roce5_del_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac,
+ u16 vlan_id);
+
+void roce5_bond_rr_set_flow(struct roce5_device *rdev, struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr);
+
+int roce5_bond_event_cfg_rdev(struct hinic5_lld_dev *lld_dev, void *uld_dev,
+ struct roce5_device **rdev, bool hold_rdev);
+void roce5_handle_bonded_port_state_event(struct roce5_device *rdev);
+
+bool roce5_get_bond_ipsurx_en(void);
+void roce5_set_bond_ipsurx_en(bool ipsurx_en);
+
+int roce5_bond_attach(struct roce5_device *rdev);
+int roce5_bond_init(void);
+void roce5_bond_exit(void);
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c
new file mode 100644
index 000000000..f295f241d
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c
@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifdef ROCE_BONDING_EN
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <net/bonding.h>
+
+#include "securec.h"
+
+#include "bond_common_defs.h"
+#include "hinic5_hw.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce_bond.h"
+#include "roce_bond_internal.h"
+#include "roce_cmd.h"
+#include "roce_mpu_cmd.h"
+#include "roce_netdev.h"
+#include "roce_infra.h"
+
+static int roce5_bond_check_slave_port(struct hinic5_lld_dev *lld_dev,
+ u8 slave_ports[], u8 slave_ports_num,
+ bool *result)
+{
+ int ret, i;
+ roce_bond_check_slave_port_req_s req = { 0 };
+ roce_bond_check_slave_port_rsp_s rsp = { 0 };
+ u16 out_size = (u16)sizeof(rsp);
+
+ for (i = 0; i < slave_ports_num; ++i) {
+ req.slave_ports[i] = slave_ports[i];
+ }
+ req.slave_ports_num = slave_ports_num;
+
+ ret = roce5_msg_to_mgmt_sync(lld_dev->hwdev,
+ ROCE_MPU_CMD_BOND_CHECK_SLAVE_PORT, &req,
+ sizeof(req), &rsp, &out_size);
+ if ((ret != 0) || (out_size == 0) || (rsp.head.status != 0)) {
+ ROCE_ERR(
+ "Failed to check slave port, ret(%d), out size(%u), status(%u)",
+ ret, out_size, rsp.head.status);
+ return -EINVAL;
+ }
+
+ *result = false;
+ for (i = 0; i < slave_ports_num; ++i) {
+ *result |= (rsp.slave_ports_result[i] != false) ? true : false;
+ }
+
+ return 0;
+}
+
+/*
+ * roce5_can_do_bond - Check if RoCE can attach to bond device
+ * @bond: bond device to check
+ * Return: true if can attach, false otherwise
+ */
+bool roce5_can_do_bond(struct bonding *bond)
+{
+ bool ret = false;
+ int slave_cnt = 0;
+ int err;
+ struct slave *slave = NULL;
+ struct list_head *iter = NULL;
+ struct hinic5_lld_dev *lld_dev = NULL;
+ char chip_name[IFNAMSIZ] = { 0 };
+ u8 slave_ports[ROCE_BOND_PORT_MAX_NUM];
+ bool result;
+
+ if (bond == NULL) {
+ ROCE_ERR("Invalid Parameter, ptr is NULL.");
+ return false;
+ }
+
+ if (roce5_bond_mode_is_supported(bond->params.mode) == false) {
+ ROCE_ERR("Invalid bond mode(%u).", bond->params.mode);
+ return false;
+ }
+
+ rcu_read_lock();
+ bond_for_each_slave_rcu(bond, slave, iter) {
+ lld_dev = hinic5_get_lld_dev_by_netdev(slave->dev);
+ if (lld_dev == NULL) {
+ goto out;
+ }
+
+ if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) {
+ ROCE_WARN("VF device %s skipped", slave->dev->name);
+ continue;
+ }
+
+ err = roce5_bond_slave_check(lld_dev, chip_name,
+ sizeof(chip_name));
+ if (err != 0) {
+ ROCE_ERR("Bond slave check err %d.", err);
+ goto out;
+ }
+
+ if (slave_cnt < ROCE_BOND_PORT_MAX_NUM) {
+ slave_ports[slave_cnt] =
+ hinic5_physical_port_id(lld_dev->hwdev);
+ }
+
+ slave_cnt++;
+ ROCE_INFO("can do bond? slave_cnt(%d), slave_name(%s).",
+ slave_cnt, slave->dev->name);
+ }
+
+ if (((slave_cnt >= ROCE_BOND_MIN_SLAVE) &&
+ (slave_cnt <= ROCE_BOND_MAX_SLAVE)) == false) {
+ goto out;
+ }
+
+ if (lld_dev == NULL) {
+ ROCE_INFO("lld dev is NULL.");
+ goto out;
+ }
+
+ err = roce5_bond_check_slave_port(lld_dev, slave_ports, slave_cnt,
+ &result);
+ if (err != 0) {
+ ROCE_ERR("Failed to check slave ports, err(%d).", err);
+ goto out;
+ }
+
+ if (result == false) {
+ ROCE_INFO("Ports don't meet requirements.");
+ goto out;
+ }
+
+ ret = true;
+
+out:
+ rcu_read_unlock();
+ return ret;
+}
+
+static int
+roce5_bond_create_transition_store(struct roce5_bond_device *bond_dev,
+ const struct bond_tracker *tracker)
+{
+ int ret;
+ struct hinic5_lld_dev *ppf_lld = NULL;
+
+ ppf_lld = roce5_bond_get_ppf_lld(tracker);
+ if (ppf_lld == NULL) {
+ ROCE_ERR("Failed to get lld of the PPF.");
+ return -EINVAL;
+ }
+
+ ret = roce5_bond_transition_store(bond_dev, ppf_lld);
+ if (ret != 0) {
+ ROCE_ERR("Failed to store resources of transition, ret(%d).",
+ ret);
+ hinic5_lld_dev_put(ppf_lld);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void
+roce5_bond_create_transition_free(struct roce5_bond_device *bond_dev)
+{
+ roce5_bond_transition_free(bond_dev);
+}
+
+/*
+ * roce5_before_bond_active - Pre-active callback for bond creation
+ * @bond_name: name of bond device
+ * @attr: bond attributes
+ * @err: error code from previous operation
+ */
+void roce5_before_bond_active(const char *bond_name, struct bond_attr *attr,
+ int err)
+{
+ struct roce5_bond_device *bond_dev = NULL;
+ struct bond_tracker tracker;
+ u16 bond_id;
+ int i, slave_cnt;
+ int ret;
+
+ if (bond_name == NULL || attr == NULL) {
+ ROCE_ERR(
+ "Invalid parameter, roce bond_name(%d) or attr(%d) is NULL.",
+ !!bond_name, !!attr);
+ return;
+ }
+
+ bond_id = attr->bond_id;
+ ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get bond tracker, bond name(%s), ret(%d).",
+ bond_name, ret);
+ goto err;
+ }
+
+ if (roce5_bond_before_active_check(&tracker, attr) == false) {
+ ROCE_ERR("Failed to check before active bond.");
+ goto err;
+ }
+
+ bond_dev = roce5_bond_create_bond_device(bond_name);
+ if (IS_ERR(bond_dev)) {
+ ROCE_ERR("Failed to create roce bond device.");
+ goto err;
+ }
+
+ /* Store transition resources during the bond creating period to prevent abnormal resources using. */
+ ret = roce5_bond_create_transition_store(bond_dev, &tracker);
+ if (ret != 0) {
+ ROCE_ERR("Failed to store resources of transition.");
+ goto err;
+ }
+
+ /* Remove the functions of slaves. */
+ for (i = 0, slave_cnt = 0;
+ i < BOND_PORT_MAX_NUM && slave_cnt < tracker.cnt; ++i) {
+ if (tracker.ndev[i] == NULL) {
+ continue;
+ }
+
+ ret = roce5_bond_add_slave(bond_dev, tracker.ndev[i],
+ &tracker.netdev_state[i], false);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to add slave, name(%s), slave name(%s).",
+ bond_dev->name, tracker.ndev[i]->name);
+ goto err_with_transition;
+ }
+
+ ++slave_cnt;
+ }
+
+ /* Free transition resources during the bond creating period. */
+ roce5_bond_create_transition_free(bond_dev);
+ bond_dev->attr = *attr;
+
+ return;
+
+err_with_transition:
+ roce5_bond_create_transition_free(bond_dev);
+err:
+ roce5_bond_destroy(bond_name, bond_id);
+}
+
+/*
+ * roce5_after_bond_active - Post-active callback for bond creation
+ * @bond_name: name of bond device
+ * @attr: bond attributes
+ * @err: error code from previous operation
+ */
+void roce5_after_bond_active(const char *bond_name, struct bond_attr *attr,
+ int err)
+{
+ struct roce5_bond_device *bond_dev = NULL;
+
+ if (bond_name == NULL || attr == NULL) {
+ ROCE_ERR(
+ "Invalid parameter, bond name(%d) or attr(%d) is NULL.",
+ !!bond_name, !!attr);
+ return;
+ }
+
+ bond_dev = roce5_get_bond_dev_by_name(bond_name);
+ if (bond_dev == NULL) {
+ ROCE_ERR("Can't find roce bond device by name(%s).", bond_name);
+ return;
+ }
+
+ roce5_update_rdev_func_val(bond_dev);
+}
+
+/*
+ * roce5_before_bond_modify - Pre-modify callback for bond modification
+ * @bond_name: name of bond device
+ * @attr: new bond attributes
+ * @err: error code from previous operation
+ */
+void roce5_before_bond_modify(const char *bond_name, struct bond_attr *attr,
+ int err)
+{
+ int ret;
+ struct roce5_bond_device *bond_dev = NULL;
+ struct bond_tracker tracker;
+
+ if (bond_name == NULL || attr == NULL) {
+ ROCE_ERR("roce bond_name or attr is NULL.");
+ return;
+ }
+
+ bond_dev = roce5_get_bond_dev_by_name(bond_name);
+ if (bond_dev == NULL) {
+ ROCE_ERR("Can't find roce bond device by name(%s).", bond_name);
+ return;
+ }
+
+ ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker);
+ if (ret != 0) {
+ ROCE_ERR("Failed to get bond tracker, bond name(%s), ret(%d).",
+ bond_name, ret);
+ return;
+ }
+
+ ret = roce5_handle_diff_slave(bond_dev, &tracker);
+ if (ret != 0) {
+ ROCE_ERR("Failed to handle different slave.");
+ return;
+ }
+
+ return;
+}
+
+/*
+ * roce5_after_bond_modify - Post-modify callback for bond modification
+ * @bond_name: name of bond device
+ * @attr: new bond attributes
+ * @err: error code from previous operation
+ */
+void roce5_after_bond_modify(const char *bond_name, struct bond_attr *attr,
+ int err)
+{
+ struct roce5_bond_device *bond_dev = NULL;
+
+ if (bond_name == NULL || attr == NULL) {
+ ROCE_ERR("roce bond_name or attr is NULL.");
+ return;
+ }
+
+ bond_dev = roce5_get_bond_dev_by_name(bond_name);
+ if (bond_dev == NULL) {
+ ROCE_ERR("can't find bond device by name(%s).", bond_name);
+ return;
+ }
+
+ bond_dev->attr = *attr;
+
+ roce5_update_rdev_func_val(bond_dev);
+}
+
+/*
+ * roce5_before_bond_deactive - Pre-deactive callback for bond deletion
+ * @bond_name: name of bond device
+ * @attr: bond attributes
+ * @err: error code from previous operation
+ */
+void roce5_before_bond_deactive(const char *bond_name, struct bond_attr *attr,
+ int err)
+{
+ struct roce5_bond_device *bond_dev = NULL;
+
+ bond_dev = roce5_get_bond_dev_by_name(bond_name);
+ if (bond_dev == NULL) {
+ ROCE_ERR("can't find bond device by name(%s).", bond_name);
+ return;
+ }
+
+ roce5_bond_destroy(bond_dev->name, attr->bond_id);
+}
+
+/*
+ * roce5_after_bond_deactive - Post-deactive callback for bond deletion
+ * @bond_name: name of bond device
+ * @attr: bond attributes
+ * @err: error code from previous operation
+ */
+void roce5_after_bond_deactive(const char *bond_name, struct bond_attr *attr,
+ int err)
+{
+ /* destroy switch thread handle, bond device may already be delete, current thread not disable care handle result */
+}
+
+/*
+ * roce5_bond_mode_is_supported - Check if bond mode is supported
+ * @bond_mode: bond mode to check
+ * Return: true if supported, false otherwise
+ */
+bool roce5_bond_mode_is_supported(u16 bond_mode)
+{
+ if (bond_mode != BOND_MODE_8023AD && bond_mode != BOND_MODE_XOR &&
+ bond_mode != BOND_MODE_ACTIVEBACKUP) {
+ ROCE_ERR(
+ "Bond mode only support 802.3ad/xor/active-backup, current bond mode(%u).",
+ bond_mode);
+ return false;
+ }
+ return true;
+}
+
+/*
+ * roce5_bond_slave_check - Check if slave device is valid for RoCE bond
+ * @lld_dev: LLD device to check
+ * @chip_name: buffer to store chip name
+ * @chip_name_size: size of chip_name buffer
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_bond_slave_check(struct hinic5_lld_dev *lld_dev, char *chip_name,
+ size_t chip_name_size)
+{
+ int err;
+ char tmp_name[IFNAMSIZ] = { 0 };
+
+ if (lld_dev == NULL) {
+ ROCE_ERR("get lld dev err.");
+ return -ENODEV;
+ }
+
+ if (roce5_func_is_vf(lld_dev->hwdev)) {
+ ROCE_ERR("Bond Slave device is VF.");
+ return -EINVAL;
+ }
+
+ err = hinic5_get_chip_name(lld_dev, tmp_name, sizeof(tmp_name));
+ if (err != 0) {
+ ROCE_ERR("Bond Slave get chip name err %d.", err);
+ return err;
+ }
+
+ if (strlen(chip_name) == 0) {
+ err = memcpy_s(chip_name, chip_name_size, tmp_name,
+ sizeof(tmp_name));
+ if (err != 0) {
+ ROCE_ERR("Failed to memcpy chip_name.(ret:%d)", err);
+ return -ENOMEM;
+ }
+ } else if (strcmp(tmp_name, chip_name) != 0) {
+ ROCE_ERR(
+ "Creating bond cross cards is not supported, bond dev chip_name %s, slave chip name %s.",
+ chip_name, tmp_name);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * roce5_bond_before_active_check - Pre-active check for bond creation
+ * @tracker: bond tracker from CFM framework
+ * @attr: bond attributes
+ * Return: true if check passes, false otherwise
+ */
+bool roce5_bond_before_active_check(struct bond_tracker *tracker,
+ struct bond_attr *attr)
+{
+ int i, slave_cnt;
+ int err;
+ struct hinic5_lld_dev *lld_dev = NULL;
+ char chip_name[IFNAMSIZ] = { 0 };
+
+ if (roce5_bond_mode_is_supported(attr->bond_mode) == false) {
+ ROCE_ERR("Invalid bond mode(%u).", attr->bond_mode);
+ return false;
+ }
+
+ if (tracker->cnt < ROCE_BOND_MIN_SLAVE ||
+ tracker->cnt > ROCE_BOND_MAX_SLAVE) {
+ ROCE_ERR("Not support the slave number(%u) of the bond.",
+ tracker->cnt);
+ return false;
+ }
+
+ for (i = 0, slave_cnt = 0;
+ i < BOND_PORT_MAX_NUM && slave_cnt < tracker->cnt; ++i) {
+ if (tracker->ndev[i] == NULL) {
+ continue;
+ }
+
+ lld_dev = hinic5_get_lld_dev_by_netdev(tracker->ndev[i]);
+ if (lld_dev == NULL) {
+ ROCE_ERR("get lld dev err.");
+ return false;
+ }
+
+ err = roce5_bond_slave_check(lld_dev, chip_name,
+ sizeof(chip_name));
+ if (err != 0) {
+ ROCE_ERR("Bond slave check err %d.", err);
+ return false;
+ }
+
+ ++slave_cnt;
+ }
+ return true;
+}
+
+#endif /* ROCE_BONDING_EN */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c
new file mode 100644
index 000000000..cb3612778
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c
@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifdef ROCE_BONDING_EN
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <net/bonding.h>
+
+#include "securec.h"
+
+#include "bond_common_defs.h"
+#include "hinic5_hw.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce_bond.h"
+#include "roce_bond_internal.h"
+#include "roce_cmd.h"
+#include "roce_netdev.h"
+#include "roce_infra.h"
+
+/* Global variables: g_roce5_bond_mutex protects g_roce5_bond_list */
+LIST_HEAD(g_roce5_bond_list);
+DEFINE_MUTEX(g_roce5_bond_mutex);
+
+static bool g_roce5_bond_ipsurx_en = true;
+
+int roce5_bond_transition_store(struct roce5_bond_device *bond_dev,
+ struct hinic5_lld_dev *ppf_lld)
+{
+ int ret = 0;
+
+ ret = hinic5_stateful_init(ppf_lld->hwdev);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to init stateful resource of the PPF, ret(%d).",
+ ret);
+ goto out;
+ }
+
+ bond_dev->transition_res.ppf_lld = ppf_lld;
+out:
+ return ret;
+}
+
+void roce5_bond_transition_free(struct roce5_bond_device *bond_dev)
+{
+ if (bond_dev->transition_res.ppf_lld == NULL) {
+ return;
+ }
+
+ hinic5_stateful_deinit(bond_dev->transition_res.ppf_lld->hwdev);
+ hinic5_lld_dev_put(bond_dev->transition_res.ppf_lld);
+ bond_dev->transition_res.ppf_lld = NULL;
+}
+
+/*
+ * Caller has to call hinic5_lld_dev_put() to free resource when finish using ppf lld.
+ * There's no need to call hinic5_lld_dev_put() to free resource when this function return NULL.
+ */
+struct hinic5_lld_dev *
+roce5_bond_get_ppf_lld(const struct bond_tracker *tracker)
+{
+ int i;
+ struct net_device *first_slave = NULL;
+ struct hinic5_lld_dev *first_slave_lld = NULL, *ppf_lld = NULL;
+
+ for (i = 0; i < BOND_PORT_MAX_NUM; ++i) {
+ if (tracker->ndev[i] != NULL) {
+ first_slave = tracker->ndev[i];
+ break;
+ }
+ }
+
+ if (first_slave == NULL) {
+ ROCE_ERR("Failed to get net device of the first slave.");
+ return NULL;
+ }
+
+ first_slave_lld = hinic5_get_lld_dev_by_netdev(first_slave);
+ if (first_slave_lld == NULL) {
+ ROCE_ERR("Failed to get lld of the first slave.");
+ return NULL;
+ }
+
+ ppf_lld = hinic5_get_ppf_lld_dev(first_slave_lld);
+ if (ppf_lld == NULL) {
+ ROCE_ERR("Failed to get lld of the PPF.");
+ return NULL;
+ }
+
+ return ppf_lld;
+}
+
+/*
+ * roce5_get_bond_dev - Get bond device by name (internal)
+ * @bond_name: name of bond device
+ * Return: pointer to roce5_bond_device if found, NULL otherwise
+ * Lock: caller must hold g_roce5_bond_mutex
+ */
+struct roce5_bond_device *roce5_get_bond_dev(const char *bond_name)
+{
+ struct roce5_bond_device *bdev = NULL;
+
+ list_for_each_entry(bdev, &g_roce5_bond_list, entry) {
+ if (strcmp(bdev->name, bond_name) == 0) {
+ return bdev;
+ }
+ }
+
+ return NULL;
+}
+
+/*
+ * roce5_get_bond_dev_by_name - Get bond device by name (public wrapper)
+ * @bond_name: name of bond device
+ * Return: pointer to roce5_bond_device if found, NULL otherwise
+ */
+struct roce5_bond_device *roce5_get_bond_dev_by_name(const char *bond_name)
+{
+ struct roce5_bond_device *bdev = NULL;
+
+ mutex_lock(&g_roce5_bond_mutex);
+ bdev = roce5_get_bond_dev(bond_name);
+ mutex_unlock(&g_roce5_bond_mutex);
+ return bdev;
+}
+
+struct roce5_bond_device *roce5_bond_create_bond_device(const char *bond_name)
+{
+ struct roce5_bond_device *rbdev = NULL;
+ errno_t ret;
+
+ if (bond_name == NULL) {
+ ROCE_ERR("Invalid parameter, ptr is NULL.");
+ return ERR_PTR(-EINVAL);
+ }
+
+ rbdev = roce5_get_bond_dev_by_name(bond_name);
+ if (rbdev != NULL) {
+ ROCE_ERR("Bond device(%s) already exists.", bond_name);
+ return ERR_PTR(-EINVAL);
+ }
+
+ rbdev = kzalloc(sizeof *rbdev, GFP_KERNEL);
+ if (rbdev == NULL) {
+ ROCE_ERR("Failed to alloc memory.");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ret = strncpy_s(rbdev->name, IFNAMSIZ, bond_name, strlen(bond_name));
+ if (ret != EOK) {
+ ROCE_ERR("Failed to copy string, ret(%d).", ret);
+ kfree(rbdev);
+ return ERR_PTR(-EFAULT);
+ }
+
+ rbdev->need_clean_on_remove = true;
+ mutex_init(&rbdev->slave_lock);
+
+ mutex_lock(&g_roce5_bond_mutex);
+ list_add_tail(&rbdev->entry, &g_roce5_bond_list);
+ mutex_unlock(&g_roce5_bond_mutex);
+
+ return rbdev;
+}
+
+static int
+roce5_bond_destroy_transition_store(struct roce5_bond_device *bond_dev)
+{
+ int ret;
+ struct hinic5_lld_dev *ppf_lld = NULL;
+
+ if (bond_dev->slave_cnt == 0) {
+ ROCE_WARN("There's no slave in roce bond.");
+ return 0;
+ }
+
+ ppf_lld = hinic5_get_ppf_lld_dev(bond_dev->slaves[0].lld_dev);
+ if (ppf_lld == NULL) {
+ ROCE_ERR("Failed to get lld of the PPF.");
+ return -EINVAL;
+ }
+
+ ret = roce5_bond_transition_store(bond_dev, ppf_lld);
+ if (ret != 0) {
+ ROCE_ERR("Failed to store resources of transition, ret(%d).",
+ ret);
+ hinic5_lld_dev_put(ppf_lld);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void
+roce5_bond_destroy_transition_free(struct roce5_bond_device *bond_dev)
+{
+ roce5_bond_transition_free(bond_dev);
+}
+/*
+ * roce5_bond_destroy - Destroy bond device
+ * @bond_name: name of bond device
+ * @bond_id: bond ID
+ */
+void roce5_bond_destroy(const char *bond_name, u16 bond_id)
+{
+ int ret;
+ u32 i;
+ struct roce5_bond_device *bond_dev = NULL;
+
+ mutex_lock(&g_roce5_bond_mutex);
+ bond_dev = roce5_get_bond_dev(bond_name);
+ if (bond_dev == NULL) {
+ /* destroy switch thread handle, bond device may already be delete, current thread not disable care handle result */
+ ROCE_INFO("bond name(%s) has been detached", bond_name);
+ mutex_unlock(&g_roce5_bond_mutex);
+ return;
+ }
+
+ list_del(&bond_dev->entry);
+ mutex_unlock(&g_roce5_bond_mutex);
+
+ if (bond_dev->bond_lld_dev != NULL) {
+ ret = roce5_bond_destroy_transition_store(bond_dev);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to store transition resources on destroy.");
+ }
+
+ bond_dev->need_clean_on_remove = false;
+ hinic5_detach_service(bond_dev->bond_lld_dev, SERVICE_T_ROCE);
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ ret = hinic5_attach_service(bond_dev->slaves[i].lld_dev,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to attach roce device, ret(%d).",
+ ret);
+ }
+ }
+ bond_dev->slave_cnt = 0;
+ roce5_bond_destroy_transition_free(bond_dev);
+ mutex_unlock(&bond_dev->slave_lock);
+ }
+
+ kfree(bond_dev);
+ return;
+}
+
+/*
+ * roce5_bond_attach - Attach bond device to RoCE
+ * @rdev: RoCE device to attach
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_bond_attach(struct roce5_device *rdev)
+{
+ u32 i;
+ int ret = 0;
+ struct roce5_bond_device *bond_dev = NULL;
+
+ mutex_lock(&g_roce5_bond_mutex);
+ list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) {
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ if (rdev->ndev != bond_dev->slaves[i].netdev) {
+ continue;
+ }
+
+ if (bond_dev->attached_rdev == NULL) {
+ bond_dev->attached_rdev = rdev;
+ rdev->bond_dev = bond_dev;
+ } else {
+ ret = -EEXIST;
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+ goto out;
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+ }
+out:
+ mutex_unlock(&g_roce5_bond_mutex);
+ return ret;
+}
+
+/*
+ * roce5_bond_is_active - Check if bond is active for RoCE device
+ * @rdev: RoCE device to check
+ * Return: true if bond is active, false otherwise
+ */
+bool roce5_bond_is_active(struct roce5_device *rdev)
+{
+ return (rdev->bond_dev != NULL);
+}
+
+void roce5_bond_detach(struct roce5_device *rdev)
+{
+ int ret;
+ u32 i;
+ struct roce5_bond_device *bond_dev = NULL;
+
+ mutex_lock(&g_roce5_bond_mutex);
+ bond_dev = rdev->bond_dev;
+ if (bond_dev == NULL) {
+ mutex_unlock(&g_roce5_bond_mutex);
+ ROCE_INFO("bond_dev is NULL.");
+ return;
+ }
+
+ if (bond_dev->need_clean_on_remove == false) {
+ mutex_unlock(&g_roce5_bond_mutex);
+ ROCE_INFO("bond_dev no need to detached on roce remove.");
+ return;
+ }
+
+ if (roce5_get_bond_dev(bond_dev->name) == bond_dev) {
+ list_del(&bond_dev->entry);
+ }
+
+ mutex_unlock(&g_roce5_bond_mutex);
+ ret = roce5_bond_destroy_transition_store(bond_dev);
+ if (ret != 0) {
+ ROCE_ERR("Failed to store transition resources on destroy.");
+ }
+ hinic5_bond_detach(bond_dev->attr.bond_id, HINIC5_BOND_USER_ROCE);
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ if (bond_dev->bond_lld_dev == bond_dev->slaves[i].lld_dev) {
+ continue;
+ }
+ ret = hinic5_attach_service(bond_dev->slaves[i].lld_dev,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Failed to attach roce device, ret(%d).", ret);
+ }
+ }
+ bond_dev->slave_cnt = 0;
+ mutex_unlock(&bond_dev->slave_lock);
+ roce5_bond_destroy_transition_free(bond_dev);
+ mutex_lock(&g_roce5_bond_mutex);
+ kfree(bond_dev);
+ rdev->bond_dev = NULL;
+ mutex_unlock(&g_roce5_bond_mutex);
+}
+/*
+ * roce5_bond_event_cfg_rdev - Configure RoCE device for bond events
+ * @lld_dev: LLD device
+ * @uld_dev: ULD device
+ * @rdev: pointer to RoCE device pointer (output)
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_bond_event_cfg_rdev(struct hinic5_lld_dev *lld_dev, void *uld_dev,
+ struct roce5_device **rdev, bool hold_rdev)
+{
+ u32 i = 0;
+ struct roce5_bond_device *bond_dev = NULL;
+
+ if (lld_dev == NULL) {
+ ROCE_ERR("lld_dev is null.");
+ return -EINVAL;
+ }
+
+ if (uld_dev != NULL) {
+ *rdev = (struct roce5_device *)uld_dev;
+ return 0;
+ }
+
+ mutex_lock(&g_roce5_bond_mutex);
+ list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) {
+ if (bond_dev == NULL) {
+ ROCE_ERR("bond_dev is null");
+ continue;
+ }
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ if (bond_dev->slaves[i].lld_dev == lld_dev) {
+ *rdev = bond_dev->attached_rdev;
+ if (hold_rdev == true && *rdev != NULL) {
+ uld5_dev_hold((*rdev)->lld_dev,
+ SERVICE_T_ROCE);
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+ goto out;
+ }
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+ }
+
+out:
+ mutex_unlock(&g_roce5_bond_mutex);
+ return *rdev ? 0 : -EINVAL;
+}
+
+/*
+ * roce5_update_rdev_func_val - Update function table values for RoCE device
+ * @bond_dev: bond device
+ */
+void roce5_update_rdev_func_val(struct roce5_bond_device *bond_dev)
+{
+ struct roce5_device *rdev = NULL;
+ u32 func_tbl_bond;
+ int ret;
+
+ if (bond_dev->bond_lld_dev == NULL) {
+ return;
+ }
+
+ rdev = hinic5_get_uld_dev(bond_dev->bond_lld_dev, SERVICE_T_ROCE);
+ if (rdev == NULL) {
+ ROCE_ERR("Failed to find roce device.");
+ return;
+ }
+
+ ret = roce5_get_func_table(rdev->hwdev, rdev->glb_func_id,
+ &func_tbl_bond);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev,
+ "Failed to get func table value, func_id(%d).",
+ rdev->glb_func_id);
+ uld5_dev_put(bond_dev->bond_lld_dev, SERVICE_T_ROCE);
+ return;
+ }
+
+ write_lock(&rdev->rwlock_func_tbl_bond);
+ rdev->func_tbl_bond = func_tbl_bond;
+ write_unlock(&rdev->rwlock_func_tbl_bond);
+
+ uld5_dev_put(bond_dev->bond_lld_dev, SERVICE_T_ROCE);
+}
+
+bool roce5_get_bond_ipsurx_en(void)
+{
+ return g_roce5_bond_ipsurx_en;
+}
+
+void roce5_set_bond_ipsurx_en(bool ipsurx_en)
+{
+ g_roce5_bond_ipsurx_en = ipsurx_en;
+}
+
+/*
+ * roce5_bond_init - Initialize RoCE bond module
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_bond_init(void)
+{
+ int ret;
+ struct bond_srv_func roce_func = {
+ .before_active = roce5_before_bond_active,
+ .after_active = roce5_after_bond_active,
+ .before_modify = roce5_before_bond_modify,
+ .after_modify = roce5_after_bond_modify,
+ .before_deactive = roce5_before_bond_deactive,
+ .after_deactive = roce5_after_bond_deactive,
+ .can_attach = roce5_can_do_bond,
+ };
+
+ ret = hinic5_bond_register(HINIC5_BOND_USER_ROCE, &roce_func);
+ if (ret != 0) {
+ ROCE_ERR("Failed to register bond, ret(%d).", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+void roce5_bond_exit(void)
+{
+ int ret;
+ struct roce5_bond_device *bond_dev = NULL;
+
+ mutex_lock(&g_roce5_bond_mutex);
+ list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) {
+ bond_dev->need_clean_on_remove = false;
+ }
+ mutex_unlock(&g_roce5_bond_mutex);
+
+ ret = hinic5_bond_unregister(HINIC5_BOND_USER_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Failed to unregister service func, ret(%d).", ret);
+ }
+}
+
+#endif /* ROCE_BONDING_EN */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h
new file mode 100644
index 000000000..fd9853215
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h
@@ -0,0 +1,97 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifndef ROCE_BOND_INTERNAL_H
+#define ROCE_BOND_INTERNAL_H
+
+#ifdef ROCE_BONDING_EN
+
+#include <net/bonding.h>
+#include <linux/types.h>
+#include <linux/netdevice.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+
+#include "hinic5_lld.h"
+#include "drv_bond_api.h"
+#include "bond_common_defs.h"
+#include "roce_bond.h"
+
+/* Slave synchronization structure for bond modify */
+struct roce5_bond_slave_sync {
+ bool exist_arr
+ [ROCE_BOND_MAX_SLAVE]; /* Indicate which slave need to be deleted */
+ struct net_device *
+ new_ndev[ROCE_BOND_MAX_SLAVE]; /* New slaves need to be added */
+ struct netdev_lower_state_info new_ndev_state[BOND_PORT_MAX_NUM];
+ u32 new_ndev_num;
+};
+
+/* Global variables declaration */
+extern struct list_head g_roce5_bond_list; //TODO: later via interface access
+extern struct mutex g_roce5_bond_mutex; //TODO: later via interface access
+extern struct workqueue_struct *g_bond_wq; //TODO: later via interface access
+
+/* CFM Interface Layer - Internal Functions */
+bool roce5_bond_mode_is_supported(u16 bond_mode);
+int roce5_bond_slave_check(struct hinic5_lld_dev *lld_dev, char *chip_name,
+ size_t chip_name_size);
+bool roce5_bond_before_active_check(struct bond_tracker *tracker,
+ struct bond_attr *attr);
+bool roce5_can_do_bond(struct bonding *bond);
+
+/* CFM Callback Functions (called by CFM framework) */
+void roce5_before_bond_active(const char *bond_name, struct bond_attr *attr,
+ int err);
+void roce5_after_bond_active(const char *bond_name, struct bond_attr *attr,
+ int err);
+void roce5_before_bond_modify(const char *bond_name, struct bond_attr *attr,
+ int err);
+void roce5_after_bond_modify(const char *bond_name, struct bond_attr *attr,
+ int err);
+void roce5_before_bond_deactive(const char *bond_name, struct bond_attr *attr,
+ int err);
+void roce5_after_bond_deactive(const char *bond_name, struct bond_attr *attr,
+ int err);
+
+/* Device Management Layer - Internal Functions */
+struct roce5_bond_device *roce5_get_bond_dev(const char *bond_name);
+struct roce5_bond_device *roce5_get_bond_dev_by_name(const char *bond_name);
+void roce5_bond_destroy(const char *bond_name, u16 bond_id);
+void roce5_update_rdev_func_val(struct roce5_bond_device *bond_dev);
+struct hinic5_lld_dev *
+roce5_bond_get_ppf_lld(const struct bond_tracker *tracker);
+struct roce5_bond_device *roce5_bond_create_bond_device(const char *bond_name);
+
+/* Slave Management Layer - Internal Functions */
+struct roce5_bond_slave *
+roce5_find_next_slave_slot(struct roce5_bond_device *rbdev);
+int roce5_get_bond_slave_index_by_ndev(struct roce5_bond_device *rbdev,
+ struct net_device *ndev);
+int roce5_bond_add_slave(struct roce5_bond_device *rbdev,
+ struct net_device *ndev,
+ struct netdev_lower_state_info *ndev_info,
+ bool is_update_table);
+int roce5_bond_del_slave(struct roce5_bond_device *rbdev, u32 slave_index);
+int roce5_find_same_slave_and_update(struct roce5_bond_device *rbdev,
+ struct bond_tracker *tracker,
+ struct roce5_bond_slave_sync *slave_sync);
+int roce5_handle_diff_slave(struct roce5_bond_device *rbdev,
+ struct bond_tracker *tracker);
+
+/* MAC Table Operation Layer - Internal Functions */
+int roce5_bond_add_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id,
+ u16 slave_func_id);
+int roce5_bond_del_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id,
+ u16 slave_func_id);
+
+/* TX Port Selection Layer - Internal Functions (declared in roce_bond.h as public) */
+
+int roce5_bond_transition_store(struct roce5_bond_device *bond_dev,
+ struct hinic5_lld_dev *ppf_lld);
+void roce5_bond_transition_free(struct roce5_bond_device *bond_dev);
+
+#endif /* ROCE_BONDING_EN */
+
+#endif /* ROCE_BOND_INTERNAL_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c
new file mode 100644
index 000000000..75264917f
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c
@@ -0,0 +1,260 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifdef ROCE_BONDING_EN
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <net/bonding.h>
+
+#include "securec.h"
+
+#include "hinic5_hw.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce_bond.h"
+#include "roce_bond_internal.h"
+#include "roce_cmd.h"
+#include "roce_netdev.h"
+#include "roce_infra.h"
+
+/*
+ * roce5_add_bond_real_slave_mac - Add real slave MAC to table
+ * @rdev: RoCE device
+ * @mac: MAC address to add
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_add_bond_real_slave_mac(struct roce5_device *rdev, u8 *mac)
+{
+ struct roce5_bond_device *bond_dev = rdev->bond_dev;
+ struct roce5_bond_slave *slave = NULL;
+ int ret;
+ u32 i;
+
+ if (bond_dev == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to find bond_dev, func_id(%u).",
+ rdev->glb_func_id);
+ return -EINVAL;
+ }
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ slave = &bond_dev->slaves[i];
+
+ ret = roce5_add_mac_tbl_mac_entry(rdev, mac, ETH_ALEN,
+ ROCE_BOND_RSVD_VLAN_ID,
+ slave->func_id);
+ if (ret != 0) {
+ mutex_unlock(&bond_dev->slave_lock);
+ ROCE_DEV_ERR(rdev,
+ "Failed to add mac_vlan entry, ret(%d).",
+ ret);
+ return ret;
+ }
+
+ if (slave->func_id != rdev->glb_func_id) {
+ /* The IPSU MAC table is used for fast forwarding. Even if the addition fails, the forwarding information
+ can still be obtained by checking the MAC table later, without judging the execution result. */
+ roce5_add_ipsu_tbl_mac_entry(rdev, mac, 0,
+ rdev->glb_func_id,
+ slave->er_id,
+ slave->func_id);
+ }
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+
+ return 0;
+}
+
+/*
+ * roce5_add_bond_vlan_slave_mac - Add VLAN slave Mac to table
+ * @rdev: RoCE device
+ * @mac: MAC address to add
+ * @vlan_id: VLAN ID
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_add_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac,
+ u16 vlan_id)
+{
+ struct roce5_bond_device *bond_dev = rdev->bond_dev;
+ struct roce5_bond_slave *slave = NULL;
+ int ret;
+ u32 i;
+
+ if (bond_dev == NULL) {
+ ROCE_DEV_ERR(rdev, "Failed to find bond_dev");
+ return -EINVAL;
+ }
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ slave = &bond_dev->slaves[i];
+ if (slave->func_id == rdev->glb_func_id) {
+ continue;
+ }
+
+ ret = roce5_add_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id,
+ slave->func_id);
+ if (ret != 0) {
+ mutex_unlock(&bond_dev->slave_lock);
+ ROCE_DEV_ERR(rdev,
+ "Failed to add mac_vlan entry, ret(%d).",
+ ret);
+ return ret;
+ }
+
+ /* The IPSU MAC table is used for fast forwarding. Even if the addition fails, the forwarding information
+ can still be obtained by checking the MAC table later, without judging the execution result. */
+ roce5_add_ipsu_tbl_mac_entry(rdev, mac, vlan_id,
+ rdev->glb_func_id, slave->er_id,
+ slave->func_id);
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+
+ return 0;
+}
+
+/*
+ * roce5_del_bond_real_slave_mac - Delete real slave Mac from table
+ * @rdev: RoCE device
+ */
+void roce5_del_bond_real_slave_mac(struct roce5_device *rdev)
+{
+ u32 i;
+ struct roce5_bond_slave *slave = NULL;
+ struct roce5_bond_device *bond_dev = rdev->bond_dev;
+
+ if (bond_dev == NULL) {
+ return;
+ }
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ slave = &bond_dev->slaves[i];
+ if (slave->func_id != rdev->glb_func_id) {
+ roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN,
+ 0, slave->er_id,
+ slave->func_id);
+ }
+
+ (void)roce5_del_mac_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN,
+ ROCE_BOND_RSVD_VLAN_ID,
+ slave->func_id);
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+}
+
+/*
+ * roce5_del_bond_vlan_slave_mac - Delete VLAN slave Mac from table
+ * @rdev: RoCE device
+ * @mac: MAC address to delete
+ * @vlan_id: VLAN ID
+ */
+void roce5_del_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac,
+ u16 vlan_id)
+{
+ u32 i;
+ struct roce5_bond_slave *slave = NULL;
+ struct roce5_bond_device *bond_dev = rdev->bond_dev;
+
+ if (bond_dev == NULL) {
+ return;
+ }
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ slave = &bond_dev->slaves[i];
+ if (slave->func_id == rdev->glb_func_id) {
+ continue;
+ }
+
+ roce5_del_ipsu_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id,
+ slave->er_id, slave->func_id);
+
+ (void)roce5_del_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id,
+ slave->func_id);
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+}
+
+/*
+ * roce5_bond_add_slave_table - Add slave table entry
+ * @rbdev: bond device
+ * @slave_er_id: slave ER ID
+ * @slave_func_id: slave function ID
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_bond_add_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id,
+ u16 slave_func_id)
+{
+ struct roce5_device *rdev = rbdev->attached_rdev;
+ struct roce5_vlan_dev_list *vlan_dev_list = NULL;
+
+ if ((rdev == NULL) || (rdev->hwdev == NULL)) {
+ ROCE_ERR("Missing attached roce device.");
+ return -EINVAL;
+ }
+
+ /* Add entry of non-vlan device */
+ roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)rdev->mac, 0,
+ rdev->glb_func_id, slave_er_id,
+ slave_func_id);
+ /* Add entries of vlan device */
+ mutex_lock(&rdev->mac_vlan_mutex);
+ if (rdev->mac_vlan_list_head.next == NULL) {
+ ROCE_ERR("mac vlan list not init.");
+ mutex_unlock(&rdev->mac_vlan_mutex);
+ return -EINVAL;
+ }
+ list_for_each_entry(vlan_dev_list, &rdev->mac_vlan_list_head, list) {
+ roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)rdev->mac,
+ vlan_dev_list->vlan_id,
+ rdev->glb_func_id, slave_er_id,
+ slave_func_id);
+ }
+ mutex_unlock(&rdev->mac_vlan_mutex);
+
+ return 0;
+}
+
+/*
+ * roce5_bond_del_slave_table - Delete slave table entry
+ * @rbdev: bond device
+ * @slave_er_id: slave ER ID
+ * @slave_func_id: slave function ID
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_bond_del_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id,
+ u16 slave_func_id)
+{
+ struct roce5_device *rdev = rbdev->attached_rdev;
+ struct roce5_vlan_dev_list *vlan_dev_list = NULL;
+
+ if ((rdev == NULL) || (rdev->hwdev == NULL)) {
+ ROCE_ERR("Missing attached roce device.");
+ return -EINVAL;
+ }
+
+ /* Delete entry of non-vlan device */
+ roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, slave_er_id,
+ slave_func_id);
+ /* Delete entries of vlan device */
+ mutex_lock(&rdev->mac_vlan_mutex);
+ if (rdev->mac_vlan_list_head.next == NULL) {
+ ROCE_ERR("mac vlan list not init.");
+ mutex_unlock(&rdev->mac_vlan_mutex);
+ return -EINVAL;
+ }
+ list_for_each_entry(vlan_dev_list, &rdev->mac_vlan_list_head, list) {
+ roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN,
+ vlan_dev_list->vlan_id,
+ slave_er_id, slave_func_id);
+ }
+ mutex_unlock(&rdev->mac_vlan_mutex);
+
+ return 0;
+}
+
+#endif /* ROCE_BONDING_EN */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c
new file mode 100644
index 000000000..a0fee96a4
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c
@@ -0,0 +1,459 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifdef ROCE_BONDING_EN
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <net/bonding.h>
+
+#include "securec.h"
+
+#include "bond_common_defs.h"
+#include "hinic5_hw.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce_bond.h"
+#include "roce_bond_internal.h"
+#include "roce_cmd.h"
+#include "roce_netdev.h"
+#include "roce_infra.h"
+
+/*
+ * roce5_find_next_slave_slot - Find next available slave slot
+ * @rbdev: bond device
+ * Return: pointer to available slave slot, NULL if no slot available
+ * Lock: caller must hold rbdev->slave_lock
+ */
+struct roce5_bond_slave *
+roce5_find_next_slave_slot(struct roce5_bond_device *rbdev)
+{
+ if (rbdev->slave_cnt >= ROCE_BOND_MAX_SLAVE) {
+ ROCE_ERR(
+ "RoCE bond device slave slots are full, max number of slaves is %d.",
+ ROCE_BOND_MAX_SLAVE);
+ return NULL;
+ }
+
+ if (rbdev->slaves[rbdev->slave_cnt].netdev != NULL) {
+ ROCE_ERR("Error occur! Slave slot is broken, slot index(%d).",
+ rbdev->slave_cnt);
+ return NULL;
+ }
+
+ return &rbdev->slaves[rbdev->slave_cnt];
+}
+
+/*
+ * roce5_get_bond_slave_index_by_ndev - Get slave index by netdev
+ * @rbdev: bond device
+ * @ndev: netdev to search for
+ * Return: slave index if found, negative error code otherwise
+ * Lock: caller must hold rbdev->slave_lock
+ */
+int roce5_get_bond_slave_index_by_ndev(struct roce5_bond_device *rbdev,
+ struct net_device *ndev)
+{
+ u32 index;
+
+ for (index = 0; index < rbdev->slave_cnt; ++index) {
+ if (rbdev->slaves[index].netdev == ndev) {
+ return (int)index;
+ }
+ }
+
+ return -ENODEV;
+}
+
+static void roce5_bond_slave_init(struct roce5_bond_slave *slave,
+ struct net_device *ndev,
+ struct hinic5_lld_dev *ndev_lld,
+ struct netdev_lower_state_info *ndev_info,
+ bool is_update_table,
+ struct roce5_bond_device *rbdev)
+{
+ int ret;
+
+ slave->netdev = ndev;
+ slave->netdev_state.link_up = ndev_info->link_up;
+ slave->netdev_state.tx_enabled = ndev_info->tx_enabled;
+
+ slave->lld_dev = ndev_lld;
+ slave->func_id = hinic5_global_func_id(slave->lld_dev->hwdev);
+ slave->er_id = hinic5_er_id(slave->lld_dev->hwdev);
+
+ hinic5_detach_service(slave->lld_dev, SERVICE_T_ROCE);
+
+ if (is_update_table) {
+ ret = roce5_bond_add_slave_table(rbdev, slave->er_id,
+ slave->func_id);
+ if (ret != 0) {
+ ROCE_WARN(
+ "Failed to add table entries of slave(%u), ret(%d).",
+ slave->func_id, ret);
+ }
+ }
+}
+
+static void roce5_bond_slave_deinit(struct roce5_bond_slave *slave,
+ bool is_update_table,
+ struct roce5_bond_device *rbdev)
+{
+ int ret;
+
+ if (is_update_table) {
+ ret = roce5_bond_del_slave_table(rbdev, slave->er_id,
+ slave->func_id);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to delete slave table entries, ret(%d).",
+ ret);
+ }
+ }
+
+ ret = hinic5_attach_service(slave->lld_dev, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to attach roce function, func_id(%u), ret(%d).",
+ slave->func_id, ret);
+ }
+}
+
+/*
+ * roce5_bond_add_slave - Add slave to bond device
+ * @rbdev: bond device
+ * @ndev: slave netdev
+ * @ndev_info: slave state info
+ * @is_update_table: whether to update MAC table
+ * Return: 0 on success, negative error code on failure
+ * Lock: caller must hold rbdev->slave_lock
+ */
+int roce5_bond_add_slave(struct roce5_bond_device *rbdev,
+ struct net_device *ndev,
+ struct netdev_lower_state_info *ndev_info,
+ bool is_update_table)
+{
+ int ret, err_ret;
+ struct hinic5_lld_dev *ndev_lld = NULL;
+ struct roce5_bond_slave *slave = NULL;
+ struct roce5_bond_slave slave_temp;
+
+ ndev_lld = hinic5_get_lld_dev_by_netdev(ndev);
+ if (ndev_lld == NULL) {
+ ROCE_ERR("Failed to get lld dev by netdev %s.", ndev->name);
+ return -ENODEV;
+ }
+
+ if (hinic5_support_roce(ndev_lld->hwdev, NULL) == false) {
+ return 0;
+ }
+
+ roce5_bond_slave_init(&slave_temp, ndev, ndev_lld, ndev_info,
+ is_update_table, rbdev);
+
+ /* Add slave to roce bond device. */
+ mutex_lock(&rbdev->slave_lock);
+
+ slave = roce5_find_next_slave_slot(rbdev);
+ if (slave == NULL) {
+ ROCE_ERR("Failed to find next slave slot.");
+ ret = -ENXIO;
+ mutex_unlock(&rbdev->slave_lock);
+ goto err_without_slot;
+ }
+
+ *slave = slave_temp;
+ ++rbdev->slave_cnt;
+
+ mutex_unlock(&rbdev->slave_lock);
+
+ /* Try to attach roce device to roce bond device. */
+ if (rbdev->attached_rdev == NULL) {
+ if (rbdev->bond_lld_dev != NULL) {
+ ROCE_ERR("Impossible situation! bond lld isn't NULL.");
+ ret = -EINVAL;
+ goto err_check;
+ }
+
+ rbdev->bond_lld_dev = ndev_lld;
+
+ ret = hinic5_attach_service(rbdev->bond_lld_dev,
+ SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to attach roce service, ret(%d), bond name(%s).",
+ ret, rbdev->name);
+ ret = -EINVAL;
+ goto err_attach;
+ }
+
+ if (rbdev->attached_rdev == NULL) {
+ ROCE_ERR(
+ "Failed to attach roce bond device with roce device, bond name(%s).",
+ rbdev->name);
+ ret = -EINVAL;
+ goto err_attach_bond;
+ }
+ }
+
+ return 0;
+
+err_attach_bond:
+ hinic5_detach_service(rbdev->bond_lld_dev, SERVICE_T_ROCE);
+err_attach:
+ rbdev->bond_lld_dev = NULL;
+err_check:
+ mutex_lock(&rbdev->slave_lock);
+ err_ret = memset_s(slave, sizeof(struct roce5_bond_slave), 0,
+ sizeof(struct roce5_bond_slave));
+ if (err_ret != 0) {
+ ROCE_ERR("Failed to clean memory of slave, ret(%d).", err_ret);
+ }
+ --rbdev->slave_cnt;
+ mutex_unlock(&rbdev->slave_lock);
+err_without_slot:
+ roce5_bond_slave_deinit(&slave_temp, is_update_table, rbdev);
+ return ret;
+}
+
+/*
+ * roce5_bond_del_slave - Delete slave from bond device
+ * @rbdev: bond device
+ * @slave_index: index of slave to delete
+ * Return: 0 on success, negative error code on failure
+ * Lock: caller must hold rbdev->slave_lock
+ */
+int roce5_bond_del_slave(struct roce5_bond_device *rbdev, u32 slave_index)
+{
+ int ret;
+ struct roce5_bond_slave *slave = NULL;
+ struct roce5_bond_slave slave_temp;
+ size_t mem_copy_len;
+
+ mutex_lock(&rbdev->slave_lock);
+
+ if (slave_index >= rbdev->slave_cnt) {
+ ROCE_ERR("Invalid parameter, slave index(%u), slave count(%u).",
+ slave_index, rbdev->slave_cnt);
+ ret = -EINVAL;
+ mutex_unlock(&rbdev->slave_lock);
+ goto err_out;
+ }
+
+ slave = &rbdev->slaves[slave_index];
+ if (rbdev->bond_lld_dev != NULL &&
+ slave->lld_dev == rbdev->bond_lld_dev) {
+ ROCE_WARN(
+ "Slave(%s) lld dev is the same as the roce bond device(%s), skip removing.",
+ slave->netdev->name,
+ (rbdev->attached_rdev != NULL) ?
+ rbdev->attached_rdev->ib_dev.name :
+ "NULL");
+ ret = 0;
+ mutex_unlock(&rbdev->slave_lock);
+ goto err_out;
+ }
+
+ slave_temp = *slave;
+
+ ret = memset_s(slave, sizeof(struct roce5_bond_slave), 0,
+ sizeof(struct roce5_bond_slave));
+ if (ret != 0) {
+ ROCE_WARN("Failed to clear slave information.");
+ }
+
+ if (slave_index != ROCE_BOND_MAX_SLAVE - 1) {
+ mem_copy_len = sizeof(struct roce5_bond_slave) *
+ (ROCE_BOND_MAX_SLAVE - slave_index - 1);
+ ret = memmove_s((void *)&rbdev->slaves[slave_index],
+ mem_copy_len,
+ (void *)&rbdev->slaves[slave_index + 1],
+ mem_copy_len);
+ if (ret != EOK) {
+ ROCE_ERR("Failed to execute memmove, ret(%d).", ret);
+ mutex_unlock(&rbdev->slave_lock);
+ goto err_out;
+ }
+ }
+
+ ret = memset_s((void *)&rbdev->slaves[ROCE_BOND_MAX_SLAVE - 1],
+ sizeof(struct roce5_bond_slave), 0,
+ sizeof(struct roce5_bond_slave));
+ if (ret != 0) {
+ ROCE_ERR("Failed to execute memmove, ret(%d).", ret);
+ mutex_unlock(&rbdev->slave_lock);
+ goto err_out;
+ }
+
+ --rbdev->slave_cnt;
+
+ mutex_unlock(&rbdev->slave_lock);
+
+ if (rbdev->slave_cnt == 0 && rbdev->attached_rdev != NULL) {
+ if (rbdev->bond_lld_dev == NULL) {
+ ROCE_ERR("Impossible situation! bond lld is NULL.");
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ hinic5_detach_service(rbdev->bond_lld_dev, SERVICE_T_ROCE);
+ if (rbdev->attached_rdev != NULL) {
+ ROCE_ERR(
+ "Failed to detach roce service, bond name(%s).",
+ rbdev->name);
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ rbdev->bond_lld_dev = NULL;
+ }
+
+ ret = roce5_bond_del_slave_table(rbdev, slave_temp.er_id,
+ slave_temp.func_id);
+ if (ret != 0) {
+ ROCE_WARN(
+ "Failed to delete table entries of slave(%u), ret(%d).",
+ slave_temp.func_id, ret);
+ ret = 0;
+ }
+
+ ret = hinic5_attach_service(slave_temp.lld_dev, SERVICE_T_ROCE);
+ if (ret != 0) {
+ ROCE_ERR("Failed to attach roce function, func_id(%u).",
+ slave_temp.func_id);
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ return ret;
+}
+
+/*
+ * roce5_find_same_slave_and_update - Find and update existing slave
+ * @rbdev: bond device
+ * @tracker: bond tracker from CFM framework
+ * @slave_sync: slave synchronization info
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_find_same_slave_and_update(struct roce5_bond_device *rbdev,
+ struct bond_tracker *tracker,
+ struct roce5_bond_slave_sync *slave_sync)
+{
+ int ret = 0;
+ int i;
+ int roce_slave_index;
+ int tracker_slave_cnt = 0;
+ struct hinic5_lld_dev *lld_dev = NULL;
+ char chip_name[IFNAMSIZ] = { 0 };
+ int err;
+
+ ret = memset_s(slave_sync, sizeof(struct roce5_bond_slave_sync), 0,
+ sizeof(struct roce5_bond_slave_sync));
+ if (ret != 0) {
+ ROCE_ERR("Failed to init slave sync info, ret(%d).", ret);
+ return ret;
+ }
+
+ mutex_lock(&rbdev->slave_lock);
+
+ if (rbdev->bond_lld_dev != NULL) {
+ err = hinic5_get_chip_name(rbdev->bond_lld_dev, chip_name,
+ sizeof(chip_name));
+ if (err != 0) {
+ ROCE_ERR("Bond dev get chip name err %d.", err);
+ mutex_unlock(&rbdev->slave_lock);
+ return err;
+ }
+ }
+
+ for (i = 0; i < BOND_PORT_MAX_NUM && tracker_slave_cnt < tracker->cnt;
+ ++i) {
+ if (tracker->ndev[i] == NULL) {
+ continue;
+ }
+
+ roce_slave_index = roce5_get_bond_slave_index_by_ndev(
+ rbdev, tracker->ndev[i]);
+ if (roce_slave_index == -ENODEV) {
+ lld_dev =
+ hinic5_get_lld_dev_by_netdev(tracker->ndev[i]);
+ err = roce5_bond_slave_check(lld_dev, chip_name,
+ sizeof(chip_name));
+ if (err != 0) {
+ ROCE_ERR("Bond slave check err %d.", err);
+ mutex_unlock(&rbdev->slave_lock);
+ return err;
+ }
+
+ slave_sync->new_ndev[slave_sync->new_ndev_num] =
+ tracker->ndev[i];
+ slave_sync->new_ndev_state[slave_sync->new_ndev_num] =
+ tracker->netdev_state[i];
+ ++slave_sync->new_ndev_num;
+ } else {
+ slave_sync->exist_arr[roce_slave_index] = true;
+
+ /* update state of the slave */
+ rbdev->slaves[roce_slave_index].netdev_state.link_up =
+ tracker->netdev_state[i].link_up;
+ rbdev->slaves[roce_slave_index].netdev_state.tx_enabled =
+ tracker->netdev_state[i].tx_enabled;
+ }
+
+ ++tracker_slave_cnt;
+ }
+
+ mutex_unlock(&rbdev->slave_lock);
+
+ return ret;
+}
+
+/*
+ * roce5_handle_diff_slave - Handle different slaves during bond modify
+ * @rbdev: bond device
+ * @tracker: bond tracker from CFM framework
+ * Return: 0 on success, negative error code on failure
+ */
+int roce5_handle_diff_slave(struct roce5_bond_device *rbdev,
+ struct bond_tracker *tracker)
+{
+ u32 i;
+ int ret = 0;
+ struct roce5_bond_slave_sync slave_sync;
+
+ ret = roce5_find_same_slave_and_update(rbdev, tracker, &slave_sync);
+ if (ret != 0) {
+ ROCE_ERR("Failed to find same netdev.");
+ return ret;
+ }
+
+ /* delete slaves */
+ for (i = rbdev->slave_cnt; i > 0; --i) {
+ if (slave_sync.exist_arr[i - 1] == false) {
+ ret = roce5_bond_del_slave(rbdev, i - 1);
+ if (ret != 0) {
+ ROCE_ERR("Failed to delete slave, ret(%d).",
+ ret);
+ return ret;
+ }
+ }
+ }
+
+ /* add new slaves */
+ for (i = 0; i < slave_sync.new_ndev_num; ++i) {
+ ret = roce5_bond_add_slave(rbdev, slave_sync.new_ndev[i],
+ &slave_sync.new_ndev_state[i], true);
+ if (ret != 0) {
+ ROCE_ERR("Failed to add new slave, ret(%d).", ret);
+ return ret;
+ }
+ }
+
+ return ret;
+}
+
+#endif /* ROCE_BONDING_EN */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c
new file mode 100644
index 000000000..dd34d319e
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c
@@ -0,0 +1,94 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved.
+
+
+#ifdef ROCE_BONDING_EN
+
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <net/bonding.h>
+#include <linux/atomic.h>
+
+#include "securec.h"
+
+#include "hinic5_hw.h"
+#include "hinic5_srv_nic.h"
+
+#include "roce_bond.h"
+#include "roce_bond_internal.h"
+#include "roce_cmd.h"
+#include "roce_netdev.h"
+#include "roce_infra.h"
+
+/*
+ * roce5_bond_get_netdev - Get bond netdev for TX
+ * @rdev: RoCE device
+ * Return: netdev with increased reference count, NULL if no bond
+ * Refcount: caller must call dev_put() when done
+ */
+struct net_device *roce5_bond_get_netdev(struct roce5_device *rdev)
+{
+ u32 i;
+ struct roce5_bond_device *bond_dev = rdev->bond_dev;
+ struct net_device *ret_dev = NULL;
+ struct slave *slave = NULL;
+
+ mutex_lock(&g_roce5_bond_mutex);
+ if (bond_dev == NULL) {
+ mutex_unlock(&g_roce5_bond_mutex);
+ return ret_dev;
+ }
+
+ mutex_lock(&bond_dev->slave_lock);
+ for (i = 0; i < bond_dev->slave_cnt; i++) {
+ rcu_read_lock();
+ slave = bond_slave_get_rcu(bond_dev->slaves[i].netdev);
+ rcu_read_unlock();
+ if (slave == NULL) {
+ continue;
+ }
+ if (bond_is_active_slave(slave)) {
+ if (netif_running(bond_dev->slaves[i].netdev) &&
+ netif_carrier_ok(bond_dev->slaves[i].netdev)) {
+ ret_dev = bond_dev->slaves[i].netdev;
+ } else if (netif_running(
+ bond_dev->slaves[(i + 1) %
+ bond_dev->slave_cnt]
+ .netdev) &&
+ netif_carrier_ok(
+ bond_dev->slaves[(i + 1) %
+ bond_dev->slave_cnt]
+ .netdev)) {
+ ret_dev = bond_dev->slaves[(i + 1) %
+ bond_dev->slave_cnt]
+ .netdev;
+ } else {
+ ret_dev = bond_dev->slaves[i].netdev;
+ }
+ dev_hold(ret_dev);
+ mutex_unlock(&bond_dev->slave_lock);
+ mutex_unlock(&g_roce5_bond_mutex);
+ return ret_dev;
+ }
+ }
+ mutex_unlock(&bond_dev->slave_lock);
+ mutex_unlock(&g_roce5_bond_mutex);
+ return ret_dev;
+}
+
+/*
+ * roce5_bond_rr_set_flow - Set flow hash for round-robin TX selection
+ * @rdev: RoCE device
+ * @rqp: RoCE QP
+ * @qp_attr: QP attributes
+ */
+void roce5_bond_rr_set_flow(struct roce5_device *rdev, struct roce5_qp *rqp,
+ roce_verbs_qp_attr_s *qp_attr)
+{
+ rqp->tx_hash_value = (u32)atomic_add_return(1, &rdev->next_port);
+ qp_attr->path_info.dw0.bs.bond_tx_hash_value =
+ (rqp->tx_hash_value & 0xf);
+}
+
+#endif /* ROCE_BONDING_EN */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h b/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h
new file mode 100644
index 000000000..6b919d753
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h
@@ -0,0 +1,25 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_DEBUG_H
+#define ROCE_DEBUG_H
+
+/*
+ * DEBUG_ONLY: statement is only valid in debug mode.
+ *
+ * RELEASE_ONLY: statement is only valid in release mode.
+ *
+ * DEBUG_REPLACE: used in functions of release to indicate the function may be replaced by another function with the
+ * same name defined in the debug mode.
+ */
+#ifdef ENABLE_DEBUG
+#define DEBUG_ONLY(statement) statement
+#define RELEASE_ONLY(statement)
+#define DEBUG_REPLACE __attribute__((weak))
+#else
+#define DEBUG_ONLY(statement)
+#define RELEASE_ONLY(statement) statement
+#define DEBUG_REPLACE
+#endif
+
+#endif /* ROCE_DEBUG_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c
new file mode 100644
index 000000000..787a86791
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c
@@ -0,0 +1,375 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved.
+
+
+#ifdef __ROCE_DFX__
+
+#include <linux/fs.h>
+#include <linux/slab.h>
+
+#include "roce.h"
+#include "hinic5_mt.h"
+#include "roce_cc_format.h"
+#include "roce_cmd.h"
+#include "roce_mpu_cmd.h"
+#include "roce_mpu_cmd_defs.h"
+#include "securec.h"
+#include "roce_dfx.h"
+
+int roce5_get_drv_version(void *buf_out, u32 *out_size)
+{
+ struct drv_version_info *ver_info = buf_out;
+ int rc;
+
+ if (!buf_out || !out_size) {
+ ROCE_ERR("Input params is NULL.");
+ return -EINVAL;
+ }
+
+ if (*out_size != sizeof(*ver_info)) {
+ ROCE_ERR("Unexpect out buf size from user :%u, expect: %lu",
+ *out_size, sizeof(*ver_info));
+ return -EINVAL;
+ }
+
+ rc = snprintf_s(ver_info->ver, sizeof(ver_info->ver),
+ sizeof(ver_info->ver) - 1, "%s %s",
+ HIROCE5_DRV_VERSION, __TIME_STR__);
+ if (rc == -1) {
+ ROCE_ERR("Snprintf roce version err");
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+void *
+roce5_adm_generic_find_cmd_func(const roce5_adm_generic_cmd_proc_t *cmd_array,
+ int elem_count, u32 target_cmd)
+{
+ int i;
+
+ if (cmd_array == NULL || elem_count <= 0) {
+ return NULL;
+ }
+
+ for (i = 0; i < elem_count; i++) {
+ if (cmd_array[i].cmd_type == target_cmd &&
+ cmd_array[i].proc_func != NULL) {
+ return cmd_array[i].proc_func;
+ }
+ }
+ return NULL;
+}
+
+static int roce5_dfx_enable_bw_ctrl(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_inbuf *inbuf,
+ struct roce5_bw_ctrl_outbuf *outbuf)
+{
+ if (rdev->hw_info.hca_type == ROCE5_2_100G_HCA) {
+ inbuf->ctrl_param.cir = ROCE5_100G_CIR;
+ inbuf->ctrl_param.pir = ROCE5_100G_PIR;
+ inbuf->ctrl_param.cnp = ROCE5_100G_CNP;
+ } else {
+ inbuf->ctrl_param.cir = ROCE5_25G_CIR;
+ inbuf->ctrl_param.pir = ROCE5_25G_PIR;
+ inbuf->ctrl_param.cnp = ROCE5_25G_CNP;
+ }
+ return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_EN, inbuf);
+}
+
+static int roce5_dfx_disable_bw_ctrl(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_inbuf *inbuf,
+ struct roce5_bw_ctrl_outbuf *outbuf)
+{
+ inbuf->ctrl_param.cir = 0;
+ inbuf->ctrl_param.pir = 0;
+ inbuf->ctrl_param.cnp = 0;
+ return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_DIS, inbuf);
+}
+
+static int roce5_dfx_change_bw_ctrl_param(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_inbuf *inbuf,
+ struct roce5_bw_ctrl_outbuf *outbuf)
+{
+ return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_RESET, inbuf);
+}
+
+static int roce5_dfx_query_bw_ctrl_param(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_inbuf *inbuf,
+ struct roce5_bw_ctrl_outbuf *outbuf)
+{
+ return roce5_query_bw_ctrl_state(rdev, &outbuf->bw_ctrl_param);
+}
+
+static const roce5_adm_dfx_bw_ctrl_cmd_proc_t
+ g_roce5_adm_dfx_bw_ctrl_cmd_proc[] = {
+ { ROCE_CMD_ENABLE_BW_CTRL, roce5_dfx_enable_bw_ctrl },
+ { ROCE_CMD_DISABLE_BW_CTRL, roce5_dfx_disable_bw_ctrl },
+ { ROCE_CMD_CHANGE_BW_CTRL_PARAM,
+ roce5_dfx_change_bw_ctrl_param },
+ { ROCE_CMD_QUERY_BW_CTRL_PARAM, roce5_dfx_query_bw_ctrl_param },
+ };
+
+int roce5_adm_dfx_bw_ctrl(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ int ret;
+ struct roce5_bw_ctrl_inbuf *inbuf =
+ (struct roce5_bw_ctrl_inbuf *)buf_in;
+ struct roce5_bw_ctrl_outbuf *outbuf =
+ (struct roce5_bw_ctrl_outbuf *)buf_out;
+ const roce5_adm_generic_cmd_proc_t *generic_array =
+ (const roce5_adm_generic_cmd_proc_t *)
+ g_roce5_adm_dfx_bw_ctrl_cmd_proc;
+ int elem_count = sizeof(g_roce5_adm_dfx_bw_ctrl_cmd_proc) /
+ sizeof(g_roce5_adm_dfx_bw_ctrl_cmd_proc[0]);
+ roce5_adm_dfx_bw_ctrl_func_t roce5_adm_dfx_bw_ctrl_func = NULL;
+
+ if (in_size < sizeof(struct roce5_bw_ctrl_inbuf)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(struct roce5_bw_ctrl_inbuf));
+ return -ENOMEM;
+ }
+ if (*out_size < sizeof(struct roce5_bw_ctrl_outbuf)) {
+ ROCE_ERR(
+ "Input parameter(out_zise: %dB) is shorter than allocated buffer(allocated: %dB)",
+ (int)*out_size,
+ (int)sizeof(struct roce5_bw_ctrl_outbuf));
+ return -ENOMEM;
+ }
+ ret = memset_s(buf_out, sizeof(struct roce5_bw_ctrl_outbuf), 0,
+ sizeof(struct roce5_bw_ctrl_outbuf));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ *out_size = sizeof(struct roce5_bw_ctrl_outbuf);
+
+ if (inbuf->cmd_type >= VM_COMPAT_TEST) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d)",
+ inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ roce5_adm_dfx_bw_ctrl_func =
+ (roce5_adm_dfx_bw_ctrl_func_t)roce5_adm_generic_find_cmd_func(
+ generic_array, elem_count, inbuf->cmd_type);
+ if (roce5_adm_dfx_bw_ctrl_func == NULL) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d)",
+ inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ return roce5_adm_dfx_bw_ctrl_func(rdev, inbuf, outbuf);
+}
+
+int roce5_adm_dfx_set_ldcp_param(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ int ret;
+ roce_ldcp_param_s ldcp_param;
+ const struct roce5_ldcp_param *inbuf =
+ (struct roce5_ldcp_param *)buf_in;
+
+ if (in_size < sizeof(struct roce5_ldcp_param)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(struct roce5_ldcp_param));
+ return -ENOMEM;
+ }
+ if (inbuf->cmd_type != ROCE_CMD_SET_LDCP_PARAM) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d)",
+ inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ ldcp_param.dw0.bs.alpha = inbuf->ldcp_param.alpha & 0xf;
+ ldcp_param.dw0.bs.beta = inbuf->ldcp_param.beta & 0xf;
+ ldcp_param.dw0.bs.gamma = inbuf->ldcp_param.gamma & 0xf;
+ ldcp_param.dw0.bs.eta = inbuf->ldcp_param.eta & 0xf;
+ ldcp_param.dw0.bs.set_flag = inbuf->ldcp_param.set_flag & 0x1;
+
+ ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id,
+ (u32 *)(void *)&ldcp_param,
+ sizeof(roce_ldcp_param_s),
+ (u16)ROCE_MPU_CMD_CC_CFG_LDCP_PARAM);
+ if (ret != 0) {
+ ROCE_ERR("Failed to set ldcp param.(ret:%d)", ret);
+ } else {
+ ROCE_WARN("Set ldcp param 0x%x", ldcp_param.dw0.value);
+ }
+ return ret;
+}
+
+int roce5_adm_dfx_port_traffic(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ roce_mpu_port_statics_cmd_inbuf_t inbuf;
+ roce_port_sta_outbuf_t *outbuf = (roce_port_sta_outbuf_t *)buf_out;
+ int ret;
+
+ memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf));
+ inbuf.cmd = *(roce_port_sta_inbuf_t *)buf_in;
+ outbuf->head.status = 0xFF;
+
+ if (in_size < sizeof(roce_port_sta_inbuf_t)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(roce_port_sta_inbuf_t));
+ return -ENOMEM;
+ }
+ if (inbuf.cmd.cmd_type != ROCE_CMD_PORT_TRAFFIC) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d).",
+ inbuf.cmd.cmd_type);
+ return -EINVAL;
+ }
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DFX_PORT_TRAFFIC,
+ (void *)&inbuf, (u16)sizeof(inbuf),
+ buf_out, (u16 *)(void *)out_size);
+ if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) {
+ ROCE_ERR(
+ "Failed to query port traffic, err(%d), out size(0x%x) status(0x%x)",
+ ret, *out_size, outbuf->head.status);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+#ifdef ROCE_CHIP_TEST
+static const roce5_adm_dfx_vnic_waterline_cmd_proc_t
+ g_roce5_adm_dfx_vnic_wl_cmd_proc[] = {
+ { ROCE_CMD_SET_VNIC_WATERLINE, roce5_set_vnic_waterline_state },
+ { ROCE_CMD_GET_VNIC_WATERLINE, roce5_get_vnic_waterline_state },
+ };
+
+int roce5_adm_dfx_vnic_wl_ctrl(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ int ret;
+ struct roce5_vnic_waterline_ctrl_buf *inbuf =
+ (struct roce5_vnic_waterline_ctrl_buf *)buf_in;
+ struct roce5_vnic_waterline_ctrl_buf *outbuf =
+ (struct roce5_vnic_waterline_ctrl_buf *)buf_out;
+ const roce5_adm_generic_cmd_proc_t *generic_array =
+ (const roce5_adm_generic_cmd_proc_t *)
+ g_roce5_adm_dfx_vnic_wl_cmd_proc;
+ int elem_count = sizeof(g_roce5_adm_dfx_vnic_wl_cmd_proc) /
+ sizeof(g_roce5_adm_dfx_vnic_wl_cmd_proc[0]);
+ roce5_adm_dfx_vnic_waterline_ctrl_func_t roce5_adm_dfx_vnic_wl_ctrl_func;
+
+ if (in_size < sizeof(struct roce5_vnic_waterline_ctrl_buf)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size,
+ (int)sizeof(struct roce5_vnic_waterline_ctrl_buf));
+ return -EINVAL;
+ }
+
+ if (inbuf->cmd_type >= VM_COMPAT_TEST) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d)",
+ inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ ret = memset_s(buf_out, sizeof(struct roce5_vnic_waterline_ctrl_buf), 0,
+ sizeof(struct roce5_vnic_waterline_ctrl_buf));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ roce5_adm_dfx_vnic_wl_ctrl_func =
+ (roce5_adm_dfx_vnic_waterline_ctrl_func_t)
+ roce5_adm_generic_find_cmd_func(
+ generic_array, elem_count, inbuf->cmd_type);
+ if (roce5_adm_dfx_vnic_wl_ctrl_func == NULL) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d)",
+ inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ return roce5_adm_dfx_vnic_wl_ctrl_func(rdev, inbuf, outbuf);
+}
+
+#endif /* ROCE_CHIP_TEST */
+
+int roce5_adm_dfx_roce_set(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+
+{
+ roce_dfx_set_cmd_s inbuf;
+ roce_dfx_set_outbuf_s *outbuf = (roce_dfx_set_outbuf_s *)buf_out;
+ int ret;
+
+ if (in_size < sizeof(roce_dfx_set_inbuf_s)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(roce_dfx_set_inbuf_s));
+ return -ENOMEM;
+ }
+
+ memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf));
+ inbuf.cmd = *(roce_dfx_set_inbuf_s *)buf_in;
+ inbuf.cmd.func_id = rdev->glb_func_id;
+
+ if (inbuf.cmd.cmd_type != ROCE_CMD_ROCE_SET) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d).",
+ inbuf.cmd.cmd_type);
+ return -EINVAL;
+ }
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DFX_ROCE_SET,
+ (void *)&inbuf, (u16)sizeof(inbuf), outbuf,
+ (u16 *)(void *)out_size);
+ if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set roce configs, err(%d), out size(0x%x) status(0x%x)",
+ ret, *out_size, outbuf->head.status);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int roce5_adm_dfx_roce_set_debug(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+
+{
+ roce_dfx_set_debug_cmd_s inbuf;
+ roce_dfx_set_debug_outbuf_s *outbuf =
+ (roce_dfx_set_debug_outbuf_s *)buf_out;
+ int ret;
+
+ if (in_size < sizeof(roce_dfx_set_debug_inbuf_s)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size, (int)sizeof(roce_dfx_set_debug_inbuf_s));
+ return -ENOMEM;
+ }
+
+ memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf));
+ inbuf.cmd = *(roce_dfx_set_debug_inbuf_s *)buf_in;
+ inbuf.cmd.func_id = rdev->glb_func_id;
+
+ if (inbuf.cmd.cmd_type != ROCE_CMD_ROCE_SET_DEBUG) {
+ ROCE_DEV_ERR(rdev, "Not support this type(%d).",
+ inbuf.cmd.cmd_type);
+ return -EINVAL;
+ }
+
+ ret = roce5_msg_to_mgmt_sync(rdev->hwdev,
+ ROCE_MPU_CMD_DFX_ROCE_SET_DEBUG,
+ (void *)&inbuf, (u16)sizeof(inbuf), outbuf,
+ (u16 *)(void *)out_size);
+ if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) {
+ ROCE_ERR(
+ "Failed to set roce configs, err(%d), out size(0x%x) status(0x%x)",
+ ret, *out_size, outbuf->head.status);
+ return -EINVAL;
+ }
+ return 0;
+}
+#endif /* __ROCE_DFX__ */
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h
new file mode 100644
index 000000000..139867581
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h
@@ -0,0 +1,378 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved.
+
+
+#ifndef ROCE_DFX_H
+#define ROCE_DFX_H
+
+#include <linux/types.h>
+
+#include "hinic5_rdma.h"
+
+#include "roce_sysfs.h"
+#include "roce.h"
+#include "roce_gid.h"
+#include "roce_compat.h"
+#include "roce_pub_cmd.h"
+#include "roce_npu_cmd_dfx_defs.h"
+#include "roce_debug.h"
+
+#include "securec.h"
+
+#define MR_KEY_2_INDEX_SHIFT 8
+
+#define ROCE_IO_DFX_CFG_VADDR_ID 0
+#define ROCE_IO_DFX_CFG_PADDR_ID 1
+#define ROCE_IO_DFX_CFG_ADDR_NUM 2
+
+#define roce5_dfx_print(fmt, args...) \
+ do { \
+ (void)printk("[roce5_dfx] : " fmt "\n", ##args); \
+ } while (0)
+
+typedef struct tag_roce_cmd_mpt_query {
+ roce_verbs_cmd_com_s com;
+} roce_cmd_mpt_query_s;
+
+typedef struct roce5_mpt_query_outbuf {
+ u32 value[16];
+} roce5_mpt_query_outbuf_s;
+
+typedef struct tag_roce_cmd_roce_cq_query {
+ roce_verbs_cmd_com_s com;
+} roce_cmd_cq_query_s;
+
+typedef struct roce5_cq_query_outbuf {
+ u32 value[32];
+} roce5_cq_query_outbuf_s;
+
+typedef struct tag_roce_cmd_qp_query {
+ roce_verbs_cmd_com_s com;
+} roce_cmd_qp_query_s;
+
+typedef struct roce5_qp_query_outbuf {
+ u32 context[128];
+} roce5_qp_query_outbuf_s;
+
+typedef struct roce5_srq_query_outbuf {
+ u32 value[24];
+} roce5_srq_query_outbuf_s;
+
+struct roce5_dfx_query_inbuf {
+ u32 cmd_type;
+ union {
+ u32 qpn;
+ u32 cqn;
+ u32 srqn;
+ u32 mpt_key;
+ u32 gid_index;
+ struct {
+ u32 qpn;
+ u32 cqn;
+ } query_pi_ci;
+ };
+};
+
+#define ROCE_DEV_NAME_LEN_MAX 64
+
+struct roce5_dfx_dbg_inbuf {
+ u32 cmd_type;
+ union {
+ u32 qpn;
+ u32 srqn;
+ };
+ char dev_name[ROCE_DEV_NAME_LEN_MAX];
+};
+
+#define ROCE_CTX_MAX_SIZE 512 // adaptation lock store referencein of structure
+
+union roce5_dfx_dbg_outbuf {
+ roce_verbs_qp_attr_s qp_attr;
+ roce_verbs_srq_attr_dfx_s srq_attr;
+ u32 buf[ROCE_CTX_MAX_SIZE];
+ u32 algo_type;
+ u32 dev_type;
+};
+
+struct roce5_dfx_common_inbuf {
+ u32 cmd_type;
+ char dev_name[ROCE_DEV_NAME_LEN_MAX];
+};
+
+union roce5_dfx_common_outbuf {
+ u32 dev_type;
+};
+
+struct roce5_dfx_pi_ci {
+ u32 qpc_sq_pi_on_chip;
+ u32 qpc_sq_pi;
+ u32 qpc_sq_load_pi;
+ u32 qpc_rq_pi_on_chip;
+ u32 qpc_rq_load_pi;
+ u32 qpc_rq_pi;
+ u32 qpc_rc_pi;
+ u32 qpc_sq_ci;
+ u32 qpc_sq_wqe_prefetch_ci;
+ u32 qpc_sq_mtt_prefetch_wqe_ci;
+ u32 qpc_sqa_ci;
+ u32 qpc_sqa_wqe_prefetch_ci;
+ u32 qpc_rq_ci;
+ u32 qpc_rq_wqe_prefetch_ci;
+ u32 qpc_rq_mtt_prefetch_wqe_ci;
+ u32 qpc_rq_base_ci;
+ u32 qpc_rc_ci;
+ u32 qpc_rc_prefetch_ci;
+ u32 cq_ci_on_chip;
+ u32 cq_ci;
+ u32 cq_load_ci;
+ u64 cq_ci_record_gpa_at_hop_num;
+ u32 cq_last_solicited_pi;
+ u32 cq_pi;
+ u32 cq_last_notified_pi;
+};
+
+struct roce5_dfx_qp_count {
+ u32 qp_alloced;
+ u32 qp_deleted;
+ u32 qp_alive;
+};
+
+#define ROCE_GID_ENTRY_NUM 17 // 1 host fe + 16 shadow fe
+union roce5_dfx_query_outbuf {
+ /* query cache context */
+ roce_verbs_qp_attr_s qp_attr;
+ roce_verbs_cq_attr_s cq_attr;
+ roce_verbs_srq_attr_s srq_attr;
+ roce_verbs_mr_attr_s mr_attr;
+ /* query host context */
+ roce5_qp_query_outbuf_s qp_query;
+ roce5_cq_query_outbuf_s cq_query;
+ roce5_srq_query_outbuf_s srq_query;
+ struct roce_mpt_context mpt;
+ struct rdma_gid_entry gid_entry[ROCE_GID_ENTRY_NUM];
+ struct roce5_dfx_pi_ci pi_ci;
+ struct roce5_dfx_qp_count qp_count;
+ u32 algo_type;
+ struct roce_uni_cmd_latch_data latch_data_outbuf;
+};
+
+enum roce5_bw_ctrl_cmd_e {
+ ROCE_BW_CTRL_DIS,
+ ROCE_BW_CTRL_EN,
+ ROCE_BW_CTRL_RESET
+};
+
+enum roce5_qos_wl_ctrl_cmd_e {
+ ROCE_QOS_WL_CTRL_CMD_SET,
+ ROCE_QOS_WL_CTRL_CMD_GET
+};
+
+struct rdma_gid_query_outbuf {
+ struct rdma_gid_entry gid_entry[ROCE_GID_ENTRY_NUM];
+};
+
+struct roce5_bw_ctrl_inbuf {
+ u32 cmd_type;
+ struct {
+ u32 cir;
+ u32 pir;
+ u32 cnp;
+ } ctrl_param;
+};
+
+struct roce5_bw_set_inbuf {
+ u32 cmd_type;
+ struct {
+ u32 mod;
+ u32 cir;
+ u32 cbs;
+ u32 pir;
+ u32 pbs;
+ } set_param;
+};
+
+struct roce5_bw_ctrl_param {
+ u8 color_type;
+ u16 ptype;
+ u8 hw_wred_mode;
+
+ u32 cir;
+ u32 pir;
+ u32 cbs;
+ u32 xbs;
+ u32 cnp;
+ u32 enable;
+};
+
+struct roce5_bw_set_param {
+ u16 ptype;
+ u8 color_type;
+ u8 hw_wred_mode;
+
+ u32 cir;
+ u32 cbs;
+ u32 pir;
+ u32 pbs;
+ u32 enable;
+};
+
+struct roce5_ldcp_param {
+ u32 cmd_type;
+ struct {
+ u8 alpha;
+ u8 beta;
+ u8 gamma;
+ u8 eta;
+ u8 set_flag;
+ } ldcp_param;
+};
+
+struct roce5_bw_ctrl_outbuf {
+ struct roce5_bw_ctrl_param bw_ctrl_param;
+};
+
+struct roce5_bw_set_outbuf {
+ struct roce5_bw_set_param bw_set_param;
+};
+
+enum roce5_dfx_io_cmd_type { ROCE_IO_CTRL_DIS, ROCE_IO_CTRL_EN };
+
+struct roce5_dfx_io_alarm {
+ enum roce5_dfx_io_cmd_type en_flag;
+ u16 pf_id;
+ u16 rsvd;
+ u16 io_latency_thd;
+ u16 exec_time;
+ u32 exp_qpn;
+ void *rcd_uaddr;
+ struct timespec64 start;
+ struct mutex io_alarm_mutex;
+};
+
+struct roce5_dfx_io_inbuf {
+ u32 cmd_type;
+ struct roce5_dfx_io_alarm io_alarm;
+};
+
+struct roce5_dfx_io_outbuf {
+ struct roce5_dfx_io_alarm io_alarm;
+};
+
+typedef struct roce5_adm_generic_cmd_proc {
+ u32 cmd_type;
+ void *proc_func;
+} roce5_adm_generic_cmd_proc_t;
+
+typedef int (*roce5_adm_func_t)(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+
+typedef struct roce5_adm_cmd_proc {
+ u32 cmd_type;
+ roce5_adm_func_t proc_func;
+} roce5_adm_cmd_proc_t;
+
+typedef int (*roce5_adm_dfx_dbg_t)(struct roce5_device *rdev,
+ const struct roce5_dfx_dbg_inbuf *inbuf,
+ union roce5_dfx_dbg_outbuf *outbuf);
+
+typedef struct roce5_adm_dfx_dbg_cmd_proc {
+ u32 cmd_type;
+ roce5_adm_dfx_dbg_t proc_func;
+} roce5_adm_dfx_dbg_cmd_proc_t;
+
+typedef int (*roce5_adm_dfx_query_t)(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf);
+
+typedef struct roce5_adm_dfx_query_cmd_proc {
+ u32 cmd_type;
+ roce5_adm_dfx_query_t proc_func;
+} roce5_adm_dfx_query_cmd_proc_t;
+
+typedef int (*roce5_adm_dfx_bw_ctrl_func_t)(struct roce5_device *rdev,
+ struct roce5_bw_ctrl_inbuf *inbuf,
+ struct roce5_bw_ctrl_outbuf *outbuf);
+
+typedef struct roce5_adm_dfx_bw_ctrl_cmd_proc {
+ u32 cmd_type;
+ roce5_adm_dfx_bw_ctrl_func_t proc_func;
+} roce5_adm_dfx_bw_ctrl_cmd_proc_t;
+
+#ifdef ROCE_CHIP_TEST
+typedef struct roce5_vnic_waterline_ctrl_param {
+ u16 red_waterline;
+ u16 yellow_waterline;
+ u32 ip_val;
+} roce5_vnic_waterline_ctrl_param_s;
+
+struct roce5_vnic_waterline_ctrl_buf {
+ u32 cmd_type;
+ roce5_vnic_waterline_ctrl_param_s waterline_param;
+};
+
+typedef int (*roce5_adm_dfx_vnic_waterline_ctrl_func_t)(
+ struct roce5_device *rdev, struct roce5_vnic_waterline_ctrl_buf *inbuf,
+ struct roce5_vnic_waterline_ctrl_buf *outbuf);
+
+typedef struct roce5_adm_dfx_vnic_waterline_cmd_proc {
+ u32 cmd;
+ roce5_adm_dfx_vnic_waterline_ctrl_func_t proc_func;
+} roce5_adm_dfx_vnic_waterline_cmd_proc_t;
+
+int roce5_adm_dfx_vnic_wl_ctrl(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+#endif /* ROCE_CHIP_TEST */
+
+/* and above next is V100 compatible data structure */
+typedef struct tag_roce_qurey_gid {
+ roce_verbs_cmd_com_s com;
+ __be32 port;
+} roce_cmd_query_gid_s;
+
+typedef int (*roce_adm_dfx_attack_func_t)(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf);
+struct roce_dfx_attack_cmd_func {
+ u32 cmd_id;
+ roce_adm_dfx_attack_func_t cmd_func;
+};
+
+DEBUG_REPLACE int roce5_get_drv_version(void *buf_out, u32 *out_size);
+void *
+roce5_adm_generic_find_cmd_func(const roce5_adm_generic_cmd_proc_t *cmd_array,
+ int elem_count, u32 target_cmd);
+int roce5_adm_dfx_query(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+int roce5_adm_dfx_bw_ctrl(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+int roce5_adm_dfx_set_ldcp_param(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+int roce5_adm_dfx_port_traffic(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+int roce5_adm_dfx_dbg(struct roce5_device *rdev, const void *buf_in,
+ void *buf_out, u32 *out_size);
+int roce5_adm_dfx_roce_set(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+int roce5_adm_dfx_roce_set_debug(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+
+int roce5_dfx_attack_func(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size);
+
+void roce5_dfx_clean_up(struct roce5_device *rdev);
+
+void *global_roce5_io_alarm_va_get(void);
+
+void global_roce5_io_alarm_va_set(u64 va);
+
+void global_roce5_io_alarm_pa_set(dma_addr_t pa);
+
+dma_addr_t global_roce5_io_alarm_pa_get(void);
+
+int roce5_dfx_query_cq(struct roce5_device *rdev, u32 cqn, u32 *context,
+ int cqc_size);
+
+int roce5_dfx_cmd_query_srq(struct roce5_device *rdev, u32 srqn,
+ roce_verbs_srq_attr_s *srq_attr);
+
+#endif // __ROCE_DFX_H__
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c
new file mode 100644
index 000000000..84cc1455b
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c
@@ -0,0 +1,828 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved.
+
+
+#include <linux/semaphore.h>
+
+#include "hinic5_hw.h"
+#include "hinic5_hwdev.h"
+#include "hisdk5_hwif.h"
+#include "hinic5_mt.h"
+#include "rdma_comp.h"
+#include "roce.h"
+#include "roce_qp.h"
+#include "roce_infra.h"
+#include "roce_mr.h"
+#include "roce_dfx.h"
+#include "roce_kernel_compat_gen.h"
+
+#define DB_BUFF_SIZE 2048
+#define DWQE_SIZE 256
+#define DWQE_TYPE_NUM 4
+#define ALIGN_UP_8B(length) (((length) + 7) >> 3)
+#define ALIGN_DOWN_8B(offset) ((offset)&0xfffffff8)
+#define IS_OFFSET_8B_ALIGN(offset) (((offset)&7) != 0)
+#define UPPER_HALF_BIT_OFFSET 32
+#define U32S_PER_U64 2
+
+static void roce5_copy_mask_attack_value_to_addr(u32 *addr, u32 attack_value,
+ u32 mask)
+{
+ u32 val;
+
+ val = *addr;
+ val &= ~mask;
+ val |= (attack_value & mask);
+ *addr = val;
+}
+
+static void roce5_copy_mask_attack_value_to_u64_addr(u64 *addr,
+ u32 attack_value, u32 mask,
+ bool is_upper_half)
+{
+ u64 val;
+
+ val = *addr;
+
+ if (is_upper_half) {
+ val &= ~((u64)mask << UPPER_HALF_BIT_OFFSET);
+ val |= ((u64)(attack_value & mask) << UPPER_HALF_BIT_OFFSET);
+ } else {
+ val &= ~mask;
+ val |= (attack_value & mask);
+ }
+
+ *addr = val;
+}
+
+static int roce5_copy_data_to_page(struct roce5_device *rdev, const u32 *val,
+ struct ib_umem *umem, u32 copy_len,
+ u32 offset, u32 attack_len, u32 mask)
+{
+ struct scatterlist *sg = NULL;
+ u32 *inbuf_tmp = NULL;
+ u32 sg_len;
+ u32 page_copy_len = 0;
+ u32 copied_len = 0;
+ u32 sgl_offset = 0;
+ u64 buf_offset = 0;
+ void *va = NULL;
+ u32 i = 0;
+ int ret;
+
+ inbuf_tmp = (u32 *)kzalloc(PAGE_SIZE, GFP_KERNEL);
+ if (inbuf_tmp == NULL) {
+ ROCE_ERR("Failed to alloc inbuf temp.");
+ return -ENOMEM;
+ }
+
+ if (mask == 0) {
+ for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) {
+ inbuf_tmp[i] = val[i % attack_len];
+ }
+ }
+
+ ROCE_INFO(
+ "copy_data_to_page va:%lx, length:%lu, offset:%u, copy_len:%u",
+ umem->address, umem->length, offset, copy_len);
+
+#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE)
+ for_each_sg(umem->sgt_append.sgt.sgl, sg, umem->sgt_append.sgt.nents, i)
+#else
+ //scatterlist of traverse use nents, orig_nents >= nents >= nmap
+ for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, i)
+#endif
+ {
+ sg_len = sg->length;
+ if (buf_offset + sg_len <=
+ offset) { //not found offset of scatterlist, accumulate sg_len, continue
往 next find
+ buf_offset += sg_len;
+ } else { //found offset of scatterlist
+ sgl_offset =
+ offset -
+ (u32)buf_offset; //get based on current scatterlist of offset
+ buf_offset =
+ offset; //found current page after, later of page of offset all max at offset
+
+ va = sg_virt(sg);
+ if (va == NULL) {
+ ROCE_ERR("Failed to get va");
+ kfree(inbuf_tmp);
+ return -ENOMEM;
+ }
+
+ va += sgl_offset;
+ page_copy_len = ((copy_len - copied_len) <
+ (sg_len - sgl_offset)) ?
+ (copy_len - copied_len) :
+ (sg_len - sgl_offset);
+
+ if (mask != 0) {
+ roce5_copy_mask_attack_value_to_addr(
+ (u32 *)va, val[0], mask);
+ ROCE_INFO(
+ "copy mask data to buf, buf_offset:%llu, sgl_offset:%u, mask_val:%u\n",
+ buf_offset, sgl_offset,
+ (val[0] & mask));
+ break;
+ }
+
+ ret = memcpy_s(va, page_copy_len, inbuf_tmp,
+ page_copy_len);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to memcpy attack value, ret(%d)",
+ ret);
+ kfree(inbuf_tmp);
+ return -EFAULT;
+ }
+
+ ROCE_INFO(
+ "copy_data_from_page buf_offset:%llu, sgl_offset:%u, copied_len:%u, page_copy_len:%u, i:%u\n",
+ buf_offset, sgl_offset, copied_len,
+ page_copy_len, i);
+
+ copied_len += page_copy_len;
+ if (copied_len >= copy_len) {
+ break;
+ }
+ }
+ }
+
+ kfree(inbuf_tmp);
+ return 0;
+}
+
+static void roce5_copy_attack_value_to_addr(u64 *addr, u64 attack_value,
+ u32 length)
+{
+ int i;
+ int count = ALIGN_UP_8B(length);
+
+ for (i = 0; i < count; i++) {
+ wmb();
+ *((u64 *)addr + i) = attack_value;
+ wmb();
+ }
+}
+
+static int roce5_dfx_attack_sqdb_hard(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_ucontext *ucontext = NULL;
+ u64 db_phy_start;
+ void __iomem *db_virt_start = NULL;
+ void __iomem *db_virt_offset = NULL;
+ void *hwdev = NULL;
+ struct roce5_qp *rqp = NULL;
+ u32 qpn = inbuf->qpn;
+ union roce_sq_db sq_db;
+ u32 db_offset;
+ u32 db_8B_offset;
+ bool is_upper_half;
+ u32 length;
+
+ hwdev = rdev->hwdev;
+ if (hwdev == NULL) {
+ ROCE_ERR("Roce hwdev is NULL");
+ return -ENOMEM;
+ }
+
+ ucontext = rdev->ucontext;
+ if (ucontext == NULL) {
+ ROCE_ERR("Roce device context is null");
+ return -ENOMEM;
+ }
+
+ db_offset = inbuf->offset;
+ if (db_offset >= DB_BUFF_SIZE) {
+ ROCE_ERR(
+ "roce attack sqdb_hard offset is too large, should be less than %u",
+ DB_BUFF_SIZE);
+ return -EINVAL;
+ }
+
+ db_8B_offset = ALIGN_DOWN_8B(db_offset);
+ db_phy_start = ucontext->db_dma_addr;
+ db_virt_start = (db_phy_start -
+ (((struct hinic5_hwdev *)hwdev)->hwif)->db_base_phy) +
+ (((struct hinic5_hwdev *)hwdev)->hwif)->db_base;
+ db_virt_offset = db_virt_start + db_8B_offset;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ if (inbuf->mask != 0) {
+ is_upper_half = IS_OFFSET_8B_ALIGN(db_offset);
+ roce5_copy_mask_attack_value_to_u64_addr((u64 *)db_virt_offset,
+ inbuf->attack_value[0],
+ inbuf->mask,
+ is_upper_half);
+ ROCE_INFO("roce attack sqdb, mask_val:0x%x, offset:%u",
+ (inbuf->attack_value[0] & inbuf->mask), db_offset);
+ } else {
+ sq_db.sq_db_val = 0;
+ sq_db.sq_db_seg.dw0.value =
+ roce5_convert_be32(inbuf->attack_value[0]);
+ sq_db.sq_db_seg.dw1.value =
+ roce5_convert_be32(inbuf->attack_value[1]);
+
+ length = (db_offset + inbuf->length > DB_BUFF_SIZE) ?
+ (DB_BUFF_SIZE - db_offset) :
+ inbuf->length;
+ roce5_copy_attack_value_to_addr((u64 *)db_virt_offset,
+ sq_db.sq_db_val, length);
+
+ ROCE_INFO("roce attack sqdb length:%u, offset:%u", length,
+ db_offset);
+ ROCE_INFO("roce attack sqdb type:%u, cos:%u, qpn:%u\n",
+ sq_db.sq_db_seg.dw0.bs.type,
+ sq_db.sq_db_seg.dw0.bs.cos,
+ sq_db.sq_db_seg.dw0.bs.qpn);
+ ROCE_INFO("roce attack sqdb mtu_shift:%u, pi:%u\n",
+ sq_db.sq_db_seg.dw1.bs.mtu_shift,
+ sq_db.sq_db_seg.dw1.bs.pi);
+ }
+
+ roce5_rqp_put(rqp);
+ return 0;
+}
+
+static int roce5_copy_attack_value_to_dwqe_addr(u64 *addr, const u32 *val,
+ u32 length, u32 attack_len)
+{
+ int i;
+ int count = ALIGN_UP_8B(length);
+ u64 *inbuf_tmp = NULL;
+ u32 step = 2;
+
+ inbuf_tmp = (u64 *)kzalloc(DWQE_SIZE, GFP_KERNEL);
+ if (inbuf_tmp == NULL) {
+ ROCE_ERR("Failed to alloc inbuf temp.");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < count * U32S_PER_U64; i += step) {
+ inbuf_tmp[i / step] = ((u64)val[(i + 1) % attack_len]
+ << UPPER_HALF_BIT_OFFSET) |
+ val[i % attack_len];
+ }
+
+ for (i = 0; i < count; i++) {
+ wmb();
+ *((u64 *)addr + i) = inbuf_tmp[i];
+ wmb();
+ }
+
+ kfree(inbuf_tmp);
+ return 0;
+}
+
+static int roce5_dfx_attack_dwqe(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_ucontext *ucontext = NULL;
+ u64 dwqe_phy_start;
+ void __iomem *dwqe_virt_start = NULL;
+ void __iomem *dwqe_virt_offset = NULL;
+ void *hwdev = NULL;
+ struct roce5_qp *rqp = NULL;
+ u32 qpn = inbuf->qpn;
+ u32 dwqe_offset;
+ u32 length;
+ u32 dwqe_8B_offset;
+ bool is_upper_half;
+ int i;
+ int ret;
+
+ hwdev = rdev->hwdev;
+ if (hwdev == NULL) {
+ ROCE_ERR("Roce hwdev is NULL");
+ return -ENOMEM;
+ }
+
+ ucontext = rdev->ucontext;
+ if (ucontext == NULL) {
+ ROCE_ERR("Roce device context is null");
+ return -ENOMEM;
+ }
+
+ dwqe_offset = inbuf->offset;
+ if (dwqe_offset >= DWQE_SIZE) {
+ ROCE_ERR(
+ "roce attack dwqe offset is too large, should be less than %u",
+ DWQE_SIZE);
+ return -EINVAL;
+ }
+
+ dwqe_phy_start = ucontext->dwqe_dma_addr;
+ //dwqe physical base address and virtual base address all is based on db perform offset of offset, so corresponding when at db of va basebase on to before offset offset
+ dwqe_virt_start =
+ (dwqe_phy_start -
+ (((struct hinic5_hwdev *)hwdev)->hwif)->db_base_phy) +
+ (((struct hinic5_hwdev *)hwdev)->hwif)->db_base;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ length = (dwqe_offset + inbuf->length > DWQE_SIZE) ?
+ (DWQE_SIZE - dwqe_offset) :
+ (inbuf->length);
+ dwqe_8B_offset = ALIGN_DOWN_8B(dwqe_offset);
+ for (i = 0; i < DWQE_TYPE_NUM; i++) {
+ dwqe_virt_offset =
+ dwqe_virt_start + i * DWQE_SIZE + dwqe_8B_offset;
+ if (inbuf->mask != 0) {
+ is_upper_half = IS_OFFSET_8B_ALIGN(dwqe_offset);
+ roce5_copy_mask_attack_value_to_u64_addr(
+ (u64 *)dwqe_virt_offset, inbuf->attack_value[0],
+ inbuf->mask, is_upper_half);
+ } else {
+ ret = roce5_copy_attack_value_to_dwqe_addr(
+ (u64 *)dwqe_virt_offset, inbuf->attack_value,
+ length, inbuf->attack_len);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to copy attack value to dwqe, ret(%d)",
+ ret);
+ return -EFAULT;
+ }
+ }
+ }
+
+ if (inbuf->mask != 0) {
+ ROCE_INFO("roce attack dwqe, mask_val:0x%x, offset:%u",
+ (inbuf->attack_value[0] & inbuf->mask), dwqe_offset);
+ } else {
+ ROCE_INFO("roce attack dwqe, length:%u, offset:%u", length,
+ dwqe_offset);
+ }
+
+ roce5_rqp_put(rqp);
+ return 0;
+}
+
+static int roce5_dfx_attack_sqwqe(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_qp *rqp = NULL;
+ int ret = 0;
+ u32 qpn = inbuf->qpn;
+ u32 sq_buf_sz;
+ u32 length;
+ u32 sq_wqe_offset;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ sq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->sq.wqebb_cnt
+ << (unsigned)rqp->sq.wqe_shift,
+ PAGE_SIZE);
+ sq_wqe_offset = inbuf->offset;
+ if (sq_wqe_offset >= sq_buf_sz) {
+ ROCE_ERR(
+ "roce attack sq wqe offset is too large, should be less than %u",
+ sq_buf_sz);
+ ret = -EINVAL;
+ goto sq_wqe_err;
+ }
+
+ length = (sq_wqe_offset + inbuf->length > sq_buf_sz) ?
+ (sq_buf_sz - sq_wqe_offset) :
+ inbuf->length;
+ ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, rqp->umem,
+ length, sq_wqe_offset, inbuf->attack_len,
+ inbuf->mask);
+ if (ret != 0) {
+ ROCE_ERR("Failed to copy to sq page, qpn:0x%x", qpn);
+ }
+
+sq_wqe_err:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int roce5_dfx_attack_rqwqe(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_qp *rqp = NULL;
+ int ret = 0;
+ u32 qpn = inbuf->qpn;
+ u32 rq_buf_sz;
+ u32 length;
+ u32 rq_attack_offset;
+ u32 rq_wqe_offset;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ rq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->rq.wqebb_cnt
+ << (unsigned)rqp->rq.wqe_shift,
+ PAGE_SIZE);
+ rq_attack_offset = inbuf->offset;
+ if (rq_attack_offset >= rq_buf_sz) {
+ ROCE_ERR(
+ "roce attack rq wqe offset is too large, should be less than %u",
+ rq_buf_sz);
+ ret = -EINVAL;
+ goto rq_wqe_err;
+ }
+
+ length = (rq_attack_offset + inbuf->length > rq_buf_sz) ?
+ (rq_buf_sz - rq_attack_offset) :
+ inbuf->length;
+ rq_wqe_offset = rqp->rq.offset + rq_attack_offset;
+ ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, rqp->umem,
+ length, rq_wqe_offset, inbuf->attack_len,
+ inbuf->mask);
+ if (ret != 0) {
+ ROCE_ERR("Failed to copy to rq page, qpn:0x%x", qpn);
+ }
+
+rq_wqe_err:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int roce5_dfx_attack_mtt(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct mtt *roce_mtt)
+{
+ struct hmm_comp_priv *comp_priv = NULL;
+ u64 *mtts = NULL;
+ u64 *mtts_offset = NULL;
+ u64 page_size;
+ u32 mtt_offset;
+ u32 length;
+ u64 val = ((u64)inbuf->attack_value[1] << 32) |
+ (u64)inbuf->attack_value[0];
+ u32 mtt_8B_offset;
+ bool is_upper_half;
+
+ if (roce_mtt == NULL) {
+ ROCE_ERR("roce qp mtt is null");
+ return -ENOMEM;
+ }
+
+ if (roce_mtt->mtt_vaddr == NULL) {
+ ROCE_ERR("roce mtt vaddr is null, layers:%u",
+ roce_mtt->mtt_layers);
+ return -ENOMEM;
+ }
+
+ comp_priv = get_hmm_comp_priv(rdev->hwdev, SERVICE_T_ROCE);
+ if (comp_priv == NULL) {
+ ROCE_ERR("Comp_priv is null");
+ return -ENOMEM;
+ }
+
+ mtts = (u64 *)hmm_em_table_find(
+ &comp_priv->mtt_em_table,
+ roce_mtt->mtt_seg[roce_mtt->mtt_layers - 1]->offset, NULL);
+ if (mtts == NULL) {
+ ROCE_ERR("Can't find va and pa of mtt entry");
+ return -ENOMEM;
+ }
+
+ page_size = 1L << roce_mtt->mtt_page_shift;
+ mtt_offset = inbuf->offset;
+ if (mtt_offset >= page_size) {
+ ROCE_ERR(
+ "roce attack mtt offset is too large, should be less than %llu",
+ page_size);
+ return -EINVAL;
+ }
+
+ mtt_8B_offset = ALIGN_DOWN_8B(mtt_offset);
+ mtts_offset = mtts + mtt_8B_offset / sizeof(u64);
+ if (inbuf->mask != 0) {
+ is_upper_half = IS_OFFSET_8B_ALIGN(mtt_offset);
+ roce5_copy_mask_attack_value_to_u64_addr(mtts_offset,
+ inbuf->attack_value[0],
+ inbuf->mask,
+ is_upper_half);
+ ROCE_INFO("roce attack mtt, mask_val:0x%x, offset:%u\n",
+ (inbuf->attack_value[0] & inbuf->mask), mtt_offset);
+ } else {
+ length = (inbuf->length + mtt_offset > page_size) ?
+ (page_size - mtt_offset) :
+ inbuf->length;
+ roce5_copy_attack_value_to_addr(mtts_offset, val, length);
+ ROCE_INFO("roce attack mtt, attack_len:%u, offset:%u", length,
+ mtt_offset);
+ }
+
+ return 0;
+}
+
+static int roce5_dfx_attack_cmtt(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_qp *rqp = NULL;
+ int ret = 0;
+ u32 qpn = inbuf->qpn;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ ret = roce5_dfx_attack_mtt(rdev, inbuf, &(rqp->mtt));
+ if (ret != 0) {
+ ROCE_ERR("Failed to attack cmtt");
+ }
+
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+struct roce5_mr *roce5_mr_find(struct roce5_device *rdev, u32 key_id)
+{
+ struct roce5_mr *mr = NULL;
+
+ mutex_lock(&rdev->device_res->mr_list.node_lock);
+ list_for_each_entry(mr, &rdev->device_res->mr_list.node, node) {
+ if (mr->rdmamr.mpt.mpt_index == key_id) {
+ mutex_unlock(&rdev->device_res->mr_list.node_lock);
+ ROCE_INFO("Find mr, it's mpt index is %u",
+ mr->rdmamr.mpt.mpt_index);
+ return mr;
+ }
+ }
+
+ mutex_unlock(&rdev->device_res->mr_list.node_lock);
+ ROCE_ERR("Don't find correct mr, key_id:%u", key_id);
+
+ return NULL;
+}
+
+static int roce5_dfx_attack_dmtt(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_mr *mr = NULL;
+ int ret = 0;
+
+ mr = roce5_mr_find(rdev, inbuf->mpt_index);
+ if (mr == NULL) {
+ ROCE_ERR(
+ "Failed to attack dmtt, don't find valid mr, mpt_index:%u",
+ inbuf->mpt_index);
+ return -EINVAL;
+ }
+
+ ret = roce5_dfx_attack_mtt(rdev, inbuf, &(mr->rdmamr.mtt));
+ if (ret != 0) {
+ ROCE_ERR("Failed to attack dmtt");
+ }
+
+ return ret;
+}
+
+static void roce5_attack_value_to_rdmarc(u32 *vaddr, u32 length, const u32 *val,
+ u32 attack_len)
+{
+ int cnt = (length + 3) >> 2;
+ int i;
+
+ for (i = 0; i < cnt; i++) {
+ vaddr[i] = val[i % attack_len];
+ }
+}
+
+static int roce5_dfx_attack_rdmarc(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_qp *rqp = NULL;
+ u32 qpn = inbuf->qpn;
+ struct rdma_rdmarc *rdmarc = NULL;
+ void *rdmarc_vaddr = NULL;
+ u32 *rdmarc_offset_vaddr = NULL;
+ u32 rdmarc_entry_num;
+ u32 rdmarc_offset;
+ u32 rdmarc_size;
+ u32 length;
+ int ret = 0;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ rdmarc = &(rqp->rdmarc);
+ rdmarc_vaddr = rdmarc->vaddr;
+ if (rdmarc_vaddr == NULL) {
+ ROCE_ERR("Rdmarc vaddr is null");
+ ret = -ENOMEM;
+ goto rdmarc_err;
+ }
+
+ rdmarc_entry_num = (1 << rdmarc->order);
+ rdmarc_size = rdmarc_entry_num * ROCE_RDMARC_ENTRY_SZ;
+ rdmarc_offset = inbuf->offset;
+ if (rdmarc_offset >= rdmarc_size) {
+ ROCE_ERR(
+ "roce attack rdmarc offset is too large, should be less than %u",
+ rdmarc_size);
+ ret = -EINVAL;
+ goto rdmarc_err;
+ }
+
+ rdmarc_offset_vaddr = (u32 *)((u8 *)rdmarc_vaddr + rdmarc_offset);
+
+ if (inbuf->mask != 0) {
+ roce5_copy_mask_attack_value_to_addr(rdmarc_offset_vaddr,
+ inbuf->attack_value[0],
+ inbuf->mask);
+ ROCE_INFO("roce attack rdmarc, mask_val:0x%x, offset:%u",
+ (inbuf->attack_value[0] & inbuf->mask),
+ rdmarc_offset);
+ } else {
+ length = (rdmarc_offset + inbuf->length > rdmarc_size) ?
+ (rdmarc_size - rdmarc_offset) :
+ inbuf->length;
+ roce5_attack_value_to_rdmarc(rdmarc_offset_vaddr, length,
+ inbuf->attack_value,
+ inbuf->attack_len);
+ ROCE_INFO("roce attack rdmarc, attack_length:%u, offset:%u",
+ length, rdmarc_offset);
+ }
+
+rdmarc_err:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int roce5_dfx_attack_cq(struct roce5_cq *cq, struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf)
+{
+ u32 ret;
+ u32 cq_attack_offset;
+ u32 cq_buf_sz;
+ u32 length;
+ u32 cq_depth = cq->cqe_mask + 1;
+ u32 cqe_size = rdev->rdma_cap.cqe_size;
+
+ cq_buf_sz = cqe_size * cq_depth;
+ if (inbuf->offset >= cq_buf_sz) {
+ ROCE_ERR(
+ "Roce attack cq cqe offset is too large, should be less than %u",
+ cq_buf_sz);
+ return -EINVAL;
+ }
+
+ cq_attack_offset = inbuf->offset;
+ length = ((cq_attack_offset + inbuf->length) > cq_buf_sz) ?
+ (cq_buf_sz - cq_attack_offset) :
+ inbuf->length;
+ ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, cq->umem,
+ length, cq_attack_offset,
+ inbuf->attack_len, inbuf->mask);
+ if (ret != 0) {
+ ROCE_ERR("Failed to copy to cq page");
+ }
+
+ return ret;
+}
+
+static int roce5_dfx_attack_cqe(struct roce5_device *rdev,
+ const struct roce_attack_inbuf *inbuf,
+ struct roce_attack_outbuf *outbuf)
+{
+ struct roce5_qp *rqp = NULL;
+ int ret = 0;
+ u32 qpn = inbuf->qpn;
+ struct roce5_cq *send_cq = NULL;
+ struct roce5_cq *recv_cq = NULL;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn);
+ return -EINVAL;
+ }
+
+ roce5_get_cqs(rqp, &send_cq, &recv_cq);
+ if (send_cq != NULL) {
+ ret = roce5_dfx_attack_cq(send_cq, rdev, inbuf);
+ if (ret != 0) {
+ ROCE_ERR("Failed to attack send_cq, qpn:0x%x", qpn);
+ goto cq_wqe_err;
+ }
+ }
+
+ if (recv_cq != NULL) {
+ ret = roce5_dfx_attack_cq(recv_cq, rdev, inbuf);
+ if (ret != 0) {
+ ROCE_ERR("Failed to attack recv_cq, qpn:0x%x", qpn);
+ goto cq_wqe_err;
+ }
+ }
+
+cq_wqe_err:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static struct roce_dfx_attack_cmd_func g_roce_attack_cmd_funcs[] = {
+ { ROCE_ATTACK_TYPE_SQDB_HARD, roce5_dfx_attack_sqdb_hard },
+ { ROCE_ATTACK_TYPE_DWQE, roce5_dfx_attack_dwqe },
+ { ROCE_ATTACK_TYPE_SQWQE, roce5_dfx_attack_sqwqe },
+ { ROCE_ATTACK_TYPE_RQWQE, roce5_dfx_attack_rqwqe },
+ { ROCE_ATTACK_TYPE_CMTT, roce5_dfx_attack_cmtt },
+ { ROCE_ATTACK_TYPE_DMTT, roce5_dfx_attack_dmtt },
+ { ROCE_ATTACK_TYPE_RDMARC, roce5_dfx_attack_rdmarc },
+ { ROCE_ATTACK_TYPE_CQE, roce5_dfx_attack_cqe },
+};
+
+void roce5_cpu_to_be32(void *data, u32 len)
+{
+ u32 chunk_sz = sizeof(u32);
+ u32 *mem = (u32 *)data;
+ u32 data_len;
+ u32 i;
+
+ if (data == NULL)
+ return;
+
+ data_len = len / chunk_sz;
+
+ for (i = 0; i < data_len; i++) {
+ *mem = cpu_to_be32(*mem);
+ mem++;
+ }
+}
+
+int roce5_dfx_attack_func(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ struct roce_attack_inbuf *inbuf = (struct roce_attack_inbuf *)buf_in;
+ struct roce_attack_outbuf *outbuf =
+ (struct roce_attack_outbuf *)buf_out;
+ roce_adm_dfx_attack_func_t func = NULL;
+ u32 cmd_type;
+ u32 i;
+
+ if ((inbuf == NULL) || (outbuf == NULL)) {
+ ROCE_ERR("Roce dfx attack buf in or buf out is null.");
+ return -EINVAL;
+ }
+
+ /* check in and out buf size */
+ if (in_size < (u32)sizeof(struct roce_attack_inbuf)) {
+ ROCE_ERR("roce dfx attack buffer in size too small");
+ return -EINVAL;
+ }
+ if (*out_size < (u32)sizeof(struct roce_attack_outbuf)) {
+ ROCE_ERR("roce dfx attack buffer out size too small");
+ return -EINVAL;
+ }
+
+ *out_size = 0;
+
+ if (inbuf->convert_endian != 0)
+ roce5_cpu_to_be32(&(inbuf->attack_value),
+ sizeof(inbuf->attack_value));
+
+ for (i = 0; i < sizeof(g_roce_attack_cmd_funcs) /
+ sizeof(g_roce_attack_cmd_funcs[0]);
+ i++) {
+ cmd_type = g_roce_attack_cmd_funcs[i].cmd_id;
+ if (cmd_type == inbuf->attack_type) {
+ func = g_roce_attack_cmd_funcs[i].cmd_func;
+ if (func == NULL) {
+ ROCE_ERR(
+ "function of attack cmd(0x%x) is not supported.",
+ cmd_type);
+ return -EINVAL;
+ }
+
+ *out_size = sizeof(struct roce_attack_outbuf);
+
+ return func(rdev, inbuf, outbuf);
+ }
+ }
+
+ ROCE_ERR("attack cmd(0x%x) does't exist.", cmd_type);
+
+ return -EINVAL;
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c
new file mode 100644
index 000000000..371fa3211
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c
@@ -0,0 +1,35 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include <linux/semaphore.h>
+#include "hinic5_mt.h"
+#include "securec.h"
+#include "roce_infra.h"
+#include "roce.h"
+
+int roce5_get_drv_version(void *buf_out, u32 *out_size)
+{
+ struct drv_version_info *ver_info = buf_out;
+ int rc;
+
+ if (!buf_out) {
+ ROCE_ERR("Buf_out is NULL.");
+ return -EINVAL;
+ }
+
+ if (*out_size != sizeof(*ver_info)) {
+ ROCE_ERR("Unexpect out buf size from user :%u, expect: %lu",
+ *out_size, sizeof(*ver_info));
+ return -EINVAL;
+ }
+
+ rc = snprintf_s(ver_info->ver, sizeof(ver_info->ver),
+ sizeof(ver_info->ver) - 1, "%s %s(debug)",
+ HIROCE5_DRV_VERSION, __TIME_STR__);
+ if (rc == -1) {
+ ROCE_ERR("Snprintf roce version err");
+ return -EFAULT;
+ }
+
+ return 0;
+}
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c
new file mode 100644
index 000000000..ef4b2da57
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c
@@ -0,0 +1,851 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved.
+
+
+#ifdef __ROCE_DFX__
+
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/kthread.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "comm_defs.h"
+#include "hinic5_hw.h"
+#include "hinic5_mt.h"
+
+#include "roce_compat.h"
+#include "roce.h"
+#include "roce_dfx.h"
+#include "roce_srq.h"
+#include "roce_qp.h"
+#include "roce_cq.h"
+#include "roce_cmd.h"
+#include "roce_mix.h"
+#include "roce_infra.h"
+#include "roce_npu_cmd.h"
+#include "roce_cqm_cmd.h"
+#include "roce_restore.h"
+#include "roce_main_extension.h"
+#include "roce_com.h"
+#include "securec.h"
+#include "hmm_common.h"
+#include "roce_npu_cmd_gid_defs.h"
+
+static int roce5_dfx_get_dev_algo(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ u32 cc_algo;
+ struct roce5_ecn_ctx ecn_ctx;
+ u32 *algo_type = &outbuf->algo_type;
+
+ if (rdev == NULL) {
+ ROCE_ERR("Failed to get roce device.");
+ return -EINVAL;
+ }
+ ecn_ctx = rdev->ecn_ctx;
+ cc_algo = ecn_ctx.cc_algo;
+ *algo_type = cc_algo;
+
+ return 0;
+}
+
+static int roce5_dfx_get_qp_count(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ struct roce5_dfx_qp_count *qp_count = &outbuf->qp_count;
+
+ qp_count->qp_alloced = rdev->qp_cnt.alloc_qp_cnt;
+ qp_count->qp_deleted = rdev->qp_cnt.del_qp_cnt;
+ qp_count->qp_alive =
+ rdev->qp_cnt.alloc_qp_cnt - rdev->qp_cnt.del_qp_cnt;
+
+ return 0;
+}
+
+/* QPC is statically allocated, so it is not necessary to judge whether it exists when querying the cache content */
+static int roce5_dfx_query_cache_qpc(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ int ret;
+ u32 qpn = inbuf->qpn;
+ roce_verbs_qp_attr_s *qp_attr = &outbuf->qp_attr;
+
+ ret = memset_s(qp_attr, sizeof(roce_verbs_qp_attr_s), 0,
+ sizeof(roce_verbs_qp_attr_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)qp_attr,
+ sizeof(roce_verbs_qp_attr_s));
+ if (ret != 0) {
+ roce5_dfx_print("Failed to query QPC from cache!");
+ roce5_dfx_print(
+ "******************From Cache: qpn(%#x) ********************",
+ qpn);
+ roce5_dfx_print(
+ ">>>>>>>>>>>>>>>> QUERY QPC FROM CACHE FAILED <<<<<<<<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Cache: qpn(%#x) ********************",
+ qpn);
+
+ return ret;
+ }
+
+ return 0;
+}
+
+static int roce5_dfx_query_host_qpc(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ struct roce5_qp *rqp = NULL;
+ int ret = 0;
+ u32 qpn = inbuf->qpn;
+ roce5_qp_query_outbuf_s *qp_query = &outbuf->qp_query;
+
+ rqp = roce5_get_rqp_by_qpn(rdev, qpn);
+ if (rqp == NULL) {
+ roce5_dfx_print("Failed to get roce qp.");
+ roce5_dfx_print(
+ "******************From Host: qpn(%#x) ********************",
+ qpn);
+ roce5_dfx_print(
+ ">>>>>>>>>>> QUERY QPC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Host: qpn(%#x) ********************",
+ qpn);
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(qp_query, sizeof(roce5_qp_query_outbuf_s),
+ rqp->qpc_info->vaddr, sizeof(roce5_qp_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy qp_ctx, ret(%d).", ret);
+ goto err_out;
+ }
+
+err_out:
+ roce5_rqp_put(rqp);
+ return ret;
+}
+
+static int roce5_dfx_uni_cmd_query_cq(struct roce5_device *rdev, u32 cqn,
+ u32 *verbs_cq_attr, int cq_attr_size)
+{
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_cq_query_s *cq_query_inbuf = NULL;
+ roce_uni_cmd_cq_query_outbuf_s *cq_query_outbuf = NULL;
+ struct roce5_cq *rcq = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_cq_query_s), &cqm_cmd_outbuf,
+ (u16)sizeof(roce_uni_cmd_cq_query_outbuf_s));
+ if (ret != 0) {
+ roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d",
+ ret);
+ return ret;
+ }
+
+ rcq = roce5_get_rcq_by_cqn(rdev, cqn);
+ if (rcq == NULL) {
+ roce5_dfx_print("Failed to get roce cq.");
+ goto err_rcq;
+ }
+ cq_query_inbuf = (roce_uni_cmd_cq_query_s *)cqm_cmd_inbuf->buf;
+ cq_query_inbuf->com.index = cpu_to_be32(cqn);
+ cq_query_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK));
+ if (rcq->shadow != 0) {
+ cq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ cq_query_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(rcq->shadow_vfid);
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_QUERY_CQ,
+ cqm_cmd_inbuf, cqm_cmd_outbuf, NULL,
+ ROCE_CMD_TIME_CLASS_B, HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ roce5_dfx_print("Failed to send cmd QUERY_CQ, ret=%d", ret);
+ ret = -1;
+ goto out;
+ }
+
+ cq_query_outbuf = (roce_uni_cmd_cq_query_outbuf_s *)cqm_cmd_outbuf->buf;
+ ret = memcpy_s((void *)verbs_cq_attr, (size_t)cq_attr_size,
+ (void *)&cq_query_outbuf->cq_info.context[0],
+ (size_t)cq_attr_size);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy verbs_cq_attr.(ret:%d)", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+out:
+ roce5_rcq_put(rcq);
+err_rcq:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+
+ return ret;
+}
+
+int roce5_dfx_query_cq(struct roce5_device *rdev, u32 cqn, u32 *context,
+ int cqc_size)
+{
+ if (ROCE5_SUPPORT_DFX_CMD_QUERY_CQ_MSG_V2(rdev) != 0) {
+ return roce5_dfx_uni_cmd_query_cq(rdev, cqn, context, cqc_size);
+ }
+ return -EINVAL;
+}
+
+/* CQC is dynamic allocation. When querying cache content, you need to determine whether it exists or not. */
+static int roce5_dfx_query_cache_cqc(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ int ret;
+ cqm_object_s *cqm_obj_cq = NULL;
+ u32 cqn = inbuf->cqn;
+ roce5_cq_query_outbuf_s *cq_query = &outbuf->cq_query;
+
+ cqm_obj_cq =
+ cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false);
+ if (cqm_obj_cq == NULL) {
+ roce5_dfx_print("Failed to get cqm_obj_cq.");
+ roce5_dfx_print(
+ "******************From Cache: cqn(%#x) ********************",
+ cqn);
+ roce5_dfx_print(
+ ">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Cache: cqn(%#x) ********************",
+ cqn);
+
+ return -EINVAL;
+ }
+
+ cqm5_object_put(cqm_obj_cq);
+
+ ret = memset_s(cq_query, sizeof(roce5_cq_query_outbuf_s), 0,
+ sizeof(roce5_cq_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset cq_context.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)cq_query,
+ sizeof(roce5_cq_query_outbuf_s));
+ if (ret != 0) {
+ roce5_dfx_print("Failed to query cq from cache!");
+ roce5_dfx_print(
+ "******************From Cache: cqn(%#x) ********************",
+ cqn);
+ roce5_dfx_print(
+ ">>>>>>>>>>>>>>>> QUERY CQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Cache: cqn(%#x) ********************",
+ cqn);
+
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_dfx_query_host_cqc(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ struct roce5_cq *rcq = NULL;
+ int ret = 0;
+ u32 cqn = inbuf->cqn;
+ roce5_cq_query_outbuf_s *cq_query = &outbuf->cq_query;
+
+ rcq = roce5_get_rcq_by_cqn(rdev, cqn);
+ if (rcq == NULL) {
+ roce5_dfx_print("Failed to get roce cq.");
+ roce5_dfx_print(
+ "******************From Host: cqn(%#x) ********************",
+ cqn);
+ roce5_dfx_print(
+ ">>>>>>>>>>> QUERY CQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Host: cqn(%#x) ********************",
+ cqn);
+
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(cq_query, sizeof(roce5_cq_query_outbuf_s),
+ rcq->cqm_cq->q_ctx_vaddr,
+ sizeof(roce5_cq_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy cq_ctx.(ret:%d)", ret);
+ goto err_out;
+ }
+
+err_out:
+ roce5_rcq_put(rcq);
+ return ret;
+}
+
+int roce5_dfx_cmd_query_srq(struct roce5_device *rdev, u32 srqn,
+ roce_verbs_srq_attr_s *srq_attr)
+{
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_srq_query_s *srq_query_inbuf = NULL;
+ roce_uni_cmd_srq_query_outbuf_s *srq_query_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_srq_query_s), &cqm_cmd_outbuf,
+ (u16)sizeof(roce_uni_cmd_srq_query_outbuf_s));
+ if (ret != 0) {
+ roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d",
+ ret);
+ return ret;
+ }
+
+ srq_query_inbuf = (roce_uni_cmd_srq_query_s *)cqm_cmd_inbuf->buf;
+ srq_query_inbuf->com.index = cpu_to_be32(srqn);
+ srq_query_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK));
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_QUERY_SRQ, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ roce5_dfx_print("Failed to send cmd QUERY_SRQ, ret=%d", ret);
+ ret = -1;
+ goto out;
+ }
+
+ srq_query_outbuf =
+ (roce_uni_cmd_srq_query_outbuf_s *)cqm_cmd_outbuf->buf;
+ ret = memcpy_s(srq_attr, sizeof(roce_verbs_srq_attr_s),
+ &srq_query_outbuf->com.srq_attr,
+ sizeof(roce_verbs_srq_attr_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy srq_attr.(ret:%d)", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+
+ return ret;
+}
+
+/* SRQC is dynamic allocation. When querying the cache content, you need to determine whether it exists or not. */
+static int roce5_dfx_query_cache_srqc(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ int ret;
+ cqm_object_s *cqm_obj_srq = NULL;
+ u32 srqn = inbuf->srqn;
+ roce_verbs_srq_attr_s *srq_attr = &outbuf->srq_attr;
+
+ cqm_obj_srq =
+ cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false);
+ if (cqm_obj_srq == NULL) {
+ roce5_dfx_print("Failed to get cqm_obj_srq");
+ roce5_dfx_print(
+ "******************From Cache: srqn(%#x) ********************",
+ srqn);
+ roce5_dfx_print(
+ ">>>>>>>>>>> QUERY SRQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Cache: srqn(%#x) ********************",
+ srqn);
+
+ return -EINVAL;
+ }
+
+ cqm5_object_put(cqm_obj_srq);
+
+ ret = memset_s(srq_attr, sizeof(roce_verbs_srq_attr_s), 0,
+ sizeof(roce_verbs_srq_attr_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset srq_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_dfx_cmd_query_srq(rdev, srqn, srq_attr);
+ if (ret != 0) {
+ roce5_dfx_print("Failed to query srq from cache");
+ roce5_dfx_print(
+ "******************From Cache: srqn(%#x) ********************",
+ srqn);
+ roce5_dfx_print(
+ ">>>>>>>>>>>>>>>> QUERY SRQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Cache: srqn(%#x) ********************",
+ srqn);
+
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int roce5_dfx_query_host_srqc(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ struct roce5_srq *rsrq = NULL;
+ int ret = 0;
+ u32 srqn = inbuf->srqn;
+ roce5_srq_query_outbuf_s *srq_query = &outbuf->srq_query;
+
+ rsrq = roce5_get_rsrq_by_srqn(rdev, srqn);
+ if (rsrq == NULL) {
+ roce5_dfx_print("Failed to get roce srq.");
+ roce5_dfx_print(
+ "******************From Host: srqn(%#x) ********************",
+ srqn);
+ roce5_dfx_print(
+ ">>>>>>>>>>> QUERY SRQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Host: srqn(%#x) ********************",
+ srqn);
+
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(srq_query, sizeof(roce5_srq_query_outbuf_s),
+ rsrq->cqm_srq->q_ctx_vaddr,
+ sizeof(roce5_srq_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy srq_ctx.(ret:%d)", ret);
+ goto err_out;
+ }
+
+err_out:
+ roce5_rsrq_put(rsrq);
+ return ret;
+}
+
+static int roce5_dfx_uni_cmd_query_mpt(struct roce5_device *rdev, u32 key,
+ struct roce_mpt_context *mpt_entry)
+{
+ int ret;
+ u32 mpt_index;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_mpt_query_s *mpt_query_inbuf = NULL;
+ roce_uni_cmd_mpt_query_outbuf_s *mpt_query_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_mpt_query_s), &cqm_cmd_outbuf,
+ (u16)sizeof(roce_uni_cmd_mpt_query_outbuf_s));
+ if (ret != 0) {
+ roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d",
+ ret);
+ return ret;
+ }
+
+ mpt_query_inbuf = (roce_uni_cmd_mpt_query_s *)cqm_cmd_inbuf->buf;
+ mpt_index = (key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF;
+ mpt_query_inbuf->com.index = cpu_to_be32(mpt_index);
+ mpt_query_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_MR_BITMASK));
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_QUERY_MPT, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ roce5_dfx_print("Failed to send cmd QUERY_MPT");
+ ret = -1;
+ goto out;
+ }
+
+ mpt_query_outbuf =
+ (roce_uni_cmd_mpt_query_outbuf_s *)cqm_cmd_outbuf->buf;
+ ret = memcpy_s(mpt_entry, sizeof(struct roce_mpt_context),
+ &mpt_query_outbuf->mpt_info.context[0],
+ sizeof(struct roce_mpt_context));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy mpt_entry.(ret:%d)", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+ return ret;
+}
+
+static int roce5_dfx_query_mpt(struct roce5_device *rdev, u32 key,
+ struct roce_mpt_context *mpt_entry)
+{
+ if (ROCE5_SUPPORT_DFX_CMD_QUERY_MPT_MSG_V2(rdev) != 0) {
+ return roce5_dfx_uni_cmd_query_mpt(rdev, key, mpt_entry);
+ }
+ return -EINVAL;
+}
+
+static int roce5_dfx_uni_cmd_query_gid(struct roce5_device *rdev, u32 port,
+ u32 gid_index,
+ struct rdma_gid_entry *gid_entry)
+{
+ int ret;
+ cqm_cmd_buf_s *cqm_cmd_inbuf = NULL;
+ cqm_cmd_buf_s *cqm_cmd_outbuf = NULL;
+ roce_uni_cmd_query_gid_s *gid_query_inbuf = NULL;
+ struct rdma_gid_query_outbuf *gid_query_outbuf = NULL;
+
+ ret = roce5_cqm_cmd_zalloc_inoutbuf(
+ rdev->hwdev, &cqm_cmd_inbuf,
+ (u16)sizeof(roce_uni_cmd_query_gid_s), &cqm_cmd_outbuf,
+ (u16)sizeof(struct rdma_gid_query_outbuf));
+ if (ret != 0) {
+ roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d",
+ ret);
+ return ret;
+ }
+
+ gid_query_inbuf = (roce_uni_cmd_query_gid_s *)cqm_cmd_inbuf->buf;
+ gid_query_inbuf->com.index = cpu_to_be32(gid_index);
+ gid_query_inbuf->com.dw0.bs.cmd_bitmask =
+ cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_GID_BITMASK));
+ gid_query_inbuf->com.opt = cpu_to_be32(port);
+
+ if (rdev->shadow != 0) {
+ gid_query_inbuf->com.dw3.bs.shadow_vfid =
+ cpu_to_be16(rdev->shadow_vfid);
+ gid_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16(
+ ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK));
+ gid_query_inbuf->gid_query.dw0.shadow.shadow_vf_num =
+ rdev->shadow_vf_num;
+ gid_query_inbuf->gid_query.dw0.value =
+ cpu_to_be32(gid_query_inbuf->gid_query.dw0.value);
+ }
+
+ ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE,
+ ROCE_CMD_QUERY_GID, cqm_cmd_inbuf,
+ cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B,
+ HINIC5_CHANNEL_ROCE);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to send cmd QUERY_GID, ret(%d)",
+ ret);
+ ret = -1;
+ goto out;
+ }
+
+ gid_query_outbuf = (struct rdma_gid_query_outbuf *)cqm_cmd_outbuf->buf;
+ ret = memcpy_s(gid_entry,
+ sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM,
+ gid_query_outbuf->gid_entry,
+ sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM);
+ if (ret != 0) {
+ ROCE_DEV_ERR(rdev, "Failed to memcpy gid_entry.(ret:%d)", ret);
+ }
+out:
+ roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf,
+ cqm_cmd_outbuf);
+
+ return ret;
+}
+
+static int roce5_dfx_query_gid(struct roce5_device *rdev, u32 port,
+ u32 gid_index, struct rdma_gid_entry *gid_entry)
+{
+ if (ROCE5_SUPPORT_DFX_CMD_QUERY_GID_MSG_V2(rdev) != 0) {
+ return roce5_dfx_uni_cmd_query_gid(rdev, port, gid_index,
+ gid_entry);
+ }
+ return -EINVAL;
+}
+
+/* MPT is statically allocated, so it is not necessary to judge whether it exists when querying the cache content */
+static int roce5_dfx_query_cache_mpt(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ int ret;
+ u32 mpt_key = inbuf->mpt_key;
+ struct roce_mpt_context *mpt_entry = &outbuf->mpt;
+
+ ret = memset_s(mpt_entry, sizeof(struct roce_mpt_context), 0,
+ sizeof(struct roce_mpt_context));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset mpt_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_dfx_query_mpt(rdev, mpt_key, mpt_entry);
+ if (ret != 0) {
+ roce5_dfx_print("Failed to query mpt from cache!");
+ roce5_dfx_print(
+ "******************From Cache: mpt_key(%#x) ********************",
+ mpt_key);
+ roce5_dfx_print(
+ ">>>>>>>>>>>>>>>> QUERY MPT FROM CACHE FAILED <<<<<<<<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Cache: mpt_key(%#x) ********************",
+ mpt_key);
+
+ return ret;
+ }
+
+ return 0;
+}
+
+static int roce5_dfx_query_host_mpt(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ u32 mpt_index = 0;
+ struct mpt *rmpt = NULL;
+ int ret = 0;
+ u32 mpt_key = inbuf->mpt_key;
+ struct roce_mpt_context *mpt_entry = &outbuf->mpt;
+
+ mpt_index = (mpt_key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF;
+ rmpt = roce5_get_mpt_by_index(rdev, mpt_index);
+ if (rmpt == NULL) {
+ roce5_dfx_print("Failed to get roce mpt.");
+ roce5_dfx_print(
+ "******************From Host: mpt_key(%#x) ********************",
+ mpt_key);
+ roce5_dfx_print(
+ ">>>>>>>>>>> QUERY MPT FROM HOST FAILED, NOT EXIST <<<<<<<<<<<");
+ roce5_dfx_print(
+ "******************From Host: mpt_key(%#x) ********************",
+ mpt_key);
+
+ return -EINVAL;
+ }
+
+ ret = memcpy_s(mpt_entry, sizeof(struct roce_mpt_context), rmpt->vaddr,
+ sizeof(struct roce_mpt_context));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memcpy mpt_entry.(ret:%d)", ret);
+ goto err_out;
+ }
+
+err_out:
+ roce5_mpt_put(rmpt);
+ return ret;
+}
+
+static int roce5_dfx_query_cache_gid(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ int ret;
+ u32 gid_index = inbuf->gid_index;
+ struct rdma_gid_entry *gid_entry = outbuf->gid_entry;
+
+ ret = memset_s(gid_entry,
+ sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM, 0,
+ sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM);
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_dfx_query_gid(rdev, 0, gid_index, gid_entry);
+ if (ret != 0) {
+ roce5_dfx_print(
+ "***************port(1), gid_index(%d) *****************",
+ gid_index);
+ roce5_dfx_print("Failed to query gid from cache!");
+ roce5_dfx_print(
+ "***************port(1), gid_index(%d) *****************",
+ gid_index);
+
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void roce5_dfx_query_pi_ci_set(const roce_verbs_qp_attr_s *qp_attr,
+ struct roce5_dfx_pi_ci *pi_ci)
+{
+ pi_ci->qpc_sq_pi_on_chip = qp_attr->chip_seg.dw5.bs.sq_pi_on_chip;
+ pi_ci->qpc_sq_pi = qp_attr->ext_seg.dfx_info.dw0.bs.sq_pi;
+ pi_ci->qpc_sq_load_pi = qp_attr->ext_seg.dfx_info.dw1.bs.sq_load_pi;
+ pi_ci->qpc_rq_pi_on_chip = qp_attr->chip_seg.dw8.bs.rq_pi_on_chip;
+ pi_ci->qpc_rq_load_pi = qp_attr->ext_seg.dfx_info.dw1.bs.rq_load_pi;
+ pi_ci->qpc_rq_pi = qp_attr->ext_seg.dfx_info.dw4.bs.rq_pi;
+ pi_ci->qpc_rc_pi = qp_attr->ext_seg.dfx_info.dw5.bs.rc_pi;
+ pi_ci->qpc_sq_ci = qp_attr->ext_seg.dfx_info.dw0.bs.sq_ci;
+ pi_ci->qpc_sq_wqe_prefetch_ci =
+ qp_attr->ext_seg.dfx_info.dw1.bs.sq_wqe_prefetch_ci;
+ pi_ci->qpc_sq_mtt_prefetch_wqe_ci =
+ qp_attr->ext_seg.dfx_info.dw3.bs.sq_mtt_prefetch_wqe_ci;
+ pi_ci->qpc_sqa_ci = qp_attr->ext_seg.dfx_info.dw2.bs.sqa_ci;
+ pi_ci->qpc_sqa_wqe_prefetch_ci =
+ qp_attr->ext_seg.dfx_info.dw2.bs.sqa_wqe_prefetch_ci;
+ pi_ci->qpc_rq_ci = qp_attr->ext_seg.dfx_info.dw4.bs.rq_ci;
+ pi_ci->qpc_rq_wqe_prefetch_ci =
+ qp_attr->ext_seg.dfx_info.dw5.bs.rq_wqe_prefetch_ci;
+ pi_ci->qpc_rq_mtt_prefetch_wqe_ci =
+ qp_attr->ext_seg.dfx_info.dw3.bs.rq_mtt_prefetch_wqe_ci;
+ pi_ci->qpc_rq_base_ci = qp_attr->chip_seg.dw4.bs.rq_base_ci;
+ pi_ci->qpc_rc_ci = qp_attr->ext_seg.dfx_info.dw5.bs.rc_ci;
+ pi_ci->qpc_rc_prefetch_ci =
+ qp_attr->ext_seg.dfx_info.dw1.bs.rc_prefetch_ci;
+}
+
+static int roce5_dfx_query_pi_ci(const struct roce5_dfx_query_inbuf *inbuf,
+ struct roce5_device *rdev,
+ union roce5_dfx_query_outbuf *outbuf)
+{
+ int ret;
+ roce_verbs_qp_attr_s be_attr; /* BE context structure */
+ roce_verbs_qp_attr_s qp_attr; /* LE context structure */
+ roce_verbs_cq_attr_s cq_attr;
+ roce5_cq_query_outbuf_s cq_query;
+ u32 qpn = inbuf->query_pi_ci.qpn;
+ u32 cqn = inbuf->query_pi_ci.cqn;
+ struct roce5_dfx_pi_ci *pi_ci = &outbuf->pi_ci;
+
+ ret = memset_s(pi_ci, sizeof(struct roce5_dfx_pi_ci), 0,
+ sizeof(struct roce5_dfx_pi_ci));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset pi_ci.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+
+ ret = memset_s(&qp_attr, sizeof(qp_attr), 0, sizeof(qp_attr));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)&be_attr,
+ sizeof(roce_verbs_qp_attr_s));
+ if (ret != 0) {
+ return ret;
+ }
+
+ roce5_be32_2_le32(&be_attr, (u32 *)(void *)&qp_attr,
+ (sizeof(roce_verbs_qp_attr_s) / sizeof(u32)));
+
+ roce5_dfx_query_pi_ci_set(&qp_attr, pi_ci);
+
+ ret = memset_s(&cq_query, sizeof(roce5_cq_query_outbuf_s), 0,
+ sizeof(roce5_cq_query_outbuf_s));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset cq_query.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)&cq_query,
+ sizeof(roce5_cq_query_outbuf_s));
+ if (ret != 0) {
+ return ret;
+ }
+
+ if (ROCE5_SUPPORT_DFX_CMD_QUERY_PI_CI_MSG_V2(rdev) != 0) {
+ (void)memcpy_s(&cq_attr, sizeof(roce_verbs_cq_attr_s),
+ &cq_query, sizeof(roce_verbs_cq_attr_s));
+ cq_attr.dw0.value = be32_to_cpu(cq_attr.dw0.value);
+ cq_attr.dfx_info.dw0.value =
+ be32_to_cpu(cq_attr.dfx_info.dw0.value);
+ cq_attr.dfx_info.dw1.value =
+ be32_to_cpu(cq_attr.dfx_info.dw1.value);
+ cq_attr.dfx_info.dw2.value =
+ be32_to_cpu(cq_attr.dfx_info.dw2.value);
+ cq_attr.dfx_info.dw3.value =
+ be32_to_cpu(cq_attr.dfx_info.dw3.value);
+ cq_attr.ci_record_gpa_at_hop_num =
+ be64_to_cpu(cq_attr.ci_record_gpa_at_hop_num);
+
+ pi_ci->cq_ci_on_chip = cq_attr.dw0.bs.ci_on_chip;
+ pi_ci->cq_ci = cq_attr.dfx_info.dw0.bs.ci;
+ pi_ci->cq_load_ci = cq_attr.dfx_info.dw2.bs.load_ci;
+ pi_ci->cq_ci_record_gpa_at_hop_num =
+ cq_attr.ci_record_gpa_at_hop_num;
+ pi_ci->cq_last_solicited_pi =
+ cq_attr.dfx_info.dw2.bs.last_solicited_pi;
+ pi_ci->cq_pi = cq_attr.dfx_info.dw1.bs.pi;
+ pi_ci->cq_last_notified_pi =
+ cq_attr.dfx_info.dw3.bs.last_notified_pi;
+ }
+ return 0;
+}
+
+static roce5_adm_dfx_query_cmd_proc_t g_roce5_adm_dfx_query_cmd_proc[] = {
+ { ROCE_CMD_GET_QPC_FROM_CACHE, roce5_dfx_query_cache_qpc },
+ { ROCE_CMD_GET_QPC_FROM_HOST, roce5_dfx_query_host_qpc },
+ { ROCE_CMD_GET_CQC_FROM_CACHE, roce5_dfx_query_cache_cqc },
+ { ROCE_CMD_GET_CQC_FROM_HOST, roce5_dfx_query_host_cqc },
+ { ROCE_CMD_GET_SRQC_FROM_CACHE, roce5_dfx_query_cache_srqc },
+ { ROCE_CMD_GET_SRQC_FROM_HOST, roce5_dfx_query_host_srqc },
+ { ROCE_CMD_GET_MPT_FROM_CACHE, roce5_dfx_query_cache_mpt },
+ { ROCE_CMD_GET_MPT_FROM_HOST, roce5_dfx_query_host_mpt },
+ { ROCE_CMD_GET_GID_FROM_CACHE, roce5_dfx_query_cache_gid },
+ { ROCE_CMD_GET_QPC_CQC_PI_CI, roce5_dfx_query_pi_ci },
+ { ROCE_CMD_GET_QP_COUNT, roce5_dfx_get_qp_count },
+ { ROCE_CMD_GET_DEV_ALGO, roce5_dfx_get_dev_algo },
+};
+
+int roce5_adm_dfx_query(struct roce5_device *rdev, const void *buf_in,
+ u32 in_size, void *buf_out, u32 *out_size)
+{
+ int ret;
+ const struct roce5_dfx_query_inbuf *inbuf =
+ (struct roce5_dfx_query_inbuf *)buf_in;
+ union roce5_dfx_query_outbuf *outbuf =
+ (union roce5_dfx_query_outbuf *)buf_out;
+ const roce5_adm_generic_cmd_proc_t *generic_array =
+ (const roce5_adm_generic_cmd_proc_t *)
+ g_roce5_adm_dfx_query_cmd_proc;
+ int elem_count = sizeof(g_roce5_adm_dfx_query_cmd_proc) /
+ sizeof(g_roce5_adm_dfx_query_cmd_proc[0]);
+ roce5_adm_dfx_query_t roce5_adm_dfx_query_func = NULL;
+
+ if (in_size < sizeof(struct roce5_dfx_query_inbuf)) {
+ ROCE_ERR(
+ "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)",
+ (int)in_size,
+ (int)sizeof(struct roce5_dfx_query_inbuf));
+ return -ENOMEM;
+ }
+ if (*out_size < sizeof(union roce5_dfx_query_outbuf)) {
+ ROCE_ERR(
+ "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)",
+ (int)*out_size,
+ (int)sizeof(union roce5_dfx_query_outbuf));
+ return -ENOMEM;
+ }
+ ret = memset_s(buf_out, sizeof(union roce5_dfx_query_outbuf), 0,
+ sizeof(union roce5_dfx_query_outbuf));
+ if (ret != 0) {
+ ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret);
+ return -ENOMEM;
+ }
+ *out_size = (u32)sizeof(union roce5_dfx_query_outbuf);
+
+ if (inbuf->cmd_type >= VM_COMPAT_TEST) {
+ roce5_dfx_print("Not support this type(%d)", inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ roce5_adm_dfx_query_func =
+ (roce5_adm_dfx_query_t)roce5_adm_generic_find_cmd_func(
+ generic_array, elem_count, inbuf->cmd_type);
+ if (roce5_adm_dfx_query_func == NULL) {
+ roce5_dfx_print("Not support this type(%d)", inbuf->cmd_type);
+ return -EINVAL;
+ }
+
+ return roce5_adm_dfx_query_func(inbuf, rdev, outbuf);
+}
+
+#endif
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c
new file mode 100644
index 000000000..7c641f422
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c
@@ -0,0 +1,83 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include <linux/slab.h>
+#include "securec.h"
+#include "roce_infra.h"
+#include "roce_obj_ext.h"
+
+int roce5_obj_ext_init(void *parent, const struct roce5_ext_reg *reg_items,
+ u8 item_num, struct roce5_obj_ext *obj_ext)
+{
+ int i, ret;
+
+ ret = memset_s(obj_ext->bufs, sizeof(void *) * ROCE_EXT_FEATURE_MAX, 0,
+ sizeof(void *) * ROCE_EXT_FEATURE_MAX);
+ if (ret != 0) {
+ ROCE_ERR(
+ "Failed to execute memset to object extension buffer, ret(%d).",
+ ret);
+ return ret;
+ }
+
+ for (i = 0; i < item_num; ++i) {
+ if (reg_items[i].ext_id >= ROCE_EXT_FEATURE_MAX) {
+ ROCE_ERR("Invalid extension id(%u).",
+ reg_items[i].ext_id);
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ if (obj_ext->bufs[reg_items[i].ext_id] != NULL) {
+ ROCE_ERR("Repeated register, extension id(%u).",
+ reg_items[i].ext_id);
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ obj_ext->bufs[reg_items[i].ext_id] =
+ kzalloc(reg_items[i].data_size, GFP_KERNEL);
+ if (obj_ext->bufs[reg_items[i].ext_id] == NULL) {
+ ROCE_ERR("Failed alloc memory for extension.");
+ ret = -ENOMEM;
+ goto err_out;
+ }
+
+ ret = reg_items[i].init_func(
+ parent, obj_ext->bufs[reg_items[i].ext_id]);
+ if (ret != 0) {
+ ROCE_ERR("Failed to init extension, ret(%d).", ret);
+ goto err_out;
+ }
+ }
+
+ return 0;
+
+err_out:
+ if (obj_ext->bufs[reg_items[i].ext_id] != NULL) {
+ kfree(obj_ext->bufs[reg_items[i].ext_id]);
+ }
+
+ for (--i; i >= 0; --i) {
+ reg_items[i].cleanup_func(parent,
+ obj_ext->bufs[reg_items[i].ext_id]);
+ kfree(obj_ext->bufs[reg_items[i].ext_id]);
+ }
+ return ret;
+}
+
+void roce5_obj_ext_deinit(void *parent, const struct roce5_ext_reg *reg_items,
+ u8 item_num, struct roce5_obj_ext *obj_ext)
+{
+ int i;
+ for (i = 0; i < item_num; ++i) {
+ if (reg_items[i].ext_id >= ROCE_EXT_FEATURE_MAX ||
+ obj_ext->bufs[reg_items[i].ext_id] == NULL) {
+ continue;
+ }
+
+ reg_items[i].cleanup_func(parent,
+ obj_ext->bufs[reg_items[i].ext_id]);
+ kfree(obj_ext->bufs[reg_items[i].ext_id]);
+ }
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h
new file mode 100644
index 000000000..3e5f7196c
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h
@@ -0,0 +1,40 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_OBJ_EXT_H
+#define ROCE_OBJ_EXT_H
+
+#include <linux/types.h>
+#include <linux/stddef.h>
+
+typedef int (*ext_init_func)(void *parent, void *ext_data);
+typedef void (*ext_cleanup_func)(void *parent, void *ext_data);
+
+enum ROCE_EXT_ID { ROCE_EXT_HYPER_ROCE, ROCE_EXT_FEATURE_MAX };
+
+struct roce5_ext_reg {
+ u8 ext_id;
+ size_t data_size;
+ ext_init_func init_func;
+ ext_cleanup_func cleanup_func;
+};
+
+struct roce5_obj_ext {
+ void *bufs[ROCE_EXT_FEATURE_MAX];
+};
+
+static inline void *roce5_obj_ext_get_buf(struct roce5_obj_ext *obj_ext,
+ enum ROCE_EXT_ID ext_id)
+{
+ if (ext_id >= ROCE_EXT_FEATURE_MAX) {
+ return NULL;
+ }
+ return obj_ext->bufs[ext_id];
+}
+
+int roce5_obj_ext_init(void *parent, const struct roce5_ext_reg *reg_items,
+ u8 item_num, struct roce5_obj_ext *obj_ext);
+void roce5_obj_ext_deinit(void *parent, const struct roce5_ext_reg *reg_items,
+ u8 item_num, struct roce5_obj_ext *obj_ext);
+
+#endif /* ROCE_OBJ_EXT_H */
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c
new file mode 100644
index 000000000..6cfed12be
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c
@@ -0,0 +1,125 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/jiffies.h>
+
+#include "roce_infra.h"
+#include "securec.h"
+#include "roce_option.h"
+
+#define ROCE_WAIT_TIME_SEC 60
+
+struct roce5_option *roce5_some(void *value)
+{
+ int ret;
+
+ struct roce5_option *opt =
+ kmalloc(sizeof(struct roce5_option), GFP_KERNEL);
+ if (opt == NULL) {
+ ROCE_ERR("Failed to alloc memory.");
+ return NULL;
+ }
+
+ ret = strcpy_s(opt->waiting_msg, ROCE_WAITING_MSG_MAX_SIZE,
+ "Waiting for object release...");
+ if (ret != EOK) {
+ ROCE_ERR("Failed to init waiting message.");
+ kfree(opt);
+ return NULL;
+ }
+
+ spin_lock_init(&opt->spin_lock);
+ kref_init(&opt->refcount);
+ opt->value = value;
+
+ return opt;
+}
+
+static void roce5_wait_release(struct kref *refcount)
+{
+ struct roce5_option *opt =
+ container_of(refcount, struct roce5_option, refcount);
+
+ opt->value = NULL;
+ complete(&opt->release_comp);
+}
+
+static void roce5_timer_print_wait_msg(struct timer_list *tl)
+{
+ struct roce5_option *opt = from_timer(opt, tl, msg_timer);
+
+ ROCE_WARN("%s", opt->waiting_msg);
+
+ mod_timer(&opt->msg_timer, jiffies + ROCE_WAIT_TIME_SEC * HZ);
+}
+
+void roce5_option_unset(struct roce5_option *opt, char *waiting_msg,
+ size_t waiting_msg_size)
+{
+ int ret;
+ unsigned long flags; /* type required by kernel API */
+
+ if (opt == NULL) {
+ ROCE_ERR("Invalid parameter, opt is NULL.");
+ return;
+ }
+
+ if (waiting_msg != NULL && waiting_msg_size > 0) {
+ ret = strcpy_s(opt->waiting_msg, ROCE_WAITING_MSG_MAX_SIZE,
+ waiting_msg);
+ if (ret != EOK) {
+ ROCE_WARN("Failed to set waiting message, ret(%d).",
+ ret);
+ }
+ }
+
+ init_completion(&opt->release_comp);
+
+ timer_setup(&opt->msg_timer, roce5_timer_print_wait_msg, 0);
+ mod_timer(&opt->msg_timer, jiffies + ROCE_WAIT_TIME_SEC * HZ);
+
+ spin_lock_irqsave(&opt->spin_lock, flags);
+ kref_put(&opt->refcount, roce5_wait_release);
+ spin_unlock_irqrestore(&opt->spin_lock, flags);
+
+ wait_for_completion(&opt->release_comp);
+
+ del_timer_sync(&opt->msg_timer);
+}
+
+void *roce5_option_get_value(struct roce5_option *opt)
+{
+ void *ans = NULL;
+ unsigned long flags; /* type required by kernel API */
+
+ if (opt == NULL) {
+ ROCE_ERR("Invalid parameter, opt is NULL.");
+ return NULL;
+ }
+
+ spin_lock_irqsave(&opt->spin_lock, flags);
+ if (opt->value != NULL) {
+ kref_get(&opt->refcount);
+ ans = opt->value;
+ }
+ spin_unlock_irqrestore(&opt->spin_lock, flags);
+
+ return ans;
+}
+
+void roce5_option_put_value(struct roce5_option *opt)
+{
+ unsigned long flags; /* type required by kernel API */
+ if (opt == NULL) {
+ ROCE_ERR("Invalid parameter, opt is NULL.");
+ return;
+ }
+
+ spin_lock_irqsave(&opt->spin_lock, flags);
+ if (opt->value != NULL) {
+ kref_put(&opt->refcount, roce5_wait_release);
+ }
+ spin_unlock_irqrestore(&opt->spin_lock, flags);
+}
\ No newline at end of file
diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h
new file mode 100644
index 000000000..dba51cd68
--- /dev/null
+++ b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h
@@ -0,0 +1,30 @@
+//SPDX-License-Identifier: GPL-2.0
+//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved.
+
+#ifndef ROCE_OPTION_H
+#define ROCE_OPTION_H
+
+#include <linux/kref.h>
+#include <linux/completion.h>
+#include <linux/timer.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+
+#define ROCE_WAITING_MSG_MAX_SIZE 100
+
+struct roce5_option {
+ spinlock_t spin_lock;
+ struct kref refcount; /* count for the value ptr. */
+ struct completion release_comp;
+ struct timer_list msg_timer;
+ char waiting_msg[ROCE_WAITING_MSG_MAX_SIZE];
+ void *value;
+};
+
+struct roce5_option *roce5_some(void *value);
+void roce5_option_unset(struct roce5_option *opt, char *waiting_msg,
+ size_t waiting_msg_size);
+void *roce5_option_get_value(struct roce5_option *opt);
+void roce5_option_put_value(struct roce5_option *opt);
+
+#endif /* ROCE_OPTION_H */
--
2.37.7
反馈： 您发送到kernel@openeuler.org的补丁/补丁集，已成功转换为PR！ PR链接地址： https://atomgit.com/openeuler/kernel/merge_requests/25657 邮件列表地址：https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/VVE... FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://atomgit.com/openeuler/kernel/merge_requests/25657 Mailing list address: https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/VVE...
participants (2)
-
Maogangpin -
patchwork bot