[PATCH OLK-6.6] infiniband/hroce5: Add hiroce5 driver

From: maogangpin <maogangpin@huawei.com<mailto:maogangpin@huawei.com>> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9708 CVE: NA --------------------------------- Add hiroce5 driver to support hi1825 chip. Signed-off-by: maogangpin <maogangpin@huawei.com<mailto:maogangpin@huawei.com>> Reviewed-by: Qiujun <roland.qiu@huawei.com<mailto:roland.qiu@huawei.com>> Reviewed-by: chiqijun <chiqijun@huawei.com<mailto:chiqijun@huawei.com>> --- .../hw/hiroce5/drv_intf/roce_drv_msg_intf.h | 14 + .../hw/hiroce5/kernel/CMakeLists.txt | 62 + drivers/infiniband/hw/hiroce5/kernel/Makefile | 543 ++++ .../hw/hiroce5/kernel/base/cq/roce_cq.h | 230 ++ .../hiroce5/kernel/base/cq/roce_cq_common.c | 299 ++ .../hw/hiroce5/kernel/base/cq/roce_cq_cqe.c | 763 ++++++ .../hiroce5/kernel/base/cq/roce_cq_create.c | 636 +++++ .../hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c | 964 +++++++ .../hiroce5/kernel/base/cq/roce_cq_destroy.c | 304 +++ .../hiroce5/kernel/base/cq/roce_cq_format.h | 29 + .../hiroce5/kernel/base/cq/roce_cq_intern.c | 98 + .../hiroce5/kernel/base/cq/roce_cq_intern.h | 24 + .../hw/hiroce5/kernel/base/event/roce_event.c | 437 +++ .../hw/hiroce5/kernel/base/event/roce_event.h | 25 + .../base/extension/roce_event_extension.c | 56 + .../base/extension/roce_main_extension.c | 170 ++ .../kernel/base/extension/roce_mr_extension.c | 40 + .../kernel/base/extension/roce_qp_extension.c | 222 ++ .../base/extension/roce_srq_extension.c | 55 + .../hw/hiroce5/kernel/base/mr/roce_mr.c | 1281 +++++++++ .../hw/hiroce5/kernel/base/mr/roce_mr.h | 113 + .../hw/hiroce5/kernel/base/qp/roce_post.h | 171 ++ .../hw/hiroce5/kernel/base/qp/roce_qp.h | 317 +++ .../hiroce5/kernel/base/qp/roce_qp_create.c | 1506 +++++++++++ .../kernel/base/qp/roce_qp_ctx_format.h | 278 ++ .../hiroce5/kernel/base/qp/roce_qp_destroy.c | 342 +++ .../hw/hiroce5/kernel/base/qp/roce_qp_exp.h | 80 + .../hw/hiroce5/kernel/base/qp/roce_qp_ext.c | 39 + .../hw/hiroce5/kernel/base/qp/roce_qp_ext.h | 19 + .../hw/hiroce5/kernel/base/qp/roce_qp_inner.h | 17 + .../hiroce5/kernel/base/qp/roce_qp_modify.c | 2398 +++++++++++++++++ .../kernel/base/qp/roce_qp_post_recv.c | 238 ++ .../kernel/base/qp/roce_qp_post_send.c | 1205 +++++++++ .../hw/hiroce5/kernel/base/qp/roce_qp_query.c | 543 ++++ .../hw/hiroce5/kernel/base/rdma/rdma_comp.c | 22 + .../hw/hiroce5/kernel/base/rdma/rdma_comp.h | 166 ++ .../hiroce5/kernel/base/rdma/rdma_comp_gid.c | 489 ++++ .../hiroce5/kernel/base/rdma/rdma_comp_init.c | 443 +++ .../kernel/base/rdma/rdma_comp_mw_mr.c | 120 + .../kernel/base/rdma/rdma_comp_mw_mr.h | 12 + .../hiroce5/kernel/base/rdma/rdma_comp_pd.c | 51 + .../hiroce5/kernel/base/rdma/rdma_comp_res.c | 140 + .../hw/hiroce5/kernel/base/rdma/roce_com.h | 78 + .../hw/hiroce5/kernel/base/rdma/roce_mix.c | 1594 +++++++++++ .../hw/hiroce5/kernel/base/rdma/roce_mix.h | 269 ++ .../hw/hiroce5/kernel/base/rdma/roce_pd.c | 138 + .../hw/hiroce5/kernel/base/rdma/roce_pd.h | 15 + .../hw/hiroce5/kernel/base/srq/roce_srq.h | 191 ++ .../hiroce5/kernel/base/srq/roce_srq_comm.c | 116 + .../hiroce5/kernel/base/srq/roce_srq_create.c | 712 +++++ .../hiroce5/kernel/base/srq/roce_srq_ctrl.c | 635 +++++ .../hiroce5/kernel/base/srq/roce_srq_format.h | 36 + .../hw/hiroce5/kernel/base/uld/roce_uld_api.c | 729 +++++ .../hiroce5/kernel/base/uld/roce_uld_inner.h | 77 + .../hw/hiroce5/kernel/base/xrc/roce_xrc.c | 169 ++ .../hw/hiroce5/kernel/base/xrc/roce_xrc.h | 17 + .../hw/hiroce5/kernel/dev/card/roce_card.c | 162 ++ .../hw/hiroce5/kernel/dev/card/roce_card.h | 27 + .../hw/hiroce5/kernel/dev/cmd/roce_cmd.c | 1270 +++++++++ .../hw/hiroce5/kernel/dev/cmd/roce_cmd.h | 82 + .../hw/hiroce5/kernel/dev/db/roce_db.c | 86 + .../hw/hiroce5/kernel/dev/db/roce_db.h | 30 + .../hw/hiroce5/kernel/dev/init/roce_main.c | 1701 ++++++++++++ .../hiroce5/kernel/dev/netdev/roce_netdev.c | 1082 ++++++++ .../hiroce5/kernel/dev/netdev/roce_netdev.h | 70 + .../hiroce5/kernel/dev/netlink/roce_netlink.c | 413 +++ .../hiroce5/kernel/dev/netlink/roce_netlink.h | 53 + .../kernel/dev/netlink/roce_netlink_process.c | 832 ++++++ .../kernel/dev/netlink/roce_netlink_process.h | 133 + .../hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c | 74 + .../hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h | 21 + .../kernel/dev/sysfs/roce_multipath_sysfs.c | 240 ++ .../kernel/dev/sysfs/roce_multipath_sysfs.h | 30 + .../hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c | 2157 +++++++++++++++ .../hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h | 241 ++ .../hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c | 472 ++++ .../hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h | 73 + .../extern/chip_ext/roce_main_chip_ext.c | 13 + .../extern/chip_ext/roce_main_chip_ext.h | 17 + .../kernel/extern/chip_ext/roce_mr_chip_ext.c | 348 +++ .../kernel/extern/chip_ext/roce_mr_chip_ext.h | 24 + .../kernel/extern/chip_ext/roce_qp_chip_ext.c | 44 + .../extern/chip_ext/roce_srq_chip_ext.c | 26 + .../extern/hyper_roce/hyper_roce_common.h | 13 + .../extern/hyper_roce/hyper_roce_netlink.c | 511 ++++ .../extern/hyper_roce/hyper_roce_netlink.h | 82 + .../kernel/extern/hyper_roce/hyper_roce_qp.c | 29 + .../kernel/extern/hyper_roce/hyper_roce_qp.h | 35 + .../extern/hyper_roce/tool/hyper_roce_tool.c | 60 + .../extern/hyper_roce/tool/hyper_roce_tool.h | 21 + .../hyper_roce/tool/hyper_roce_tool_pktdrop.c | 142 + .../hyper_roce/tool/hyper_roce_tool_pktdrop.h | 23 + .../hw/hiroce5/kernel/include/roce.h | 665 +++++ .../hw/hiroce5/kernel/include/roce_compat.h | 87 + .../kernel/include/roce_event_extension.h | 17 + .../hw/hiroce5/kernel/include/roce_gid.h | 48 + .../hw/hiroce5/kernel/include/roce_infra.h | 77 + .../kernel/include/roce_kernel_compat_gen.h | 6 + .../kernel/include/roce_main_extension.h | 77 + .../kernel/include/roce_mr_extension.h | 14 + .../hiroce5/kernel/include/roce_os_compat.h | 237 ++ .../kernel/include/roce_qp_extension.h | 61 + .../hw/hiroce5/kernel/include/roce_restore.h | 88 + .../kernel/include/roce_srq_extension.h | 38 + .../hw/hiroce5/kernel/include/roce_user.h | 100 + .../kernel/include/roce_verbs_compat.h | 24 + .../hw/hiroce5/kernel/util/bond/roce_bond.h | 79 + .../hiroce5/kernel/util/bond/roce_bond_cfm.c | 490 ++++ .../kernel/util/bond/roce_bond_device.c | 490 ++++ .../kernel/util/bond/roce_bond_internal.h | 97 + .../hiroce5/kernel/util/bond/roce_bond_mac.c | 260 ++ .../kernel/util/bond/roce_bond_slave.c | 459 ++++ .../hiroce5/kernel/util/bond/roce_bond_tx.c | 94 + .../hw/hiroce5/kernel/util/debug/roce_debug.h | 25 + .../hw/hiroce5/kernel/util/dfx/roce_dfx.c | 375 +++ .../hw/hiroce5/kernel/util/dfx/roce_dfx.h | 378 +++ .../hiroce5/kernel/util/dfx/roce_dfx_attack.c | 828 ++++++ .../hiroce5/kernel/util/dfx/roce_dfx_debug.c | 35 + .../hiroce5/kernel/util/dfx/roce_dfx_query.c | 851 ++++++ .../kernel/util/obj_ext/roce_obj_ext.c | 83 + .../kernel/util/obj_ext/roce_obj_ext.h | 40 + .../hiroce5/kernel/util/option/roce_option.c | 125 + .../hiroce5/kernel/util/option/roce_option.h | 30 + 123 files changed, 36652 insertions(+) create mode 100644 drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt create mode 100644 drivers/infiniband/hw/hiroce5/kernel/Makefile create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h diff --git a/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h b/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h new file mode 100644 index 000000000..4875a1f1a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +#ifndef ROCE_DRV_MSG_INTF_H +#define ROCE_DRV_MSG_INTF_H + +#include "udk_usrnl.h" + +typedef struct { + usrnl_drv_msg_hdr_s common_hdr; + unsigned long long card_id; +} roce5_drv_msg_hdr_s; + +#endif /* ROCE_DRV_MSG_INTF_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt b/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt new file mode 100644 index 000000000..e67756615 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt @@ -0,0 +1,62 @@ +option(ENABLE_DEBUG "Enable debug in RoCE kernel driver" OFF) +option(ENABLE_HYPER_ROCE "Enable hyper roce" OFF) + +if("${BUILD_VERSION}" MATCHES "_debug$") + set(ENABLE_DEBUG ON) +endif() + +# ubx kernel hyper_roce使能宏 +if (DEFINED ENV{build_roce_mode}) + if ($ENV{build_roce_mode} STREQUAL "hyperroce") + set(ENABLE_HYPER_ROCE ON) + endif() +endif() + +if("${BUILD_VERSION}" MATCHES "ub_ascend") + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core-v1) +elseif("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(UBUS_DIR ${TOP_DIR}/drivers/ubus) + set(KDIR ${KERNEL_WORK_DIR}/../linux-4.19) + set(UMMU_CORE_BUILD_DIR ${UBUS_DIR}/kernel/ummu-core-v1) +else() + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core) +endif() + +set(UBUS_UBC_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ubus) +set(HISDK5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/sdk/knldk/lld) +set(HINIC5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/nic/linux) +set(HIROCE5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/roce/linux/kernel) + +if("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(hiroce5_depends hisdk hinic) + add_device_ko(LOCAL_MODULE hiroce5 + KO_SRC_FOLDER ${CMAKE_CURRENT_SOURCE_DIR} + MAKE_ARGS "IS_ASCEND=true" + TARGETE_DPENDS "${hiroce5_depends}") +else() + add_custom_target(hiroce5_ko + COMMENT echo "build ${CMAKE_CURRENT_SOURCE_DIR} start." + COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/Makefile ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${MAKE} -j64 + -C ${KDIR} + M=${CMAKE_CURRENT_BINARY_DIR} + src=${CMAKE_CURRENT_SOURCE_DIR} + UBUS_UBC_BUILD_DIR=${UBUS_UBC_BUILD_DIR} + HI1823_TRUNK_DIR=${TOP_DIR}/ChipSolution + HI1823_BUILD_DIR=${TOP_DIR}/ChipSolution/build + CONFIG_UBUS_DEVICE=y + HI1823_OS_TYPE=openEuler24.03 + VERBOSE=1 + HISDK5_SYMVERS=${HISDK5_BUILD_DIR}/Module.symvers + HINIC5_SYMVERS=${HINIC5_BUILD_DIR}/Module.symvers + CHIP_VERSION=hi1825v100 + VERSION=ROCE_STANDARD + ENABLE_DEBUG=${ENABLE_DEBUG} + ENABLE_HYPER_ROCE=${ENABLE_HYPER_ROCE} + COMMAND cp -f *.ko ${CMAKE_INSTALL_PREFIX}/ko + DEPENDS kernel + ) + add_dependencies(hiroce5_ko hisdk5_ko) + add_dependencies(hiroce5_ko hinic5_ko) + add_dependencies(hiroce5_ko ubus_ko) +endif() \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/Makefile b/drivers/infiniband/hw/hiroce5/kernel/Makefile new file mode 100644 index 000000000..0527b1b8a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/Makefile @@ -0,0 +1,543 @@ +ifeq ($(KERNELRELEASE), ) + +.PHONY: all kernel install_kernel install clean clean_kernel + +CWD=$(shell pwd) +TOP_DIR=$(CWD)/../../../../../../.. +INSTALL_MOD_PATH=$(TOP_DIR)/ci/bin + +KERN_VERSION=$(shell uname -r) +KSRC=/lib/modules/$(shell uname -r)/build +ifeq ($(CONFIG_UB_UNIFIED_UBUS), y) + KERN_VERSION ?= 6.6.0-132.0.0.111.oe2403sp3.aarch64 + KSRC=${UBUS_BUILD_KERNEL_DIR} +endif + +ENABLE_DEBUG ?= OFF +ENABLE_HYPER_ROCE ?= ON + +PANGEA_EULER_2107=4.19.90-vhulk2107.1.0.h699.eulerosv2r10.aarch64 +PANGEA_EULER_2108=4.19.90-vhulk2108.6.0.h832.eulerosv2r10.aarch64 +PANGEA_EULER_2202=4.19.90-vhulk2202.2.0.h1064.eulerosv2r10.aarch64 +EULER_V2R12_H1032=5.10.0-136.12.0.86.h1032.eulerosv2r12.x86_64 +EULER_V2R12_ARM_H1032=5.10.0-136.12.0.86.h1032.eulerosv2r12.aarch64 +EULER_V2R12_ARM_H1258=5.10.0-136.12.0.86.h1258.eulerosv2r12.aarch64 +EULER_V2R11_X86=5.10.0-60.18.0.50.h509.eulerosv2r11.x86_64 +EULER_V2R11_ARM=5.10.0-60.18.0.50.h665.eulerosv2r11.aarch64 +HCE_2_ARM_R865_35=5.10.0-60.18.0.50.r865_35.hce2.aarch64 +HCE_2_ARM_R1526_92=5.10.0-136.12.0.86.r1526_92.hce2.aarch64 +HCE_3_ARM_R1040_72=6.6.0-72.0.0.76.r1040_72.hce3.aarch64 +HCE_3_ARM_R1245_84=6.6.0-72.0.0.76.r1245_84.hce3.aarch64 +OPEN_EULER_153_2203_ARM=5.10.0-153.12.0.92.oe2203sp2.aarch64 +OPEN_EULER_136_2203_ARM=5.10.0-136.12.0.86.oe2203sp1.aarch64 +OPEN_EULER_2403_X86=6.6.0-28.0.0.34.oe2403.x86_64 +OPEN_EULER_2403_ARM=6.6.0-28.0.0.34.oe2403.aarch64 +OPEN_EULER_2403_SP1_ARM=6.6.0-72.0.0.76.oe2403sp1.aarch64 +OPEN_EULER_2403_SP2_ARM=6.6.0-98.0.0.103.oe2403sp2.aarch64 +OPEN_EULER_2403_SP3_ARM=6.6.0-132.0.0.111.oe2403sp3.aarch64 +EULER_V2R9_ARM=4.19.90-vhulk2009.2.0.h310.eulerosv2r9.aarch64 + +OS_NAME := $(shell grep -E '^NAME=' /etc/os-release | cut -d= -f2 | tr -d '"') +OS_VERSION_ID := $(shell grep -E '^VERSION_ID=' /etc/os-release | cut -d= -f2 | tr -d '"') +OS_VERSION := $(shell grep -E '^VERSION=' /etc/os-release | cut -d= -f2 | tr -d '"') + +KBUILD_EXTRA_SYMBOLS+=$(TOP_DIR)/ci/lib/SDK_Module.symvers +KBUILD_EXTRA_SYMBOLS+=$(TOP_DIR)/ci/lib/NIC_Module.symvers + +# Set CFLAGS based on public macro definations +CFLAGS += -DHW_CONVERT_ENDIAN +CFLAGS += -DROCE_SERVICE +CFLAGS += -DROCE_DRIVER_COMPILE +CFLAGS += -D__ROCE_DFX__ +CFLAGS += -DROCE_CC_EN +CFLAGS += -DRDMA_ROCE_MODULE +CFLAGS += $(HI1823_FLAGS) +CFLAGS += -Werror -Wall +CFLAGS += -DROCE_DWQE_EN + +# Set CFLAGS, KBUILD_EXTRA_SYMBOLS and other THIRDPARTY* variables +# from OS and kernel version information +ifeq ($(OFED_VERSION), MLNX_OFED) + THIRDPARTY = /opt/buildtools/mlnx-ofa_kernel-5.8 + KBUILD_EXTRA_SYMBOLS+=/opt/buildtools/mlnx-ofa_kernel-5.8/Module.symvers +else ifeq ($(KERN_VERSION),$(filter $(KERN_VERSION), $(PANGEA_EULER_2107) $(PANGEA_EULER_2108) $(PANGEA_EULER_2202))) + KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/pangea/output/aarch64/euleros/ofed_knl/include/Module.symvers + THIRDPARTY_INC = $(TOP_DIR)/pangea/output/aarch64/euleros/ofed_knl/ + CFLAGS += -DCONFIG_INFINIBAND_USER_MEM + CFLAGS += -DHAVE_GET_USER_PAGES_GUP_FLAGS + CFLAGS += -DCONFIG_MODULE_UPGRADE + CFLAGS += -DCONFIG_NVMF_AA +else ifeq ($(KERN_VERSION),$(filter $(KERN_VERSION), $(EULER_V2R12_H1032) $(EULER_V2R12_ARM_H1032) $(EULER_V2R12_ARM_H1258) \ + $(EULER_V2R11_ARM) $(OPEN_EULER_153_2203_ARM) $(HCE_2_ARM_R865_35) $(HCE_2_ARM_R1526_92) $(OPEN_EULER_136_2203_ARM) \ + $(EULER_V2R9_ARM) $(OPEN_EULER_2403_X86) $(OPEN_EULER_2403_ARM) $(OPEN_EULER_2403_SP1_ARM) $(OPEN_EULER_2403_SP2_ARM) $(OPEN_EULER_2403_SP3_ARM) \ + $(HCE_3_ARM_R1040_72) $(HCE_3_ARM_R1245_84))) + THIRDPARTY_INC = /usr/src/kernels/$(KERN_VERSION)/include +else ifeq ($(OS_NAME),openEuler) + ifeq ($(OS_VERSION_ID),24.03) + ifneq (,$(findstring SP4,$(OS_VERSION))) + THIRDPARTY_INC = /usr/src/kernels/$(KERN_VERSION)/include + endif + endif +else ifeq ($(KERN_VERSION), $(filter $(KERN_VERSION), $(EULER_V2R11_X86))) + THIRDPARTY =/usr/src/compat-rdma-4.17 + KBUILD_EXTRA_SYMBOLS+=/usr/src/compat-rdma/Module.symvers + THIRDPARTY_INC = /usr/src/compat-rdma +else + ifeq ($(CROSS_COMPILE),) + THIRDPARTY =/usr/src/compat-rdma-4.17 + KBUILD_EXTRA_SYMBOLS+=/usr/src/compat-rdma/Module.symvers + THIRDPARTY_INC = /usr/src/compat-rdma +# centos compat 417 need 6 para + CFLAGS += -DHAVE_NLA_PARSE_6_PARAMS + else + THIRDPARTY =/arm/cross_compile/install/sysroot/usr/src/compat-rdma-4.8 + KBUILD_EXTRA_SYMBOLS+=/arm/cross_compile/install/sysroot/usr/src/compat-rdma/Module.symvers + THIRDPARTY_INC = /arm/cross_compile/install/sysroot/usr/src/compat-rdma + endif +endif + +# Set CFLAGS from compiled driver version +ifeq ($(VERSION),ROCE_STANDARD) + CFLAGS += -DROCE_STANDARD + CFLAGS += -DROCE_BONDING_EN + CFLAGS += -DSIGN_MTT_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond + # CFLAGS += -DROCE_LATENCY_PROF_EN # 时延统计工具，需手动开启 +else ifeq ($(VERSION),ROCE_VBS) + CFLAGS += -DROCE_STANDARD + CFLAGS += -DROCE_BONDING_EN + CFLAGS += -DSIGN_MTT_EN + CFLAGS += -DROCE_VBS_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond + CFLAGS += -I$(TOP_DIR)/test/at_test/host/service/vbs/extend/roce_kernel/ +else ifeq ($(VERSION),CHIP_TEST) + CFLAGS += -DROCE_CHIP_TEST + CFLAGS += -DSIGN_MTT_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/chip_ext +else ifeq ($(VERSION),ROCE_COMPUTE) + CFLAGS += -DROCE_STANDARD + CFLAGS += -DROCE_COMPUTE +else ifeq ($(VERSION),ROCE_VROCE) + CFLAGS += -DROCE_VROCE_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_extended_library/host/service/roce/linux/kernel/extern/vroce/include +endif + +ifeq ($(ENABLE_DEBUG),ON) + CFLAGS += -DENABLE_DEBUG +endif + +ifeq ($(ENABLE_HYPER_ROCE),ON) + CFLAGS += -DENABLE_HYPER_ROCE + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce/tool + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/cqm +endif + +# Set CFLAGS from default file directories +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce/hi182x +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/drv_intf +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/cq +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/event +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/mr +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/qp +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/rdma +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/srq +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/xrc +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/uld +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/card +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/cmd +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/db +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netdev +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netlink +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sdk_api +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sysfs +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/dfx +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/option +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/debug +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/obj_ext +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_cfm_intf/hmm +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/urts +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/crm +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/rdma +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ccp +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg +CFLAGS += -I$(TOP_DIR)/platform/huawei_secure_c/include +CFLAGS += -I$(TOP_DIR)/platform/huawei_secure_c/src +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/roce +CFLAGS += -DSECUREC_EXPORT_KERNEL_SYMBOL=0 + +##Safe compilation options +ifneq ($(HI1823_OS_TYPE), BCLINUX) + CFLAGS += -fstack-protector-all -Wl,-z,noexecstack +endif + +# Set CFLAGS from Driver version info, +# Currently not in use. +CFLAGS += -D HIROCE5_DRV_VER=\"$(HIROCE5_DRV_VERSION)\" + +export KERN_VERSION +export KBUILD_EXTRA_SYMBOLS + +DEPMOD = /sbin/depmod +INSTALL_MOD_DIR ?= $(shell test -f /etc/redhat-release && echo extra/ofa_kernel || echo updates) + +##Define compilation target +all: kernel + +install: install_kernel +install_kernel: install_modules + +##Test whether there are autoconf.h, kconfig.h files in the kernel directory, and if so, refer to the header file later +autoconf_h=$(shell /bin/ls -1 $(KSRC)/include/*/autoconf.h 2> /dev/null | head -1) +kconfig_h=$(shell /bin/ls -1 $(KSRC)/include/*/kconfig.h 2> /dev/null | head -1) + +ifneq ($(kconfig_h),) + KCONFIG_H = -include $(kconfig_h) +endif + +##Test whether V variable is defined. If so, "make" will print some information during subsequent compilation +V ?= 0 + +##KBUILD_NOCMDDEP=1, when "make" again after "make", it will not recompile +##LINUXINCLUDE, reconfigure kernel module compilation options +##__OFED_BUILD__, use macro definition configured by OFED instead of that configured by kernel + +linux_include := -include $(autoconf_h) \ + $(KCONFIG_H) \ + $(BACKPORT_INCLUDES) \ + -I$(THIRDPARTY)/include \ + -I$(THIRDPARTY)/include/uapi \ + $$(if $$(CONFIG_XEN),-I$$(srctree)/arch/x86/include/mach-xen) \ + $$(if $$(CONFIG_XEN),-D__XEN_INTERFACE_VERSION__=$$(CONFIG_XEN_INTERFACE_VERSION)) \ + -I$$(srctree)/arch/$$(SRCARCH)/include \ + -I$$(srctree)/arch/$$(SRCARCH)/include/uapi \ + -Iarch/$$(SRCARCH)/include/generated \ + -Iarch/$$(SRCARCH)/include/generated/uapi \ + -Iinclude \ + $$(if $$(KBUILD_SRC),-Iinclude2 -I$$(srctree)/include) \ + -I$$(srctree)/include/uapi \ + -I$$(srctree)/arch/$$(SRCARCH)/include \ + -Iinclude/generated/uapi \ + -Iarch/$$(SRCARCH)/include/generated + +######################### +# make kernel # +######################### +#NB: The LINUXINCLUDE value comes from main kernel Makefile +# with local directories prepended. This eventually affects +# CPPFLAGS in the kernel Makefile +kernel: $(COMPAT_CONFIG) $(COMPAT_AUTOCONF) + @echo '============== INFO ==============' + @echo 'Building kernel modules' + @echo 'Driver Version: $(VERSION)' + @echo 'Kernel Version: $(KERN_VERSION)' + @echo 'Modules Directory: $(INSTALL_MOD_PATH)/$(MODULES_DIR)' + @echo 'Kernel Sources: $(KSRC)' + @echo 'Thirdparty Source: $(THIRDPARTY)' + @echo 'Thirdparty Symbols: $(KBUILD_EXTRA_SYMBOLS)' + @echo 'TOP_DIR: $(TOP_DIR)' + @echo 'HI1823_TRUNK_DIR: $(HI1823_TRUNK_DIR)' + @echo '==================================' + $(MAKE) -C $(KSRC) M="$(CWD)" V=$(V) KBUILD_NOCMDDEP=1 $(WITH_MAKE_PARAMS) EXTRA_CFLAGS="$(CFLAGS)" \ + LINUXINCLUDE='$(linux_include)' ENABLE_DEBUG=$(ENABLE_DEBUG) ENABLE_HYPER_ROCE=$(ENABLE_HYPER_ROCE) modules + +######################### +# Install kernel # +######################### +install_modules: + @echo "Installing kernel modules" + + $(MAKE) -C $(KSRC) M="$(CWD)" \ + INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) \ + INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \ + $(WITH_MAKE_PARAMS) modules_install; + if [ ! -n "$(INSTALL_MOD_PATH)" ]; then $(DEPMOD) $(KERN_VERSION);fi; + +clean: clean_kernel + +clean_kernel: + @echo "clean kernel modules" + $(MAKE) -C $(KSRC) M="$(CWD)" $(WITH_MAKE_PARAMS) clean + @/bin/rm -f $(clean-files) + +clean-files := Module.symvers modules.order Module.markers compat/modules.order +clean-files += $(COMPAT_CONFIG) $(COMPAT_AUTOCONF) + +help: + @echo + @echo "kernel: build kernel modules" + @echo "all: build kernel modules" + @echo + @echo "install_kernel: install kernel modules under $(INSTALL_MOD_PATH)/$(MODULES_DIR)" + @echo "install: run install_kernel" + @echo + @echo "clean: delete kernel modules binaries" + @echo "clean_kernel: delete kernel modules binaries" + @echo + +else + +obj-m += hiroce5.o + +ENABLE_DEBUG ?= OFF +$(info kernel ENABLE_DEBUG: $(ENABLE_DEBUG)) + +ifeq ($(CONFIG_UBUS_DEVICE), y) +TOP_DIR=$(HI1823_TRUNK_DIR) + +GLOBAL_VERSION=$(shell cat $(HI1823_TRUNK_DIR)/src/GLOBAL_VERSION_NEW | grep driver | awk -F ':' '{print $$2}') +ccflags-y += -DGLOBAL_VERSION_STR=\"$(GLOBAL_VERSION)\" +ccflags-y += -DHW_CONVERT_ENDIAN +ccflags-y += -DROCE_SERVICE +ccflags-y += -DROCE_DRIVER_COMPILE +ccflags-y += -D__ROCE_DFX__ +ccflags-y += -DROCE_CC_EN +ccflags-y += -DRDMA_ROCE_MODULE +ccflags-y += $(HI1823_FLAGS) +# ccflags-y += -Werror -Wall +ccflags-y += -Wall +ccflags-y += -DROCE_DWQE_EN + +KBUILD_EXTRA_SYMBOLS += $(HISDK5_SYMVERS) +KBUILD_EXTRA_SYMBOLS += $(HINIC5_SYMVERS) +# ccflags-y = /usr/src/kernels/$(KERN_VERSION)/include + +ccflags-y += -DROCE_STANDARD +ccflags-y += -DROCE_BONDING_EN +ccflags-y += -DSIGN_MTT_EN + +ifeq ($(ENABLE_DEBUG),ON) + ccflags-y += -DENABLE_DEBUG +endif +ifeq ($(ENABLE_HYPER_ROCE),ON) + ccflags-y += -DENABLE_HYPER_ROCE + ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce + ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce/tool + ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/cqm +endif + +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce/hi182x +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/drv_intf +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/cq +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/event +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/mr +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/qp +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/rdma +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/srq +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/xrc +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/uld +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/card +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/cmd +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/db +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netdev +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netlink +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sdk_api +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sysfs +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/dfx +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/option +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/debug +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/obj_ext +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_cfm_intf/hmm +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/urts +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/hwsdk +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/ +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/rdma +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/crm +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ccp +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/roce +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg +ccflags-y += -I$(TOP_DIR)/platform/huawei_secure_c/include +ccflags-y += -I$(TOP_DIR)/platform/huawei_secure_c/src +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/knldk/rdma +ccflags-y += -I$(TOP_DIR)/..//opensource/openeuler/kernel/olk-6.6/include/linux/ +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface +ccflags-y += -I$(TOP_DIR)/../opensource/openeuler/kernel/olk-6.6/include +ccflags-y += -nostdinc +endif + +# List files of release here. +ccflags-y += -DSECUREC_EXPORT_KERNEL_SYMBOL=0 +hiroce5-y := ../../../../../../../platform/huawei_secure_c/src/securecutil.o \ + ../../../../../../../platform/huawei_secure_c/src/secureinput_a.o \ + ../../../../../../../platform/huawei_secure_c/src/secureprintoutput_a.o \ + ../../../../../../../platform/huawei_secure_c/src/memset_s.o \ + ../../../../../../../platform/huawei_secure_c/src/memcpy_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strncpy_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strncat_s.o \ + ../../../../../../../platform/huawei_secure_c/src/sprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/vsprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/vsnprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/snprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/vsscanf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/sscanf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/memmove_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strcpy_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strcat_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strtok_s.o \ + ../../../../cfm/hmm/hmm_buddy.o \ + ../../../../cfm/hmm/hmm_common.o \ + ../../../../cfm/hmm/hmm_em.o \ + ../../../../cfm/hmm/hmm_init.o \ + ../../../../cfm/hmm/hmm_mpt.o \ + ../../../../cfm/hmm/hmm_mr.o \ + ../../../../cfm/hmm/hmm_mtt.o \ + ../../../../cfm/hmm/hmm_umem.o \ + ../../../../cfm/rdma/rdma_bitmap.o \ + base/cq/roce_cq_common.o \ + base/cq/roce_cq_cqe.o \ + base/cq/roce_cq_create.o \ + base/cq/roce_cq_ctrl.o \ + base/cq/roce_cq_destroy.o \ + base/cq/roce_cq_intern.o \ + base/extension/roce_event_extension.o \ + base/extension/roce_main_extension.o \ + base/extension/roce_mr_extension.o \ + base/extension/roce_qp_extension.o \ + base/extension/roce_srq_extension.o \ + base/mr/roce_mr.o \ + base/qp/roce_qp_create.o \ + base/qp/roce_qp_destroy.o \ + base/qp/roce_qp_ext.o \ + base/qp/roce_qp_modify.o \ + base/qp/roce_qp_post_recv.o \ + base/qp/roce_qp_post_send.o \ + base/qp/roce_qp_query.o \ + base/rdma/rdma_comp.o \ + base/rdma/rdma_comp_res.o \ + base/rdma/rdma_comp_gid.o \ + base/rdma/rdma_comp_init.o \ + base/rdma/rdma_comp_pd.o \ + base/rdma/rdma_comp_mw_mr.o \ + base/rdma/roce_pd.o \ + base/rdma/roce_mix.o \ + base/srq/roce_srq_comm.o \ + base/srq/roce_srq_create.o \ + base/srq/roce_srq_ctrl.o \ + base/xrc/roce_xrc.o \ + base/event/roce_event.o \ + base/uld/roce_uld_api.o \ + dev/card/roce_card.o \ + dev/db/roce_db.o \ + dev/cmd/roce_cmd.o \ + dev/init/roce_main.o \ + dev/netdev/roce_netdev.o \ + dev/netlink/roce_netlink_process.o \ + dev/netlink/roce_netlink.o \ + dev/sdk_api/roce_cqm_cmd.o \ + dev/sysfs/roce_sysfs.o \ + dev/sysfs/roce_ucc_sysfs.o \ + dev/sysfs/roce_multipath_sysfs.o \ + util/dfx/roce_dfx.o \ + util/dfx/roce_dfx_query.o \ + util/option/roce_option.o \ + util/obj_ext/roce_obj_ext.o + +ifeq ($(ENABLE_DEBUG),ON) + # List debug files here. + hiroce5-y += util/dfx/roce_dfx_debug.o + hiroce5-y += util/dfx/roce_dfx_attack.o +endif + +ifeq ($(ENABLE_HYPER_ROCE),ON) + # List hyper roce files here. + hiroce5-y += extern/hyper_roce/hyper_roce_netlink.o + hiroce5-y += extern/hyper_roce/hyper_roce_qp.o + hiroce5-y += extern/hyper_roce/tool/hyper_roce_tool.o + hiroce5-y += extern/hyper_roce/tool/hyper_roce_tool_pktdrop.o +endif + +# Add hiroce5 objects from compiled driver version +# from dpu_extended_library directories +ifeq ($(VERSION),ROCE_STANDARD) + hiroce5-y +=util/bond/roce_bond_mac.o + hiroce5-y +=util/bond/roce_bond_slave.o + hiroce5-y +=util/bond/roce_bond_cfm.o + hiroce5-y +=util/bond/roce_bond_device.o + hiroce5-y +=util/bond/roce_bond_tx.o +else ifeq ($(VERSION),ROCE_COMPUTE) + # hiroce5-y +=util/bond/roce_bond_common.o +else ifeq ($(VERSION),CHIP_TEST) +# CHIP_TEST or ROCE_VBS, as VBS driver currently runs on chip test + hiroce5-y +=extern/chip_ext/roce_main_chip_ext.o + hiroce5-y +=extern/chip_ext/roce_mr_chip_ext.o + hiroce5-y +=extern/chip_ext/roce_qp_chip_ext.o + hiroce5-y +=extern/chip_ext/roce_srq_chip_ext.o +else ifeq ($(VERSION),ROCE_VROCE) + hiroce5-y +=util/bond/roce_bond_mac.o + hiroce5-y +=util/bond/roce_bond_slave.o + hiroce5-y +=util/bond/roce_bond_cfm.o + hiroce5-y +=util/bond/roce_bond_device.o + hiroce5-y +=util/bond/roce_bond_tx.o + hiroce5-y +=../../../../../../../src/dpu_extended_library/host/roce/vroce/roce_cmd_ext.o +else ifeq ($(VERSION),ROCE_VBS) + hiroce5-y +=util/bond/roce_bond_mac.o + hiroce5-y +=util/bond/roce_bond_slave.o + hiroce5-y +=util/bond/roce_bond_cfm.o + hiroce5-y +=util/bond/roce_bond_device.o + hiroce5-y +=util/bond/roce_bond_tx.o + hiroce5-y +=../../../../../../../test/at_test/host/service/vbs/extend/roce_kernel/roce_vbs_qp.o +endif + +# Driver Extension +# DPU_HIROCE5_DRV_EXTEND_MK, DPU_DRV_MK_DIR, DPU_DRV_HIROCE5_EXTEND_OBJS对外提供用户使用，修改需要评审 +ifneq ($(DPU_HIROCE5_DRV_EXTEND_MK),) + $(info Using roce extension $(DPU_HIROCE5_DRV_EXTEND_MK)) + export DPU_DRV_MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + include $(DPU_HIROCE5_DRV_EXTEND_MK) + hiroce5-y += $(DPU_DRV_HIROCE5_EXTEND_OBJS) +endif + +build_info: + mkdir -p build + @echo "CURDIR=$(CURDIR)" > build/build_src.txt + @for obj in $(hiroce5-y); do \ + echo $$(realpath $$obj) >> build/build_src.txt; \ + done + +endif + +include $(HI1823_BUILD_DIR)/host/linux/Makefile.ko \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h new file mode 100644 index 000000000..37562da9c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQ_H +#define ROCE_CQ_H + +#include <rdma/ib_verbs.h> +#include <linux/slab.h> + +#include "hinic5_hw.h" +#include "hinic5_rdma.h" + +#include "roce.h" +#include "roce_srq.h" + +#include "roce_option.h" +#include "hinic5_hmm.h" + +#include "securec.h" + +/* DB type of ARM_CQ */ +enum { ROCE_CQ_DB_REQ_NOT_SOL = 1, ROCE_CQ_DB_REQ_NOT = 2 }; + +#define CQ_DW_TO_BYTE 4 +#define ROCE_CQE_UP_THRESHOLD 1024 + +/* +1.Same type of operation as SQ WQE +8'h00-Send +8'h01-Send with Invalidate +8'h02-Send with Immediate Data +8'h03-reserved + +8'h04-RDMA Write +8'h05-RDMA Write with Immediate Data +8'h06-reserved +8'h07-reserved + +8'h08-RDMA Read +8'h09-reserved +8'h0a-reserved +8'h0b-reserved + +8'h0c-Atomic Compare & Swap +8'h0d-Atomic Fetch & Add +8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation) +8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation) + +8'h10-Fast Register PMR +8'h11-Local Invalidate +8'h12-Bind Memory Window Type1/2 +8'h13-Local opreation(Extended for further local opreation) +other-Reserved + +2.Receive +00000 - RDMA Write with Immediate +00001 - Send +00010 - Send with Immediate +00011 - Send & Invalidate + +3.The following are general +11110 Error coding +10110 Resize coding +*/ +enum roce5_cqe_send_opcode { + ROCE_OPCODE_SEND = 0x0, + ROCE_OPCODE_SEND_WITH_INV = 0x1, + ROCE_OPCODE_SEND_WITH_IMM = 0x2, + /* 0x3 reserved */ + + ROCE_OPCODE_RDMA_WRITE = 0x4, + ROCE_OPCODE_RDMA_WRITE_WITH_IMM = 0x5, + /* 0x6 and 0x7 reserved */ + + ROCE_OPCODE_RDMA_READ = 0x8, + /* 0x9~0xb reserved */ + + ROCE_OPCODE_ATOMIC_COMP_AND_SWP = 0xc, + ROCE_OPCODE_ATOMIC_FETCH_AND_ADD = 0xd, + ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP = 0xe, + ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD = 0xf, + + ROCE_OPCODE_FAST_REG_PMR = 0x10, + ROCE_OPCODE_LOCAL_INV = 0x11, + ROCE_OPCODE_BIND_TYPE2_MW = 0x12, + ROCE_OPCODE_REG_SIG_MR = 0x13, + + ROCE_OPCODE_RESIZE_CQE = 0x16, + ROCE_OPCODE_ERR = 0x1e, + ROCE_OPCODE_CQE_UNUSED = 0x1f /* Be used in new CQ buf when reszie cq */ +}; + +enum roce5_cqe_recv_opcode { + ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM = 0x0, + ROCE_RECV_OPCODE_SEND = 0x1, + ROCE_RECV_OPCODE_SEND_WITH_IMM = 0x2, + ROCE_RECV_OPCODE_SEND_WITH_INV = 0x3 +}; + +/* Define the state type of the CQ */ +enum cq_state { + ROCE_CQ_STATE_INVALID = 0x0, + ROCE_CQ_STATE_ERR = 0x1, + ROCE_CQ_STATE_OVERFLOW = 0x2, + ROCE_CQ_STATE_VALID = 0xf, + ROCE_CQ_STATE_MEM_INIT = 0xa /* Initial value of Host Memory */ +}; + +#define ROCE_CQ_TIME_OUT_CHECK_VALUE 0xe +#define ROCE_CQ_STATE_CHECK_VALUE 0x0 + +#define ROCE_CQE_RQ_INLINE 1 +#define ROCE_CQE_RQ_NORMAL 0 +#define ROCE_CQE_SEND_COMP 1 +#define ROCE_CQE_RECV_COMP 0 + +#define ATOMIC_DATA_LEN 8 /* Specified as 8B by protocol */ + +#define ROCE_CQE_INVALID_VALUE 0xff +#define ROCE_CQ_RESIZE_POLL_TIMES 100 + +/* roce Commands, bufs and data structures related to cq */ +struct roce5_cq_buf { + struct tag_cqm_buf *buf; /* pointer to describe the buf structure b cqm */ + struct mtt + mtt; /* the mtt struct used by kernel mode and user mode to discribe buf */ + int entry_size; /* the size of cqe */ + int buf_size; /* the size of cq_buf */ +}; + +struct roce5_cq_resize_buf { + struct roce5_cq_buf buf; /* the size of buf that was resized */ + int cqe; /* the number of resized cqe */ +}; + +#define ROCE_RCQ_ULD_DEF_LEN 64 +#define ROCE_RCQ_MAGIC 0x5a5ae0e0 +struct roce5_rcq_uld_def { + u8 uld_def[ROCE_RCQ_ULD_DEF_LEN]; +}; + +struct roce5_cq { + struct ib_cq ibcq; + cqm_queue_s *cqm_cq; /* Save the handle obtained from cqm */ + struct roce5_db + db; /* The address information of the software DB that stores the CI user mode/kernel mode */ + __be32 *set_ci_db; /* Kernel-mode software DB */ + u32 cons_index; /* consumer pointer */ + + u32 arm_sn; /* the serial number of arm */ + int arm_flag; /* Used to determine whether the arm command has been sent, to avoid repeated sending */ + u32 cqn; + unsigned int vector; /* associated to the eq used */ + struct roce5_cq_buf buf; /* pointer describing the buf structure of cqm */ + struct roce5_cq_resize_buf *resize_buf; /* resize buf struction */ + + spinlock_t lock; /* Need to lock when operating cq */ + struct mutex resize_mutex; /* resize the mutex of cq */ + struct ib_umem *umem; /* record the infomation mapped by User-mode buf */ + struct ib_umem * + resize_umem; /* record the infomation mapped by buf which the User-mode resized */ + + struct list_head send_qp_list; /* send queue qp */ + struct list_head recv_qp_list; /* receive queue qp */ + + int reset_notify_added; + struct list_head reset_notify; + u32 cqe_mask; /* mask of the cq size */ + + /* + * 'left_num' and 'wqebb_index' are only valid in the merged CQE within the RQ. + * 'left_num' records the remaining count of the last merged CQE. + * 'wqebb_index' records the wqebb index of the current CQE. + */ + u32 left_num; + u32 wqebb_index; + struct roce5_option *opt; + u8 shadow; + u32 shadow_vfid; + + void (*reset_flow_comp)(struct roce5_cq *); + + struct roce5_rcq_uld_def + uld_def; /* must placed at cq_magic before, preventstop uld access memory stomping, support up to 64B space */ + u32 rcq_magic; +}; + +/* Used when destroy QP */ +void roce5_cq_clean(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq); +void roce5_cq_async_event(struct roce5_device *rdev, struct roce5_cq *cq, + int type); +void roce5_cq_clean_process(struct roce5_cq *cq, u32 qpn, + struct roce5_srq *srq); +void roce5_cq_put_umem(struct roce5_device *rdev, struct roce5_cq_buf *buf, + struct ib_umem **umem, u8 shadow_cq); +int roce5_create_cq_common(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce5_cq *rcq, + u32 index); +int roce5_cq_get_umem(struct roce5_device *rdev, struct ib_ucontext *context, + struct roce5_cq *rcq, u64 buf_addr, int cqe, + bool is_resize); +void *roce5_get_sw_cqe(struct roce5_cq *cq, unsigned int n); +void *roce5_get_cqe_from_buf(struct roce5_cq_buf *buf, unsigned int n); +void roce5_cq_set_ci(struct roce5_cq *cq); +void roce5_cq_buf_init(struct roce5_cq_buf *buf); +void *roce5_get_cqe(struct roce5_cq *cq, unsigned int n); +void roce5_reset_flow_comp(struct roce5_cq *rcq); + +void roce5_lock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __acquires(&roce5_send_cq->lock) __acquires(&roce5_recv_cq->lock); +void roce5_unlock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __releases(&roce5_send_cq->lock) __releases(&roce5_recv_cq->lock); + +struct roce5_cq *roce5_get_rcq_by_cqn(struct roce5_device *rdev, u32 cqn); +void roce5_cq_completion(void *svc_hd, u32 cqn, void *cq_handler); + +static inline void roce5_rcq_put(struct roce5_cq *rcq) +{ + if (rcq == NULL) { + return; + } + roce5_option_put_value(rcq->opt); + cqm5_object_put(&rcq->cqm_cq->object); +} + +#endif // ROCE_CQ_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c new file mode 100644 index 000000000..f212acecc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c @@ -0,0 +1,299 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "hinic5_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_user.h" +#include "roce_cq.h" +#include "roce_xqe_format.h" +#include "roce_option.h" +#include "hinic5_hmm.h" +#include "securec.h" +#include "roce_os_compat.h" +#include "roce_aeq_format.h" + +/* **************************************************************************** + Prototype : roce5_cq_async_event + Description : roce5_cq_async_event + Input : struct roce5_device *rdev + struct roce5_cq *cq + int type + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_async_event(struct roce5_device *rdev, struct roce5_cq *cq, + int type) +{ + struct ib_cq *ibcq = &cq->ibcq; + struct ib_event event; + int ret; + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset event.(ret:%d)", + ret); + } + if (type != ROCE_EVENT_TYPE_CQ_ERROR) { + ROCE_DEV_WARN_LIMIT( + rdev, + "Unexpected event type(0x%x) on CQ(%06x), func_id(%d)", + type, cq->cqn, rdev->glb_func_id); + return; + } + + if (ibcq->event_handler) { + event.device = ibcq->device; + event.event = IB_EVENT_CQ_ERR; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } +} + +/* **************************************************************************** + Prototype : roce5_cq_buf_init + Description : roce5_cq_buf_init + Input : struct roce5_cq_buf *buf + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_buf_init(struct roce5_cq_buf *buf) +{ + /* optype was initialized into ROCE_OPCODE_CQE_UNUSED(0x1f) */ + int ret; + ret = memset_s(buf->buf->direct.va, + (unsigned long)((unsigned)buf->buf_size), + ROCE_CQE_INVALID_VALUE, + (unsigned long)((unsigned)buf->buf_size)); + if (ret != 0) { + ROCE_ERR("Failed to memset va.(ret:%d)", ret); + } +} + +/* **************************************************************************** + Prototype : roce5_cq_put_umem + Description : Release umem and corresponding mtt, corresponding to put + Input : struct roce5_device *rdev + struct roce5_cq_buf *buf + struct ib_umem **umem + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_put_umem(struct roce5_device *rdev, struct roce5_cq_buf *buf, + struct ib_umem **umem, u8 shadow_cq) +{ + if (shadow_cq != 0) { + return; + } + /* free MTT of Buffer */ + hmm_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + + /* release umem */ + ib_umem_release(*umem); +} + +static void *roce5_cq_buf_offset(struct tag_cqm_buf *buf, unsigned int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce5_get_cqe_from_buf(struct roce5_cq_buf *buf, unsigned int n) +{ + return roce5_cq_buf_offset(buf->buf, (n * buf->entry_size)); +} + +void *roce5_get_cqe(struct roce5_cq *cq, unsigned int n) +{ + return roce5_get_cqe_from_buf(&cq->buf, n); +} + +/* **************************************************************************** + Prototype : roce5_get_sw_cqe + Description : roce5_get_sw_cqe + Input : struct roce5_cq *cq + unsigned int n + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void *roce5_get_sw_cqe(struct roce5_cq *cq, unsigned int n) +{ + struct roce_cqe *cqe = NULL; + struct roce_cqe tmp_cqe; + unsigned int tmp_val; + + cqe = (struct roce_cqe *)roce5_get_cqe( + cq, (n & ((unsigned int)cq->cqe_mask))); + if (cqe == NULL) { + ROCE_ERR("Get sw cqe failed."); + return NULL; + } + tmp_cqe.dw0.value = roce5_convert_cpu32(cqe->dw0.value); + tmp_cqe.dw1.value = roce5_convert_cpu32(cqe->dw1.value); + + /* Add judgment condition: the optype of CQE cannot be UNUSED, UNUSED means that it has been initialized in resize + cq */ + tmp_val = ((n & ((unsigned int)cq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) == + 0) ? + 1 : + 0; + if ((ROCE_LIKELY(tmp_cqe.dw1.bs.op_type != ROCE_OPCODE_CQE_UNUSED)) && + ((tmp_cqe.dw0.bs.owner ^ tmp_val) != 0)) { + return cqe; + } + + return NULL; +} + +int roce5_cq_get_umem(struct roce5_device *rdev, struct ib_ucontext *context, + struct roce5_cq *rcq, u64 buf_addr, int cqe, + bool is_resize) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + int cqe_size = (int)rdev->rdma_cap.cqe_size; + struct roce5_cq_buf *buf = NULL; + struct ib_umem **umem = NULL; + + if (rcq->shadow != 0) { + return 0; + } + if (is_resize) { + buf = &rcq->resize_buf->buf; + umem = &rcq->resize_umem; + } else { + buf = &rcq->buf; + umem = &rcq->umem; + } + + *umem = roce5_umem_get(&rdev->ib_dev, context, buf_addr, + (unsigned long)(cqe * cqe_size), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(*umem)) { + ROCE_DEV_ERR(rdev, "Failed to get ib_umem, func_id(%d)", + rdev->glb_func_id); + return (int)PTR_ERR(*umem); + } + + page_shift = (int)roce5_umem_get_buf_page_shift(rdev, *umem, buf_addr); + npages = roce5_umem_get_mtt_npages(rdev, *umem, page_shift); + + buf->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &buf->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma_mtt, func_id(%d)", + rdev->glb_func_id); + goto err_buf; + } + + ret = roce5_umem_write_mtt(rdev, &buf->mtt, *umem, true); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt for umem, func_id(%d)", + rdev->glb_func_id); + goto err_mtt; + } + + return 0; + +err_mtt: + hmm_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + +err_buf: + ib_umem_release(*umem); + + return ret; +} + +void roce5_reset_flow_comp(struct roce5_cq *rcq) +{ + struct ib_cq *ibcq = &rcq->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +/** + * @brief Get roce cq by cqn. When querying succeeded, you have to call roce5_rcq_put() to free + the cqm_obj and opt. But there is no need to call roce5_rcq_put() if querying failed. + * @return Success: return the roce cq. Failure: return NULL. + */ +struct roce5_cq *roce5_get_rcq_by_cqn(struct roce5_device *rdev, u32 cqn) +{ + cqm_object_s *cqm_obj = NULL; + cqm_queue_s *cqm_queue = NULL; + struct roce5_option *opt = NULL; + struct roce5_cq *rcq = NULL; + + cqm_obj = cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find rcq according to cqn(0x%x), func_id(%d)", + cqn, rdev->glb_func_id); + goto err_out; + } + + cqm_queue = container_of(cqm_obj, cqm_queue_s, object); + opt = (struct roce5_option *)cqm_queue->priv; + if (opt == NULL) { + ROCE_DEV_ERR(rdev, "Failed to get option, priv ptr is NULL."); + goto err_out_cqm_put; + } + + rcq = (struct roce5_cq *)roce5_option_get_value(opt); + if (rcq == NULL) { + ROCE_DEV_ERR(rdev, "Failed to get rcq from option."); + goto err_out_cqm_put; + } + + return rcq; + +err_out_cqm_put: + cqm5_object_put(cqm_obj); +err_out: + return NULL; +} + +void roce5_cq_completion(void *svc_hd, u32 cqn, void *cq_handler) +{ + struct roce5_option *opt = NULL; + struct roce5_cq *rcq = NULL; + + if (cq_handler == NULL) { + ROCE_ERR("Cq_handler is null"); + return; + } + + opt = (struct roce5_option *)cq_handler; + rcq = (struct roce5_cq *)roce5_option_get_value(opt); + if (rcq == NULL) { + ROCE_WARN("Failed to get rcq from option."); + return; + } + + ++rcq->arm_sn; + rcq->arm_flag = 0; + + rcq->ibcq.comp_handler(&rcq->ibcq, rcq->ibcq.cq_context); + + roce5_option_put_value(opt); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c new file mode 100644 index 000000000..b79aad2dc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c @@ -0,0 +1,763 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "hinic5_hw.h" +#include "securec.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" +#include "roce_restore.h" +#include "roce_aeq_format.h" +#include "roce_xqe_format.h" + +#include "roce_main_extension.h" + +/* **************************************************************************** + Prototype : roce5_next_cqe_sw + Description : roce5_next_cqe_sw + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static struct roce_cqe *roce5_next_cqe_sw(struct roce5_cq *cq) +{ + return (struct roce_cqe *)roce5_get_sw_cqe(cq, cq->cons_index); +} + +static int g_error_cqe_to_wc[ROCE_COMPLETE_EVENT_MAX] = { + [ROCE_COMPLETE_EVENT_LOC_LEN_ERR] = IB_WC_LOC_LEN_ERR, + [ROCE_COMPLETE_EVENT_LOC_QP_OPERATION_ERR] = IB_WC_LOC_QP_OP_ERR, + [ROCE_COMPLETE_EVENT_LOC_PROTECTION_ERR] = IB_WC_LOC_PROT_ERR, + [ROCE_COMPLETE_EVENT_WR_FLUSH_ERR] = IB_WC_WR_FLUSH_ERR, + + [ROCE_COMPLETE_EVENT_MW_BIND_ERR] = IB_WC_MW_BIND_ERR, + [ROCE_COMPLETE_EVENT_BAD_RESP_ERR] = IB_WC_BAD_RESP_ERR, + + [ROCE_COMPLETE_EVENT_LOC_ACCESS_ERR] = IB_WC_LOC_ACCESS_ERR, + [ROCE_COMPLETE_EVENT_REM_INV_REQ_ERR] = IB_WC_REM_INV_REQ_ERR, + [ROCE_COMPLETE_EVENT_REM_ACCESS_ERR] = IB_WC_REM_ACCESS_ERR, + [ROCE_COMPLETE_EVENT_REM_OPERATION_ERR] = IB_WC_REM_OP_ERR, + [ROCE_COMPLETE_EVENT_RETRY_CTR_EXCEED_ERR] = IB_WC_RETRY_EXC_ERR, + + [ROCE_COMPLETE_EVENT_RNR_RETRY_CTR_EXCEED_ERR] = + IB_WC_RNR_RETRY_EXC_ERR, + + [ROCE_COMPLETE_EVENT_REM_ABORTED_ERR] = IB_WC_REM_ABORT_ERR, + [ROCE_COMPLETE_EVENT_XRC_VIOLATION_ERR] = IB_WC_REM_INV_RD_REQ_ERR, +}; + +/* **************************************************************************** + Prototype : roce5_handle_error_cqe + Description : roce5_handle_error_cqe + Input : struct roce_err_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_error_cqe(struct roce_err_cqe *cqe, struct ib_wc *wc) +{ + if (cqe->dw7.bs.syndrome >= ROCE_COMPLETE_EVENT_MAX) { + wc->status = IB_WC_GENERAL_ERR; + } else if (cqe->dw7.bs.syndrome > 0) { + wc->status = g_error_cqe_to_wc[cqe->dw7.bs.syndrome]; + if (wc->status == 0) { + wc->status = IB_WC_GENERAL_ERR; + } + } + + wc->vendor_err = cqe->dw7.bs.vendor_err; +} + +static void roce5_get_local_opcode_from_cqe(const struct roce_cqe *cqe, + struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case ROCE_OPCODE_REG_SIG_MR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + ROCE_WARN("Unknown cqe optype"); + break; + } +} + +static void roce5_get_opcode_type_part_1(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + + case ROCE_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_SEND: + wc->opcode = IB_WC_SEND; + break; + + case ROCE_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + break; + default: + roce5_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +static void roce5_get_opcode_type_part_2(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = cqe->byte_cnt; + break; + + case ROCE_OPCODE_ATOMIC_COMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + case ROCE_OPCODE_FAST_REG_PMR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + roce5_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +/* **************************************************************************** + Prototype : roce5_get_opcode_from_scqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_get_opcode_from_scqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->wc_flags = 0; + + if (cqe->dw1.bs.op_type < ROCE_OPCODE_RDMA_READ) { + roce5_get_opcode_type_part_1(cqe, wc); + } else { + roce5_get_opcode_type_part_2(cqe, wc); + } +} + +/* **************************************************************************** + Prototype : roce5_get_opcode_from_rcqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_get_opcode_from_rcqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->byte_len = cqe->byte_cnt; + + switch (cqe->dw1.bs.op_type) { + case ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + /* The driver does not perform big or small endian conversion for immediate data, but the incoming CQE + has been converted to the CPU endian, so it needs to be converted back */ + wc->ex.imm_data = roce5_convert_be32(cqe->imm_invalid_rkey); + break; + + case ROCE_RECV_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = + roce5_convert_be32(cqe->imm_invalid_rkey); + break; + + case ROCE_RECV_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + + case ROCE_RECV_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + /* The driver does not perform big or small endian conversion for immediate data, but the incoming CQE + has been converted to the CPU endian, so it needs to be converted back */ + wc->ex.imm_data = roce5_convert_be32(cqe->imm_invalid_rkey); + break; + + default: + ROCE_WARN("Not supported"); + break; + } +} + +/* **************************************************************************** + Prototype : roce5_cq_poll_and_resize + Description : roce5_cq_poll_and_resize + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_poll_and_resize(struct roce5_device *rdev, + struct roce5_cq *cq) +{ + if (ROCE_LIKELY(cq->resize_buf != NULL)) { + /* Release the original Buffer of CQ */ + cqm5_object_resize_free_old(&cq->cqm_cq->object); + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + } +} + +/* **************************************************************************** + Prototype : roce5_cq_get_cur_qp + Description : roce5_cq_get_cur_qp + Input : struct roce5_cq *cq + struct roce5_qp **cur_qp + struct roce_cqe *cqe + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_get_cur_qp(struct roce5_cq *cq, struct roce5_qp **cur_qp, + struct roce_cqe *cqe) +{ + struct roce5_device *rdev = to_roce5_dev(cq->ibcq.device); + + if ((*cur_qp == NULL) || (cqe->dw0.bs.qpn != (u32)(*cur_qp)->qpn)) { + if (*cur_qp != NULL) { + roce5_rqp_put(*cur_qp); + } + + *cur_qp = roce5_get_rqp_by_qpn(rdev, cqe->dw0.bs.qpn); + if (*cur_qp == NULL) { + ROCE_DEV_ERR( + rdev, + "CQ(%06x) with entry for unknown QPN(%06x), func_id(%d)", + cq->cqn, cqe->dw0.bs.qpn, rdev->glb_func_id); + return -EINVAL; + } + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_get_xrc_srq + Description : roce5_cq_get_xrc_srq + Input : struct roce5_cq *cq + struct roce5_srq **srq + struct roce_cqe *cqe + u32 qp_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_get_xrc_srq(struct roce5_cq *cq, struct roce5_srq **srq, + struct roce_cqe *cqe, u32 qp_type) +{ + u32 srqn = cqe->dw6.bs.srqn_rqpn; + struct roce5_device *rdev = to_roce5_dev(cq->ibcq.device); + + if (qp_type == IB_QPT_XRC_TGT) { + if (*srq != NULL) { + roce5_rsrq_put(*srq); + } + + *srq = roce5_get_rsrq_by_srqn(rdev, cqe->dw6.bs.srqn_rqpn); + if (*srq == NULL) { + ROCE_DEV_ERR( + rdev, + "CQ(%06x) with entry for unknown SRQN(%06x), func_id(%d)", + cq->cqn, srqn, rdev->glb_func_id); + return -EINVAL; + } + } + + return 0; +} + +static int roce5_poll_recv_cqe(struct roce5_cq *rcq, struct roce5_qp *cur_qp, + struct roce5_srq *srq, struct ib_wc *wc) +{ + u32 wqe_ctr = rcq->wqebb_index; + unsigned int tail = 0; + struct roce5_srq *srq_tmp = NULL; + + if (cur_qp->ibqp.srq) { + srq_tmp = to_roce5_srq(cur_qp->ibqp.srq); + /* Determine whether to overflow */ + if (wqe_ctr > (int)srq_tmp->max_depth) { + ROCE_ERR("Get wqe index(0x%x) from cqe", wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq_tmp->wrid[wqe_ctr]; + roce5_free_srq_wqe(srq_tmp, wqe_ctr); + } else if (srq != NULL) { + /* Determine whether to overflow */ + if (wqe_ctr > srq->max_depth) { + ROCE_ERR("Get wqe index(0x%x) from cqe", wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq->wrid[wqe_ctr]; + roce5_free_srq_wqe(srq, wqe_ctr); + } else { + struct roce5_wq *wq = &(cur_qp->rq); + tail = (u32)(wq->tail & ((unsigned int)wq->wqebb_cnt - 1)); + wc->wr_id = wq->wrid[tail]; + ++wq->tail; + } + + return 0; +} + +static void roce5_poll_move_tail(const struct roce5_qp *cur_qp, + struct roce5_wq *wq, + struct roce_cqe *trans_cqe) +{ + u16 wqe_ctr = 0; + if (cur_qp->sq_signal_bits == 0) { + wqe_ctr = trans_cqe->dw7.bs.wqe_cnt; + wq->tail += (u16)(wqe_ctr - (u16)wq->tail); + } +} + +static void roce5_poll_one_get_av_info(const struct roce5_qp *cur_qp, + struct ib_wc *wc, + struct roce_cqe trans_cqe, + struct roce_cqe *cqe) +{ + int ret; + + if ((cur_qp->qp_type == IB_QPT_UD) || (cur_qp->qp_type == IB_QPT_GSI)) { + wc->sl = trans_cqe.dw4.bs.vlan_pri; + wc->src_qp = trans_cqe.dw6.bs.srqn_rqpn; + wc->network_hdr_type = trans_cqe.dw6.bs.stp; + wc->smac[0] = + (u8)(trans_cqe.dw4.bs.smac_h >> + 8); /* Take the first Byte data, shift 8bit */ + wc->smac[1] = (u8)(trans_cqe.dw4.bs.smac_h & 0xff); + ret = memcpy_s(&wc->smac[2], sizeof(cqe->smac_l), &cqe->smac_l, + sizeof(cqe->smac_l)); //2 : smac array idx + if (ret != 0) { + ROCE_ERR("Failed to memcpy smac.(ret:%d)", ret); + } + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_SMAC); + if (trans_cqe.dw6.bs.vlan_pre != 0) { + wc->vlan_id = trans_cqe.dw4.bs.vlan_id; + wc->wc_flags = + (int)((u32)wc->wc_flags | IB_WC_WITH_VLAN); + } + } +} + +static int roce5_poll_one_get_qp_and_srq(struct roce5_cq *cq, + struct roce5_qp **cur_qp, + struct ib_wc *wc, + struct roce_cqe *trans_cqe, + struct roce5_srq **srq) +{ + int ret = 0; + ret = roce5_cq_get_cur_qp(cq, cur_qp, trans_cqe); + if (ret != 0) { + ROCE_ERR("Failed to get current qp"); + return ret; + } + + wc->qp = &((*cur_qp)->ibqp); + + ret = roce5_cq_get_xrc_srq(cq, srq, trans_cqe, wc->qp->qp_type); + if (ret != 0) { + ROCE_ERR("Failed to get xrc srq"); + return ret; + } + + return ret; +} + +static void roce5_bytes_trans(u32 *addr, int dw_num) +{ + u32 *dw = addr; + int i = 0; + + for (i = 0; i < dw_num; i++) { + *dw = roce5_convert_cpu32(*dw); + dw++; + } +} + +static int roce5_get_and_trans_cqe(struct roce5_device *rdev, + struct roce5_cq *cq, + struct roce_cqe *trans_cqe, int *is_send, + struct roce_cqe **cqe) +{ + int ret; + + do { + *cqe = roce5_next_cqe_sw(cq); + if (*cqe == NULL) { + return -EAGAIN; + } + + /* Later, it is necessary to perform big and small end conversion on CQE. + In order to avoid turning back after processing, copy a copy */ + ret = memcpy_s((void *)trans_cqe, sizeof(struct roce_cqe), + (void *)*cqe, sizeof(struct roce_cqe)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy trans_cqe.(ret:%d).", ret); + return -ENOMEM; + } + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + /* Convert the CQE into the CPU endian corresponding to the running environment */ + roce5_bytes_trans( + (u32 *)(void *)trans_cqe, + (int)(sizeof(struct roce_cqe) / CQ_DW_TO_BYTE)); + + *is_send = trans_cqe->dw1.bs.s_r; + /* Resize CQ operation in progress */ + if (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == + ROCE_OPCODE_RESIZE_CQE)) { + roce5_cq_poll_and_resize(rdev, cq); + } + } while (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == + ROCE_OPCODE_RESIZE_CQE)); + + return 0; +} + +static void roce5_poll_send_cqe(struct roce5_qp *cur_qp, + struct roce_cqe *trans_cqe, struct ib_wc *wc) +{ + struct roce5_wq *wq = &(cur_qp->sq); + /* QP of type IB_SIGNAL_REQ_WR, tail needs to skip several WRs that do not generate CQE */ + roce5_poll_move_tail(cur_qp, wq, trans_cqe); + + wc->wr_id = wq->wrid[wq->tail & ((unsigned int)wq->wqebb_cnt - 1)]; + ++wq->tail; + + if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce5_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe, + wc); + return; + } + + roce5_get_opcode_from_scqe(trans_cqe, wc); +} + +static void roce5_init_wc(struct roce_cqe *trans_cqe, struct ib_wc *wc) +{ + roce5_get_opcode_from_rcqe(trans_cqe, wc); + wc->wc_flags = (int)((unsigned int)wc->wc_flags | IB_WC_GRH); + wc->pkey_index = 0; + wc->vlan_id = 0xffff; + wc->sl = 0; + /* avoid cm_req_handler()->ib_lid_be16() trigger call trace */ + wc->slid = 0; + return; +} + +static int roce5_poll_one_cqe(struct roce5_cq *rcq, struct roce5_qp *cur_qp, + int is_send, struct roce_cqe *trans_cqe, + struct roce_cqe *cqe, struct roce5_srq *srq, + struct ib_wc *wc) +{ + int ret = 0; + + wc->status = IB_WC_SUCCESS; + if (is_send != 0) { + roce5_poll_send_cqe(cur_qp, trans_cqe, wc); + return 0; + } + + ret = roce5_poll_recv_cqe(rcq, cur_qp, srq, wc); + if (ret != 0) { + return ret; + } + + if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce5_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe, + wc); + return 0; + } + + roce5_init_wc(trans_cqe, wc); + + roce5_poll_one_get_av_info((cur_qp), wc, *trans_cqe, cqe); + + /* Kernel mode does not support receiving inline */ + if (trans_cqe->dw1.bs.inline_r == ROCE_CQE_RQ_INLINE) { + ROCE_ERR("Receive inline not supported in kernel space"); + return -EINVAL; + } + + return 0; +} + +static int roce5_update_cons_index(struct roce5_device *rdev, + struct roce5_cq *rcq, + struct roce_cqe *trans_cqe) +{ + if (trans_cqe->dw1.bs.flush_op != 0) { + if (trans_cqe->dw1.bs.s_r == ROCE_CQE_SEND_COMP) { + ROCE_DEV_ERR(rdev, "Unexpected flush operation in SQ."); + ++rcq->cons_index; + return -EBADE; + } + + if (rcq->left_num == 0) { + rcq->left_num = trans_cqe->wqe_num; + rcq->wqebb_index = trans_cqe->dw1.bs.wqebb_cnt; + } + } else { + rcq->left_num = 1; + rcq->wqebb_index = trans_cqe->dw1.bs.wqebb_cnt; + } + + if (rcq->left_num == 1) { + ++rcq->cons_index; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_poll_one + Description : roce5_poll_one + Input : struct roce5_cq *cq + struct roce5_qp **cur_qp + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_poll_one(struct roce5_device *rdev, struct roce5_cq *cq, + struct roce5_qp **cur_qp, struct ib_wc *wc) +{ + int ret = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe trans_cqe; + struct roce5_srq *srq = NULL; + int is_send = 0; + bool need_poll = true; + + while (need_poll) { + ret = roce5_get_and_trans_cqe(rdev, cq, &trans_cqe, &is_send, + &cqe); + if (ret != 0) { + goto err_out; + } + + ret = roce5_update_cons_index(rdev, cq, &trans_cqe); + if (ret != 0) { + goto err_out; + } + + ret = roce5_poll_one_get_qp_and_srq(cq, cur_qp, wc, &trans_cqe, + &srq); + if (ret != 0) { + goto err_out; + } + + /* Poll cq exception handling, generally cq judges user state information, xrc judges wrid */ + if ((*cur_qp) == NULL) { + ROCE_ERR("cur_qp is NULL when poll cq"); + ret = -EACCES; + goto err_out; + } + if ((*cur_qp)->umem || + ((*cur_qp)->ibqp.xrcd && srq != NULL && !(srq->wrid))) { + ROCE_ERR( + "qp(%u) create in user space , but poll cq in kernel. NOT PERMIT!", + (*cur_qp)->qpn); + ret = -EACCES; + goto err_out; + } + + if (trans_cqe.dw1.bs.fake != 0) { + ROCE_INFO("Fake cqe go repoll."); + continue; + } + + ret = roce5_poll_one_cqe(cq, *cur_qp, is_send, &trans_cqe, cqe, + srq, wc); + if (ret != 0) { + goto err_out; + } + + --cq->left_num; + need_poll = false; + } + +err_out: + roce5_rsrq_put(srq); + return ret; +} + +/* the format of cq_ci DB:the bits of ci less than 24bit */ +void roce5_cq_set_ci(struct roce5_cq *cq) +{ + *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); +} + +int roce5_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct roce5_device *rdev = NULL; + struct roce5_cq *rcq = to_roce5_cq(ibcq); + struct roce5_qp *cur_qp = NULL; + unsigned long flags = 0; /* type required by kernel API */ + int npolled = 0; + int ret = 0; + + spin_lock_irqsave(&rcq->lock, flags); + + rdev = to_roce5_dev(ibcq->device); + + for (npolled = 0; npolled < num_entries; ++npolled) { + ret = roce5_poll_one(rdev, rcq, &cur_qp, wc + npolled); + if (ret != 0) { + break; + } + } + + roce5_cq_set_ci(rcq); + + spin_unlock_irqrestore(&rcq->lock, flags); + + if ((ret == 0) || (ret == -EAGAIN)) { + ret = npolled; + goto err_out; + } + +err_out: + roce5_rqp_put(cur_qp); + return ret; +} + +static void roce5_cq_arm(struct roce5_cq *cq, u32 cmd, void __iomem *uar_page) +{ + struct roce_db_cq_arm db_value; + int ret; + + ret = memset_s(&db_value, sizeof(db_value), 0, sizeof(db_value)); + if (ret != 0) { + ROCE_ERR("Failed to memset db_value.(ret:%d)", ret); + } + + db_value.dw0.bs.type = 3; + db_value.dw0.bs.cos = 0; /* arm_cq don't need cos */ + db_value.dw0.bs.cp = 1; /* 1 for cp */ + db_value.dw0.bs.non_filter = 1; + db_value.dw0.bs.cqc_type = 0; + db_value.dw0.bs.cqn = cq->cqn; + db_value.dw0.value = roce5_convert_be32(db_value.dw0.value); + + db_value.dw1.bs.cmd_sn = cq->arm_sn; + db_value.dw1.bs.cmd = cmd; + db_value.dw1.bs.ci = cq->cons_index; + db_value.dw1.value = roce5_convert_be32(db_value.dw1.value); + + /* + * Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + wmb(); + + roce5_write64((u32 *)(void *)&db_value, uar_page); +} + +int roce5_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + unsigned long lock_flags = 0; /* type required by kernel API */ + struct roce5_cq *rcq = NULL; + + if (CHECK_2LAYER_IS_NULL(ibcq, device)) { + return -EINVAL; + } + + rcq = to_roce5_cq(ibcq); + + spin_lock_irqsave(&rcq->lock, lock_flags); + if (rcq->arm_flag != 0) { + spin_unlock_irqrestore(&rcq->lock, lock_flags); + return 0; + } + + rcq->arm_flag = 1; + spin_unlock_irqrestore(&rcq->lock, lock_flags); + + /* Only 64-bit is supported, 64-bit writes are atomic, no need to lock */ + roce5_cq_arm(rcq, + (((unsigned int)flags & IB_CQ_SOLICITED_MASK) == + IB_CQ_SOLICITED) ? + ROCE_CQ_DB_REQ_NOT_SOL : + ROCE_CQ_DB_REQ_NOT, + to_roce5_dev(ibcq->device)->kernel_db_map); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c new file mode 100644 index 000000000..0df5d34e4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c @@ -0,0 +1,636 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "comm_defs.h" +#include "roce_cq.h" +#include "roce_npu_cmd.h" +#include "roce_cqm_cmd.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_os_compat.h" +#include "roce_user.h" +#include "roce_option.h" +#include "roce_cq_intern.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" + +#define ROCE_CQE_BASE_SIZE_BYTES 16 + +/* **************************************************************************** + Prototype :roce5_cq_attr_assign + Description : Assign value to CQC + Input : struct roce_cq_attr *cq_attr + struct roce5_cq *rcq + int eqn + int page_shift + Output : None + + 1.Date : 2019/10/13 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_attr_assign(roce_verbs_cq_attr_s *cq_attr, + struct roce5_cq *rcq, int eqn, int page_shift) +{ + /* Assign value to CQ */ + /* dw0 */ + cq_attr->dw0.bs.size = (u32)ROCE_ILOG2( + (unsigned)(rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)); + cq_attr->dw0.bs.page_size = (u32)page_shift; + cq_attr->dw0.bs.cqe_size = (u32)ROCE_ILOG2( + (unsigned)(rcq->buf.entry_size / ROCE_CQE_BASE_SIZE_BYTES)); + cq_attr->dw0.bs.mtt_page_size = + rcq->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + cq_attr->dw0.bs.tss_timer_num = + 7; /* 7 : The maximum number of timers supported by cq */ + cq_attr->dw0.bs.arm_timer_en = 0; + cq_attr->dw0.bs.timer_mode = 0; + cq_attr->dw0.bs.cnt_adjust_en = 1; + cq_attr->dw0.bs.cnt_clear_en = 1; + cq_attr->dw0.bs.ci_on_chip = 0; + cq_attr->dw0.value = cpu_to_be32(cq_attr->dw0.value); + + /* dw1 */ + cq_attr->dw1.bs.dma_attr_idx = 0; + cq_attr->dw1.bs.so_ro = 0; + cq_attr->dw1.bs.state = ROCE_CQ_STATE_VALID; + cq_attr->dw1.value = cpu_to_be32(cq_attr->dw1.value); + + /* dw2 */ + cq_attr->dw2.bs.idle_max_count = 0; + cq_attr->dw2.bs.cqecnt_lth = 10; /* update ci threshhold: 2^10=1024 */ + cq_attr->dw2.bs.cqecnt_rctl_en = 0; + cq_attr->dw2.bs.ceqe_en = 1; + cq_attr->dw2.bs.arm_ceqe_en = 1; + cq_attr->dw2.bs.ceqn = (u8)eqn; + cq_attr->dw2.value = cpu_to_be32(cq_attr->dw2.value); + + /* dw3 */ + cq_attr->dw3.bs.timeout = + 0; /* The timeout mechanism is disabled by default when CQ is created */ + cq_attr->dw3.bs.max_cnt = + 0; /* The overtime mechanism is disabled by default when CQ is created */ + cq_attr->dw3.value = cpu_to_be32(cq_attr->dw3.value); + + /* dw4 - dw5 */ + cq_attr->cqc_l0mtt_gpa = rcq->buf.mtt.mtt_paddr; + cq_attr->cqc_l0mtt_gpa = cpu_to_be64(cq_attr->cqc_l0mtt_gpa); + + /* dw6 - dw7 */ + cq_attr->ci_record_gpa_at_hop_num = cpu_to_be64( + (rcq->db.dma & (~0x3uLL)) | rcq->buf.mtt.mtt_layers); +} + +static int roce5_cq_fill_create_inbuf(struct roce5_device *rdev, + struct roce5_cq *rcq, int vector, + int page_shift, + cqm_cmd_buf_s *cqm_cmd_inbuf) +{ + int eqn = 0; + roce_verbs_cq_attr_s *cq_attr = NULL; + roce_uni_cmd_create_cq_s *cq_sw2hw_inbuf = NULL; + + cq_sw2hw_inbuf = (roce_uni_cmd_create_cq_s *)cqm_cmd_inbuf->buf; + cq_sw2hw_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (rcq->shadow != 0) { + cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_sw2hw_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rcq->shadow_vfid); + } + cq_attr = &cq_sw2hw_inbuf->cq_attr; + + /* Get the EQN of the CEQ based on the Vector index */ + eqn = hinic5_vector_to_eqn(rdev->hwdev, SERVICE_T_ROCE, vector); + if (eqn < 0) { + ROCE_DEV_ERR(rdev, + "Failed to get eqn from hinic vector, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + roce5_cq_attr_assign(cq_attr, rcq, eqn, page_shift); + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_sw2hw + Description : Send the cqc configuration command + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int vector + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_sw2hw(struct roce5_device *rdev, struct roce5_cq *rcq, + int vector, int page_shift) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_create_cq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + ret = roce5_cq_fill_create_inbuf(rdev, rcq, vector, page_shift, + cqm_cmd_inbuf); + if (ret != 0) { + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_SW2HW_CQ, + cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send SW2HW_CQ command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + + /* CMDq times out or CMDq does not work, update the device status, notify the PCIe module to reset + the device through OFED */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto err_send_cmd; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + rcq->cons_index = 0; + rcq->arm_sn = 1; + rcq->arm_flag = 0; + rcq->vector = (u32)vector; + + return 0; + +err_send_cmd: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_create_cq_check + Description : roce5_create_cq_check + Input : struct ib_device *ibdev + int entries + int vector + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_cq_check(struct ib_device *ibdev, int entries, + int vector) +{ + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + ROCE_DEV_ERR( + rdev, + "Cqe number invalid, entries(%d), max_cqes(%d), func_id(%d)", + entries, (int)rdev->rdma_cap.max_cqes, + rdev->glb_func_id); + return -EINVAL; + } + + if (vector > (int)rdev->rdma_cap.num_comp_vectors) { + ROCE_DEV_ERR(rdev, "Vector over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +static int roce5_check_cq_create_flags(u32 flags) +{ + /* + * It returns non-zero value for unsupported CQ + * create flags, otherwise it returns zero. + */ + return (int)(flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN | + IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)); +} + +static int roce5_cq_cqc_cfg(struct roce5_device *rdev, struct roce5_cq *rcq, + int vector, struct ib_udata *udata, u64 cq_buf_addr) +{ + int ret = 0; + int page_shift = (int)roce5_umem_get_buf_page_shift(rdev, rcq->umem, + cq_buf_addr) - + PAGE_SHIFT_4K; + + /* configurate CQC */ + ret = roce5_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to handle cq sw2hw, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + /* User mode requires outgoing CQN */ + if (ib_copy_to_udata(udata, &rcq->cqn, sizeof(u32)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + return ret; + } + + return 0; +} + +static int roce5_create_user_cq(struct roce5_device *rdev, struct roce5_cq *rcq, + int cqe_num, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret = 0; + int vector = attr->comp_vector; + struct roce5_create_cq_cmd ucmd = { 0 }; + roce_user_handle_t hdl = { 0 }; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + goto err_out; + } + + ROCE_CONSTUCT_ULD_HDL_RCQ(&hdl, rdev, rcq, udata, &rcq->buf.mtt, + &rcq->db.dma, cqe_num); + ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rcq->shadow, &rcq->shadow_vfid, + NULL); + ret = roce5_uld_cq_create((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_ERR("Failed to execute uld cq create, ret(%d)", ret); + goto err_out; + } + /* Entries has been decreased by one when it was input by User Mode, and it is the expected value exactly + after increased by one by Kernel Mode */ + ret = roce5_cq_get_umem(rdev, ibcontext, rcq, ucmd.buf_addr, cqe_num, + false); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get umem, func_id(%d)", + rdev->glb_func_id); + goto err_destroy_uld; + } + + ret = roce5_db_map_user(to_roce5_ucontext(ibcontext), + (unsigned long)ucmd.db_addr, &rcq->db, + rcq->shadow); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to map kernel_mem to user_mem, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + ret = roce5_cq_cqc_cfg(rdev, rcq, vector, udata, ucmd.buf_addr); + if (ret != 0) { + goto err_unmap_db; + } + + return 0; + +err_unmap_db: + roce5_db_unmap_user(to_roce5_ucontext(ibcontext), &rcq->db, + rcq->shadow); + +err_free_mtt: + roce5_cq_put_umem(rdev, &rcq->buf, &rcq->umem, rcq->shadow); + rcq->umem = NULL; + +err_destroy_uld: + roce5_uld_cq_destroy((roce_uld_handle)&hdl); + +err_out: + + return ret; +} + +static void roce5_fill_rcq_info(struct roce5_cq *rcq) +{ + rcq->cqn = rcq->cqm_cq->index; + rcq->buf.buf = &rcq->cqm_cq->q_room_buf_1; + + /* Initialize buf to unused */ + roce5_cq_buf_init(&rcq->buf); + + /* Software DB assignment */ + rcq->set_ci_db = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr + ->doorbell_record); + *rcq->set_ci_db = 0; + rcq->db.db_record = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr + ->doorbell_record); + rcq->db.dma = rcq->cqm_cq->q_header_paddr; +} + +/* **************************************************************************** + Prototype : roce5_create_kernel_cq + Description : roce5_create_kernel_cq + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + int vector + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_kernel_cq(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries, int vector, + u32 index) +{ + int ret = 0; + int page_shift = 0; + + rcq->buf.buf_size = entries * (int)rdev->rdma_cap.cqe_size; + rcq->cqm_cq = cqm5_object_rdma_queue_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_SCQ, + (u32)rcq->buf.buf_size, rcq->opt, true, index, 0, 0); + if (rcq->cqm_cq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma_queue from cqm_object, func_id(%d), index(%d)", + rdev->glb_func_id, index); + return (-ENOMEM); + } + + /* Buffer is obtained from room1 when the queue is just created */ + if (rcq->cqm_cq->current_q_room != CQM_RDMA_Q_ROOM_1) { + ROCE_DEV_ERR(rdev, "Not use CQM_RDMA_Q_ROOM_1, func_id(%d)", + rdev->glb_func_id); + ret = -EINVAL; + goto err_free_cqm_cq; + } + + roce5_fill_rcq_info(rcq); + + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size); + + /* allocate MTT */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, rcq->buf.buf->buf_number, + (u32)page_shift, &rcq->buf.mtt, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_cqm_cq; + } + + /* Write the PA of the CQ Buffer to the MTT */ + ret = roce5_buf_write_mtt(rdev, &rcq->buf.mtt, rcq->buf.buf); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + page_shift = (int)ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size) - + PAGE_SHIFT_4K; + + /* configurate CQC */ + ret = roce5_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to configure CQC, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + return 0; + +err_free_mtt: + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_free_cqm_cq: + cqm5_object_delete(&rcq->cqm_cq->object); + + return ret; +} + +static int roce5_cq_depth_init(struct roce5_device *rdev, struct roce5_cq *rcq, + const struct ib_cq_init_attr *attr, int *cqe_num) +{ + u32 entries = (u32)attr->cqe; + + /* The CQE queue should reserve four special CQE for the HOST CI can be updated to the CQC CI */ + entries += ROCE_CQE_UP_THRESHOLD; + entries = ROCE_ROUNDUP_POW_OF_TWO(entries) & 0xffffffff; + /* Chip Constraints: Minimum queue depth needs to be page-aligned */ + if ((entries * rdev->rdma_cap.cqe_size) < PAGE_SIZE) { + entries = (PAGE_SIZE >> ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + } + if (entries == 0) { + ROCE_DEV_ERR(rdev, "Invalid entries after align, func_id(%u)", + rdev->glb_func_id); + return -EINVAL; + } + rcq->cqe_mask = entries - 1; + /* Check if max spec is exceeded */ + if (rdev->rdma_cap.max_cqes > UINT_MAX - ROCE_CQE_UP_THRESHOLD) { + ROCE_DEV_ERR( + rdev, + "max_cqes overflow detected, max_cqes(%u), func_id(%u)", + rdev->rdma_cap.max_cqes, rdev->glb_func_id); + return -EINVAL; + } + if (entries > (rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR( + rdev, + "Over range after align, entries(%u), max_cqes(%u), func_id(%u)", + entries, + rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD, + rdev->glb_func_id); + return -EINVAL; + } + + if ((int)entries > INT_MAX) { + ROCE_DEV_ERR(rdev, "Bug detected: Int Overflow!"); + return -EINVAL; + } + + *cqe_num = (int)entries; + rcq->ibcq.cqe = (int)entries - ROCE_CQE_UP_THRESHOLD; + return 0; +} + +static int roce5_do_create_cq(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce5_cq *rcq, + u32 index) +{ + struct roce5_device *rdev = NULL; + struct roce5_ucontext *context = NULL; + int ret = 0; + int vector = attr->comp_vector; + int cqe_num = 0; + + rdev = to_roce5_dev(ibdev); + ret = roce5_cq_depth_init(rdev, rcq, attr, &cqe_num); + if (ret != 0) { + return ret; + } + + mutex_init(&rcq->resize_mutex); + spin_lock_init(&rcq->lock); + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + rcq->buf.entry_size = (int)rdev->rdma_cap.cqe_size; + rcq->rcq_magic = ROCE_RCQ_MAGIC; + + rcq->opt = roce5_some(rcq); + if (rcq->opt == NULL) { + ROCE_DEV_ERR(rdev, "Failed to init some, cqn(%u).", rcq->cqn); + return -ENOMEM; + } + + INIT_LIST_HEAD(&rcq->send_qp_list); + INIT_LIST_HEAD(&rcq->recv_qp_list); + if (udata != NULL) { + ret = roce5_create_cq_alloc_cqn(rdev, rcq, index); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm cq resource, func_id(%d)", + rdev->glb_func_id); + return ret; + } + context = roce5_queue_to_ucontext((void *)(&rcq->ibcq), udata, + ROCE_Q_TYPE_CQ); + ret = roce5_create_user_cq(rdev, rcq, cqe_num, attr, + &context->ibucontext, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create user_cq, func_id(%d)", + rdev->glb_func_id); + cqm5_object_delete(&rcq->cqm_cq->object); + return ret; + } + } else { + ret = roce5_create_kernel_cq(rdev, rcq, cqe_num, vector, index); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create kernel_cq, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + + rcq->reset_flow_comp = roce5_reset_flow_comp; + + return 0; +} + +int roce5_create_cq_common(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce5_cq *rcq, + u32 index) +{ + int ret; + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + + if (roce5_check_cq_create_flags(attr->flags) != 0) { + ROCE_ERR("Not support the cq_create flag(%x)", attr->flags); + return -EOPNOTSUPP; + } + + ret = roce5_create_cq_check(ibdev, entries, vector); + if (ret != 0) { + ROCE_ERR("Failed to check cq information"); + return ret; + } + + return roce5_do_create_cq(ibdev, attr, udata, rcq, index); +} + +int roce5_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) +{ + struct roce5_cq *rcq = NULL; + + if ((ibcq == NULL) || (attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rcq = to_roce5_cq(ibcq); + return roce5_create_cq_common(ibcq->device, attr, udata, rcq, + ROCE_CQN_INVLD); +} + +struct ib_cq *roce5_create_cq_ofa(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret; + struct roce5_cq *rcq = NULL; + struct ib_uobject uobject; + + if ((ibdev == NULL) || (attr == NULL)) { + ROCE_ERR("ibdev or attr is NULL"); + return ERR_PTR(-EINVAL); + } + + uobject.context = ibcontext; + + rcq = (struct roce5_cq *)kzalloc(sizeof(*rcq), GFP_KERNEL); + if (rcq == NULL) { + ret = -ENOMEM; + ROCE_ERR("Failed to alloc rcq"); + goto out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + rcq->ibcq.device = ibdev; +#if defined(IB_CQ_HAVE_UOBJECT) + rcq->ibcq.uobject = ((udata != NULL) ? &uobject : NULL); +#endif + ret = roce5_create_cq(&rcq->ibcq, attr, udata); + if (ret != 0) { + kfree(rcq); + rcq = NULL; + goto out; + } + +out: + /* disassociate anyway */ + if (rcq != NULL) { + rcq->ibcq.uobject = NULL; + } + return ((ret != 0) ? (struct ib_cq *)ERR_PTR((long)ret) : &rcq->ibcq); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c new file mode 100644 index 000000000..eaf8e22f4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c @@ -0,0 +1,964 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "comm_defs.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_npu_cmd.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_infra.h" +#include "roce_compat.h" +#include "roce_os_compat.h" +#include "roce_cqe_format.h" +#include "roce_user.h" +#include "roce_com.h" + +/* and above next is V100 adaptation data structure */ +typedef struct tag_roce_cmd_resize_cq { + roce_verbs_cmd_com_s com; + + u32 rsvd; + u32 page_size; /* Size of the resize buf page. */ + u32 log_cq_size; /* Cq depth after resize */ + u32 mtt_layer_num; /* Number of mtt levels after resize */ + /* DW4~5 */ + union { + u64 mtt_base_addr; /* Start address of mr or mw */ + u32 cqc_l0mtt_gpa[2]; + }; + u32 mtt_page_size; /* Size of the mtt page after resize. */ + roce_xq_mtt_info_s mtt_info; +} roce_cmd_resize_cq_s; + +typedef struct tag_roce_cmd_modify_cq { + roce_verbs_cmd_com_s com; + u32 max_cnt; + u32 timeout; +} roce_cmd_modify_cq_s; + +static int roce5_uni_cmd_cq_modify(struct roce5_device *rdev, + struct roce5_cq *cq, u32 cnt, u32 period) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_modify_cq_s *cq_modify_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_cq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_modify_inbuf = (roce_uni_cmd_modify_cq_s *)cqm_cmd_inbuf->buf; + cq_modify_inbuf->com.index = cpu_to_be32(cq->cqn); + cq_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); + if (cq->shadow != 0) { + cq_modify_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_modify_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(cq->shadow_vfid); + } + cq_modify_inbuf->cq_modify.max_cnt = cpu_to_be32(cnt); + cq_modify_inbuf->cq_modify.timeout = cpu_to_be32(period); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_MODIFY_CQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_CQ, cqn(0x%x), ret(%d), func_id(%hu)", + cq->cqn, ret, rdev->glb_func_id); + /* CMDq may return a positive number, so its return value cannot be used directly */ + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int roce5_cq_modify(struct roce5_device *rdev, struct roce5_cq *cq, + u32 cnt, u32 period) +{ + if (ROCE5_SUPPORT_CQ_MODIFY_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_cq_modify(rdev, cq, cnt, period); + } + return -EINVAL; +} + +/* **************************************************************************** + Prototype : roce5_modify_cq + Description : OFED_3_12 + Input : struct ib_cq *ibcq + u16 cq_count + u16 cq_period + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period) +{ + int ret = 0; + struct roce5_cq *cq = NULL; + struct roce5_device *rdev = NULL; + + if (ibcq == NULL) { + ROCE_ERR(" Ibcq is null"); + return -EINVAL; + } + + cq = to_roce5_cq(ibcq); + rdev = to_roce5_dev(ibcq->device); + + ret = roce5_cq_modify(rdev, cq, cq_count, cq_period); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to modify cq, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_alloc_resize_buf + Description : Kernel mode applies for resize_buf, and its corresponding MTT + When there is an error in execution, this function ensures that all intermediate resources are released; + when the execution is successful, this function can confirm that all resources are acquired. + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_alloc_resize_buf(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries) +{ + int ret = 0; + int page_shift = 0; + + if (rcq->resize_buf != NULL) { + ROCE_DEV_ERR(rdev, "Resize buffer is busy, func_id(%d)", + rdev->glb_func_id); + return -EBUSY; + } + + /* Apply for resize_buf and assign it */ + rcq->resize_buf = (struct roce5_cq_resize_buf *)kmalloc( + sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc resize buffer, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + ret = cqm5_object_resize_alloc_new(&rcq->cqm_cq->object, + (u32)rcq->resize_buf->buf.buf_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to resize cq buffer, func_id(%d)", + rdev->glb_func_id); + goto err_resize_buf_alloc; + } + + /* If the application is successful, record the buffer information of the new buffer */ + rcq->resize_buf->cqe = entries - ROCE_CQE_UP_THRESHOLD; + rcq->resize_buf->buf.buf = &rcq->cqm_cq->q_room_buf_2; + + /* Initialize new buf to unused state */ + roce5_cq_buf_init(&rcq->resize_buf->buf); + + page_shift = (int)ROCE_ILOG2(rcq->resize_buf->buf.buf->buf_size); + + /* Apply MTT for resize_buf */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, rcq->resize_buf->buf.buf->buf_number, + (u32)page_shift, &rcq->resize_buf->buf.mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_mtt_alloc; + } + + /* configurate MTT, write PA of resize_buf to MTT */ + ret = roce5_buf_write_mtt(rdev, &rcq->resize_buf->buf.mtt, + rcq->resize_buf->buf.buf); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_mtt_write; + } + + return 0; + +err_mtt_write: + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_mtt_alloc: + /* free resize_buf */ + cqm5_object_resize_free_new(&rcq->cqm_cq->object); + +err_resize_buf_alloc: + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_cq_free_resize_buf + Description : Kernel Mode releases resize_buf and its corresponding MTT + Input : struct roce5_device *rdev + struct roce5_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_free_resize_buf(struct roce5_device *rdev, + struct roce5_cq *rcq) +{ + /* Release the MTT of resize_buf first */ + hmm_mtt_free(rdev->hwdev, &(rcq->resize_buf->buf.mtt), SERVICE_T_ROCE); + + /* free resize_buf */ + cqm5_object_resize_free_new(&(rcq->cqm_cq->object)); + + /* free the resize_buf pointer itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* **************************************************************************** + Prototype : roce5_cq_alloc_resize_umem + Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_alloc_resize_umem(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_resize_cq_cmd ucmd = { 0 }; + struct roce5_ucontext *context = roce5_queue_to_ucontext( + (void *)(&rcq->ibcq), udata, ROCE_Q_TYPE_CQ); + + /* If a resize is being executed, it is not allowed to execute another resize at the same time */ + if (rcq->resize_umem != NULL) { + ROCE_DEV_ERR(rdev, "Resize_umem is busy, func_id(%d)", + rdev->glb_func_id); + return -EBUSY; + } + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + return -EFAULT; + } + + rcq->resize_buf = (struct roce5_cq_resize_buf *)kmalloc( + sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to alloc cq resize buffer, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + ret = roce5_cq_get_umem(rdev, &context->ibucontext, rcq, ucmd.buf_addr, + entries, true); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get umem, func_id(%d)", + rdev->glb_func_id); + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; + } + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + rcq->resize_buf->cqe = entries - ROCE_CQE_UP_THRESHOLD; + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_free_resize_umem + Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf + Input : struct roce5_device *rdev + struct roce5_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_free_resize_umem(struct roce5_device *rdev, + struct roce5_cq *rcq) +{ + /* Free MTT and umem of resize_buf */ + roce5_cq_put_umem(rdev, &rcq->resize_buf->buf, &rcq->resize_umem, 0); + rcq->resize_umem = NULL; + + /* free resize_buf itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* **************************************************************************** + Prototype : roce5_cq_get_outstanding_cqes + Description : roce5_cq_get_outstanding_cqes + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_get_outstanding_cqes(struct roce5_cq *cq) +{ + u32 i = 0; + + i = cq->cons_index; + while (roce5_get_sw_cqe(cq, i)) { + ++i; + } + + return (int)(i - cq->cons_index); +} + +static int roce5_cq_get_next_cqe(struct roce_cqe **cqe, struct roce5_cq *cq, + unsigned int *i, u32 *times, + const struct roce_cqe *start_cqe) +{ + do { + *cqe = (struct roce_cqe *)roce5_get_sw_cqe(cq, ++(*i)); + if (*cqe == NULL) { + ROCE_DELAY(); + --(*times); + } + } while ((*cqe == NULL) && (*times != 0)); + + if ((*cqe == start_cqe) || (*cqe == NULL)) { + ROCE_ERR("Failed to get resize CQE, CQN(0x%x)", cq->cqn); + return -ENOMEM; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_resize_copy_cqes + Description : roce5_cq_resize_copy_cqes + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_resize_copy_cqes(struct roce5_cq *cq) +{ + struct roce_cqe *cqe = NULL; + struct roce_cqe *new_cqe = NULL; + struct roce_cqe *start_cqe = NULL; + u32 times; + unsigned int i = 0; + int ret = 0; + + i = cq->cons_index; + + times = ROCE_CQ_RESIZE_POLL_TIMES; + do { + cqe = (struct roce_cqe *)roce5_get_sw_cqe(cq, i); + if (cqe == NULL) { + ROCE_DELAY(); + --times; + } + } while ((cqe == NULL) && (times != 0)); + + if (cqe == NULL) { + ROCE_ERR("Failed to get resize CQE, CQN(0x%x)", cq->cqn); + return -ENOMEM; + } + + start_cqe = cqe; + + /* r_cqe:resize_cqe + CI + +--------+-----+-----+-----+----------+ + old_buf: | | CQE | CQE |R_CQE| | + +--------+-----+-----+-----+----------+ + | | + | | + +--------+-----+-----+-----+----------+ + new_buf: | | CQE | CQE | | | + +--------+-----+-----+-----+----------+ + */ + /* Convert the fields needed by CQE to little endian */ + cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value); + while (cqe->dw1.bs.op_type != ROCE_OPCODE_RESIZE_CQE) { + /* The resized PI can be inherited, and the index of the original CQE remains unchanged in resize_buf */ + new_cqe = (struct roce_cqe *)roce5_get_cqe_from_buf( + &cq->resize_buf->buf, + (i & ((unsigned)cq->resize_buf->cqe))); + + ret = memcpy_s((void *)new_cqe, sizeof(struct roce_cqe), + (void *)cqe, sizeof(struct roce_cqe)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy new_cqe.(ret:%d)", ret); + return -ENOMEM; + } + + /* If the CQE has been wrapped in resize_buf, the corresponding Owner bit needs to be modified + to be owned by the software. + * The rule is: when the owner bit is owned by CQE software, the O bit is opposite to the high bit of ci; + when the owner bit is owned by hardware, the O bit is the same as the high bit of CI. + */ + new_cqe->dw0.bs.owner = ((i & ((unsigned)cq->resize_buf->cqe + + ROCE_CQE_UP_THRESHOLD)) != 0) ? + 1 : + 0; + + /* After processing, turn the DW0/DW1 of CQE back to big endian */ + cqe->dw0.value = roce5_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_be32(cqe->dw1.value); + + /* Convert DW0/DW1 of the CQE in the new queue back to big endian */ + new_cqe->dw0.value = roce5_convert_be32(new_cqe->dw0.value); + new_cqe->dw1.value = roce5_convert_be32(new_cqe->dw1.value); + + /* Get the next CQE */ + ret = roce5_cq_get_next_cqe(&cqe, cq, &i, ×, start_cqe); + if (ret != 0) { + return ret; + } + + /* Convert DW0/DW1 of CQE to little endian */ + cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value); + } + + return 0; +} + +static int roce5_uni_cmd_cq_resize_inbuf_fill(struct roce5_device *rdev, + struct roce5_cq *rcq, + int page_shift, + cqm_cmd_buf_s **cqm_cmd_inbuf) +{ + int ret = 0; + struct dev_roce_svc_own_cap *rdma_own_cap = NULL; + u32 cq_size = 0; + u32 mtt_layer = 0; + u64 mtt_paddr = 0; + u32 mtt_page_size = 0; + roce_uni_cmd_resize_cq_s *cq_resize_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_resize_cq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + rdma_own_cap = &rdev->rdma_cap.dev_rdma_cap.roce_own_cap; + cq_size = (u32)rcq->resize_buf->cqe + ROCE_CQE_UP_THRESHOLD; + cq_size = (u32)ROCE_ILOG2(cq_size); + mtt_layer = rcq->resize_buf->buf.mtt.mtt_layers; + mtt_paddr = rcq->resize_buf->buf.mtt.mtt_paddr; + mtt_page_size = rcq->resize_buf->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + + cq_resize_inbuf = (roce_uni_cmd_resize_cq_s *)(*cqm_cmd_inbuf)->buf; + cq_resize_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_resize_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (rcq->shadow != 0) { + cq_resize_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_resize_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rcq->shadow_vfid); + } + cq_resize_inbuf->cq_resize.page_size = cpu_to_be32((u32)page_shift); + cq_resize_inbuf->cq_resize.log_cq_size = cpu_to_be32(cq_size); + cq_resize_inbuf->cq_resize.mtt_layer_num = cpu_to_be32(mtt_layer); + cq_resize_inbuf->cq_resize.mtt_base_addr = mtt_paddr; + cq_resize_inbuf->cq_resize.mtt_base_addr = + cpu_to_be64(cq_resize_inbuf->cq_resize.mtt_base_addr); + cq_resize_inbuf->cq_resize.mtt_page_size = cpu_to_be32(mtt_page_size); + + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_flags = 0; + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_num = 0; + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_own_cap->cmtt_cl_start); + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_own_cap->cmtt_cl_end); + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_size = + cpu_to_be32(rdma_own_cap->cmtt_cl_sz); + return 0; +} + +static int roce5_cq_resize(struct roce5_device *rdev, struct roce5_cq *rcq, + int page_shift) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + if (ROCE5_SUPPORT_IB_CMD_RESIZE_CQ_MSG_V2(rdev) != 0) { + ret = roce5_uni_cmd_cq_resize_inbuf_fill(rdev, rcq, page_shift, + &cqm_cmd_inbuf); + } + + if (ret != 0) { + return ret; + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_RESIZE_CQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RESIZE_CQ command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + + /* When CMDq times out or CMDq cannot work, update the device status and notify the PCIe module to reset + the device through OFED */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +/* **************************************************************************** + Prototype : roce5_user_cq_resize + Description : roce5_user_cq_resize + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_user_cq_resize(struct roce5_device *rdev, struct roce5_cq *rcq, + int entries, struct ib_udata *udata) +{ + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "Over range after align, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + ret = roce5_cq_alloc_resize_umem(rdev, rcq, entries, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc resize_umem, func_id(%d)", + rdev->glb_func_id); + return ret; + } +#if defined(UMEM_HAVE_PAGE_SHIFT) + page_shift = rcq->resize_umem->page_shift - PAGE_SHIFT_4K; +#else + page_shift = PAGE_SHIFT - PAGE_SHIFT_4K; +#endif + + /* Send the cq_resize command to configure CQC. After the configuration is successful, the new CQE is written to + resize_buf, and the old buffer may still retain the old CQE. */ + ret = roce5_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to resize cq, func_id(%d)", + rdev->glb_func_id); + roce5_cq_free_resize_umem(rdev, rcq); + return ret; + } + + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_resize_user_cq + Description : roce5_resize_user_cq + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_resize_user_cq(struct roce5_device *rdev, struct roce5_cq *rcq, + int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_resize_cq_cmd ucmd = { 0 }; + int cqe_entries = entries; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + return -EFAULT; + } + + mutex_lock(&rcq->resize_mutex); + + if (ucmd.stage == 1) { + goto release_flow; + } + + /* The CQE queue should reserve four special CQE for the HOST CI can be updated to the CQC CI */ + cqe_entries += ROCE_CQE_UP_THRESHOLD; + cqe_entries = + (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); + //Minimum queue depth needs to be aligned by page + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) { + cqe_entries = + (int)(PAGE_SIZE >> (unsigned int)ROCE_ILOG2( + rdev->rdma_cap.cqe_size)); + } + if (cqe_entries == 0) { + ROCE_DEV_ERR(rdev, + "Invalid cqe_entries after align, func_id(%d)", + rdev->glb_func_id); + ret = -EINVAL; + goto out; + } + rcq->cqe_mask = cqe_entries - 1; + + /* The new depth of CQ cannot be the same as the old depth. Four special CQE space is reserved when CQ is created, + so the value of ibcq->cqe is 4 smaller than the actual value */ + if (cqe_entries == (rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "No need to resize cq, func_id(%d)", + rdev->glb_func_id); + ret = 0; + goto out; + } + + ret = roce5_user_cq_resize(rdev, rcq, cqe_entries, udata); + if (ret != 0) { + goto out; + } + + mutex_unlock(&rcq->resize_mutex); + + return 0; + +release_flow: + /* free old MTT */ + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + rcq->buf = rcq->resize_buf->buf; + ib_umem_release(rcq->umem); + rcq->umem = rcq->resize_umem; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_kernel_cq_resize + Description : roce5_kernel_cq_resize + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_kernel_cq_resize(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries) +{ + int outst_cqe = 0; + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "Over range after align, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + /* The number of resized CQEs cannot be smaller than the number of outstanding CQEs */ + outst_cqe = roce5_cq_get_outstanding_cqes(rcq); + if (entries < (outst_cqe + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR( + rdev, + "Can't resize, because smaller than the number of outstanding CQES, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + ret = roce5_cq_alloc_resize_buf(rdev, rcq, entries); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc resize buffer, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + page_shift = (int)ROCE_ILOG2(rcq->cqm_cq->q_room_buf_2.buf_size) - + PAGE_SHIFT_4K; + + /* Send the cq_resize command to configure CQC. After the configuration is successful, the new CQE is written to + resize_buf, and the old buffer may still retain the old CQE. */ + ret = roce5_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to resize cq, func_id(%d)", + rdev->glb_func_id); + roce5_cq_free_resize_buf(rdev, rcq); + return ret; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_resize_kernel_cq + Description : roce5_resize_kernel_cq + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_resize_kernel_cq(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries) +{ + int ret = 0; + int old_cqe = 0; + struct mtt old_mtt; + int cqe_entries = entries; + + mutex_lock(&rcq->resize_mutex); + + cqe_entries += ROCE_CQE_UP_THRESHOLD; + cqe_entries = + (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); + /* Minimum queue depth needs to be aligned by page */ + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) { + cqe_entries = (PAGE_SIZE >> (unsigned int)ROCE_ILOG2( + rdev->rdma_cap.cqe_size)); + } + + /* The new depth of CQ cannot be the same as the old depth. Four special CQE space is reserved when CQ is created, + so the value of ibcq->cqe is four smaller than the actual value */ + if (cqe_entries == (rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "No need to resize cq, func_id(%d)", + rdev->glb_func_id); + mutex_unlock(&rcq->resize_mutex); + return 0; + } + + ret = roce5_kernel_cq_resize(rdev, rcq, cqe_entries); + if (ret != 0) { + goto out; + } + + /* free old MTT */ + old_mtt = rcq->buf.mtt; + + /* When copying CQE from the old buffer to resize_buf, the user may be polling cqe from the old buf, so + it needs to be locked.If CQE of the old buffer has been polled in the polling process, polling process will + free old buffer and switch to new buffer. + */ + spin_lock_irq(&rcq->lock); + + /* If the new buf has been switched to, nothing needs to be done */ + if (rcq->resize_buf != NULL) { + ret = roce5_cq_resize_copy_cqes(rcq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Have not polled resize cqe(fuc:%d, cqn:%u, ret:%d)", + rdev->glb_func_id, rcq->cqn, ret); + roce5_cq_free_resize_buf(rdev, rcq); + spin_unlock_irq(&rcq->lock); + mutex_unlock(&rcq->resize_mutex); + return ret; + } + old_cqe = rcq->ibcq.cqe; + rcq->buf = rcq->resize_buf->buf; + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + } + + spin_unlock_irq(&rcq->lock); + + /* Non-0 means that the above resize_buf non-empty branch has been entered, the old buffer needs to be released */ + if (old_cqe != 0) { + hmm_mtt_free(rdev->hwdev, &old_mtt, SERVICE_T_ROCE); + cqm5_object_resize_free_old(&rcq->cqm_cq->object); + } + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +static int roce5_resize_cq_check(struct ib_cq *ibcq, int entries, + const struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + + if (ibcq == NULL) { + ROCE_ERR("Ibcq is null"); + return -EINVAL; + } + + if ((ibcq->uobject != NULL) && (udata == NULL)) { + ROCE_ERR("Udata is null, but uobject is not null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibcq->device); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + ROCE_DEV_ERR(rdev, + "Resize CQEs invalid. entries(%d), func_id(%d)", + entries, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_resize_cq + Description : roce5_resize_cq + Input : struct ib_cq *ibcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_cq *rcq = NULL; + struct roce5_device *rdev = NULL; + + ret = roce5_resize_cq_check(ibcq, entries, udata); + if (ret != 0) { + ROCE_ERR("Failed to check resize_cq"); + return ret; + } + + rcq = to_roce5_cq(ibcq); + rdev = to_roce5_dev(ibcq->device); + + if (ibcq->uobject) { + ret = roce5_resize_user_cq(rdev, rcq, entries, udata); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to resize user cq, func_id(%u) ret(%d)", + (u32)rdev->glb_func_id, ret); + return ret; + } + } else { + ret = roce5_resize_kernel_cq(rdev, rcq, entries); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to resize kernel cq, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +void roce5_lock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __acquires(&roce5_send_cq->lock) __acquires(&roce5_recv_cq->lock) +{ + if (roce5_send_cq == roce5_recv_cq) { + spin_lock_irq(&roce5_send_cq->lock); + __acquire(&roce5_recv_cq->lock); + } else if (roce5_send_cq->cqn < roce5_recv_cq->cqn) { + spin_lock_irq(&roce5_send_cq->lock); + spin_lock_nested(&roce5_recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&roce5_recv_cq->lock); + spin_lock_nested(&roce5_send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +void roce5_unlock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __releases(&roce5_send_cq->lock) __releases(&roce5_recv_cq->lock) +{ + if (roce5_send_cq == roce5_recv_cq) { + __release(&roce5_recv_cq->lock); + spin_unlock_irq(&roce5_send_cq->lock); + } else if (roce5_send_cq->cqn < roce5_recv_cq->cqn) { + spin_unlock(&roce5_recv_cq->lock); + spin_unlock_irq(&roce5_send_cq->lock); + } else { + spin_unlock(&roce5_send_cq->lock); + spin_unlock_irq(&roce5_recv_cq->lock); + } +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c new file mode 100644 index 000000000..c4e442598 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_npu_cmd.h" +#include "hinic5_hmm.h" +#include "securec.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_compat.h" +#include "roce_os_compat.h" +#include "roce_cq_intern.h" +#include "roce_com.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" + +/* and above next is V100 adaptation data structure */ +typedef struct tag_roce_cmd_cq_cache_invalidate { + roce_verbs_cmd_com_s com; + roce_xq_mtt_info_s mtt_info; +} roce_cmd_cq_cache_invalidate_s; +#define CQC_CHECK_RESCHED_LOOP_TIMES 0xffffU +static int roce5_check_cqc_data_state(struct roce5_device *rdev, + const struct roce5_cq *cq, + struct roce5_cq_context *cqc_data, + u32 check_state) +{ + int read_back_flag = 0; + u32 times = (u32)rdev->try_times; + struct roce5_cq_context check_cqc_data; + while ((times--) != 0) { + check_cqc_data.dw2.value = be32_to_cpu(cqc_data->dw2.value); + if (check_cqc_data.dw2.bs.state == check_state) { + read_back_flag = 1; + break; + } + ROCE_DELAY(); + + // limit schedule cond_resched count, preventstop thread schedule frequency past high + if ((times & CQC_CHECK_RESCHED_LOOP_TIMES) == 0) { + cond_resched(); + } + } + + if (read_back_flag == 0) { + ROCE_DEV_ERR( + rdev, + "Failed to read back after try %d times, Cqn(0x%x), CQ state(0x%x), func_id(%u)", + rdev->try_times, cq->cqn, cqc_data->dw2.value, + (u32)rdev->glb_func_id); + return -1; + } + + return 0; +} +/* **************************************************************************** + Prototype : roce5_cq_hw2sw + Description : roce5_cq_hw2sw + Input : struct roce5_device *rdev + struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_cq_hw2sw(struct roce5_device *rdev, struct roce5_cq *cq) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + struct roce5_cq_context *cqc_data = NULL; + roce_uni_cmd_cq_hw2sw_s *cq_hw2sw_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_cq_hw2sw_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_hw2sw_inbuf = (roce_uni_cmd_cq_hw2sw_s *)cqm_cmd_inbuf->buf; + cq_hw2sw_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); + if (cq->shadow != 0) { + cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_hw2sw_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(cq->shadow_vfid); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_HW2SW_CQ, + cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send HW2SW_CQ command, func_id(%u), cqn(0x%x), (ret:%d)", + (u32)rdev->glb_func_id, cq->cqn, ret); + + ret = (-EINVAL); + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + + /* CMDq may return a positive number, so its return value cannot be used directly */ + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + /* HI1825_V100 will retry load cqc in cmdq */ + if (ROCE5_SUPPORT_RETRY_LOAD_CQC_MSG_FEATURE(rdev) != 0) { + cqc_data = (struct roce5_cq_context *)((void *)cq->cqm_cq + ->q_ctx_vaddr); + return roce5_check_cqc_data_state(rdev, cq, cqc_data, + ROCE_CQ_TIME_OUT_CHECK_VALUE); + } + return 0; +} + +static int roce5_uni_cmd_cq_cache_out(struct roce5_device *rdev, + struct roce5_cq *cq, + cqm_cmd_buf_s **cqm_cmd_inbuf) +{ + int ret = 0; + struct dev_roce_svc_own_cap *roce_own_cap = NULL; + roce_uni_cmd_cq_hw2sw_s *cq_cache_invld_inbuf = NULL; + + /* Send the cache out command */ + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_cq_hw2sw_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce_own_cap = &rdev->rdma_cap.dev_rdma_cap.roce_own_cap; + cq_cache_invld_inbuf = (roce_uni_cmd_cq_hw2sw_s *)(*cqm_cmd_inbuf)->buf; + cq_cache_invld_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (cq->shadow != 0) { + cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_cache_invld_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(cq->shadow_vfid); + } + cq_cache_invld_inbuf->cmtt_cache.mtt_flags = 0; + cq_cache_invld_inbuf->cmtt_cache.mtt_num = 0; + cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_start = + cpu_to_be32(roce_own_cap->cmtt_cl_start); + cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_end = + cpu_to_be32(roce_own_cap->cmtt_cl_end); + cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_size = + cpu_to_be32(roce_own_cap->cmtt_cl_sz); + + return 0; +} + +static int roce5_cq_cache_out(struct roce5_device *rdev, struct roce5_cq *cq) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + if (ROCE5_SUPPORT_CQ_CACHE_OUT_MSG_V2(rdev) != 0) { + ret = roce5_uni_cmd_cq_cache_out(rdev, cq, &cqm_cmd_inbuf); + } + + if (ret != 0) { + return ret; + } + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_MISC_CQ_CACHE_INVLD, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send CQ_CACHE_INVLD command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + return 0; + } + + return -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +int roce5_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + struct roce5_cq *rcq = NULL; + struct roce5_ucontext *ucontext = NULL; + char waiting_msg[ROCE_WAITING_MSG_MAX_SIZE]; + roce_user_handle_t hdl = { 0 }; + + if (ibcq == NULL) { + ROCE_ERR("Ibcq is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibcq->device); + rcq = to_roce5_cq(ibcq); + + ret = sprintf_s(waiting_msg, ROCE_WAITING_MSG_MAX_SIZE, + "Waiting for roce cq(%u) release...", rcq->cqn); + if (ret < 0) { + ROCE_DEV_WARN(rdev, "Failed to setup waiting message, ret(%d).", + ret); + } + + roce5_option_unset(rcq->opt, waiting_msg, sizeof(waiting_msg)); + + /* Modify CQC to be owned by the software */ + ret = roce5_cq_hw2sw(rdev, rcq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify CQC, ret(%d), func_id(%d), cqn(0x%x)", + ret, rdev->glb_func_id, rcq->cqn); + return ret; + } + + /* Send the cache out command */ + ret = roce5_cq_cache_out(rdev, rcq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send cache out command, ret(%d), func_id(%hu)", + ret, rdev->glb_func_id); + return ret; + } + + ROCE_CONSTUCT_ULD_HDL_RCQ(&hdl, rdev, rcq, NULL, NULL, NULL, 0); + roce5_uld_cq_destroy((roce_uld_handle)&hdl); + + /* If it is user mode, you also need to cancel the mapping of the corresponding software DB */ + if (ibcq->uobject != NULL) { + ucontext = roce5_queue_to_ucontext((void *)ibcq, udata, + ROCE_Q_TYPE_CQ); + roce5_db_unmap_user(ucontext, &rcq->db, rcq->shadow); + roce5_cq_put_umem(rdev, &rcq->buf, &rcq->umem, rcq->shadow); + } else { + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + } + + roce5_destroy_cq_dealloc_cqn(rdev, rcq); + return 0; +} + +int roce5_destroy_cq_ofa(struct ib_cq *ibcq) +{ + int ret = 0; + struct roce5_cq *rcq = NULL; + + if (ibcq == NULL) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + ret = roce5_destroy_cq(ibcq, NULL); + if (ret == 0) { + rcq = to_roce5_cq(ibcq); + kfree(rcq); + } + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h new file mode 100644 index 000000000..3c1cb6640 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQ_FORMAT_H +#define ROCE_CQ_FORMAT_H + +#include <linux/types.h> + +#include "roce_qp_ctx_format.h" + +typedef struct roce5_cq_context { + u32 rsvd[2]; + union { + struct { + u32 rsvd : 28; + u32 state : 4; + } bs; + u32 value; + } dw2; + + u32 rsvd1[12]; + + struct roce5_qpc_timer_seg timer_seg; + + u32 rsvd2[8]; +} roce5_cq_context_s; + +#endif /* ROCE_CQ_FORMAT_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c new file mode 100644 index 000000000..119fafba9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c @@ -0,0 +1,98 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "hinic5_hw.h" +#include "securec.h" + +#include "roce.h" +#include "roce_user.h" +#include "roce_cq.h" +#include "roce_os_compat.h" +#include "roce_cq_intern.h" + +/* + * high performance cq, based on reserved of cq, first to firstget get. + * 特 point: supports 23V200 & 25V100 two types NIC, according to device type different to allocate different of cq. + * later get of cq so bind qp performance will has so next decrease. +*/ + +#define ROCE_CQN_PERF_BASE 1 + +int roce5_create_cq_alloc_cqn(struct roce5_device *rdev, struct roce5_cq *rcq, + u32 index) +{ + u32 cqn = index; + u32 offset = ROCE_CQN_PERF_BASE; + u32 step = rdev->cqn_ctrl.rsvd_cq_step; + u32 rsvd_cqs = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs; + + mutex_lock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + if (rdev->cqn_ctrl.alloc_cq_cnt == rdev->cqn_ctrl.high_perf_cq_cnt) { + /* The reserved high performance CQs have been fully allocated. */ + goto cqn_alloc; + } + + while (rdev->cqn_ctrl.alloc_cq_cnt < rdev->cqn_ctrl.high_perf_cq_cnt && + (offset < rsvd_cqs)) { + if (test_and_set_bit(offset, &rdev->cqn_ctrl.cq_bitmap) != 0) { + offset += step; + continue; + } + cqn = offset; + rdev->cqn_ctrl.alloc_cq_cnt++; + ROCE_DEV_INFO( + rdev, + "Alloc High performance CQ, func_id(%d), total_high_perf_cq(%u), cqn(%d)", + rdev->glb_func_id, rdev->cqn_ctrl.alloc_cq_cnt, cqn); + break; + } + +cqn_alloc: + mutex_unlock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + rcq->cqm_cq = cqm5_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SCQ, 0, + rcq->opt, false, cqn, 0, 0); + if (rcq->cqm_cq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue from cqm object, func_id(%d), index(%d)", + rdev->glb_func_id, index); + return -ENOMEM; + } + + rcq->cqn = rcq->cqm_cq->index; + + return 0; +} + +void roce5_destroy_cq_dealloc_cqn(struct roce5_device *rdev, + struct roce5_cq *rcq) +{ + u32 cqn = rcq->cqn; + u32 rsvd_cqs = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs; + /* Call the CQM interface to release CQN and CQC. Since there is no cq_buf and software DB in user mode, + it does not need to be released; since kernel mode has both, it needs to be released */ + cqm5_object_delete(&rcq->cqm_cq->object); + kfree(rcq->opt); + rcq->opt = NULL; + + mutex_lock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + if ((cqn < rsvd_cqs) && + test_and_clear_bit(cqn, &rdev->cqn_ctrl.cq_bitmap) != 0 && + rdev->cqn_ctrl.alloc_cq_cnt > 0) { + rdev->cqn_ctrl.alloc_cq_cnt--; + ROCE_DEV_INFO( + rdev, + "Dealloc High performance CQ, func_id(%d), total_high_perf_cq(%u), cqn(%d)", + rdev->glb_func_id, rdev->cqn_ctrl.alloc_cq_cnt, cqn); + } + mutex_unlock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + return; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h new file mode 100644 index 000000000..a2cb56943 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQ_INTERN_H +#define ROCE_CQ_INTERN_H + +#include <rdma/ib_verbs.h> + +#include "hinic5_hw.h" +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_cq.h" + +#define ROCE_CQN_INVLD CQM_INDEX_INVALID + +int roce5_create_cq_alloc_cqn(struct roce5_device *rdev, struct roce5_cq *rcq, + u32 index); +void roce5_destroy_cq_dealloc_cqn(struct roce5_device *rdev, + struct roce5_cq *rcq); + +#endif //ROCE_CQ_INTERN_H \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c new file mode 100644 index 000000000..0ce3ffb25 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c @@ -0,0 +1,437 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_event_extension.h" +#include "roce_aeq_format.h" +#include "hinic5_mt.h" +#include "roce_restore.h" +#include "roce_infra.h" +#include "roce_cq.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif +#include "roce_event.h" + +static void roce5_event_report_common(const struct roce5_device *rdev, + int event_str_index) +{ + switch (event_str_index) { + case OFED_ET_PATH_MIG: + ROCE_DEV_INFO_LIMIT(rdev, + "OFED event: Path mig. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_COMM_EST: + ROCE_INFO_ONCE( + "OFED event: Communication establish. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_SQ_DRAINED: + ROCE_DEV_ERR_LIMIT(rdev, + "OFED event: Sq drained. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_SRQ_QP_LAST_WQE: + ROCE_ERR_ONCE("OFED event: Srq/qp last wqe. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_WQ_CATAS_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Wq catas error. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_PATH_MIG_FAILED: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Path mig failed. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_WQ_INVAL_REQ_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Wq inval req error. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_WQ_ACCESS_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Wq access error. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_CQ_ERR: + ROCE_DEV_ERR_LIMIT(rdev, + "OFED event: Cq error. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_SRQ_LIMIT: + ROCE_DEV_ERR_LIMIT(rdev, + "OFED event: Srq limit. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_SRQ_CATAS_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Srq catas error. function id: %u", + rdev->glb_func_id); + break; + case NON_OFED_ET_QPC_LOOKUP_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, + "[non ofed event type] Qpc lookup error. function id: %u", + rdev->glb_func_id); + break; + case NON_OFED_ET_OTHER_TYPE_ERR: + ROCE_ERR_ONCE( + "[non ofed event type] other type error. function id: %u", + rdev->glb_func_id); + break; + default: + break; + } +} + +/* **************************************************************************** + Prototype : roce5_event_report + Description : roce5_event_report + Input : struct roce5_device *rdev + int event_str_index + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_event_report(struct roce5_device *rdev, u8 *val, + int event_str_index) +{ + if (event_str_index <= NON_OFED_ET_OTHER_TYPE_ERR) { + roce5_event_report_common(rdev, event_str_index); + } else { + roce5_event_report_extend(rdev, val, event_str_index); + } +} + +/* **************************************************************************** + Prototype : to_qp_event_str_index + Description : to_qp_event_str_index + Input : u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int to_qp_event_str_index(u8 event_type) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + return OFED_ET_PATH_MIG; + + case ROCE_EVENT_TYPE_COMM_EST: + return OFED_ET_COMM_EST; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + return OFED_ET_SQ_DRAINED; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + return OFED_ET_SRQ_QP_LAST_WQE; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + return OFED_ET_WQ_CATAS_ERR; + + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + return OFED_ET_PATH_MIG_FAILED; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + return OFED_ET_WQ_INVAL_REQ_ERR; + + default: + return OFED_ET_WQ_ACCESS_ERR; + } +} + +/* **************************************************************************** + Prototype : roce5_handle_qp_async_event + Description : roce5_handle_qp_async_event + Input : struct roce5_device *rdev + u32 qpn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_qp_async_event(struct roce5_device *rdev, u32 qpn, + u8 event_type) +{ + struct roce5_qp *rqp = NULL; + cqm_object_s *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm5_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, + qpn, false); + if (cqm_obj_queue == NULL) { + ROCE_DEV_ERR_LIMIT( + rdev, + "Failed to get cqm obj queue, func_id(%d), qpn:%u, event_type:%u", + rdev->glb_func_id, qpn, event_type); + return; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_queue); + roce5_qp_async_event(rdev, rqp, (int)event_type); + cqm5_object_put(cqm_obj_queue); +} + +/* **************************************************************************** + Prototype : roce5_handle_cq_async_event + Description : roce5_handle_cq_async_event + Input : struct roce5_device *rdev + u32 cqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_cq_async_event(struct roce5_device *rdev, u32 cqn, + u8 event_type) +{ + struct roce5_cq *rcq = NULL; + + rcq = roce5_get_rcq_by_cqn(rdev, cqn); + if (rcq == NULL) { + ROCE_DEV_ERR(rdev, "Failed to get roce cq by cqn(%u).", cqn); + return; + } + + roce5_cq_async_event(rdev, rcq, (int)event_type); + roce5_rcq_put(rcq); +} + +/* **************************************************************************** + Prototype : roce5_handle_srq_async_event + Description : roce5_handle_srq_async_event + Input : struct roce5_device *rdev + u32 srqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_srq_async_event(struct roce5_device *rdev, u32 srqn, + u8 event_type) +{ + struct roce5_srq *rsrq = NULL; + cqm_object_s *cqm_obj_queue = NULL; + + cqm_obj_queue = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_queue == NULL) { + ROCE_DEV_ERR_LIMIT( + rdev, + "Failed to get cqm obj queue, func_id(%d), srqn:%u, event_type:%u", + rdev->glb_func_id, srqn, event_type); + return; + } + + rsrq = cqmobj_to_roce5_srq(cqm_obj_queue); + roce5_srq_async_event(rdev, rsrq, (int)event_type); + cqm5_object_put(cqm_obj_queue); +} + +static bool roce5_async_event_need_reset(u8 event_status) +{ + if ((event_status == API_FORMAT_ERROR) || + (event_status == UNKNOWN_RESERVED_ERROR1)) { + return true; + } + + return false; +} + +u8 roce5_async_event_level(void *svc_hd, u8 event_type, u8 *val) +{ + u8 event_level = FAULT_LEVEL_MAX; + u8 event_status; + u64 param = *((u64 *)val); + + if (svc_hd == NULL) { + ROCE_ERR_LIMIT("Svc_hd is null"); + return FAULT_LEVEL_SUGGESTION; + } + + event_status = param & 0xff; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_SRQ_LIMIT: + case ROCE_EVENT_TYPE_DEBUG: + event_level = FAULT_LEVEL_SUGGESTION; + break; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: //ROCE_AEQE_EVENT_QP_REQ_ERR + if (event_status == API_FORMAT_ERROR) { + return FAULT_LEVEL_SERIOUS_RESET; + } + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: //ROCE_AEQE_EVENT_WQE_CATAS + if (roce5_async_event_need_reset(event_status)) { + return FAULT_LEVEL_SERIOUS_RESET; + } + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: //CQ ERR + if (roce5_async_event_need_reset(event_status)) { + return FAULT_LEVEL_SERIOUS_RESET; + } + break; + + default: + event_level = FAULT_LEVEL_GENERAL; + break; + } + + return event_level; +} + +static int roce5_async_event_handle_common(u8 event_type, u8 *val, + struct roce5_device *rdev) +{ + u32 xid = 0; + int event_str_index = 0; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + xid = *((u32 *)val); + roce5_handle_qp_async_event(rdev, xid, event_type); + event_str_index = to_qp_event_str_index(event_type); + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: + xid = *((u32 *)val); + roce5_handle_cq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_CQ_ERR; + break; + + case ROCE_EVENT_TYPE_SRQ_LIMIT: + xid = *((u32 *)val); + roce5_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_LIMIT; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + xid = *((u32 *)val); + roce5_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_CATAS_ERR; + break; + + default: + event_str_index = to_unkown_event_str_index(event_type, val); + break; + } + + return event_str_index; +} + +/* **************************************************************************** + Prototype : roce5_async_event + Description : roce5_async_event + Input : void *svc_hd + u8 event_type + u64 val + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_async_event(void *svc_hd, u8 event_type, u8 *val) +{ + int event_str_index = 0; + struct roce5_device *rdev = NULL; + + if (svc_hd == NULL) { + ROCE_ERR_LIMIT("Svc_hd is null"); + return; + } + + rdev = (struct roce5_device *)svc_hd; + if (event_type <= ROCE_EVENT_TYPE_ODP_PAGE_FAULT) { + event_str_index = + roce5_async_event_handle_common(event_type, val, rdev); + } else { + event_str_index = + roce5_async_event_handle_extend(event_type, val, rdev); + } + + roce5_event_report(rdev, val, event_str_index); +} + +#ifdef ROCE_BONDING_EN +void roce5_handle_bonded_port_state_event(struct roce5_device *rdev) +{ + struct net_device *master = netdev_master_upper_dev_get_rcu(rdev->ndev); + enum ib_port_state bonded_port_state = IB_PORT_NOP; + struct roce5_bond_device *bond_dev = NULL; + enum ib_port_state curr_port_state; + enum ib_event_type event; + struct roce5_bond_slave *slave = NULL; + u32 i; + + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + ROCE_DEV_ERR(rdev, "No bond_dev found"); + return; + } + + if (master == NULL) { + bonded_port_state = IB_PORT_DOWN; + } else { + if (netif_running(master) != 0) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->netdev == NULL) { + continue; + } + + curr_port_state = + (netif_running(slave->netdev) && + netif_carrier_ok(slave->netdev)) ? + IB_PORT_ACTIVE : + IB_PORT_DOWN; + bonded_port_state = + (bonded_port_state != IB_PORT_ACTIVE) ? + curr_port_state : + IB_PORT_ACTIVE; + } + mutex_unlock(&bond_dev->slave_lock); + } else { + bonded_port_state = IB_PORT_DOWN; + } + } + + if (rdev->port_state != bonded_port_state) { + event = (bonded_port_state == IB_PORT_ACTIVE) ? + IB_EVENT_PORT_ACTIVE : + IB_EVENT_PORT_ERR; + if (rdev->port_state != IB_PORT_NOP) { + roce5_ifconfig_up_down_event_report(rdev, event); + } + rdev->port_state = bonded_port_state; + } +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h new file mode 100644 index 000000000..b5acc9b15 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_EVENT_H +#define ROCE_EVENT_H + +#include <linux/types.h> + +#define ROCE_M_DELAY 100 +#define API_FORMAT_ERROR 0x2 +#define UNKNOWN_RESERVED_ERROR1 0xe +#define QPC_LOOKUP_ERR_VALUE 0x300d00016 + +struct roce5_aeqe_event_data { + u32 xid; + + //microcode side by ROCE_AEQE_EVENT construct, driver side byte order differs from microcode side + u32 err_code : 8; + u32 type : 12; + u32 subtype : 8; + u32 module_type : 4; +}; + +#endif /* ROCE_EVENT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c new file mode 100644 index 000000000..c33bab11a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c @@ -0,0 +1,56 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#include <linux/types.h> + +#include "roce_event_extension.h" +#include "roce_aeq_format.h" + +int to_unkown_event_str_index(uint8_t event_type, const uint8_t *val) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_OFED_NO_DEF: + if (*((u64 *)val) == QPC_LOOKUP_ERR_VALUE) { + return NON_OFED_ET_QPC_LOOKUP_ERR; + } + return NON_OFED_ET_OTHER_TYPE_ERR; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + return NON_OFED_ET_OTHER_TYPE_ERR; + + case ROCE_EVENT_TYPE_DEBUG: + return NON_OFED_ET_DEBUG_ERR; + + default: + return INVAL_ET_ERR; + } +} + +void roce5_event_report_extend(const struct roce5_device *rdev, u8 *val, + int event_str_index) +{ + if (event_str_index == NON_OFED_ET_DEBUG_ERR) { + struct roce5_aeqe_event_data *data = + (struct roce5_aeqe_event_data *)val; + /* debug subtype according to ROCE_AEQE_DEBUG_SUBTYPE_E */ + ROCE_ERR_ONCE( + "[non ofed event type] Debug extend event subtype 0x%x. function id: %u", + data->subtype, rdev->glb_func_id); + return; + } + + ROCE_ERR_ONCE( + "[non ofed event type] Invalid extend event type error. function id: %u", + rdev->glb_func_id); +} + +int roce5_async_event_handle_extend(u8 event_type, u8 *val, + struct roce5_device *rdev) +{ + int event_str_index = 0; + + event_str_index = to_unkown_event_str_index(event_type, val); + + return event_str_index; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c new file mode 100644 index 000000000..759ba598f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c @@ -0,0 +1,170 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_main_extension.h" +#include "roce_infra.h" +#include "roce_os_compat.h" +#include "roce_kernel_compat_gen.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" + +static struct mutex g_rdev_mutex; +#endif + +void roce5_service_init_pre(void) +{ +#ifdef ROCE_BONDING_EN + mutex_init(&g_rdev_mutex); +#endif + + return; +} + +void roce5_lock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_lock(&g_rdev_mutex); +#endif +} + +void roce5_unlock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_unlock(&g_rdev_mutex); +#endif +} + +int roce5_get_rdev_by_uld(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, + struct hinic5_event_info *event) +{ +#ifdef ROCE_BONDING_EN + int ret; + + ret = roce5_bond_event_cfg_rdev(lld_dev, uld_dev, rdev, false); + if (ret != 0) { + return -1; + } +#else + *rdev = (struct roce5_device *)uld_dev; +#endif + return 0; +} + +#ifndef ROCE_CHIP_TEST +void roce5_init_dev_ext_handlers(struct roce5_device *rdev) +{ + return; +} +#endif + +int roce5_board_cfg_check(struct roce5_device *rdev) +{ + int ret = 0; + int port_num = 0; + int port_speed = 0; + + port_num = rdev->board_info.port_num; + port_speed = rdev->board_info.port_speed; + if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_100G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_100G_HCA; + } else if ((port_num == ROCE5_4_PORT_NUM) && + (port_speed == ROCE5_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_4_25G_HCA; + } else if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_25G_HCA; + } else if ((port_num == ROCE5_1_PORT_NUM) && + (port_speed == ROCE5_100G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_1_100G_HCA; + } else if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_200G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_200G_HCA; + } else if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_400G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_400G_HCA; + } else if ((port_num == ROCE5_4_PORT_NUM) && + (port_speed == ROCE5_200G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_4_200G_HCA; + } else { + ROCE_DEV_ERR( + rdev, + "Invalid fw cfg, port_num(%d), port_speed(%d), hca_type(%d)", + port_num, port_speed, rdev->hw_info.hca_type); + ret = (-EINVAL); + } + + return ret; +} + +void *roce5_rdev_alloc_ext(void) +{ +#if defined(HAVE__IB_ALLOC_DEVICE) + return (void *)ib_alloc_device(roce5_device, ib_dev); +#else + return (void *)ib_alloc_device(sizeof(struct roce5_device)); +#endif +} + +int roce5_set_comm_event(const struct roce5_device *rdev, + const struct hinic5_event_info *event) +{ + ROCE_DEV_INFO(rdev, "Comm event unsupported, func_id(%d), event(%d)", + rdev->glb_func_id, event->type); + return -1; +} + +struct roce5_ucontext * +roce5_queue_to_ucontext(void *ibq, struct ib_udata *udata, int q_type) +{ + struct roce5_ucontext *rctx = NULL; + +#if defined(UVERBS_IOCTL_HAVE_RDMA_UDATA_TO_DRV_CONTEXT) + rctx = rdma_udata_to_drv_context(udata, struct roce5_ucontext, + ibucontext); +#else + struct ib_qp *ibqp = NULL; + struct ib_cq *ibcq = NULL; + struct ib_srq *ibsrq = NULL; + switch (q_type) { + case ROCE_Q_TYPE_QP: + ibqp = (struct ib_qp *)ibq; + rctx = to_roce5_ucontext(ibqp->uobject->context); + break; + case ROCE_Q_TYPE_CQ: + ibcq = (struct ib_cq *)ibq; + rctx = to_roce5_ucontext(ibcq->uobject->context); + break; + case ROCE_Q_TYPE_SRQ: + ibsrq = (struct ib_srq *)ibq; + rctx = to_roce5_ucontext(ibsrq->uobject->context); + break; + default: + ROCE_ERR("Unknown queue type(%d)", q_type); + break; + } +#endif + return rctx; +} + +#ifndef ROCE_VROCE_EN +void roce5_fill_gid_entry_ext(const struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + return; +} + +void roce5_cfg_info_update_ext(struct roce5_device *rdev, + roce_get_cfg_info_resp_s *resp) +{ + return; +} + +int roce5_add_check_ext(const struct hinic5_lld_dev *lld_dev) +{ + return 0; +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c new file mode 100644 index 000000000..7f2e14669 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + +#include "roce_mr.h" +#include "roce_kernel_compat_gen.h" +#include "roce_main_extension.h" +#include "roce_mr_extension.h" + +#ifndef ROCE_CHIP_TEST +int roce5_check_alloc_mr_type(enum ib_mr_type mr_type) +{ + int ret = 0; + + if (mr_type != IB_MR_TYPE_MEM_REG) { + ret = -EINVAL; + ROCE_ERR("mr_type is invalid. mr_type:%u", mr_type); + } + + return ret; +} + +enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type) +{ + switch (ib_mr_type) { +#if defined(IB_MR_TYPE_HAVE_USER) && defined(IB_MR_TYPE_HAVE_INTEGRITY) + case IB_MR_TYPE_MEM_REG: + return RDMA_FRMR; + case IB_MR_TYPE_USER: + return RDMA_USER_MR; + case IB_MR_TYPE_INTEGRITY: + return RDMA_SIG_MR; +#else + case IB_MR_TYPE_MEM_REG: + return RDMA_DMA_MR; +#endif + default: + return RDMA_DMA_MR; + } +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c new file mode 100644 index 000000000..0e0170e9d --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c @@ -0,0 +1,222 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_qp_extension.h" +#include "roce_ctx_format.h" +#include "roce_restore.h" + +#ifdef ROCE_EXTEND +#include "roce_pd.h" +#endif + +int to_roce5_qp_type(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return ROCE_QP_ST_RC; + + case IB_QPT_UC: + return ROCE_QP_ST_UC; + + case IB_QPT_UD: + return ROCE_QP_ST_UD; + + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + return ROCE_QP_ST_XRC; + + case IB_QPT_GSI: + return ROCE_QP_ST_UD; + + default: + return -1; + } +} + +bool roce5_check_qp_modify_ok(enum ib_qp_state cur_state, + enum ib_qp_state next_state, enum ib_qp_type type, + enum ib_qp_attr_mask mask, + enum rdma_link_layer ll) +{ +#if defined(IB_MODIFY_QP_IS_OK_HAVE_RDMA_LINK_LAYER) + return (bool)ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll); +#else + return (bool)ib_modify_qp_is_ok(cur_state, next_state, type, mask); +#endif +} + +int roce5_is_qp_normal(struct roce5_qp *rqp, struct ib_qp_init_attr *init_attr) +{ + return 1; +} + +#ifdef ROCE_EXTEND +static struct roce5_qp * +roce5_cdev_lookup_and_check_rqp(struct roce5_device *rdev, u32 qpn) +{ + struct roce5_qp *rqp = NULL; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Can't find rqp according to qpn(0x%x), func_id(%d)", + qpn, rdev->glb_func_id); + return NULL; + } + + if (rqp->qpn >= QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC) { + ROCE_DEV_ERR(rdev, + "qpn[%u] more than sqpc num offset(%d) for sqpc.", + rqp->qpn, QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + roce5_rqp_put(rqp); + return NULL; + } + + return rqp; +} + +static int roce5_alloc_sqpc(struct roce5_device *rdev, struct roce5_qp *rqp, + struct roce5_vbs_qp *vbs_rqp) +{ + cqm_qpc_mpt_s *qpc_info = NULL; + + /* Call the CQM interface to allocate QPNs and QPCs */ + /* For SQPC, do not need to allocate QPN, QPN is directly spcified in rsvd segment */ + qpc_info = cqm5_object_qpc_mpt_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, NULL, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, 0, 0); + if (qpc_info == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create qpc by cqm object, func_id(%d), qpn(%u)", + rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + return -ENOMEM; + } + + if (qpc_info->xid != (rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC)) { + ROCE_DEV_ERR( + rdev, + "Create qpc error, func_id(%d), expect qpn(%d), actual qpn(%d)", + rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, + qpc_info->xid); + cqm5_object_delete(&(qpc_info->object)); + return -EFAULT; + } + + vbs_rqp->vbs_sqpc_info = qpc_info; + rqp->vbs_qp_ptr = (void *)vbs_rqp; + + return 0; +} + +long roce5_vbs_create_sqpc(struct roce5_device *rdev, void *buf) +{ + struct roce5_modify_qp_vbs_cmd cmd; + struct roce5_vbs_qp *vbs_rqp = NULL; + struct roce5_qp *rqp = NULL; + struct roce5_pd *pd = NULL; + int ret; + + ret = (int)copy_from_user(&cmd, buf, sizeof(cmd)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to copy data from user"); + return (long)ret; + } + + rqp = roce5_cdev_lookup_and_check_rqp(rdev, cmd.qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + vbs_rqp = (struct roce5_vbs_qp *)kzalloc(sizeof(*vbs_rqp), GFP_KERNEL); + if (vbs_rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc vbs_rqp, func_id(%d)", + rdev->glb_func_id); + ret = -ENOMEM; + goto put_rqp; + } + + pd = roce5_get_pd(rqp); + ret = roce5_db_map_user(to_roce5_ucontext(pd->ibpd.uobject->context), + cmd.ci_record_addr, &vbs_rqp->db, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to map db page to user, func_id(%d)", + rdev->glb_func_id); + goto free_rqp; + } + + ret = roce5_alloc_sqpc(rdev, rqp, vbs_rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc sqpc, func_id(%d)", + rdev->glb_func_id); + goto free_rqp; + } + + roce5_rqp_put(rqp); + return 0; + +free_rqp: + kfree(vbs_rqp); +put_rqp: + roce5_rqp_put(rqp); + return (long)ret; +} +#endif + +#ifndef ROCE_CHIP_TEST +void roce5_set_qp_dif_attr(struct roce5_qp *rqp, + const struct ib_qp_init_attr *init_attr, + const struct roce5_device *rdev) +{ + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EN) != 0) { + rqp->signature_en = true; + ROCE_DEV_INFO(rdev, + "func(%d) qp(%u) roce5_create_qp signature_en.", + rdev->glb_func_id, rqp->qpn); + } + return; +} +#endif + +#ifndef ROCE_VBS_EN +void roce5_set_attr_ext(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, int attr_mask) +{ + return; +} + +void roce5_transmit_ci_record_addr_ext(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr) +{ + return; +} +#endif + +#ifndef ROCE_VBS_EN +int roce5_modify_qp_pre_ext(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + return 0; +} + +void roce5_destroy_pcqc_ext(struct roce5_qp *rqp) +{ + return; +} + +#endif + +#ifndef ROCE_VROCE_EN + +void roce5_cos_update_ext(struct roce5_device *rdev, u8 *cos, + enum ib_qp_type qp_type) +{ + *cos = *cos & 0x7; +} +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c new file mode 100644 index 000000000..7067e8c80 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c @@ -0,0 +1,55 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include "roce_srq_extension.h" + +u8 roce5_get_container_sz(u32 container_mode) +{ + switch (container_mode) { + case ROCE_SRQ_MODE_0: + return ROCE_SRQ_CONT_NUM_MODE0; + case ROCE_SRQ_MODE_1: + return ROCE_SRQ_CONT_NUM_MODE1; + case ROCE_SRQ_MODE_2: + return ROCE_SRQ_CONT_NUM_MODE2; + case ROCE_SRQ_MODE_3: + return ROCE_SRQ_CONT_NUM_MODE3; + default: + return ROCE_SRQ_CONT_NUM_MODE3; + } +} + +u8 roce5_calculate_cont_th(u32 srq_limit) +{ + u8 cont_th = 0; + u32 srq_limit_tmp = srq_limit; + + srq_limit_tmp >>= 1; + while (srq_limit_tmp != 0) { + srq_limit_tmp >>= 1; + ++cont_th; + } + + return cont_th; +} + +#ifndef ROCE_CHIP_TEST +void roce5_srq_container_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, struct roce5_device *rdev) +{ + rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC); + + if (rsrq->xrc_en != 0) { + rsrq->container_flag = rdev->cfg_info.srq_container_en; + } + rsrq->container_mode = (rsrq->xrc_en != 0) ? + rdev->cfg_info.xrc_srq_container_mode : + rdev->cfg_info.srq_container_mode; + rsrq->container_warn_th = + roce5_calculate_cont_th(init_attr->attr.srq_limit); + rsrq->rqe_cnt_th = rdev->cfg_info.warn_th; + rsrq->container_size = roce5_get_container_sz(rsrq->container_mode); +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c new file mode 100644 index 000000000..f7f9117f4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c @@ -0,0 +1,1281 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_mr.h" +#include "roce_mr_extension.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "rdma_comp_mw_mr.h" +#include "hmm_common.h" +#include "roce_os_compat.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" + +/* **************************************************************************** + Prototype : get_key_from_index + Description : get_key_from_index + Input : u32 mpt_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static u32 get_key_from_index(u32 mpt_index) +{ + /* Algorithm of mr key is obtained by index shift calculation */ + return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) | + (mpt_index << MR_KEY_LEFT_SHIFT_OFS); +} + +/* **************************************************************************** + Prototype : convert_ib_access + Description : convert ib access to rdma comp access + Input : int access_flag + Output : None + + 1.Date : 2015/7/16 + Modification : Created function + +**************************************************************************** */ +static u32 convert_ib_access(int access_flag) +{ + u32 u_access_flag = (u32)access_flag; + u32 access; + + access = (u32)((((u_access_flag & IB_ACCESS_REMOTE_ATOMIC) != 0) ? + RDMA_IB_ACCESS_REMOTE_ATOMIC : + 0) | + (((u_access_flag & IB_ACCESS_REMOTE_WRITE) != 0) ? + RDMA_IB_ACCESS_REMOTE_WRITE : + 0) | + (((u_access_flag & IB_ACCESS_REMOTE_READ) != 0) ? + RDMA_IB_ACCESS_REMOTE_READ : + 0) | + (((u_access_flag & IB_ACCESS_LOCAL_WRITE) != 0) ? + RDMA_IB_ACCESS_LOCAL_WRITE : + 0) | + (((u_access_flag & IB_ACCESS_MW_BIND) != 0) ? + RDMA_IB_ACCESS_MW_BIND : + 0) | + (((u_access_flag & IB_ZERO_BASED) != 0) ? + RDMA_IB_ACCESS_ZERO_BASED : + 0)); + + return access; +} + +/* **************************************************************************** + Prototype : check_ib_access + Description : check remote access without local write + Input : int access_flag + Output : None + + 1.Date : 2017/11/18 + Modification : Created function + +**************************************************************************** */ +static int check_ib_access(int access_flag) +{ + /* + * Local write permission is required if remote write or + * remote atomic permission is also requested. + */ + + if (((((u32)access_flag) & + (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE)) != 0) && + ((((u32)access_flag) & IB_ACCESS_LOCAL_WRITE) == 0)) { + return -EINVAL; + } + + return 0; +} + +u32 roce5_get_mpt_xid_mode(struct roce5_device *rdev) +{ + u8 mode = rdev->mpt_xid_mode; + + if (mode == ROCE_MPT_XID_SEQUENTIAL) + return CQM_INDEX_INVALID; + + return CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, CQM_XID_LOW_BIT_0_0_1, + (u32)mode); +} + +/* **************************************************************************** + Prototype : roce5_alloc_tpt + Description : alloc mpt and mtt + Input : struct roce5_device *rdev + struct hmm_rdma *mr + u32 npages + u32 page_shift + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_alloc_tpt(struct roce5_device *rdev, struct hmm_rdma *mr, u32 npages, + u32 page_shift, u32 mpt_xid_mode) +{ + int ret = 0; + + ret = hmm_mpt_alloc_templated(rdev->hwdev, &mr->mpt, SERVICE_T_ROCE, + mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + return ret; + } + + mr->enabled = RDMA_MPT_EN_SW; + + /* npages = 0 is a legal case, when npages = 0 or npages = 1, MTT does not need to do address translation */ + ret = hmm_mtt_alloc(rdev->hwdev, npages, page_shift, &mr->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mtt, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto err_alloc_mtt; + } + + return 0; + +err_alloc_mtt: + hmm_mpt_free(rdev->hwdev, &mr->mpt); + mr->enabled = RDMA_MPT_DISABLED; + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_free_tpt + Description : free mpt and mtt + Input : struct roce5_device *rdev + struct hmm_rdma *mr + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +void roce5_free_tpt(struct roce5_device *rdev, struct hmm_rdma *mr) +{ + hmm_mtt_free(rdev->hwdev, &mr->mtt, SERVICE_T_ROCE); + + hmm_mpt_free(rdev->hwdev, &mr->mpt); + + mr->enabled = RDMA_MPT_DISABLED; +} + +/* **************************************************************************** + Prototype : roce5_set_rdma_mr + Description : set the member of rdma_mr + Input : struct hmm_rdma *mr + enum rdma_mr_type mr_type + u32 pdn + u64 iova + u64 size + u32 access + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +void roce5_set_rdma_mr(struct hmm_rdma *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access) +{ + mr->iova = iova; + mr->size = size; + mr->pdn = pdn; + mr->access = convert_ib_access(( + int)access); /* Convert ib permissions to rdma component permissions */ + mr->key = get_key_from_index( + mr->mpt.mpt_index); /* Convert from mpt index to key */ + mr->type = roce5_mr_type2hmm(mr_type); +} + +/* **************************************************************************** + Prototype : roce5_set_rdma_mw + Description : set the member of rdma_mw + Input : struct rdma_mw *mw + u32 pdn + enum ib_mw_type type + Output : None + + 1.Date : 2016/6/25 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_rdma_mw(struct rdma_mw *mw, u32 pdn, enum ib_mw_type type) +{ + mw->enabled = RDMA_MPT_EN_SW; + mw->pdn = pdn; + mw->key = get_key_from_index(mw->mpt.mpt_index); + mw->type = (type == IB_MW_TYPE_1 ? RDMA_MW_TYPE_1 : RDMA_MW_TYPE_2); +} +#ifdef __ROCE_DFX__ +static void roce5_store_mr_list(struct roce5_device *rdev, + struct list_head *node) +{ + struct roce5_list *roce_list = NULL; + + if (rdev->device_res == NULL) { + ROCE_INFO("Hasn't been created resource, don't need store."); + return; + } + + roce_list = &rdev->device_res->mr_list; + mutex_lock(&roce_list->node_lock); + list_add(node, &roce_list->node); + mutex_unlock(&roce_list->node_lock); +} + +static void roce5_delete_mr_list(struct roce5_device *rdev, + struct list_head *node) +{ + struct roce5_list *roce_list = NULL; + + if (rdev->device_res == NULL) { + ROCE_INFO("Resource has been release, don't need delete."); + return; + } + + roce_list = &rdev->device_res->mr_list; + mutex_lock(&roce_list->node_lock); + list_del(node); + mutex_unlock(&roce_list->node_lock); +} +#endif //__ROCE_DFX__ + +/* **************************************************************************** + Prototype : roce5_get_dma_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + int access + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +struct ib_mr *roce5_get_dma_mr(struct ib_pd *ibpd, int access) +{ + int ret = 0; + u32 mpt_xid_mode; + struct roce5_mr *mr = NULL; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if ((ibpd == NULL) || (check_ib_access(access) != 0)) { + ret = -EINVAL; + ROCE_ERR("Invalid Param.p1:%d", access); + goto err_out; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + mr = (struct roce5_mr *)kzalloc(sizeof *mr, GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, + "Failed to alloc memory for dma mr, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, 0, 0, mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, func_id(%d)", + rdev->glb_func_id); + goto err_alloc_tpt; + } + + /* Set the content in rdma_mr */ + roce5_set_rdma_mr(&mr->rdmamr, RDMA_DMA_MR, pd->pdn, 0ULL, + ROCE_DMA_MR_SIZE, (u32)access); + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(mr->rdmamr), + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of DMA mr, func_id(%d)", + rdev->glb_func_id); + goto err_enable_mpt; + } +#ifdef __ROCE_DFX__ + roce5_store_mr_list(rdev, &mr->node); +#endif + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + mr->ibmr.device = ibpd->device; + + ROCE_DEV_INFO(rdev, "Roce reg dma mr success, mr mpt index is %u", + mr->rdmamr.mpt.mpt_index); + + return &mr->ibmr; + +err_enable_mpt: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce5_alloc_priv_pages(struct ib_device *ibdev, struct roce5_mr *rmr, + u32 max_pages) +{ + int ret; + + rmr->page_map_size = + roundup(max_pages * sizeof(u64), ROCE5_MR_PAGES_ALIGN); + rmr->pages = (__be64 *)(void *)(uintptr_t)get_zeroed_page(GFP_KERNEL); + if (rmr->pages == NULL) { + ROCE_ERR("Failed to alloc rmr->pages"); + ret = -ENOMEM; + goto err_out; + } + + rmr->page_map = dma_map_single(ibdev->dev.parent, rmr->pages, + rmr->page_map_size, DMA_TO_DEVICE); + if (dma_mapping_error(ibdev->dev.parent, rmr->page_map) != 0) { + ROCE_ERR("Failed to do dma mapping"); + ret = -ENOMEM; + goto err_free_pages; + } + + return 0; + +err_free_pages: + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + +err_out: + return ret; +} + +void roce5_free_frmr_umem(struct ib_umem **fumem, int fumem_nr) +{ + int i; + + ROCE_INFO("%d frmr umems to be free.", fumem_nr); + for (i = 0; i < fumem_nr; i++) { + ib_umem_release(fumem[i]); + } +} + +static void roce5_free_priv_pages(struct roce5_mr *rmr) +{ + struct ib_device *ib_dev = rmr->ibmr.device; + + if (rmr->pages != NULL) { + ib_dev = rmr->ibmr.device; + dma_unmap_single(ib_dev->dev.parent, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + rmr->pages = NULL; + } +} + +static int roce5_alloc_mr_param_validate(const struct ib_pd *ibpd, + enum ib_mr_type mr_type, + u32 max_num_sg) +{ + int ret = 0; + + if (ibpd == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibpd is null"); + goto err_out; + } + + ret = roce5_check_alloc_mr_type(mr_type); + if (ret != 0) { + ret = -EINVAL; + ROCE_ERR("mr_type is invalid"); + goto err_out; + } + + if (max_num_sg >= ROCE_FRMR_MAX_PAGES) { + ROCE_ERR("Invalid max_num_sg(%d)", max_num_sg); + ret = -EINVAL; + goto err_out; + } + + return 0; + +err_out: + return ret; +} + +static int roce5_alloc_mr_param_check(struct roce5_device *rdev, + struct roce5_mr **mr, u32 max_num_sg) +{ + *mr = (struct roce5_mr *)kzalloc(sizeof(struct roce5_mr), GFP_KERNEL); + if ((*mr) == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc mr memory, func(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + (*mr)->max_pages = max_num_sg; + return 0; +} +/* **************************************************************************** + Prototype : roce5_alloc_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + enum ib_mr_type mr_type + u32 max_num_sg + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +struct ib_mr *roce5_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, + u32 max_num_sg) +{ + int ret = 0; + u16 sign_val = 0; + u32 mpt_xid_mode; + struct roce5_mr *mr = NULL; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + ret = roce5_alloc_mr_param_validate(ibpd, mr_type, max_num_sg); + if (ret != 0) { + goto err_out; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + ret = roce5_alloc_mr_param_check(rdev, &mr, max_num_sg); + if (ret != 0) { + goto err_out; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, max_num_sg, 0, + mpt_xid_mode); //alloc mptc and mtt + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc tpt, func(%d)", + rdev->glb_func_id); + goto err_alloc_tpt; + } + + ret = roce5_alloc_priv_pages(ibpd->device, mr, + max_num_sg); //alloc memory to store pa + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc memory, func(%u)", + rdev->glb_func_id); + goto err_alloc_priv_pages; + } + + roce5_set_rdma_mr(&mr->rdmamr, roce5_get_mrtype(mr_type), pd->pdn, 0ULL, + max_num_sg * (1 << PAGE_SHIFT), + ROCE5_MR_DEFAULT_ACCESS); + +#ifdef SIGN_MTT_EN + /* + MTT table entry of bit[11:2] used for fill in addr_signature, to hardware used for check MTT table entry complete validity. [1:1] is vmr, indicates current page whether be swap out. + bit[0:0] is vld bit indicates pointed by of next one level DMTT or Data Buffer of GPA whether valid. use sign_val record [11:0] this 12bit data, exist mr in. + send down frmr wqe after chip will do dma use pbl table fill MTT table, here need in advance calculate check value, in roce5_set_page() in write pbl table. + */ + sign_val = (u16)(hmm_gen_mtt_sign(mr->rdmamr.mtt.mtt_paddr, + mr->rdmamr.mtt.mtt_type) & + 0xffff); +#endif + mr->sign_val = ((sign_val << 1) | MTT_PA_VALID); + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(mr->rdmamr), + HINIC5_CHANNEL_ROCE); // send to chip by cmdq + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to enable mpt, func(%d)", + rdev->glb_func_id); + goto err_enale_mpt; + } + +#ifdef __ROCE_DFX__ + roce5_store_mr_list(rdev, &mr->node); +#endif + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_enale_mpt: + roce5_free_priv_pages(mr); + +err_alloc_priv_pages: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce5_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct roce5_mr *rmr = NULL; + + if (ibmr == NULL) { + ROCE_ERR("Ibmr is null"); + return -EINVAL; + } + + rmr = to_roce5_mr(ibmr); + if (ROCE_UNLIKELY(rmr->npages >= rmr->max_pages)) { + ROCE_ERR("Invalid npages(0x%x), can't set page", rmr->npages); + return -ENOMEM; + } + + rmr->pages[rmr->npages] = cpu_to_be64(addr | rmr->sign_val); + rmr->npages++; + + return 0; +} + +/* RoCE supports FRMR, for at privileged user, can via post send WR of way for at at free status of MR perform fast register. + here need in advance via input parameter of sgl available physical page 整 理 in pbl table 里, later via frmr wqe send down. */ +int roce5_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + int ret; + struct roce5_mr *rmr = NULL; + + /* sg_offset can be null */ + if ((ibmr == NULL) || (sg == NULL)) { + ROCE_ERR("Ibmr or sg is null"); + return -EINVAL; + } + + rmr = to_roce5_mr(ibmr); + rmr->npages = 0; + + if (ROCE_UNLIKELY(((u32)sg_nents) > rmr->max_pages)) { + ROCE_ERR("Invalid sg_nents(0x%x), max(0x%x)", sg_nents, + rmr->max_pages); + return -EINVAL; + } + + ib_dma_sync_single_for_cpu(ibmr->device, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + + ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, roce5_set_page); + + ib_dma_sync_single_for_device(ibmr->device, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + + return ret; +} + +static int roce5_umem_write_mtt_check(const struct roce5_device *rdev, + const struct mtt *mtt, + const struct ib_umem *umem) +{ + if ((rdev == NULL) || (mtt == NULL) || (umem == NULL)) { + ROCE_ERR("Rdev or mtt or umem is null"); + return -EINVAL; + } + + return 0; +} + +static int roce5_umem_write_mtt_update(struct roce5_device *rdev, + struct mtt *mtt, struct ib_umem *umem, + u64 *page_list, bool use_huge) +{ + int ret = 0; + u32 npages = 0; /* The number of recorded pages */ + u32 start_index = 0; /* page need to be written to mtt */ + struct ib_block_iter biter; + + rdma_umem_for_each_dma_block(umem, &biter, BIT(mtt->buf_page_shift)) { + page_list[npages++] = rdma_block_iter_dma_address(&biter); + /* The size of the memory pointed to by page_list is one page, which can store up to PAGE_SIZE / sizeof(u64) + pas. After a page is full, mtt needs to be written */ + if (npages == BIT(mtt->mtt_page_shift) / PA_SIZE) { + ret = hmm_mtt_write(rdev->hwdev, mtt, start_index, + npages, page_list, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to write mtt, ret(%d), start_index(%u), func_id(%u).", + ret, start_index, rdev->glb_func_id); + goto out; + } + start_index += npages; + npages = 0; + } + } + + if (npages != 0) { + ret = hmm_mtt_write(rdev->hwdev, mtt, start_index, npages, + page_list, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to write mtt, ret(%d), start_index(%u), func_id(%u).", + ret, start_index, rdev->glb_func_id); + goto out; + } + } + +out: + return ret; +} + +int roce5_umem_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct ib_umem *umem, bool use_huge) +{ + int ret = 0; + u64 *page_list = NULL; /* page_list to write to mtt */ + + ret = roce5_umem_write_mtt_check(rdev, mtt, umem); + if (ret != 0) { + return ret; + } + + page_list = (u64 *)kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc memory for page list, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + ret = roce5_umem_write_mtt_update(rdev, mtt, umem, page_list, use_huge); + if (ret != 0) { + ROCE_ERR("Failed to write mtt."); + } + + kfree(page_list); + return ret; +} + +/* **************************************************************************** + Prototype : roce5_buf_write_mtt + Description : roce5_buf_write_mtt + Input : struct roce5_device *rdev + struct mtt *mtt + struct tag_cqm_buf *buf + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +int roce5_buf_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct tag_cqm_buf *buf) +{ + u64 *page_list = NULL; + int ret = 0; + u32 i = 0; + + if ((rdev == NULL) || (mtt == NULL) || (buf == NULL)) { + ROCE_ERR("Rdev or mtt or buf is null"); + return (-EINVAL); + } + + page_list = + (u64 *)kzalloc(buf->buf_number * sizeof *page_list, GFP_KERNEL); + if (page_list == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc memory for page list, func_id(%d)", + rdev->glb_func_id); + return (-ENOMEM); + } + + /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */ + for (i = 0; i < buf->buf_number; ++i) { + page_list[i] = buf->buf_list[i].pa; + } + + ret = hmm_mtt_write(rdev->hwdev, mtt, 0, buf->buf_number, page_list, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + rdev->glb_func_id); + } + + kfree(page_list); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_is_umem_dma_contiguous + Description : Determine whether the input ib_umem occupies a contiguous + region in DMA address space, i.e., all scatter-gather entries + are adjacent in DMA addresses and the total length matches + umem->length. + Input : struct roce5_device *rdev + struct ib_umem *umem + Output : bool - true if the umem is DMA-contiguous, false otherwise + 1.Date : 2026/3/12 + Modification : Created function +**************************************************************************** */ +static inline bool roce5_is_umem_dma_contiguous(struct roce5_device *rdev, + struct ib_umem *umem) +{ + struct scatterlist *sg = NULL; + int i; + dma_addr_t expected_dma, dma; + uint64_t total_len = 0, len; + +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + sg = umem->sgt_append.sgt.sgl; +#else + sg = umem->sg_head.sgl; +#endif + + expected_dma = sg_dma_address(sg); + +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + for_each_sg(umem->sgt_append.sgt.sgl, sg, umem->sgt_append.sgt.nents, i) +#else + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) +#endif + { + dma = sg_dma_address(sg); + len = sg_dma_len(sg); + + if (i > 0 && dma != expected_dma) { + ROCE_DEV_WARN( + rdev, + "Enable huge page, but discontinuity at sg[%d]: expected 0x%llx, got 0x%llx\n", + i, (u64)expected_dma, (u64)dma); + return false; + } + + total_len += len; + expected_dma = dma + len; + } + + /* Check the total length. */ + if (total_len != umem->length) { + ROCE_DEV_WARN(rdev, "Total length %llu != umem length(%zu)\n", + total_len, umem->length); + return false; + } + + return true; +} + +static int roce5_get_page_info(struct roce5_device *rdev, struct ib_umem *umem, + u32 *npages, u32 *page_shift, u64 virt_addr) +{ + *page_shift = roce5_umem_get_buf_page_shift(rdev, umem, virt_addr); + *npages = roce5_umem_get_mtt_npages(rdev, umem, *page_shift); + + if (rdev->huge_page_en && *page_shift > PAGE_SHIFT) { + if (*page_shift != PAGE_SHIFT_2MB) { + ROCE_DEV_WARN( + rdev, + "RoCE reg huge MR expect page_size(2MB), but found %lu.", + BIT(*page_shift)); + } + ROCE_DEV_INFO(rdev, + "RoCE reg huge MR: npages(%u), page_shift(%u).", + *npages, *page_shift); + return ROCE_USE_HUGEPAGE; + } + + return ROCE_NOT_USE_HUGEPAGE; +} + +int roce5_user_mr_reg(struct roce5_device *rdev, struct roce5_mr *mr, u32 pdn, + u64 virt_addr, u64 length, int access) +{ + int ret = 0; + u32 npages = 0; + u32 page_shift = PAGE_SHIFT; + u32 mpt_xid_mode; + int use_huge; + struct hinic5_hwdev *hwdev = (struct hinic5_hwdev *)rdev->hwdev; + + if (hwdev == NULL) { + ROCE_ERR("hwdev is null"); + return 0; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + + use_huge = roce5_get_page_info(rdev, mr->umem, &npages, &page_shift, + virt_addr); + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift, + mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, func_id(%d)", + hinic5_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + roce5_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length, + (u32)access); + + ret = roce5_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem, use_huge); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + hinic5_global_func_id(hwdev)); + goto err_write_mtt; + } + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, hwdev, &mr->rdmamr, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of user mr, func_id(%d)", + hinic5_global_func_id(hwdev)); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + return ret; +} + +/* **************************************************************************** + Prototype : roce5_reg_user_mr + Description : register MR for user + Input : struct ib_pd *ibpd + u64 start + u64 length + u64 virt_addr + int access + struct ib_udata *udata + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +struct ib_mr *roce5_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, + u64 virt_addr, int access, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_mr *mr = NULL; + struct roce5_device *rdev = NULL; + struct roce5_pd *rpd = to_roce5_pd(ibpd); + roce_user_handle_t *handle = NULL; + + if (ibpd == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibpd is null"); + goto err_out; + } + + rdev = to_roce5_dev(ibpd->device); + + mr = (struct roce5_mr *)kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, + "Failed to alloc memory for roce mr, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } +#if defined(IB_UMEM_GET_PEER) + mr->umem = roce5_mr_umem_get_peer(rdev, (size_t)length, access, start, + udata, true); +#else + mr->umem = roce5_umem_get(&rdev->ib_dev, ibpd->uobject->context, start, + (size_t)length, access, 0); +#endif + if (IS_ERR(mr->umem)) { + handle = (roce_user_handle_t *)kzalloc( + sizeof(roce_user_handle_t), GFP_KERNEL); + if (handle == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR( + rdev, + "Failed to alloc memory for handle, func_id(%d)", + rdev->glb_func_id); + goto err_get_umem; + } + handle->magic = ROCE_ULD_HANDLE_MAGIC_NUM; + handle->rdev = rdev; + handle->mr = mr; + handle->virt_addr = virt_addr; + handle->length = length; + handle->pdn = rpd->pdn; + handle->access = access; + + mr->david_en = true; + mr->private_data = (void *)handle; + + /* call ULD interface */ + ret = roce5_uld_reg_user_mr((roce_uld_handle)handle); + if (ret != 0) { + ROCE_DEV_WARN( + rdev, + "Failed to reg user mr uld, func_id(%d), ret(%d), err(%d)", + rdev->glb_func_id, ret, (int)PTR_ERR(mr->umem)); + ret = (int)PTR_ERR(mr->umem); + kfree(handle); + mr->private_data = NULL; + goto err_get_umem; + } + + mr->umem = NULL; + } else { + ret = roce5_user_mr_reg(rdev, mr, rpd->pdn, virt_addr, length, + access); + if (ret != 0) { + goto err_mr_update; + } + } +#ifdef __ROCE_DFX__ + roce5_store_mr_list(rdev, &mr->node); +#endif + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + ROCE_DEV_INFO(rdev, "Roce reg user mr success, mr mpt index is %u", + mr->rdmamr.mpt.mpt_index); + + return &mr->ibmr; + +err_mr_update: + ib_umem_release(mr->umem); + +err_get_umem: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +int roce5_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_mr *mr = NULL; + struct roce5_device *rdev = NULL; + + if (ibmr == NULL) { + ROCE_ERR("Ibmr is null"); + return -EINVAL; + } + + mr = to_roce5_mr(ibmr); + rdev = to_roce5_dev(ibmr->device); + + if (mr->rdmamr.type == HMM_RDMA_FRMR) { + roce5_free_priv_pages(mr); + } + + if (mr->david_en) { + if (mr->private_data == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid parameter, private data is NULL."); + return -EINVAL; + } + + ret = roce5_uld_dereg_user_mr( + (roce_uld_handle)mr->private_data); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "roce dereg user mr uld failed, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + kfree(mr->private_data); + mr->private_data = NULL; + } else { + ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, + &mr->rdmamr, SERVICE_T_ROCE, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Mr may has mw bind on it, ret(%d), mr_key(%#x), func_id(%d)", + ret, mr->rdmamr.key, rdev->glb_func_id); + if (ret == (-CMDQ_TIMEOUT)) { + rdev->dev_status_to_ofed = + ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + return ret; + } + + roce5_free_tpt(rdev, &mr->rdmamr); + + ib_umem_release(mr->umem); + } + + if (mr->fumem_nr != 0) { + roce5_free_frmr_umem(mr->fumem, mr->fumem_nr); + mr->fumem_nr = 0; + } +#ifdef __ROCE_DFX__ + roce5_delete_mr_list(rdev, &mr->node); +#endif + + kfree(mr); + + return 0; +} + +int roce5_dereg_mr_ofa(struct ib_mr *ibmr) +{ + return roce5_dereg_mr(ibmr, NULL); +} + +static void roce5_err_enable_mpt_handler(void *hwdev, struct roce5_mw *mw) +{ + hmm_mpt_free(hwdev, &mw->rdmamw.mpt); + mw->rdmamw.enabled = RDMA_MPT_DISABLED; + return; +} + +static void roce5_mw2hmm_adapter(const struct rdma_mw *rdmamw, + struct hmm_rdma *hmmrdma) +{ + hmmrdma->mpt = rdmamw->mpt; + hmmrdma->key = rdmamw->key; + hmmrdma->pdn = rdmamw->pdn; + hmmrdma->type = (rdmamw->type == RDMA_MW_TYPE_1) ? HMM_RDMA_MW_TYPE_1 : + HMM_RDMA_MW_TYPE_2; + hmmrdma->enabled = rdmamw->enabled; +} + +int roce5_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) +{ + int ret; + u32 mpt_xid_mode; + struct roce5_mw *mw = NULL; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + struct hmm_rdma hmmrdma = { 0 }; + + if (ibmw == NULL) { + ROCE_ERR("Ibmw is null"); + return -EINVAL; + } + + mw = to_roce5_mw(ibmw); + pd = to_roce5_pd(ibmw->pd); + rdev = to_roce5_dev(ibmw->device); + + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + ret = hmm_mpt_alloc_templated(rdev->hwdev, &mw->rdmamw.mpt, + SERVICE_T_ROCE, mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt, func_id(%d)", + rdev->glb_func_id); + goto err; + } + + roce5_set_rdma_mw(&mw->rdmamw, pd->pdn, ibmw->type); + roce5_mw2hmm_adapter(&mw->rdmamw, &hmmrdma); + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &hmmrdma, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of mw, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + if (ret == (-CMDQ_TIMEOUT)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + goto err_enable_mpt; + } + + mw->rdmamw.enabled = hmmrdma.enabled; + mw->ibmw.rkey = mw->rdmamw.key; + return 0; +err_enable_mpt: + roce5_err_enable_mpt_handler(rdev->hwdev, mw); +err: + return ret; +} + +struct ib_mw *roce5_alloc_mw_ofa(struct ib_pd *ibpd, enum ib_mw_type type, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_mw *mw = NULL; + + if (ibpd == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibpd is null"); + goto out; + } + + mw = (struct roce5_mw *)kzalloc(sizeof *mw, GFP_KERNEL); + if (mw == NULL) { + ret = -ENOMEM; + ROCE_ERR("Failed to alloc memory for roce mw"); + goto out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + mw->ibmw.pd = ibpd; + mw->ibmw.device = ibpd->device; + mw->ibmw.type = type; + ret = roce5_alloc_mw(&mw->ibmw, udata); + if (ret != 0) { + kfree(mw); + } + +out: + return (ret != 0 ? (struct ib_mw *)ERR_PTR((long)ret) : &mw->ibmw); +} + +/* **************************************************************************** + Prototype : roce5_dealloc_mw + Description : dealloc mw and release related resource + Input : struct ib_mw *ibmw + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +int roce5_dealloc_mw(struct ib_mw *ibmw) +{ + int ret = 0; + struct roce5_mw *mw = NULL; + struct roce5_device *rdev = NULL; + struct hmm_rdma hmmrdma; + + if (ibmw == NULL) { + ROCE_ERR("Ibmw is null"); + return -EINVAL; + } + + mw = to_roce5_mw(ibmw); + rdev = to_roce5_dev(ibmw->device); + roce5_mw2hmm_adapter(&mw->rdmamw, &hmmrdma); + + ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, &hmmrdma, + SERVICE_T_ROCE, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to disable mpt of mw, func_id(%d)", + rdev->glb_func_id); + if (ret == (-CMDQ_TIMEOUT)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + return ret; + } + mw->rdmamw.enabled = hmmrdma.enabled; + + hmm_mpt_free(rdev->hwdev, &mw->rdmamw.mpt); + mw->rdmamw.enabled = RDMA_MPT_DISABLED; + + return ret; +} + +int roce5_dealloc_mw_ofa(struct ib_mw *ibmw) +{ + int ret = 0; + struct roce5_mw *mw = NULL; + + ret = roce5_dealloc_mw(ibmw); + if (ret == 0) { + mw = to_roce5_mw(ibmw); + kfree(mw); + } + + return ret; +} + +int roce5_check_mr_status(struct ib_mr *ibmr, u32 check_mask, + struct ib_mr_status *mr_status) +{ + int ret = 0; + struct roce5_mr *mr = NULL; + + if ((ibmr == NULL) || (mr_status == NULL)) { + ROCE_ERR("ibmr or mr_status is NULL ptr."); + return -EINVAL; + } + + mr = to_roce5_mr(ibmr); + if ((check_mask & (~IB_MR_CHECK_SIG_STATUS)) != 0) { + ROCE_ERR("check_mask(%#x) invalid", check_mask); + return -EINVAL; + } + + if (mr->signature_en == 0) { + ROCE_ERR("mr not configure dif info"); + ret = -EINVAL; + return ret; + } + + mr->sig.sig_status_checked = true; + mr_status->fail_status = 0; + + /* If err does not exist, return 0 */ + if (!mr->sig.sig_err_exists) { + return 0; + } + + ret = memcpy_s((void *)&mr_status->sig_err, sizeof(struct ib_sig_err), + (void *)&mr->sig.sig_err_item, + sizeof(struct ib_sig_err)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy sig_err.(ret:%d)", ret); + return -ENOMEM; + } + mr->sig.sig_err_exists = false; + mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; + + return 0; +} + +/** + * @brief Get mpt by index. When querying succeeded, you have to call roce5_mpt_put() to free + the cqm_obj_mpt. But there is no need to call roce5_mpt_put() if querying failed. + * @return Success: return the mpt. Failure: return NULL. + */ +struct mpt *roce5_get_mpt_by_index(struct roce5_device *rdev, u32 mpt_index) +{ + cqm_object_s *cqm_obj_mpt = NULL; + cqm_qpc_mpt_s *cqm_mpt = NULL; + struct mpt *mpt = NULL; + + cqm_obj_mpt = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_MPT, mpt_index, false); + if (cqm_obj_mpt == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find mpt according to index(0x%x), func_id(%d)", + mpt_index, rdev->glb_func_id); + return NULL; + } + + cqm_mpt = container_of(cqm_obj_mpt, cqm_qpc_mpt_s, object); + mpt = (struct mpt *)cqm_mpt->priv; + if (mpt == NULL) { + ROCE_DEV_ERR(rdev, "Failed to convert obj to mpt."); + cqm5_object_put(cqm_obj_mpt); + } + + return mpt; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h new file mode 100644 index 000000000..3124c2470 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MR_H +#define ROCE_MR_H + +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/vmalloc.h> + +#include "roce.h" + +#include "hinic5_hmm.h" +#include "hinic5_rdma.h" +#include "rdma_comp.h" + +#include "securec.h" + +#define ROCE5_MR_DEFAULT_ACCESS \ + (IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE | \ + IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_ATOMIC) + +#define ROCE5_MR_PAGES_ALIGN 64U +#define ROCE5_FRMR_UMEM_NUM 512 +#define ROCE5_SIZE_2M (2 * 1024 * 1024) + +struct roce5_sig_ctx { + struct ib_sig_attrs sig_attrs; /* recorded when submitting WR */ + struct ib_sig_err + sig_err_item; /* Get information configuration from CQE when polling CQE */ + bool sig_status_checked; /* Indicates whether the status is checked */ + bool sig_err_exists; /* Indicates if an error exists */ + u32 sig_err_count; /* Count the number of wrong CQEs */ +}; + +/* uld service can extension structure space */ +#define ROCE_MR_ULD_DEF_LEN 24 +#define ROCE_NOT_USE_HUGEPAGE 0 +#define ROCE_USE_HUGEPAGE 1 + +/* used for verify uld access whether memory stomping */ +#define ROCE_RMR_MAGIC 0x1317F73a +struct roce5_mr_uld_def { + u8 self_def[ROCE_MR_ULD_DEF_LEN]; +}; + +struct roce5_mr { + struct ib_mr ibmr; /* mr structure provided by ofed */ + struct ib_umem * + umem; /* Maintain the structure where the user-mode allocated buffer */ + struct ib_umem **fumem; /* used to frmr only */ + struct hmm_rdma rdmamr; /* mr structure provided by rdma component */ + struct roce5_sig_ctx sig; /* Record signature related information */ + u16 signature_en; /* Whether the logo contains signature information */ + u16 sign_val; /* MTT entry [11:0] */ + __be64 *pages; + u32 npages; + u32 fumem_nr; + u32 max_pages; + size_t page_map_size; + dma_addr_t page_map; +#ifdef __ROCE_DFX__ + struct list_head node; +#endif // __ROCE_DFX__ + bool david_en; /* Indicate whether the mr is a david mr */ + void *private_data; /* private data for driver */ + u32 user_data_len; /* length of user data which stored in struct hmm_rdma */ + struct roce5_mr_uld_def uld_def; + u32 rmr_magic; /* preventstop uld access memory stomping, support up to 24B space */ +}; + +struct roce5_mw { + struct ib_mw ibmw; /* mw structure provided by ofed */ + struct rdma_mw rdmamw; /* mw structure provided by rdma component */ +}; + +int roce5_alloc_tpt(struct roce5_device *rdev, struct hmm_rdma *mr, u32 npages, + u32 page_shift, u32 mpt_xid_mode); +void roce5_free_tpt(struct roce5_device *rdev, struct hmm_rdma *mr); +void roce5_set_rdma_mr(struct hmm_rdma *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access); +void roce5_free_frmr_umem(struct ib_umem **fumem, int fumem_nr); +struct mpt *roce5_get_mpt_by_index(struct roce5_device *rdev, u32 mpt_index); +static inline void roce5_mpt_put(struct mpt *rmpt) +{ + if (rmpt == NULL) { + return; + } + cqm5_object_put(&((cqm_qpc_mpt_s *)(rmpt->mpt_object))->object); +} + +static inline int roce5_mr_type2hmm(enum rdma_mr_type mr_type) +{ + int mr_type_convert_table[] = { + [RDMA_DMA_MR] = HMM_RDMA_DMA_MR, + [RDMA_USER_MR] = HMM_RDMA_USER_MR, + [RDMA_FRMR] = HMM_RDMA_FRMR, + [RDMA_FMR] = HMM_RDMA_FMR, + [RDMA_PHYS_MR] = HMM_RDMA_PHYS_MR, + [RDMA_RSVD_LKEY] = HMM_RDMA_RSVD_LKEY, + [RDMA_SIG_MR] = HMM_RDMA_SIG_MR, + [RDMA_INDIRECT_MR] = HMM_RDMA_INDIRECT_MR, + [RDMA_ODP_IMPLICIT_MR] = HMM_RDMA_ODP_IMPLICIT_MR, + [RDMA_ODP_EXPLICIT_MR] = HMM_RDMA_ODP_EXPLICIT_MR, + }; + + return mr_type_convert_table[mr_type]; +} + +#endif // __ROCE_MR_H_ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h new file mode 100644 index 000000000..28a092b29 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h @@ -0,0 +1,171 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#ifndef ROCE_POST_H +#define ROCE_POST_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "roce_wqe_format.h" + +#define ROCE_SQ_DOORBELL_TYPE_2 2 +#define ROCE_SQ_DOORBELL_TYPE_23 23 +#define ROCE_SQ_DOORBELL_TYPE_21 21 + +#define ROCE_DWQE_DDB_TYPE 3 /* RoCE direct wqe of DDB type */ + +#define ROCE_IMM_EXT_LEN 4 +#define ROCE_TASK_SEG_ALIGN 8 + +enum roce_tsl_8_byte_aligned_size_e { + ROCE_SEND_LOCAL_WQE_TSL = 2, + ROCE_RDMA_WQE_TSL = 4, + ROCE_ATOMIC_CWP_WQE_TSL = 6, + ROCE_UD_WQE_COM_TSL = 8 +}; + +enum { + ROCE_WQE_OPCODE_SEND = 0x00, + ROCE_WQE_OPCODE_SEND_INVAL = 0x01, + ROCE_WQE_OPCODE_SEND_IMM = 0x02, + + ROCE_WQE_OPCODE_RDMA_WRITE = 0x04, + ROCE_WQE_OPCODE_RDMA_WRITE_IMM = 0x05, + + ROCE_WQE_OPCODE_RDMA_READ = 0x08, + + ROCE_WQE_OPCODE_ATOMIC_CMP_SWP = 0x0c, + ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD = 0x0d, + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP = 0x0e, + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD = 0x0f, + + ROCE_WQE_OPCODE_FRMR = 0x10, + ROCE_WQE_OPCODE_LOCAL_INVAL = 0x11, + ROCE_WQE_OPCODE_BIND_MW = 0x12, + ROCE_WQE_OPCODE_REG_SIG_MR = + 0x13, /* Extended for further local opreation */ +}; + +#define ROCE_KERNEL_OPCODE_NUM (ROCE_WQE_OPCODE_REG_SIG_MR + 1) + +enum { + ROCE_DWQE_DB_SUBTYPE_SEND = 0x1, + ROCE_DWQE_DB_SUBTYPE_SEND_IMM = 0x2, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE = 0x3, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE_IMM = 0x4, + ROCE_DWQE_DB_SUBTYPE_RDMA_READ = 0x5, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_CMP_SWP = 0x6, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_FETCH_ADD = 0x7 +}; + +/* UD send WQE task seg1 */ +struct roce_wqe_ud_tsk_seg_cycle1 { + roce_wqe_tsk_com_seg_u common; + + /* DW0 */ + u32 data_len; + + /* DW1 */ + u32 immdata_invkey; + + /* DW2 */ + union { + struct { + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; /* 0: No limit on the static rate (100% port speed) + 1-6: reserved + 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s. + 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. */ + u32 rsvd1 : 3; + u32 fl : 1; + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { + u32 hop_limit : 8; + u32 sgid_idx : 7; + u32 rsvd0 : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tc : 8; + } bs; + u32 value; + } dw3; + + /* DW4 */ + union { + struct { + u32 flow_label : 20; + u32 rsvd0 : 4; + u32 smac_index : 3; + u32 rsvd1 : 5; + } bs; + u32 value; + } dw4; + + /* DW5~8 */ + u8 dgid[16]; + + /* DW9 */ + union { + struct { + u32 dst_qp : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw9; + + /* DW10 */ + u32 qkey; +}; + +/* UD send WQE task seg2; */ +struct roce_wqe_ud_tsk_seg_cycle2 { + /* DW0 */ + union { + struct { + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 cfi : 1; + u32 vlan_prio : 3; + } bs; + u32 value; + } dw0; + + /* DW1 */ + u32 dmac_l32; +}; + +struct roce5_post_send_normal_param { + struct roce5_wqe_ctrl_seg *ctrl_seg; + union roce5_wqe_tsk_com_seg *tsk_com_seg; + u32 wqe_size; + u8 *wqe; + struct roce_wqe_data_seg *dseg; + u32 wr_num; /* record posted WR numbers */ + u32 index; /* WQEBB id */ + int inline_flag; /* Inline flag */ + u32 data_len; + u32 *data_len_addr; + u32 sq_rmd_size; + s32 opcode; + s32 cycle; + u32 sq_wqe_ssn; + struct roce5_wqe_ctrl_seg ctrl_seg_tmp; + + unsigned long flags; /* type required by kernel API */ +}; + +typedef struct tag_roce_opcode_attr { + uint32_t rsvd : 31; + uint32_t remote_opcode : 1; +} roce_opcode_attr_s; + +#endif /* ROCE_POST_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h new file mode 100644 index 000000000..7088cd550 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_H +#define ROCE_QP_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_cq.h" +#include "roce_db.h" +#include "roce_pd.h" +#include "roce_compat.h" +#include "roce_npu_cmd_qp_defs.h" +#include "roce_wqe_base_format.h" +#include "roce_post.h" +#include "roce_mix.h" +#include "roce_xqe_format.h" +#include "roce_qp_ctx_format.h" +#include "roce_cq_format.h" +#include "roce_srq_format.h" +#include "roce_com.h" +#include "roce_obj_ext.h" +#include "roce_kernel_compat_gen.h" + +#define ROCE_RQ_MIN_SGE 4 +#define ROCE_RQ_MID_SGE 8 +#define ROCE_RQ_MAX_SGE 16 + +#define ROCE_WQE_NEXT_SGE_INVALID (1UL << 31) + +#define ROCE_RDMARC_DESTROY_CHECK_VALUE 0xcece5a5a + +#define RDMARC_NUM_PER_CACHELINE 8 +#define RDMARC_TABLE_ENTRY_SIZE 32 +#define ROCE_QP_MAX_PFETCH_MTT_LAYER 3 +#define ROCE_QP_MAX_TIMER_NUM 5 + +#define ROCE_QP_SMI_QP_NUM 0x0 +#define ROCE_QP_GSI_QP_NUM 0x1 +#define ROCE_QP_INVLID_QP_NUM CQM_INDEX_INVALID + +#define ROCE_QP_MAX_DWQE_SIZE 256 +#define ROCE_QP_DEFAULT_WQE_SHIFT 6 + +#define ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE 512 + +#define ROCE_QP_STATE_MEM_INIT 0xa + +#define ROCE_UD_MAX_INLINE_LEN_SUB 48 + +#define ROCE_QP_GPA_SIG_LEN 3 +#define ROCE_WR_MIN_NUM 2 + +#define ROCE_QPC_PMTU_TRANSLATE(pmtu) (((1U << ((pmtu)-1)) - 1) & 0xf) + +#define ROCE_QP_OPTPAR_ALT_ADDR_PATH (1 << 0) +#define ROCE_QP_OPTPAR_RRE (1 << 1) +#define ROCE_QP_OPTPAR_RAE (1 << 2) +#define ROCE_QP_OPTPAR_RWE (1 << 3) +#define ROCE_QP_OPTPAR_PKEY_INDEX (1 << 4) +#define ROCE_QP_OPTPAR_Q_KEY (1 << 5) +#define ROCE_QP_OPTPAR_RNR_TIMEOUT (1 << 6) +#define ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH (1 << 7) +#define ROCE_QP_OPTPAR_SRA_MAX (1 << 8) +#define ROCE_QP_OPTPAR_RRA_MAX (1 << 9) +#define ROCE_QP_OPTPAR_PM_STATE (1 << 10) +#define ROCE_QP_OPTPAR_RETRY_COUNT (1 << 11) +#define ROCE_QP_OPTPAR_RNR_RETRY (1 << 12) +#define ROCE_QP_OPTPAR_ACK_TIMEOUT (1 << 13) +#define ROCE_QP_OPTPAR_SCHED_QUEUE (1 << 14) +#define ROCE_QP_OPTPAR_COUNTER_INDEX (1 << 15) + +#define roce5_LB1_MASK_1825V100 0x7 + +enum roce_qp_state { + ROCE_QP_STATE_RST = 0, + ROCE_QP_STATE_INIT = 1, + ROCE_QP_STATE_RTR = 2, + ROCE_QP_STATE_RTS = 3, + ROCE_QP_STATE_SQER = 4, + ROCE_QP_STATE_SQD = 5, + ROCE_QP_STATE_ERR = 6, + ROCE_QP_STATE_SQ_DRAINING = 7, + ROCE_QP_STATE_NUM +}; + +enum { + ROCE_QP_PM_MIGRATED = 0x3, + ROCE_QP_PM_ARMED = 0x0, + ROCE_QP_PM_REARM = 0x1 +}; + +enum { ROCE_QP_NO_SRQ, ROCE_QP_HAS_SRQ }; + +enum { ROCE_QP = 0, ROCE_QP_EXT }; + +struct roce5_wq { + u64 *wrid; + spinlock_t lock; + u32 wqebb_cnt; + u32 max_post; + u32 max_sge; + u32 offset; + u32 wqe_shift; + u32 head; + u32 tail; +}; + +struct roce5_sqp_attr { + int pkey_index; + u32 qkey; + u32 send_psn; +}; + +#define ROCE_RQP_ULD_DEF_LEN 64 +#define ROCE_RQP_MAGIC 0x5a5b5c5d +struct roce5_rqp_uld_def { + u8 uld_def[ROCE_RQP_ULD_DEF_LEN]; +}; + +struct roce5_qp { + struct ib_qp ibqp; + u32 qpn; + struct roce5_db db; + cqm_queue_s *qp_buf_info; + cqm_qpc_mpt_s *qpc_info; + struct roce5_wq rq; + u32 sq_signal_bits; + unsigned int sq_next_wqe; + u32 sq_max_wqes_per_wr; + struct roce5_wq sq; + struct ib_umem *umem; + struct mtt mtt; + struct roce5_buf buf; + int buf_size; + struct mutex mutex; + u32 xrcdn; + u8 port; + u8 atomic_rd_en; + u8 qp_state; + u8 qp_ext; + u32 qp_type; + u32 max_inline_data; + int max_dwqe_size; + u32 sq_wqe_ssn; + struct rdma_rdmarc rdmarc; + u16 rsp_depth; + u8 sl; + bool has_rq; + u8 *sq_head_addr; + u8 *sq_tail_addr; + bool signature_en; + u8 pi_on_chip; + u8 double_sgl_mode; + u8 ext_mtu; + u8 ext_mtu_mode; + + u8 db_sgid_index; + u8 db_path_mtu; + u8 db_cos; + u8 shadow; + + struct roce5_sqp_attr *sqp_attr; + + u32 qid; + u32 local_comm_id; /* used by NOF AA to reuse qpn when disconnection happens */ + u32 remote_comm_id; + void *wb_va; /* used for xrc */ + dma_addr_t wb_dma; /* used for xrc */ + u32 shadow_vfid; + + /* hot-plug record */ + struct list_head qps_list; + struct list_head cq_recv_list; + struct list_head cq_send_list; +#if defined(ROCE_VBS_EN) || defined(ROCE_EXTEND) + void *vbs_qp_ptr; +#endif +#ifdef ROCE_BONDING_EN + u32 tx_hash_value; +#endif + + struct roce5_obj_ext obj_ext; + + struct roce5_rqp_uld_def + uld_def; /* must placed at qp_magic before, preventstop uld access memory stomping, support up to 64B space */ + u32 rqp_magic; +}; + +typedef struct tag_roce_modify_qp_param { + u32 optpar; + roce_verbs_qp_attr_s qp_attr; + roce_verbs_cmd_qp_modify_rsp_s modify_rsp; +} roce_modify_qp_param_s; + +void roce5_qp_async_event(struct roce5_device *rdev, struct roce5_qp *qp, + int type); + +void roce5_free_opt_rdmarc(struct roce5_qp *rqp); +void roce5_set_opt_rdmarc(struct rdma_rdmarc *rdmarc, u16 depth, + roce_verbs_qp_attr_s *qp_attr); +void roce5_get_cqs(struct roce5_qp *rqp, struct roce5_cq **send_cq, + struct roce5_cq **recv_cq); +int roce5_qp_modify_2rst_cmd(struct roce5_device *rdev, u32 qpn); +int roce5_qp_cache_out_cmd(struct roce5_device *rdev, struct roce5_qp *rqp); +struct roce5_pd *roce5_get_pd(struct roce5_qp *rqp); +bool roce5_sqp_check(const struct roce5_qp *rqp); +void *roce5_get_wqe(struct roce5_qp *rqp, u32 offset); +int roce5_wq_overflow(struct roce5_wq *wq, u32 wr_num, struct ib_cq *ibcq); +void roce5_set_data_seg(struct roce_wqe_data_seg *dseg, struct ib_sge *sge); +void roce5_qpc_to_be(roce_verbs_qp_attr_s *qp_attr, struct roce5_qp *rqp, + u32 *be_ctx); +void roce5_qp_rst2init(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr); + +int roce5_send_qp_lb_cmd(u32 qpn, struct roce5_device *rdev, u8 cmd, + cqm_cmd_buf_s *buf_in, cqm_cmd_buf_s *buf_out, + u32 timeout); + +int roce5_qp_query(struct roce5_device *rdev, u32 qpn, u32 *context, + int qpc_size); +int roce5_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +void qpc_seg_to_le32(struct roce5_qp_context *be_ctx, + struct roce5_qp_context *le_ctx, u32 srq_vld); +void roce5_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size); + +int roce5_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); + +int roce5_post_send_ofa(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr); +struct roce5_qp *roce5_get_rqp_by_qpn(struct roce5_device *rdev, u32 qpn); +static inline void roce5_rqp_put(struct roce5_qp *rqp) +{ + if (rqp == NULL) { + return; + } + cqm5_object_put(&rqp->qpc_info->object); +} + +#define ROCE_UD_MTU_SHIFT 4 /* 4K mtu */ +static inline void roce5_set_sq_db(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct roce_sq_db_seg *db, + unsigned int index) +{ + db->dw0.bs.qpn = rqp->qpn & 0xfffff; + db->dw0.bs.ctx_size = 1; /* 512B */ + db->dw0.bs.cp_flag = 0; + db->dw0.bs.cos = rqp->db_cos; + if (ROCE5_SUPPORT_SQ_DB_TYPE_21(rdev) != 0) { + db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_21; + } else if (ROCE5_SUPPORT_SQ_DB_TYPE_MSG_V2(rdev) != 0) { + db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_23; + } else { + db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_2; + } + + db->dw1.bs.pi = ((index & 0xffff) >> 8) & 0xff; /* 8bits */ + db->dw1.bs.sub_type = 0; + db->dw1.bs.mtu_shift = rqp->db_path_mtu & 0x7; + db->dw1.bs.sgid_index = rqp->db_sgid_index; + if (rqp->qp_type == IB_QPT_RC) { + } else if (rqp->qp_type == IB_QPT_UD || rqp->qp_type == IB_QPT_GSI) { + db->dw1.bs.mtu_shift = ROCE_UD_MTU_SHIFT; + } else if (rqp->qp_type == IB_QPT_XRC_INI || + rqp->qp_type == IB_QPT_XRC_TGT) { + db->dw1.bs.xrc_vld = 1; + } + + db->dw0.value = roce5_convert_be32(db->dw0.value); + db->dw1.value = roce5_convert_be32(db->dw1.value); +} + +static inline int roce5_write_sq_db(struct roce5_device *rdev, + struct roce5_qp *rqp, u32 index, + struct ib_udata *udata) +{ + int ret = 0; + union roce_sq_db sq_db; + + ret = memset_s(&sq_db.sq_db_val, sizeof(u64), 0, sizeof(u64)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memset sq_db.(ret:%d)", ret); + return -ENOMEM; + } + + roce5_set_sq_db(rdev, rqp, &sq_db.sq_db_seg, index); + + if (udata == NULL) { + *(rqp->db.db_record) = cpu_to_be32(index & 0xffff); + } + + wmb(); + ret = cqm5_ring_hardware_db(rdev->hwdev, SERVICE_T_ROCE, index & 0xff, + sq_db.sq_db_val); + + /* + * Make sure doorbells don't leak out of SQ spinlock + * and reach the HCA out of order. + */ + wmb(); + return ret; +} + +#if defined(IB_QP_CREATE_FLAGS_HAVE_INTEGRITY_EN) +#define IB_QP_CREATE_SIGNATURE_EN IB_QP_CREATE_INTEGRITY_EN +#endif +#endif // ROCE_QP_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c new file mode 100644 index 000000000..6adbe787c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c @@ -0,0 +1,1506 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_infra.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" +#include "roce_aeq_format.h" +#include "roce_restore.h" +#include "roce_os_compat.h" +#include "roce_cc_format.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_qp_inner.h" +#include "roce_com.h" +#include "roce_qp_ext.h" + +#include "hinic5_hmm.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#include "securec.h" + +/* **************************************************************************** + Prototype : roce5_check_rq_size + Description : roce5_check_rq_size + Input : struct roce5_device *rdev + struct roce5_qp *rqp + struct ib_qp_cap *cap + bool has_rq + Output : None + + 1.Date : 2017/4/27 + Modification : Created function +**************************************************************************** */ +static int roce5_check_rq_size(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr, bool has_rq) +{ + u32 max_sge_num; + struct ib_qp_cap *cap = &init_attr->cap; + + max_sge_num = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg; + if ((cap->max_recv_wr > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) || + (cap->max_recv_sge > max_sge_num)) { + ROCE_DEV_ERR( + rdev, + "Failed to check rq size, over range, func_id(%d), attr max_recv_wr(%d), cap max_wqes(%d), \ + attr max_recv_sge(%d), cap max_sge_num(%d)", + rdev->glb_func_id, cap->max_recv_wr, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + cap->max_recv_sge, max_sge_num); + return -EINVAL; + } + + if (has_rq && ((cap->max_recv_wr == 0) || (cap->max_recv_sge == 0))) { + ROCE_DEV_ERR( + rdev, + "Has rq, cap->max_recv_wr(%d), cap->max_recv_sge(%d), func_id(%d)", + cap->max_recv_wr, cap->max_recv_sge, rdev->glb_func_id); + return -EINVAL; + } + + if ((!has_rq) && (cap->max_recv_wr != 0)) { + ROCE_DEV_ERR(rdev, + "Not has rq, cap->max_recv_wr(%d), func_id(%d).", + cap->max_recv_wr, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_set_rq_size + Description : roce5_set_rq_size + Input : struct roce5_device *rdev + struct roce5_qp *rqp + struct ib_qp_cap *cap + bool is_user + bool has_rq + Output : None + + 1.Date : 2015/4/29 + Modification : Created function +**************************************************************************** */ +/* **************************************************************************** +* +* 1 there is SQ or RQ +* SQ/RQ depth must be power of 2 +* if max_send_wr=0, no need to alloc buf for SQ/RQ + +* RQ WQE SIZE equals 64 or128 +* SQ WQE SIZE equals multi of wqebb size + +* SQ max_gs, max 8 and min 0 +* RQ max_gs, can be 4 or 8, but can not be 0. + +* 2 there is no SQ or RQ +* all related params should be 0 +* + +**************************************************************************** */ +static int roce5_set_rq_size(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr, bool is_user, + bool has_rq) +{ + int ret; + u32 wqebb_num = 0; + u32 sge_total_len = 0; + u32 page_size; + struct ib_qp_cap *cap = &init_attr->cap; + + page_size = roce5_get_page_size(rdev, true); + + ret = roce5_check_rq_size(rdev, rqp, init_attr, has_rq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check rq size, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (has_rq) { + wqebb_num = ROCE_MAX(2U, cap->max_recv_wr); + rqp->rq.wqebb_cnt = + (u32)(ROCE_ROUNDUP_POW_OF_TWO(wqebb_num) & 0xffffffff); + + /* RQ SGE range [0,4] select 4,[5,8] select 8 */ + rqp->rq.max_sge = + (cap->max_recv_sge <= ROCE_RQ_MIN_SGE) ? + ROCE_RQ_MIN_SGE : + ((cap->max_recv_sge <= ROCE_RQ_MID_SGE) ? + ROCE_RQ_MID_SGE : + ROCE_RQ_MAX_SGE); + + sge_total_len = (u32)((u32)rqp->rq.max_sge * + sizeof(struct roce_wqe_data_seg)); + rqp->rq.wqe_shift = (u32)ROCE_ILOG2(sge_total_len); + + if ((u32)((u32)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift) < page_size) { + rqp->rq.wqebb_cnt = + (page_size >> (unsigned)rqp->rq.wqe_shift); + } + + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + rqp->rq.max_post = + ROCE_MIN(rdev->rdma_cap.dev_rdma_cap + .roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + cap->max_recv_sge = (u32)rqp->rq.max_sge; + } else { + rqp->rq.max_post = + ROCE_MIN(rdev->rdma_cap.dev_rdma_cap + .roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + cap->max_recv_sge = + ROCE_MIN((u32)rqp->rq.max_sge, + rdev->rdma_cap.dev_rdma_cap + .roce_own_cap.max_rq_sg); + } + } else { + rqp->rq.wqebb_cnt = 0; + rqp->rq.max_sge = 0; + rqp->rq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->rq.max_post = 0; + cap->max_recv_sge = 0; + } + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + return 0; +} + +static int roce5_set_user_sq_size_check(struct roce5_device *rdev, + const struct ib_qp_cap *cap, + const struct roce5_qp *rqp) +{ + u32 ud_max_inline_size; + + if (cap->max_send_wr > + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes)) { + ROCE_DEV_ERR(rdev, "Log_sq_bb_count over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (rqp->qp_type == IB_QPT_UD) { + ud_max_inline_size = + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz > + ROCE_UD_MAX_INLINE_LEN_SUB) ? + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz - + ROCE_UD_MAX_INLINE_LEN_SUB) : + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz; + } else { + ud_max_inline_size = rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz; + } + + /* RC:192B inline data, UD:176B */ + if (cap->max_inline_data > ud_max_inline_size) { + ROCE_DEV_ERR(rdev, "SQ max inline data over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} +/* **************************************************************************** + Prototype : roce5_set_user_sq_size + Description : roce5_set_user_sq_size + Input : struct roce5_device *rdev + struct ib_qp_cap *cap + struct roce5_qp *rqp + struct roce5_create_qp *ucmd + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +static int roce5_set_user_sq_size(struct roce5_device *rdev, + struct ib_qp_cap *cap, struct roce5_qp *rqp, + struct create_qp_cmd *ucmd) +{ + u32 tmp_max_sq_wqe_sz = 0; + u32 sq_buf_sz = 0; + int ret = 0; + u32 page_size; + + page_size = roce5_get_page_size(rdev, true); + + ret = roce5_set_user_sq_size_check(rdev, cap, rqp); + if (ret != 0) { + return ret; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + /* between max and min */ + tmp_max_sq_wqe_sz = + (u32)(ROCE_ROUNDUP_POW_OF_TWO( + rdev->rdma_cap.max_sq_desc_sz) & + 0xffffffff); + if ((ucmd->log_sq_stride > ROCE_ILOG2(tmp_max_sq_wqe_sz)) || + (ucmd->log_sq_stride < + ROCE_ILOG2(rdev->rdma_cap.wqebb_size))) { + ROCE_DEV_ERR( + rdev, + "WQE size invalid, func_id(%d) log_sq_stride:%d wqebb_size:%d tmp_max_sq_wqe_sz:%d", + rdev->glb_func_id, ucmd->log_sq_stride, + rdev->rdma_cap.wqebb_size, tmp_max_sq_wqe_sz); + return -EINVAL; + } + + rqp->sq.wqebb_cnt = (u32)(1U << ucmd->log_sq_bb_count); + rqp->sq.wqe_shift = ucmd->log_sq_stride; + + if ((u32)((u64)rqp->sq.wqebb_cnt + << (unsigned)rqp->sq.wqe_shift) < page_size) { + rqp->sq.wqebb_cnt = + (page_size >> (unsigned)rqp->sq.wqe_shift); + } + + sq_buf_sz = (u32)ROCE_ALIGN( + (u64)rqp->sq.wqebb_cnt << (unsigned)rqp->sq.wqe_shift, + page_size); + rqp->buf_size = + (int)ROCE_ALIGN(((unsigned)sq_buf_sz + + ((unsigned)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift)), + page_size); + + rqp->sq.offset = 0; + rqp->rq.offset = sq_buf_sz; + + rqp->max_inline_data = (u32)cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ucmd->log_sq_stride; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_send_wqe_overhead + Description : roce5_send_wqe_overhead + Input : enum ib_qp_type qp_type + Output : None + + 1.Date : 2015/5/7 + Modification : Created function + +**************************************************************************** */ +static int roce5_send_wqe_overhead(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_frmr_tsk_seg); + + case IB_QPT_UC: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_send_tsk_seg); + + case IB_QPT_UD: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_ud_tsk_seg); + + case IB_QPT_GSI: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_ud_tsk_seg); + + case IB_QPT_XRC_TGT: + return 0; + + case IB_QPT_XRC_INI: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_frmr_tsk_seg); + + default: + ROCE_ERR("Not supported this qp_type(%d)", qp_type); + return -1; + } +} + +static int roce5_set_kernel_sq_size_check(const struct roce5_device *rdev, + const struct ib_qp_cap *cap, + int *sq_max_wqe_size, + enum ib_qp_type qp_type) +{ + int wqe_overhead = 0; + int ret = 0; + + if (cap->max_send_wr > + (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) { + ROCE_DEV_ERR(rdev, "SQ WR over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_send_sge > rdev->rdma_cap.max_sq_sg) { + ROCE_DEV_ERR(rdev, "SQ SGE over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_inline_data > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz) { + ROCE_DEV_ERR(rdev, "SQ max inline data over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + wqe_overhead = roce5_send_wqe_overhead(qp_type); + if (wqe_overhead < 0) { + ROCE_DEV_ERR(rdev, "Invalid service type(%d), func_id(%d)", + (int)qp_type, rdev->glb_func_id); + return -EINVAL; + } + + *sq_max_wqe_size = ROCE_MAX((int)(cap->max_send_sge * + sizeof(struct roce_wqe_data_seg)), + (int)cap->max_inline_data) + + wqe_overhead; + if (*sq_max_wqe_size > (int)rdev->rdma_cap.max_sq_desc_sz) { + ROCE_DEV_ERR( + rdev, + "SQ WQE size over range, func_id(%d) max_send_sge:%u,max_inline_data:%u," + "wqe_overhead:%d,sq_max_wqe_size:%d,max_sq_desc_sz:%u", + rdev->glb_func_id, cap->max_send_sge, + cap->max_inline_data, wqe_overhead, *sq_max_wqe_size, + rdev->rdma_cap.max_sq_desc_sz); + return -EINVAL; + } + + return ret; +} +/* **************************************************************************** + Prototype : roce5_set_kernel_sq_size + Description : roce5_set_kernel_sq_size + Input : struct roce5_device *rdev + struct ib_qp_cap *cap + enum roce5_qp_type qp_type + struct roce5_qp *rqp + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +static int roce5_set_kernel_sq_size(struct roce5_device *rdev, + struct ib_qp_cap *cap, + enum ib_qp_type qp_type, + struct roce5_qp *rqp) + +{ + u32 sq_buf_sz = 0; + int sq_max_wqe_size = 0; + int ret = 0; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + ret = roce5_set_kernel_sq_size_check(rdev, cap, + &sq_max_wqe_size, qp_type); + if (ret != 0) { + return ret; + } + + rqp->sq.wqe_shift = (u32)ROCE_ILOG2(rdev->rdma_cap.wqebb_size); + rqp->sq_max_wqes_per_wr = (u32)ROCE_DIV_ROUND_UP( + (unsigned)sq_max_wqe_size, + (u32)(1UL << (unsigned)rqp->sq.wqe_shift)); + + /* + * As one wqebb ought to be reserved for invalidate wqebb, 1 is added to sq.wqebb_cnt here + * to satisfy the actual IB_MAD_QP_SEND_SIZE set by applications + */ + rqp->sq.wqebb_cnt = + (u32)((u32)cap->max_send_wr * rqp->sq_max_wqes_per_wr + + 1); + rqp->sq.wqebb_cnt = + (u32)(ROCE_ROUNDUP_POW_OF_TWO(rqp->sq.wqebb_cnt) & + 0xffffffff); + + if ((u32)((u64)rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift) < + PAGE_SIZE) { + rqp->sq.wqebb_cnt = + (PAGE_SIZE >> ((u32)rqp->sq.wqe_shift)); + } + + rqp->sq.max_sge = + (u32)(((u64)rqp->sq_max_wqes_per_wr + << (unsigned)rqp->sq.wqe_shift) - + (unsigned)roce5_send_wqe_overhead(qp_type)); + rqp->sq.max_sge = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_desc_sz, + rqp->sq.max_sge) / + (u32)sizeof(struct roce_wqe_data_seg); + + sq_buf_sz = (u32)ALIGN((u64)rqp->sq.wqebb_cnt + << (unsigned)rqp->sq.wqe_shift, + PAGE_SIZE); + rqp->buf_size = (int)ALIGN(((unsigned)sq_buf_sz + + ((unsigned)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift)), + PAGE_SIZE); + + rqp->sq.offset = 0u; + rqp->rq.offset = sq_buf_sz; + + if (((rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr) == 1) { + ROCE_DEV_ERR(rdev, + "Only one WR,please check,func_id(%d)", + rdev->glb_func_id); + rqp->sq.max_post = ROCE_WR_MIN_NUM; + cap->max_send_wr = rqp->sq.max_post; + } else { + rqp->sq.max_post = (rqp->sq.wqebb_cnt - 1) / + rqp->sq_max_wqes_per_wr; + cap->max_send_wr = (u32)rqp->sq.max_post; + } + + cap->max_send_sge = (u32)ROCE_MIN(rqp->sq.max_sge, + rdev->rdma_cap.max_sq_sg); + rqp->max_inline_data = (u32)cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_qp_async_event + Description : roce5_qp_async_event + Input : struct roce5_device *rdev + struct roce5_qp *qp + int type + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +void roce5_qp_async_event(struct roce5_device *rdev, struct roce5_qp *qp, + int type) +{ + struct ib_event event; + struct ib_qp *ibqp = &qp->ibqp; + int ret; + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset dmac.(ret:%d)", ret); + return; + } + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + + switch (type) { + case ROCE_EVENT_TYPE_COMM_EST: + event.event = IB_EVENT_COMM_EST; + break; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + break; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + + default: + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static void *roce5_buf_offset(struct roce5_buf *buf, u32 offset) +{ + return (void *)((char *)buf->direct.buf + offset); +} + +void *roce5_get_wqe(struct roce5_qp *rqp, u32 offset) +{ + return roce5_buf_offset(&rqp->buf, offset); +} + +static void *roce5_get_send_wqe_head(struct roce5_qp *rqp) +{ + return roce5_get_wqe(rqp, rqp->sq.offset); +} + +/* **************************************************************************** + Prototype : roce5_get_send_wqe_tail + Description : roce5_get_send_wqe_tail + Input : struct roce5_qp *rqp + Output : None + + 1.Date : 2015/8/8 + Modification : Created function + +**************************************************************************** */ +static void *roce5_get_send_wqe_tail(struct roce5_qp *rqp) +{ + return roce5_get_wqe(rqp, rqp->sq.offset + (rqp->sq.wqebb_cnt + << (u32)rqp->sq.wqe_shift)); +} + +/* **************************************************************************** + Prototype : roce5_sq_buf_init + Description : roce5_sq_buf_init + Input : struct roce5_qp *qp + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** */ +static void roce5_sq_buf_init(struct roce5_qp *qp) +{ + u32 entries = qp->rq.offset / (1U << qp->sq.wqe_shift); + u8 *wqe = NULL; + struct roce5_wqe_ctrl_seg *wqe_ctrl = NULL; + u32 i; + + for (i = 0; i < entries; i++) { + wqe = (u8 *)qp->buf.direct.buf + + (u32)(i * (1U << ((u32)qp->sq.wqe_shift))); + wqe_ctrl = (struct roce5_wqe_ctrl_seg *)((void *)wqe); + wqe_ctrl->dw0.value = be32_to_cpu(wqe_ctrl->dw0.value); + wqe_ctrl->dw0.bs.owner = 1; + wqe_ctrl->dw0.value = cpu_to_be32(wqe_ctrl->dw0.value); + } +} + +static int roce5_check_qp_udata(struct roce5_device *rdev, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct roce5_qp *rqp, + struct create_qp_cmd *ucmd) +{ + int ret = 0; + + if ((udata == NULL) || + (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0)) { + ret = -EFAULT; + return ret; + } + + ret = roce5_set_user_sq_size(rdev, &init_attr->cap, rqp, ucmd); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set user sq_size, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return ret; +} + +static bool +roce5_need_qpn_lb1_consistent_srqn(const struct roce5_qp *rqp, + const struct roce5_device *rdev, + const struct ib_qp_init_attr *init_attr) +{ + if ((init_attr->srq == NULL) || (rdev->cfg_info.lb_en == 0)) { + return false; + } + + if (ROCE5_SUPPORT_QP_SRQN_LB_MSG_V2(rdev) != 0) { + return true; + } + + return false; +} + +static bool +roce5_need_qpn_lb1_consistent_scqn(struct roce5_device *rdev, + struct ib_qp_init_attr *init_attr) +{ + if ((init_attr->send_cq == NULL) || (rdev->cfg_info.lb_en == 0)) { + return false; + } + + if (ROCE5_SUPPORT_ALLOC_QPC_MSG_FEATURE(rdev) != 0) { + return false; + } + return true; +} + +static int roce5_find_free_lb_qpn(struct roce5_device *rdev) +{ + int idx = 0; + for (; idx < ROCE_LB_QPN_MAX; idx++) { + if (rdev->qp_lb_scqn_ctrl.qp[idx] == 0) { + return idx; + } + } + return -1; +} + +static int roce5_alloc_qpc(struct roce5_qp *rqp, struct roce5_device *rdev, + u32 qpn, struct ib_qp_init_attr *init_attr) +{ + struct roce5_srq *rsrq = NULL; + struct roce5_cq *rcq = NULL; + u32 qpn_tmp = qpn; + u32 roce_lb_mask = 0; + u32 cqm_lb_mode = 0; + int idx = -1; + + if ((rqp == NULL) || (rdev == NULL) || (init_attr == NULL)) { + ROCE_ERR("rqp, rdev or init_attr is NULL"); + return -EINVAL; + } + + if (ROCE_IS_DEV_1825V100(rdev)) { + cqm_lb_mode = CQM_XID_LOW_BIT_1_1_1; + roce_lb_mask = roce5_LB1_MASK_1825V100; + } else { + ROCE_DEV_ERR(rdev, "Unknown device type(%u), func_id(%u).", + rdev->device_type, rdev->glb_func_id); + return -EINVAL; + } + mutex_lock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + + if (roce5_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + rsrq = to_roce5_srq(init_attr->srq); + qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, cqm_lb_mode, + rsrq->srqn); + } else if ((qpn != ROCE_QP_GSI_QP_NUM) && + roce5_need_qpn_lb1_consistent_scqn(rdev, init_attr)) { + rcq = to_roce5_cq(init_attr->send_cq); + //if not found free of qpn, idx return -1, not need do LB + idx = roce5_find_free_lb_qpn(rdev); + if (idx >= 0) { + qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, + cqm_lb_mode, rcq->cqn); + } + } + + rqp->qpc_info = cqm5_object_qpc_mpt_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, rqp, + qpn_tmp, 0, 0); + + if (rqp->qpc_info == NULL && rdev->full_qp_spec) { + /* For subsequent QPs, XID alignment is not mandatory. */ + rqp->qpc_info = cqm5_object_qpc_mpt_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, + rqp, qpn, 0, 0); + } else if (rqp->qpc_info != NULL) { + if (idx >= 0) { //idx >= 0 indicates has emptyremaining of lb qpn bit position + rdev->qp_lb_scqn_ctrl.qp[idx] = rqp->qpc_info->xid; + ROCE_DEV_INFO( + rdev, + "RoCE create lb qpn func_id(%d), qpn(%d), idx(%d)", + rdev->glb_func_id, rqp->qpc_info->xid, idx); + } + } + mutex_unlock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + if (rqp->qpc_info == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create qpc mpt by cqm object, func_id(%d), qpn(%u)", + rdev->glb_func_id, qpn_tmp); + return -ENOMEM; + } + + rqp->qpn = rqp->qpc_info->xid; + if (roce5_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + if (rsrq != NULL && ((rsrq->srqn & roce_lb_mask) != + (rqp->qpn & roce_lb_mask))) { + ROCE_DEV_ERR( + rdev, + "Failed to create qpc LB srqn, func_id(%d), qpn(%u) srqn(%u) mask:0x%x", + rdev->glb_func_id, rqp->qpn, rsrq->srqn, + roce_lb_mask); + } + } + + return 0; +} + +static int roce5_create_qp_user_umem_mtt(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct ib_pd *ibpd, + struct create_qp_cmd *ucmd) +{ + u32 npages = 0; + int ret; + + if (rqp->shadow != 0) { + return 0; + } + rqp->umem = roce5_umem_get(&rdev->ib_dev, ibpd->uobject->context, + ucmd->buf_addr, (size_t)rqp->buf_size, 0, 0); + if (IS_ERR(rqp->umem)) { + ROCE_DEV_ERR(rdev, "Failed to get ib_umem, func_id(%d)", + rdev->glb_func_id); + ret = (int)PTR_ERR(rqp->umem); + return ret; + } + + rqp->buf.page_shift = (int)roce5_umem_get_buf_page_shift( + rdev, rqp->umem, ucmd->buf_addr); + npages = roce5_umem_get_mtt_npages(rdev, rqp->umem, + (u32)rqp->buf.page_shift); + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)rqp->buf.page_shift, + &rqp->mtt, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc rdma_mtt, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_umem; + } + + ret = roce5_umem_write_mtt(rdev, &rqp->mtt, rqp->umem, 1); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +err_umem: + ib_umem_release(rqp->umem); + return ret; +} + +void roce5_destroy_qp_user_umem_mtt(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + if (rqp->shadow != 0) { + return; + } + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + ib_umem_release(rqp->umem); +} + +/* **************************************************************************** + Prototype : roce5_create_qp_user + Description : create qp resource for user space qp + Input : struct roce5_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + struct ib_udata *udata + struct roce5_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_qp_user(struct roce5_device *rdev, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct roce5_qp *rqp) +{ + int ret = 0; + struct roce5_ucontext *roce5_uctx = NULL; + struct create_qp_cmd ucmd = { 0 }; + u32 qpn = ROCE_QP_INVLID_QP_NUM; + + roce5_uctx = to_roce5_ucontext(ibpd->uobject->context); + + ret = roce5_check_qp_udata(rdev, init_attr, udata, rqp, &ucmd); + if (ret != 0) { + return ret; + } + + ret = roce5_create_qp_user_umem_mtt(rdev, rqp, ibpd, &ucmd); + if (ret != 0) { + return ret; + } + + if (init_attr->qp_type != IB_QPT_XRC_TGT) { + ret = roce5_db_map_user(roce5_uctx, ucmd.db_addr, &rqp->db, + rqp->shadow); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to map db page to user, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_map_db; + } + } + + ret = roce5_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) { + goto err_alloc_qpc; + } + + return 0; + +err_alloc_qpc: + if (init_attr->qp_type != IB_QPT_XRC_TGT) { + roce5_db_unmap_user(roce5_uctx, &rqp->db, rqp->shadow); + } + +err_map_db: + roce5_destroy_qp_user_umem_mtt(rdev, rqp); + + return ret; +} + +static void roce5_free_wrid(struct roce5_qp *rqp) +{ + if (rqp->rq.wrid) { + kvfree(rqp->rq.wrid); + } + + if (rqp->sq.wrid) { + kvfree(rqp->sq.wrid); + } +} + +static int roce5_get_wrid(struct roce5_qp *rqp, struct roce5_device *rdev) +{ + int ret = 0; + + rqp->sq.wrid = (u64 *)kzalloc((unsigned)rqp->sq.wqebb_cnt * sizeof(u64), + GFP_KERNEL); + if (rqp->sq.wrid == NULL) { + rqp->sq.wrid = (u64 *)(void *)vzalloc( + (size_t)(rqp->sq.wqebb_cnt * sizeof(u64))); + } + + rqp->rq.wrid = (u64 *)kzalloc((unsigned)rqp->rq.wqebb_cnt * sizeof(u64), + GFP_KERNEL); + if (rqp->rq.wrid == NULL) { + rqp->rq.wrid = (u64 *)(void *)vzalloc( + (size_t)(rqp->rq.wqebb_cnt * sizeof(u64))); + } + + if ((rqp->sq.wrid == NULL) || (rqp->rq.wrid == NULL)) { + ROCE_DEV_ERR(rdev, + "Failed to alloc wrid of sq or rq, func_id(%d)", + rdev->glb_func_id); + ret = -ENOMEM; + } + + return ret; +} + +static void roce5_create_qp_kernel_set_attr(struct roce5_qp *rqp, + cqm_queue_s *qp_buf_info) +{ + int ret; + ret = memset_s(qp_buf_info->q_room_buf_1.direct.va, + (size_t)((unsigned)rqp->buf_size), 0, + (size_t)((unsigned)rqp->buf_size)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_buf_info.(ret:%d)", ret); + return; + } + rqp->buf.direct.buf = qp_buf_info->q_room_buf_1.direct.va; + rqp->buf.direct.map = qp_buf_info->q_room_buf_1.direct.pa; + rqp->buf.page_shift = + (int)ROCE_ILOG2(qp_buf_info->q_room_buf_1.buf_size); + + roce5_sq_buf_init(rqp); + + rqp->db.db_record = (__be32 *)(void *)(&qp_buf_info->q_header_vaddr + ->doorbell_record); + rqp->db.dma = qp_buf_info->q_header_paddr; + + qp_buf_info->q_header_vaddr->doorbell_record = 0; + + rqp->qp_buf_info = qp_buf_info; + + rqp->sq_head_addr = (u8 *)roce5_get_send_wqe_head(rqp); + rqp->sq_tail_addr = (u8 *)roce5_get_send_wqe_tail(rqp); + rqp->max_dwqe_size = 0; + rqp->sq_wqe_ssn = 0; + + if (rqp->max_inline_data != 0) { + rqp->max_dwqe_size = ROCE_QP_MAX_DWQE_SIZE; + } +} + +static int roce5_create_qp_kernel_normal(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + int ret; + cqm_queue_s *qp_buf_info = NULL; + + qp_buf_info = cqm5_object_rdma_queue_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_QP, + (u32)rqp->buf_size, rqp, true, ROCE_QP_INVLID_QP_NUM, 0, 0); + if (qp_buf_info == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue by cqm object, func_id(%d)", + rdev->glb_func_id); + return -1; + } + + roce5_create_qp_kernel_set_attr(rqp, qp_buf_info); + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, qp_buf_info->q_room_buf_1.buf_number, + (u32)rqp->buf.page_shift, &rqp->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_alloc_mtt; + } + ret = roce5_buf_write_mtt(rdev, &rqp->mtt, + &rqp->qp_buf_info->q_room_buf_1); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_write_mtt; + } + + ret = roce5_get_wrid(rqp, rdev); + if (ret != 0) { + goto err_alloc_wrid; + } + + return 0; + +err_alloc_wrid: + roce5_free_wrid(rqp); + +err_write_mtt: + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + +err_alloc_mtt: + cqm5_object_delete(&rqp->qp_buf_info->object); + + return ret; +} + +static void roce5_destroy_qp_kernel_normal(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + roce5_free_wrid(rqp); + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + cqm5_object_delete(&rqp->qp_buf_info->object); +} + +/* **************************************************************************** + Prototype : roce5_create_qp_kernel + Description : create qp resource for kernel space qp + Input : struct roce5_device *rdev + struct ib_qp_init_attr *init_attr + struct roce5_qp *rqp + u32 qpn + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_qp_kernel(struct roce5_device *rdev, + struct ib_qp_init_attr *init_attr, + struct roce5_qp *rqp, u32 qpn) +{ + int ret = 0; + + ret = roce5_set_kernel_sq_size(rdev, &init_attr->cap, + (enum ib_qp_type)rqp->qp_type, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set sq buffer size in kernel, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + ret = roce5_create_qp_kernel_normal(rdev, rqp); + if (ret != 0) { + return ret; + } + } + + ret = roce5_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) { + goto err_alloc_qpc; + } + + return 0; + +err_alloc_qpc: + if (rqp->qp_type != IB_QPT_XRC_TGT) { + roce5_destroy_qp_kernel_normal(rdev, rqp); + } + + return ret; +} + +static void roce5_qp_add_cq_list(struct ib_qp_init_attr *init_attr, + struct roce5_qp *rqp, + struct roce5_device *rdev) +{ + struct roce5_cq *send_rcq = to_roce5_cq(init_attr->send_cq); + struct roce5_cq *recv_rcq = to_roce5_cq(init_attr->recv_cq); + unsigned long flags = 0; /* type required by kernel API */ + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce5_lock_cqs(send_rcq, recv_rcq); + list_add_tail(&rqp->qps_list, &rdev->qp_list); + list_add_tail(&rqp->cq_send_list, &send_rcq->send_qp_list); + list_add_tail(&rqp->cq_recv_list, &recv_rcq->recv_qp_list); + roce5_unlock_cqs(send_rcq, recv_rcq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); +} + +static int roce5_alloc_xrc_extra_dma(const struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + dma_addr_t wb_dma = 0; + void *wb_va = NULL; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + return 0; + } + + wb_va = dma_alloc_coherent(rdev->dev, (size_t)4UL, &wb_dma, GFP_KERNEL); + if (wb_va == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc wb_va, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + rqp->wb_dma = wb_dma; + rqp->wb_va = wb_va; + + return 0; +} + +#define ROCE_MAX_PI_ON_CHIP_QPN 1024 /* temporarily when set PI is 1024(count word), later debug add */ +static int roce5_set_pi_on_chip(const struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + /* shadow scenario next, actual use of is vf, and pi on chip function in shadow scenario not already verify, default disable */ + if (ROCE5_SUPPORT_SET_PI_ON_CHIP_MSG_V2(rdev) != 0 && + rqp->shadow == 0) { + /* becauseslice on store pi resource has limit, disable VF of pi_on_chip, such as need enabled, need modify configuration file in chip store store PI data, Note 审monitorslice on resource whether satisfy requirement */ + rqp->pi_on_chip = (roce5_func_is_vf(rdev->hwdev) == false) && + (rqp->qpn < ROCE_MAX_PI_ON_CHIP_QPN); + } + + return 0; +} + +static int roce5_do_create_qp(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret = 0; + + mutex_init(&rqp->mutex); + + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + + rqp->qp_state = IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + + ret = roce5_alloc_xrc_extra_dma(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc xrc extra dma, func_id(%d), ret(%d).", + rdev->glb_func_id, ret); + return ret; + } + + if (ibpd != NULL && ibpd->uobject) { + ret = roce5_set_rq_size(rdev, rqp, init_attr, true, + rqp->has_rq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set rq size, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + + ret = roce5_create_qp_user(rdev, ibpd, init_attr, udata, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to create qp user, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + } else { + ret = roce5_set_rq_size(rdev, rqp, init_attr, false, + rqp->has_rq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set rq size, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + + ret = roce5_create_qp_kernel(rdev, init_attr, rqp, + ROCE_QP_INVLID_QP_NUM); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to create qp kernel, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + + roce5_qp_add_cq_list(init_attr, rqp, rdev); + } + + ret = roce5_set_pi_on_chip(rdev, rqp); + if (ret != 0) { + goto create_qp_err; + } + + ret = roce5_create_qp_ext_init(rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init qp extension, ret(%d).", + ret); + goto create_qp_err; + } + + return ret; +create_qp_err: + if (rqp->wb_va != NULL) { + dma_free_coherent(rdev->dev, sizeof(u32), rqp->wb_va, + rqp->wb_dma); + } + return ret; +} + +/* **************************************************************************** + Prototype : roce5_create_qp_gsi + Description : roce5_create_qp_gsi + Input : struct roce5_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_qp_gsi(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr) +{ + int ret = 0; + + rqp->sqp_attr = (struct roce5_sqp_attr *)kzalloc( + sizeof(struct roce5_sqp_attr), GFP_KERNEL); + if (rqp->sqp_attr == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to allocate memory for sqp attribute."); + return -ENOMEM; + } + + mutex_init(&rqp->mutex); + + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + + rqp->qp_state = (u8)IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + rqp->has_rq = !init_attr->srq; + + ret = roce5_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set rq size, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_out; + } + + ret = roce5_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_GSI_QP_NUM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create qp kernel, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_out; + } + + rqp->qpn = ROCE_QP_GSI_QP_NUM; + + roce5_qp_add_cq_list(init_attr, rqp, rdev); + + return 0; + +err_out: + kfree(rqp->sqp_attr); + + return ret; +} + +static int roce5_create_check_init_attr(const struct ib_pd *ibpd, + const struct ib_qp_init_attr *init_attr, + const struct ib_udata *udata) +{ + if (init_attr == NULL) { + ROCE_ERR("init_attr is null"); + return (-EINVAL); + } + + if ((ibpd == NULL) && (init_attr->qp_type != IB_QPT_XRC_TGT)) { + ROCE_ERR( + "Ibpd is null and qp_type is not IB_QPT_XRC_TGT, qp_type(0x%x)", + init_attr->qp_type); + return (-EINVAL); + } + + if ((udata) && (init_attr->qp_type == IB_QPT_GSI)) { + ROCE_ERR("Udata is not null and qp_type is IB_QPT_GSI"); + return (-EINVAL); + } + + return 0; +} + +static int roce5_set_qp_attr(struct ib_qp_init_attr *init_attr, + struct roce5_qp *rqp, struct roce5_device *rdev, + struct ib_pd **ibpd, int qp_ext) +{ + u16 xrcdn = 0; + + rqp->qp_ext = (u8)qp_ext; + if (init_attr->qp_type == IB_QPT_XRC_TGT) { + *ibpd = to_roce5_xrcd(init_attr->xrcd)->pd; + xrcdn = (u16)to_roce5_xrcd(init_attr->xrcd)->xrcdn; + init_attr->send_cq = to_roce5_xrcd(init_attr->xrcd)->cq; + init_attr->recv_cq = init_attr->send_cq; + + rqp->has_rq = false; + rqp->xrcdn = xrcdn; + } else if (init_attr->qp_type == IB_QPT_XRC_INI) { + init_attr->recv_cq = init_attr->send_cq; + rqp->has_rq = false; + } else if ((init_attr->qp_type == IB_QPT_RC) || + (init_attr->qp_type == IB_QPT_UC) || + (init_attr->qp_type == IB_QPT_UD)) { + rqp->has_rq = !init_attr->srq; + } else { + ROCE_DEV_ERR(rdev, "Unknown QP Type(%u), func_id(%d)", + init_attr->qp_type, rdev->glb_func_id); + return (-EINVAL); + } + + return 0; +} + +static int roce5_create_gsi_qp_result(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr) +{ + int ret; + + ret = roce5_create_qp_gsi(rdev, rqp, init_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create gsi qp, func_id(%d), ret(%d).", + rdev->glb_func_id, ret); + return ret; + } + + rqp->ibqp.qp_num = rqp->qpn; + return 0; +} + +static int roce5_create_qp_common(struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int qp_ext) +{ + int ret; + int uld_ret; + //when init_attr->qp_type == IB_QPT_XRC_TGT, ibpd is NULL. + struct ib_pd *ibpd = rqp->ibqp.pd; + struct roce5_device *rdev = + to_roce5_dev(ibpd ? ibpd->device : init_attr->xrcd->device); + roce_user_handle_t hdl = { 0 }; + + if (rdev->ecn_ctx.cc_algo == ROCE_CC_LDCP_ALGO && + (init_attr->qp_type == IB_QPT_XRC_TGT || + init_attr->qp_type == IB_QPT_XRC_INI)) { + ROCE_DEV_ERR( + rdev, + "XRC QP cant be created when LDCP is enabled, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (init_attr->qp_type == IB_QPT_GSI) { + return roce5_create_gsi_qp_result(rdev, rqp, init_attr); + } + + rqp->rqp_magic = ROCE_RQP_MAGIC; + ret = roce5_set_qp_attr(init_attr, rqp, rdev, &ibpd, qp_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set qp attribute, ret(%d).", ret); + goto err_set_attr; + } + + ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, NULL, udata, &rqp->mtt, + &rqp->db.dma); + ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rqp->shadow, &rqp->shadow_vfid, + NULL); + ret = roce5_uld_qp_create((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_ERR("Failed to execute uld qp creation, ret(%d)", ret); + goto err_set_attr; + } + ret = roce5_do_create_qp(rdev, rqp, ibpd, init_attr, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create qp in common process, ret(%d).", + ret); + goto err_create_qp; + } + + roce5_set_qp_dif_attr(rqp, init_attr, rdev); + rqp->ibqp.qp_num = rqp->qpn; + rqp->rsp_depth = 0; + + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.alloc_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + + return 0; + +err_create_qp: + uld_ret = roce5_uld_qp_destroy((roce_uld_handle)&hdl); + if (uld_ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to destroy uld qp in common process, uld_ret(%d).", + uld_ret); + } +err_set_attr: + return ret; +} + +struct ib_qp *roce5_create_qp_compat(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct roce5_qp *rqp = NULL; + int ret; + + ret = roce5_create_check_init_attr(ibpd, init_attr, udata); + if (ret != 0) { + ROCE_ERR("Failed to check init attr"); + return (struct ib_qp *)ERR_PTR((long)ret); + } + + rqp = (struct roce5_qp *)kzalloc(sizeof(*rqp), GFP_KERNEL); + if (rqp == NULL) { + ROCE_ERR("Failed to alloc roce qp."); + return (struct ib_qp *)ERR_PTR((long)-ENOMEM); + } + + rqp->ibqp.pd = ibpd; + + ret = roce5_create_qp_common(rqp, init_attr, udata, ROCE_QP); + if (ret != 0) { + ROCE_ERR("Failed to execute common qp creation, ret(%d)", ret); + goto err_create_qp; + } + + return &(rqp->ibqp); +err_create_qp: + kfree(rqp); + return (struct ib_qp *)ERR_PTR((long)-ENOMEM); +} + +int roce5_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_qp *rqp = NULL; + + if (ibqp == NULL) { + ROCE_ERR("Invalid parameter, ibqp is NULL."); + return -EINVAL; + } + + ret = roce5_create_check_init_attr(ibqp->pd, init_attr, udata); + if (ret != 0) { + ROCE_ERR("Failed to check init attr"); + return ret; + } + + rqp = to_roce5_qp(ibqp); + ret = roce5_create_qp_common(rqp, init_attr, udata, ROCE_QP); + if (ret != 0) { + ROCE_ERR("Failed to execute common qp creation, ret(%d)", ret); + return ret; + } + + return 0; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h new file mode 100644 index 000000000..c435e7090 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h @@ -0,0 +1,278 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_CTX_FOMRAT_H +#define ROCE_QP_CTX_FOMRAT_H + +#include <linux/types.h> + +/* Align each field with 4bytes. */ +#pragma pack(push) +#pragma pack(4) + +struct roce5_chip_seg_rrwc { + u32 dw0; + + union { + u64 rrw_curt_sge_va; + struct { + u32 rrw_curt_sge_va_hi; + u32 rrw_curt_sge_va_lo; + } dw1; + }; + + u32 rsvd[5]; +}; + +struct roce5_chip_seg_rcc { + u32 dw0; + + union { + u64 rc_curt_sge_va; + struct { + u32 rc_curt_sge_va_hi; + u32 rc_curt_sge_va_lo; + } dw1; + }; + + u32 rsvd[5]; +}; + +struct roce5_chip_seg_sqc { + u32 rsvd[9]; + + union { + u64 sq_curt_sge_va; + struct { + u32 sq_curt_sge_va_hi; + u32 sq_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_sqac { + u32 rsvd[9]; + + union { + u64 sqa_curt_sge_va; + struct { + u32 sqa_curt_sge_va_hi; + u32 sqa_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_rqc { + u32 rsvd[9]; + + union { + u64 rq_curt_sge_va; + struct { + u32 rq_curt_sge_va_hi; + u32 rq_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_srqc { + u32 rsvd[9]; + + union { + u64 srq_curt_sge_va; + struct { + u32 srq_curt_sge_va_hi; + u32 srq_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_qpcc { + union { + u64 sq_rq_l0mtt_gpa; + struct { + u32 sq_rq_l0mtt_gpa_hi; + u32 sq_rq_l0mtt_gpa_lo; + } dw0; + }; + + union { + u64 sq_rq_pi_record_gpa_at_hop_num; + struct { + u32 sq_rq_pi_record_gpa_hi; + u32 sq_rq_pi_record_gpa_lo_at_hop_num; + } dw2; + }; + + u32 dw4; + + union { + struct { + u32 rsvd1 : 22; + u32 qp_rre : 1; + u32 qp_rae : 1; + u32 qp_rwe : 1; + u32 rsvd : 7; + } bs; + u32 value; + } dw5; + + u32 rsvd[2]; +}; + +typedef struct roce5_qpc_chip_seg { + struct roce5_chip_seg_rrwc rrwc; + struct roce5_chip_seg_rcc rcc; + struct roce5_chip_seg_sqc sqc; + struct roce5_chip_seg_sqac sqac; + union { + struct roce5_chip_seg_rqc rqc; + struct roce5_chip_seg_srqc srqc; + }; + struct roce5_chip_seg_qpcc qpcc; +} roce5_qpc_chip_seg_s; + +struct roce5_qpc_timer_seg { + u32 rsvd[8]; +}; + +typedef struct roce5_qpc_drv_seg { + union { + struct { + u32 dest_qp : 24; + u32 state : 4; + u32 rsvd : 4; + } bs; + u32 value; + } dw0; + + u32 dw1; + + union { + struct { + u32 pmtu : 3; + u32 rsvd : 13; + u32 to_retry_limit : 3; + u32 rnr_retry_limit : 3; + u32 min_rnr_nak : 5; + u32 ack_to : 5; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 rsvd : 26; + u32 rra_max : 3; + u32 sra_max : 3; + } bs; + u32 value; + } dw3; +} roce5_qpc_drv_seg_s; + +typedef struct roce5_drv_path_info { + u32 rsvd[2]; + + u8 dgid[16]; + + union { + struct { + u32 flow_label : 20; + u32 tclass : 8; + u32 rsvd : 4; + } bs; + u32 value; + } dw6; + + union { + struct { + u32 hoplmt : 8; + u32 sgid_index : 7; + u32 rsvd : 14; + u32 sl : 3; + } bs; + u32 value; + } dw7; +} roce5_drv_path_info_s; + +typedef struct roce5_ucode_common_ctx { + union { + struct { + u32 rsvd1 : 20; + u32 port : 2; + u32 rsvd : 10; + } bs; + u32 value; + } dw0; + + u32 dw1; + + union { + u32 qkey; + u32 value; + } dw2; + + u32 dw3; +} roce5_ucode_common_ctx_s; + +typedef struct roce5_ucode_sq_ctx { + union { + struct { + u32 next_send_psn : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw8; + + u32 rsvd[12]; +} roce5_ucode_sq_ctx_s; + +typedef struct roce5_ucode_rq_ctx { + union { + struct { + u32 next_rcv_psn : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw20; + + u32 rsvd[9]; +} roce5_ucode_rq_ctx_s; + +typedef struct roce5_drv_ucode_all_info { + /* DW0~DW3 */ + roce5_ucode_common_ctx_s common; + + /* DW4~DW16 */ + roce5_ucode_sq_ctx_s sq_ctx; + + /* DW17~DW26 */ + roce5_ucode_rq_ctx_s rq_ctx; + + u32 rsvd; +} roce5_drv_ucode_all_info_s; + +typedef struct roce5_qpc_sw_seg { + roce5_qpc_drv_seg_s drv_seg; + roce5_drv_path_info_s path_seg; + roce5_drv_ucode_all_info_s ucode_seg; + u32 rsvd[8]; +} roce5_qpc_sw_seg_s; + +typedef struct roce5_qp_context { + struct roce5_qpc_chip_seg chip_seg; + struct roce5_qpc_timer_seg timer_seg; + struct roce5_qpc_sw_seg sw_seg; +} roce5_qp_context_s; + +#pragma pack(pop) + +#endif /* ROCE_QP_CTX_FOMRAT_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c new file mode 100644 index 000000000..71a77ada0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c @@ -0,0 +1,342 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_qp_ext.h" +#include "roce_qp_extension.h" +#include "roce_restore.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif +#include "hinic5_hmm.h" +#include "securec.h" +#include "roce_os_compat.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_qp_inner.h" + +static void roce5_clear_lb_qpn(struct roce5_device *rdev, u32 qpn) +{ + int idx = 0; + mutex_lock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + for (; idx < ROCE_LB_QPN_MAX; idx++) { + if (rdev->qp_lb_scqn_ctrl.qp[idx] == qpn) { + rdev->qp_lb_scqn_ctrl.qp[idx] = 0; + ROCE_DEV_INFO( + rdev, + "RoCE destroy lb qpn func_id(%d), qpn(%d), idx(%d)", + rdev->glb_func_id, qpn, idx); + break; + } + } + mutex_unlock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); +} + +static void roce5_clean_qp_user(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_udata *udata) +{ + struct roce5_ucontext *ucontext = NULL; + + roce5_free_opt_rdmarc(rqp); + + cqm5_object_delete(&(rqp->qpc_info->object)); + + roce5_clear_lb_qpn(rdev, rqp->qpn); + + roce5_destroy_pcqc_ext(rqp); + + ucontext = roce5_queue_to_ucontext((void *)(&rqp->ibqp), udata, + ROCE_Q_TYPE_QP); + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + roce5_db_unmap_user(ucontext, &rqp->db, rqp->shadow); + } + + roce5_destroy_qp_user_umem_mtt(rdev, rqp); +} + +/* **************************************************************************** + Prototype : roce5_cq_clean_process + Description : roce5_cq_clean_process + Input : struct roce5_cq *cq + u32 qpn + struct roce5_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_clean_process(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq) +{ + u32 prod_index = 0; + int nfreed = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe *dest = NULL; + u8 owner_bit = 0; + int ret; + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->cons_index; roce5_get_sw_cqe(cq, prod_index); + ++prod_index) { + if (prod_index == (u32)((int)cq->cons_index + cq->ibcq.cqe)) { + break; + } + } + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + /* xcqe:qpn's cqe. + CI + +--------+------+------+------+----------+ + b4_clean: | | cqe1 | xcqe2| cqe3 | | + +--------+------+------+------+----------+ + \ + \ + +---------------+------+------+----------+ + af_clean: | | cqe1 | cqe3 | | + +---------------+------+------+----------+ + */ + while ((int)--prod_index - (int)cq->cons_index >= 0) { + cqe = (struct roce_cqe *)roce5_get_cqe( + cq, prod_index & (u32)cq->cqe_mask); + + cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value); + cqe->dw7.value = roce5_convert_cpu32(cqe->dw7.value); + + if (cqe->dw0.bs.qpn == qpn) { + if (srq && (cqe->dw1.bs.s_r == ROCE_CQE_RECV_COMP)) { + roce5_free_srq_wqe(srq, cqe->dw7.bs.wqe_cnt); + } + + ++nfreed; + } else if (nfreed != 0) { + dest = (struct roce_cqe *)roce5_get_cqe( + cq, (unsigned int)((int)prod_index + nfreed) & + (unsigned int)cq->cqe_mask); + dest->dw0.value = roce5_convert_cpu32(dest->dw0.value); + owner_bit = dest->dw0.bs.owner; + ret = memcpy_s((void *)dest, sizeof(struct roce_cqe), + (void *)cqe, sizeof(struct roce_cqe)); + if (ret != 0) { + ROCE_ERR("[Failed to memcpy dest.(ret:%d)", + ret); + return; + } + dest->dw0.bs.owner = owner_bit; + dest->dw0.value = roce5_convert_be32(dest->dw0.value); + dest->dw1.value = roce5_convert_be32(dest->dw1.value); + dest->dw7.value = roce5_convert_be32(dest->dw7.value); + } else { + ROCE_INFO("Nothing need to do"); + } + + cqe->dw0.value = roce5_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_be32(cqe->dw1.value); + cqe->dw7.value = roce5_convert_be32(cqe->dw7.value); + } + + if (nfreed != 0) { + cq->cons_index += (u32)nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + + roce5_cq_set_ci(cq); + } +} + +/* **************************************************************************** + Prototype : roce5_clean_qp_kernel + Description : roce5_clean_qp_kernel + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Created function + +**************************************************************************** */ +static void roce5_clean_qp_kernel(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + unsigned long flags = 0; /* type required by kernel API */ + + roce5_free_opt_rdmarc(rqp); + + roce5_get_cqs(rqp, &send_cq, &recv_cq); + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce5_lock_cqs(send_cq, recv_cq); + /* del from lists under both locks above to protect reset flow paths */ + list_del(&rqp->qps_list); + list_del(&rqp->cq_send_list); + list_del(&rqp->cq_recv_list); + + roce5_cq_clean_process(recv_cq, rqp->qpn, + rqp->ibqp.srq ? to_roce5_srq(rqp->ibqp.srq) : + NULL); + + if (send_cq != recv_cq) { + roce5_cq_clean_process(send_cq, rqp->qpn, NULL); + } + + roce5_unlock_cqs(send_cq, recv_cq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); + + cqm5_object_delete(&(rqp->qpc_info->object)); + + roce5_clear_lb_qpn(rdev, rqp->qpn); + + if (rqp->sq.wrid) { + kvfree(rqp->sq.wrid); + rqp->sq.wrid = NULL; + } + + if (rqp->rq.wrid) { + kvfree(rqp->rq.wrid); + rqp->rq.wrid = NULL; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + cqm5_object_delete(&(rqp->qp_buf_info->object)); + } + + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +} + +static int roce5_qp_destroy(struct roce5_qp *rqp, struct roce5_device *rdev) +{ + int ret = 0; + + ret = roce5_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) to RESET, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce5_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "QP(0x%06x) cache invalid, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce5_destroy_qp_common(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_pd *pd = roce5_get_pd(rqp); + roce_user_handle_t hdl = { 0 }; + + roce5_destroy_qp_ext_deinit(rqp); + + if (rqp->wb_va != NULL) { + dma_free_coherent(rdev->dev, sizeof(u32), rqp->wb_va, + rqp->wb_dma); + rqp->wb_va = NULL; + } + + if (rqp->qp_state != IB_QPS_RESET) { + ret = roce5_qp_destroy(rqp, rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to destroy qp, ret(%d).", + ret); + return ret; + } + + rqp->qp_state = IB_QPS_RESET; + } + + if (pd->ibpd.uobject) { + roce5_clean_qp_user(rdev, rqp, udata); + } else { + roce5_clean_qp_kernel(rdev, rqp); + + if (roce5_sqp_check(rqp) != 0 && rqp->sqp_attr != NULL) { + kfree(rqp->sqp_attr); + rqp->sqp_attr = NULL; + } + } + + ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, NULL, NULL, NULL, NULL); + ret = roce5_uld_qp_destroy((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to destroy uld qp, ret(%d).", ret); + return ret; + } + + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.del_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + return 0; +} + +int roce5_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + int ret; + + if (ibqp == NULL) { + ROCE_ERR("Ibqp is null"); + return -EINVAL; + } + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + ret = roce5_destroy_qp_common(rdev, rqp, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Error occur when destroy qp! ret(%d).", + ret); + return ret; + } + +#if !defined(IB_DEVICE_OPS_HAVE_IB_QP_OBJ) + kfree(rqp); +#endif + return 0; +} + +int roce5_destroy_qp_ofa(struct ib_qp *ibqp) +{ + return roce5_destroy_qp(ibqp, NULL); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h new file mode 100644 index 000000000..646717c2a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h @@ -0,0 +1,80 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_EXP_H +#define ROCE_QP_EXP_H + +#include <linux/io-mapping.h> +#include <linux/list.h> + +#include <rdma/ib_verbs.h> + +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_db.h" +#include "roce_cq.h" +#include "roce_qp.h" + +#define EXP_OK (0) +#define EXP_PARAM_ERR (1) +#define EXP_FUNC_ERR (2) + +#define QPC_MIN_SIZE (512) +#define CONTX_SZ (sizeof(struct roce_qp_context) / QPC_MIN_SIZE) + +#define BUFFER_SZ (sizeof(struct roce_qp_context) / sizeof(u32)) + +#define TYPE_LEN (8) +#define DEV_NUM (32) +#define QPN_MAX (0xFFFFF) +#define QPN_MIN (0) +#define IB_QP_EXP_CMD (1 << 23) +#define ROCE_OPTPAR_EXP (1 << 16) +#define VA_OFFSET (8) + +struct roce5_device_list { + struct ib_device *ib_dev[DEV_NUM]; + int ib_dev_num; + struct mutex mutex; +}; + +struct ib_qp_info { + dma_addr_t sq_pg_base_addr[64]; + dma_addr_t hw_doorbell_addr; + dma_addr_t sw_doorbell_addr; + + u32 sq_depth; + u32 qpn; + u8 sq_pg_cnt; + u8 qp_cos; + u8 cntx_sz; + u8 sq_wqe_size; +}; + +struct ib_qp_attr_data { + u32 data_type; //plog:1 + u32 data_len; + u8 *data_buf; +}; + +struct ib_qp_attr_exp { + struct ib_qp_attr qp_attr; + struct ib_qp_attr_data qp_attr_data; +}; + +struct roce5_device_list *roce5_get_plog_device_info(void); + +int ib_get_qp_info(char *dev_name, int qpn, struct ib_qp_info *qp_info); +struct ib_qp *ib_get_qp(const char *dev_name, int qpn); +int ib_put_qp(struct ib_qp *ibqp); +int roce5_qp_modify_exp(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state, + u16 vlan_id); +int roce5_modify_extend_qp(struct ib_qp_attr *attr, int attr_mask, + struct ib_qp *ibqp); + +#endif //ROCE_QP_EXP_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c new file mode 100644 index 000000000..a548667ee --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c @@ -0,0 +1,39 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "roce_qp.h" +#include "roce_infra.h" +#include "roce_obj_ext.h" +#ifdef ENABLE_HYPER_ROCE +#include "hyper_roce_qp.h" +#endif + +const struct roce5_ext_reg g_rqp_exts[] = { +#ifdef ENABLE_HYPER_ROCE + { + .ext_id = ROCE_EXT_HYPER_ROCE, + .data_size = sizeof(struct roce5_qp_hyper_extend), + .init_func = roce5_hyper_qp_init, + .cleanup_func = roce5_hyper_qp_cleanup, + }, +#endif +}; + +const u8 g_rqp_exts_num = sizeof(g_rqp_exts) / sizeof(struct roce5_ext_reg); + +int roce5_create_qp_ext_init(struct roce5_qp *rqp) +{ + int ret = roce5_obj_ext_init((void *)rqp, g_rqp_exts, g_rqp_exts_num, + &rqp->obj_ext); + if (ret != 0) { + ROCE_ERR("Failed to init qp extension, ret(%d).", ret); + return ret; + } + return 0; +} + +void roce5_destroy_qp_ext_deinit(struct roce5_qp *rqp) +{ + roce5_obj_ext_deinit((void *)rqp, g_rqp_exts, g_rqp_exts_num, + &rqp->obj_ext); +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h new file mode 100644 index 000000000..8b3e95565 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_QP_EXT_H +#define ROCE_QP_EXT_H + +#include "roce_qp.h" +#include "roce_obj_ext.h" + +static inline void *roce5_qp_get_ext_buf(struct roce5_qp *rqp, + enum ROCE_EXT_ID ext_id) +{ + return roce5_obj_ext_get_buf(&rqp->obj_ext, ext_id); +} + +int roce5_create_qp_ext_init(struct roce5_qp *rqp); +void roce5_destroy_qp_ext_deinit(struct roce5_qp *rqp); + +#endif /* ROCE_QP_EXT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h new file mode 100644 index 000000000..c757ce2ba --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef ROCE_QP_INNER_H +#define ROCE_QP_INNER_H + +#include <rdma/ib_verbs.h> +#include "roce.h" +#include "roce_qp.h" + +int roce5_destroy_qp_common(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_udata *udata); +void roce5_destroy_qp_user_umem_mtt(struct roce5_device *rdev, + struct roce5_qp *rqp); + +#endif /* ROCE_QP_INNER_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c new file mode 100644 index 000000000..5a6a4ec7b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c @@ -0,0 +1,2398 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> +#include <linux/sched.h> + +#include "comm_defs.h" +#include "roce_compat.h" +#include "hinic5_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_qp_exp.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "securec.h" +#include "roce_ctx_format.h" +#include "roce_npu_cmd.h" +#include "roce_npu_cmd_qp_defs.h" +#include "roce_npu_cmd_ext_defs.h" +#include "roce_cmd.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" +#include "roce_netdev.h" +#include "roce_os_compat.h" + +#ifdef ROCE_ANTI_ATTACK +#include "roce_anti_attack.h" +#endif + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#define ROCE_MTT_GPA_VALID 0x1 +#define ROCE_WQE_PREFETCH_MAXNUM 5 //5: 8(count word) WQEBB +#define ROCE_SQ_WQEBB_SIZE_SHIFT 4 +#define ROCE_SQ_WQEBB_SIZE_MASK 0x7 +#define ROCE_SQ_PD_MASK 0x3ffff +#define ROCE_SQ_PAGE_SIZE_MASK 0xf +#define ROCE_SQ_WQECNT_LTH 4 +#define ROCE_SQ_PREFETCH_MINNUM 2 +#define ROCE_SQ_PREFETCH_MAXNUM 4 +#define ROCE_SQ_SIZE_MASK 0x1f +#define ROCE_SQ_CQN_MASK 0xfffff +#define ROCE_RQ_WQECNT_LTH 0xe +#define ROCE_RQ_MTT_PREFETCH_MAXLEN0 3 +#define ROCE_RQ_MTT_PREFETCH_MAXWQE 7 +#define ROCE_RQ_WQE_PREFETCH_MAXNUM 7 +#define ROCE_SRQ_INIT_STATE 0xf +#define ROCE_MAX_PI_ON_CHIP_QPN 1024 + +enum { + BIT_LENGTH_IB_MTU_256 = 8, + BIT_LENGTH_IB_MTU_512, + BIT_LENGTH_IB_MTU_1024, + BIT_LENGTH_IB_MTU_2048, + BIT_LENGTH_IB_MTU_4096, + BIT_LENGTH_IB_MTU_8192, +}; + +enum { + CACHE_INVLAID_ALLOC, + CACHE_INVLAID_CMDQ, + CACHE_INVLAID_RDMARC_CHECK, + CACHE_INVLAID_QP_CHECK, + MODIFY_2RST_CMDQ, + CMDQ_SDKCALL_BUTT +}; + +#define ROCE_CMDQ_TIMEOUT 1000 +#define ROCE_CMDQ_TIMEOUT_10MS 10 + +#ifdef ROCE_SIGN_EN +enum mtt_check_type_e { MTT_CHECK_TYPE_0 = 0, MTT_CHECK_TYPE_1 }; + +#define ROCE_QP_QPCC_SIGN_WIDTH 5 +#define ROCE_QP_QPCC_SIGN_SPLITNUM 4 +#define ROCE_QP_QPCC_SIGN_CHECKBITS \ + (ROCE_QP_QPCC_SIGN_WIDTH * ROCE_QP_QPCC_SIGN_SPLITNUM) + +#define ROCE_QP_SUB_SIGN_WIDTH 3 +#define ROCE_QP_SUB_SIGN_SPLITNUM 5 +#define ROCE_QP_SUB_SIGN_CHECKBITS \ + (ROCE_QP_SUB_SIGN_WIDTH * ROCE_QP_SUB_SIGN_SPLITNUM) + +#define ROCE_QP_GPA_SIGN_WIDTH 3 +#define ROCE_QP_GPA_SIGN_SPLITNUM 11 +#define ROCE_QP_GPA_SIGN_CHECKBITS \ + (ROCE_QP_GPA_SIGN_WIDTH * ROCE_QP_GPA_SIGN_SPLITNUM) + +#define ROCE_SQ_WQE_SIGN_WIDTH 8 +#define ROCE_SQ_WQE_SIGN_SPLITNUM 9 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_1 32 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_2 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_12 \ + (ROCE_SQ_WQE_SIGN_CHECKBITS_2 + ROCE_SQ_WQE_SIGN_CHECKBITS_1) +#define ROCE_SQ_WQE_SIGN_CHECKBITS_3 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_123 \ + (ROCE_SQ_WQE_SIGN_CHECKBITS_3 + ROCE_SQ_WQE_SIGN_CHECKBITS_12) +#define ROCE_SQ_WQE_SIGN_CHECKBITS \ + (ROCE_SQ_WQE_SIGN_WIDTH * ROCE_SQ_WQE_SIGN_SPLITNUM) + +#define ROCE_MTT_SIGN_WIDTH 11 +#define ROCE_MTT_SIGN_SPLITNUM 5 +#define ROCE_MTT_SIGN_CHECKBITS_1 22 +#define ROCE_MTT_SIGN_CHECKBITS_2 33 +#define ROCE_MTT_SIGN_CHECKBITS_12 \ + (ROCE_MTT_SIGN_CHECKBITS_1 + ROCE_MTT_SIGN_CHECKBITS_2) +#define ROCE_MTT_SIGN_CHECKBITS (ROCE_MTT_SIGN_WIDTH * ROCE_MTT_SIGN_SPLITNUM) + +#define ROCE_MTT_BLOCK_SIGN_WIDTH 5 +#define ROCE_MTT_BLOCK_SIGN_SPLITNUM 11 + +/* **************************************************************************** + Prototype : roce5_calculate_sign_bit + Description : roce5_calculate_signature + Input : u32 *check_bit + u32 sign_width + u32 sign_split + u32 *sign_bit + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static void roce5_calculate_sign_bit(u32 *check_bit, u32 sign_width, + u32 sign_split, u32 *sign_bit) +{ + u32 i = 0; + u32 j = 0; + + for (i = 0; i < sign_width; i++) { + sign_bit[i] = 0; + for (j = 0; j < sign_split; j++) { + sign_bit[i] = sign_bit[i] ^ + check_bit[i + sign_width * j]; + } + + sign_bit[i] = (~sign_bit[i]) & 0x1; + } +} + +/* **************************************************************************** + Prototype : roce5_calculate_signature + Description : roce5_calculate_signature + Input : u32 *sign_bit + u32 sign_width + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_calculate_signature(u32 *sign_bit, u32 sign_width) +{ + u32 i = 0; + u32 signature = 0; + + for (i = 0; i < sign_width; i++) { + signature |= sign_bit[i] << i; + } + + return (signature & ((1U << sign_width) - 1)); +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_rcc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_rcc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_sqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_sqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_sqac_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_sqac_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_rqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_rqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_rrwc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_rrwc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_sq_wqe_sign + Description : Signature 8bit + CheckedBits = {QPn[19:0], WQECI[19:0], WQE.Header[31:0]} + Input : u32 qpn + u32 wqe_ci + u32 wqe_header + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_sq_wqe_sign(u32 qpn, u32 wqe_ci, u32 wqe_header) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_SQ_WQE_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_SQ_WQE_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_SQ_WQE_SIGN_CHECKBITS_1; i++) { + if (wqe_header & (1U << i)) { + check_bit[i] = 1; + } + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_1; + i < ROCE_SQ_WQE_SIGN_CHECKBITS_12; i++) { + if (wqe_ci & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_1))) { + check_bit[i] = 1; + } + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_12; + i < ROCE_SQ_WQE_SIGN_CHECKBITS_123; i++) { + if (qpn & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_12))) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_SQ_WQE_SIGN_WIDTH, + ROCE_SQ_WQE_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_SQ_WQE_SIGN_WIDTH); + + return signature; +} +#endif + +#define ROCE_BASE_GID_IDX \ + 1 /* current IP mode supports precise push GID, reserved this field compatible old version firmware */ +static void roce5_qp_set_path(const struct roce5_qp *rqp, + struct rdma_ah_attr *ah_attr, + roce_verbs_qpc_attr_path_s *path_info) +{ + u8 *dmac = ah_attr->roce.dmac; + int ret; + + path_info->dw7.bs.hoplmt = (u8)ah_attr->grh.hop_limit; + path_info->dw6.bs.tclass = + (u8)(ah_attr->grh.traffic_class | IP_ECN_PERMIT); + path_info->dw6.bs.flow_label = ah_attr->grh.flow_label & + 0xfffff; /* flow_label: 20bit */ + path_info->dw7.bs.sgid_index = ah_attr->grh.sgid_index & + 0x7f; /* sgid_index: 7bit */ + + path_info->dw7.bs.base_sgid_n = + (path_info->dw7.bs.sgid_index != ROCE_BASE_GID_IDX); + + ret = memcpy_s((void *)path_info->dgid, sizeof(path_info->dgid), + (void *)ah_attr->grh.dgid.raw, sizeof(path_info->dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + } + + ret = memcpy_s((void *)&path_info->dmac_l32, + sizeof(path_info->dmac_l32), + (void *)&dmac[ROCE_RAH_DMAC_L32_START], + sizeof(path_info->dmac_l32)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac_l32.(ret:%d)", ret); + } + path_info->dmac_l32 = cpu_to_be32(path_info->dmac_l32); + path_info->dw0.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | + dmac[1]; //shift dmac[0] left by 8 bits + path_info->dw7.bs.sl = (ah_attr->sl & 0x7); /* sl: 3bit */ + path_info->dw0.bs.udp_src_port_h = + ((rqp->qpn >> ROCE_UDP_SRC_PORT_H8_SHIFT) & 0xff); + path_info->dw7.bs.udp_src_port = (rqp->qpn & 0xff); +#ifdef ROCE_BONDING_EN + path_info->dw0.bs.bond_tx_hash_value = (rqp->tx_hash_value & 0xf); +#endif +} + +struct roce5_pd *roce5_get_pd(struct roce5_qp *rqp) +{ + if (rqp->qp_type == IB_QPT_XRC_TGT) { + return to_roce5_pd(to_roce5_xrcd(rqp->ibqp.xrcd)->pd); + } else { + return to_roce5_pd(rqp->ibqp.pd); + } +} + +void roce5_get_cqs(struct roce5_qp *rqp, struct roce5_cq **send_cq, + struct roce5_cq **recv_cq) +{ + switch (rqp->qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = to_roce5_cq(to_roce5_xrcd(rqp->ibqp.xrcd)->cq); + *recv_cq = *send_cq; + break; + + case IB_QPT_XRC_INI: + *send_cq = to_roce5_cq(rqp->ibqp.send_cq); + *recv_cq = *send_cq; + break; + + default: + *send_cq = to_roce5_cq(rqp->ibqp.send_cq); + *recv_cq = to_roce5_cq(rqp->ibqp.recv_cq); + break; + } +} + +/* **************************************************************************** + Prototype : roce5_set_access_flags + Description : roce5_set_access_flags + Input : struct roce5_qp *rqp + roce_verbs_qp_attr_s *qp_attr + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + 2.Date : 2016/2/26 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_access_flags(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr, + struct ib_qp_attr *attr, int attr_mask) +{ + u16 dest_rd_atomic = 0; + u32 access_flags = 0; + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + dest_rd_atomic = attr->max_dest_rd_atomic; + } else { + dest_rd_atomic = rqp->rsp_depth; + } + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) { + access_flags = (u32)attr->qp_access_flags; + } else { + access_flags = rqp->atomic_rd_en; + } + + if (dest_rd_atomic == 0) { + access_flags &= IB_ACCESS_REMOTE_WRITE; + } + + if ((access_flags & IB_ACCESS_REMOTE_READ) != 0) { + qp_attr->com_info.dw0.bs.rre = 1; /* rc_rre: 1bits */ + } + + if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) != 0) { + qp_attr->com_info.dw0.bs.rae = 1; /* rc_rae: 1bits */ + } + + if ((access_flags & IB_ACCESS_REMOTE_WRITE) != 0) { + qp_attr->com_info.dw0.bs.rwe = 1; /* rrw_rwe: 1bits */ + } +} + +bool roce5_sqp_check(const struct roce5_qp *rqp) +{ + if (((rqp->qp_type == IB_QPT_GSI) && + (rqp->qpn == ROCE_QP_SMI_QP_NUM)) || + (rqp->qpn == ROCE_QP_GSI_QP_NUM)) { + return true; + } + + return false; +} + +/* **************************************************************************** + Prototype : store_sqp_attrs + Description : save the special + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + int attr_mask + Output : int status + + 1.Date : 2015/7/24 + Modification : Created function + +**************************************************************************** */ +static int store_sqp_attrs(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + int attr_mask) +{ + if (rqp->sqp_attr == NULL) { + ROCE_ERR("Error! sqp attribute is NULL, qpn(%u).", rqp->qpn); + return -EINVAL; + } + + if (((u32)attr_mask & IB_QP_PKEY_INDEX) != 0) { + rqp->sqp_attr->pkey_index = attr->pkey_index; + } + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) { + rqp->sqp_attr->qkey = attr->qkey; + } + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) { + rqp->sqp_attr->send_psn = attr->sq_psn; + } + + return 0; +} + +/* **************************************************************************** + Prototype : qpc_seg_to_be + Input : struct roce_qp_context *be_ctx + struct roce_qp_context *le_ctx + struct roce5_qp *rqp + Output : None + + 1.Date : 2015/7/29 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +static void qpc_seg_to_be(roce_verbs_qp_attr_s *be_ctx, + roce_verbs_qp_attr_s *le_ctx, struct roce5_qp *rqp) +{ + /* DRV Seg */ + int ret; + + ret = memcpy_s((void *)be_ctx->path_info.dgid, + sizeof(le_ctx->path_info.dgid), + (void *)le_ctx->path_info.dgid, + sizeof(le_ctx->path_info.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + } + + /* CHIP Seg */ + be_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa = + cpu_to_be64(le_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa); + + be_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = cpu_to_be64( + le_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num); +} + +/* **************************************************************************** + Prototype : roce5_qpc_to_be + Description : translate big endian + Input : roce_verbs_qp_attr_s *qp_attr + struct roce5_qp *rqp + u32 *be_ctx + Output : None + + 1.Date : 2015/7/29 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +void roce5_qpc_to_be(roce_verbs_qp_attr_s *qp_attr, struct roce5_qp *rqp, + u32 *be_ctx) +{ + u32 *ctx = NULL; + u32 *ctx1 = NULL; + u32 i = 0; + u32 ctx_size = 0; + + ctx = be_ctx; + ctx1 = (u32 *)qp_attr; + ctx_size = sizeof(roce_verbs_qp_attr_s) / sizeof(u32); + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) { + *ctx = cpu_to_be32(*ctx1); + } + + qpc_seg_to_be((roce_verbs_qp_attr_s *)((void *)be_ctx), qp_attr, rqp); +} + +/* **************************************************************************** + Prototype : roce5_alloc_opt_rdmarc + Description : alloc rdmarc when modify qp + Input : struct roce5_qp *rqp + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static int roce5_alloc_opt_rdmarc(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + u32 response_depth = 0; + + rdev = to_roce5_dev(rqp->ibqp.device); + response_depth = (attr->max_dest_rd_atomic == 0) ? + 1 : + (u32)ROCE_ROUNDUP_POW_OF_TWO( + (u32)attr->max_dest_rd_atomic); + if (rqp->rsp_depth != response_depth) { + if (rqp->rsp_depth > 0) { + roce5_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + } + + ret = roce5_rdma_rdmarc_alloc(rdev->hwdev, response_depth, + &rqp->rdmarc); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc rdma rdmarc, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + rqp->rsp_depth = (u16)response_depth; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_free_opt_rdmarc + Description : free rdmarc when modify qp failed or destory qp + Input : struct roce5_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +void roce5_free_opt_rdmarc(struct roce5_qp *rqp) +{ + struct roce5_device *rdev = NULL; + + rdev = to_roce5_dev(rqp->ibqp.device); + + if (rqp->rsp_depth > 0) { + roce5_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + } + + rqp->rsp_depth = 0; +} + +/* **************************************************************************** + Prototype : roce5_set_opt_rdmarc + Description : set opt rdmarc field + Input : struct roce5_device *rdev + struct roce5_qp *rqp + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** */ +void roce5_set_opt_rdmarc(struct rdma_rdmarc *rdmarc, u16 depth, + roce_verbs_qp_attr_s *qp_attr) +{ + qp_attr->chip_seg.dw10.bs.rc_page_gpa_h = + (rdmarc->dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & + 0xffffff; /* high:24bit, 64-24= 40 */ + qp_attr->chip_seg.rc_page_gpa_l = + (u32)(rdmarc->dma_addr >> + ROCE5_RDMARC_GPA_L_SHIFT); //bit[39:8] + qp_attr->chip_seg.dw4.bs.rc_size = rdmarc->order; + qp_attr->com_info.dw1.bs.rra_max = + ((u8)fls(depth - 1)) & + ROCE5_RDMARC_RRA_MAX_MASK; /* rra_max:3bit */ + qp_attr->chip_seg.dw4.bs.rc_max_size = + rdmarc->ext_order - ROCE5_RDMARC_EXT_ORDER_SHIFT; +} + +static int roce5_set_opt_mtu(struct roce5_device *rdev, + roce_verbs_qp_attr_s *qp_attr, + const struct ib_qp_attr *attr) +{ + enum roce5_mtu rmtu = (enum roce5_mtu)attr->path_mtu; + if (rmtu == ROCE_MTU_8192 && ROCE5_SUPPORT_8K_MTU(rdev) == 0) { + ROCE_DEV_ERR(rdev, "Not support 8K MTU."); + return -EINVAL; + } + + qp_attr->com_info.dw3.bs.pmtu = (u32)rmtu & 0x7; /* 1,2,3,4,5 */ + qp_attr->com_info.dw5.bs.mtu_code = + ROCE_QPC_PMTU_TRANSLATE(rmtu); /* 0,1,3,7,15 */ + if (qp_attr->com_info.dw3.bs.pmtu == IB_MTU_1024) { + qp_attr->com_info.dw3.bs.base_mtu_n = 0; + } else { + qp_attr->com_info.dw3.bs.base_mtu_n = 1; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_set_opt_sra + Description : set opt sra + Input : roce_verbs_qp_attr_s *qp_attr + struct ib_qp_attr *attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_opt_sra(roce_verbs_qp_attr_s *qp_attr, + struct ib_qp_attr *attr) +{ + u8 initiator_depth; + + initiator_depth = + (attr->max_rd_atomic == 0) ? + 1 : + (u8)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_rd_atomic); + qp_attr->com_info.dw1.bs.sra_max = + ((u32)ROCE_FLS(initiator_depth - 1)) & 0x7; +} + +static void roce5_set_opt_attr_av(struct roce5_qp *rqp, + struct roce5_device *rdev, + struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, u32 *optpar) +{ + u8 cos; + roce5_get_cos_inbuf_s inbuf = { 0 }; + + roce5_qp_set_path(rqp, &attr->ah_attr, &qp_attr->path_info); + qp_attr->com_info.dw3.bs.ext_mtu = rqp->ext_mtu & 0x1; + qp_attr->com_info.dw3.bs.ext_md = rqp->ext_mtu_mode & 0x1; + *optpar |= ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH; + + inbuf.sl = attr->ah_attr.sl; + inbuf.sgid_index = attr->ah_attr.grh.sgid_index; + inbuf.port_num = attr->ah_attr.port_num; + inbuf.traffic_class = attr->ah_attr.grh.traffic_class; + + (void)roce5_get_dcb_cfg_cos(rdev, &inbuf, &cos, rqp->qp_type); + + qp_attr->com_info.dw5.bs.cos = cos; + + rqp->db_sgid_index = attr->ah_attr.grh.sgid_index & 0x7f; + rqp->db_cos = cos; +} + +static void roce5_set_opt_field_attr(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + int attr_mask, + roce_verbs_qp_attr_s *qp_attr, u32 *optpar) +{ + /* RTR2RTS/SQD2SQD RETRY_CNT */ + if ((((u32)attr_mask) & IB_QP_RETRY_CNT) != 0) { + qp_attr->com_info.dw1.bs.to_retry_limit = + attr->retry_cnt & 0x7; /* to_retry_limit:3 */ + *optpar |= ROCE_QP_OPTPAR_RETRY_COUNT; + } + + /* RTR2RTS/SQD2SQD RNR_RETRY */ + if ((((u32)attr_mask) & IB_QP_RNR_RETRY) != 0) { + qp_attr->com_info.dw1.bs.rnr_retry_limit = + attr->rnr_retry & 0x7; /* rnr_retry_limit:3bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_RETRY; + } + + /* INIT2RTR/SQD2SQD/RTR2RTS/RTS2RTS MIN_RNR_TIMER */ + if ((((u32)attr_mask) & IB_QP_MIN_RNR_TIMER) != 0) { + qp_attr->com_info.dw2.bs.min_rnr_nak = + attr->min_rnr_timer & 0x1f; /* min_rnr_nak:5bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) { + qp_attr->com_info.dw8.bs.next_send_psn = + attr->sq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_RQ_PSN) != 0) { + qp_attr->com_info.dw9.bs.next_rcv_psn = + attr->rq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_DEST_QPN) != 0) { + qp_attr->com_info.dw0.bs.dest_qp = attr->dest_qp_num & + 0xffffff; /* dest_qp:24bit */ + } + + roce5_set_attr_ext(rqp, attr, qp_attr, attr_mask); +} + +static s32 roce5_set_db_path_mtu(const struct ib_qp_attr *attr, + struct roce5_qp *rqp, + struct roce5_device *rdev) +{ + switch ((enum roce5_mtu)attr->path_mtu) { + case ROCE_MTU_256: + case ROCE_MTU_512: + ROCE_DEV_ERR( + rdev, + "Only support mtu larger than 1K.func_id(%d) mtu:%u", + rdev->glb_func_id, attr->path_mtu); + return -EINVAL; + case ROCE_MTU_1024: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_1024; + break; + case ROCE_MTU_2048: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_2048; + break; + case ROCE_MTU_4096: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_4096; + break; + case ROCE_MTU_8192: + if (ROCE5_SUPPORT_8K_MTU(rdev) != 0) { + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_8192; + break; + } + + ROCE_DEV_ERR(rdev, "Not support 8K MTU."); + return -EINVAL; + default: + return -EINVAL; + } + + return 0; +} + +static int roce5_set_opt_field(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, + roce_modify_qp_param_s *modify_para) +{ + int ret = 0; + roce_verbs_qp_attr_s *qp_attr = &modify_para->qp_attr; + u32 *optpar = &modify_para->optpar; + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) { + qp_attr->com_info.q_key = attr->qkey; + *optpar |= ROCE_QP_OPTPAR_Q_KEY; + } + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + roce5_set_opt_attr_av(rqp, rdev, attr, qp_attr, optpar); + } + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) { + qp_attr->com_info.dw5.bs.port = attr->port_num & + 0x7; /* port: 3bits */ + *optpar |= ROCE_QP_OPTPAR_SCHED_QUEUE; + } + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + roce5_set_opt_rdmarc(&rqp->rdmarc, rqp->rsp_depth, qp_attr); + *optpar |= ROCE_QP_OPTPAR_RRA_MAX; + } + + /* RTR2RTS/SQD2SQD SRA_MAX */ + if ((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) { + roce5_set_opt_sra(qp_attr, attr); + *optpar |= ROCE_QP_OPTPAR_SRA_MAX; + } + + /* RWE/RRE/RAE,RST2INIT */ + if ((((u32)attr_mask) & + (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) != 0) { + roce5_set_access_flags(rqp, qp_attr, attr, attr_mask); + *optpar |= ROCE_QP_OPTPAR_RWE | ROCE_QP_OPTPAR_RRE | + ROCE_QP_OPTPAR_RAE; + } + + /* RTR2RTS/SQD2SQD ACK TIMEOUT */ + if ((((u32)attr_mask) & IB_QP_TIMEOUT) != 0) { + qp_attr->com_info.dw2.bs.ack_to = attr->timeout & + 0x1f; /* ack_to:5bits */ + *optpar |= ROCE_QP_OPTPAR_ACK_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) { + ret = roce5_set_db_path_mtu(attr, rqp, rdev); + if (ret != 0) { + return ret; + } + ret = roce5_set_opt_mtu(rdev, qp_attr, attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set MTU."); + return ret; + } + } + + roce5_set_opt_field_attr(rqp, attr, attr_mask, qp_attr, optpar); + return ret; +} + +static void roce5_set_qp_srq_attr(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr, + struct roce5_pd *pd, + const struct roce5_cq *recv_cq) +{ + struct roce5_srq *rsrq = to_roce5_srq(rqp->ibqp.srq); + struct roce5_srq_context temp; + + if (rsrq->container_flag != 0) { + struct roce5_srq_context *srqc = + (struct roce5_srq_context *)((void *)rsrq->cqm_srq + ->q_ctx_vaddr); + temp.dw0.value = be32_to_cpu(srqc->dw0.value); + qp_attr->chip_seg.dw12.bs.srq_pd = temp.dw0.bs.pdn & + 0x3ffff; /* srq_pd:18bits */ + qp_attr->chip_seg.dw12.bs.srq_wqebb_size = + temp.dw0.bs.wqebb_size; + qp_attr->chip_seg.dw12.bs.srq_page_size = temp.dw0.bs.page_size; + qp_attr->chip_seg.dw12.bs.srq_size = temp.dw0.bs.size; + + temp.dw1.value = be32_to_cpu(srqc->dw1.value); + qp_attr->com_info.dw5.bs.so_ro = temp.dw1.bs_c.so_ro; + qp_attr->com_info.dw5.bs.dma_attr_idx = + temp.dw1.bs_c.dma_attr_idx; + + qp_attr->chip_seg.dw9.bs.container_sz = + MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode; + qp_attr->chip_seg.dw9.bs.container_en = 1; + + qp_attr->com_info.dw3.bs.srq_container = 1; + + if (rsrq->rqe_cnt_th > 0) { + qp_attr->chip_seg.dw9.bs.srq_warth_flag = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = + rsrq->rqe_cnt_th & + 0xf; /* srq_rqecnt_th:4bits */ + } + + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & + 0x3ffff; /* srqn:18bits */ + } else { + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & + 0x3ffff; /* srqn:18bits */ + } + + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw0.bs.rkey_en = 1; + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & + 0xfffff; /* rq_cqn:20bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; +} + +static void roce5_set_qp_rst2init_attr(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr, + struct roce5_pd *pd, + struct roce5_cq *recv_cq) +{ + qp_attr->chip_seg.dw4.bs.rq_base_ci = + (u32)(ROCE_ILOG2(rqp->rq.offset >> rqp->rq.wqe_shift)); + qp_attr->chip_seg.dw19.bs.qp_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + qp_attr->com_info.dw3.bs.dsgl_en = rqp->double_sgl_mode & 0x1; + + if (rqp->ibqp.srq != NULL) { + roce5_set_qp_srq_attr(rqp, qp_attr, pd, recv_cq); + } else { + qp_attr->chip_seg.dw15.bs.rq_size = + (rqp->rq.wqebb_cnt > 0) ? + (((u32)ROCE_ILOG2(rqp->rq.wqebb_cnt)) & 0x1f) : + 0; + qp_attr->chip_seg.dw15.bs.rq_page_size = + ((u32)((u32)rqp->buf.page_shift - PAGE_SHIFT_4K)) & + ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw15.bs.rq_pd = pd->pdn & + 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw8.bs.rq_wqebb_size = + (u32)(rqp->rq.wqe_shift - BYTES_TO_16B_SHIFT) & 0x7; + qp_attr->com_info.dw7.bs.rq_cqn = recv_cq->cqn & + 0xfffff; /* rq_cqn:20bits */ + qp_attr->chip_seg.dw15.bs.rq_rkey_en = 1; + + qp_attr->chip_seg.dw8.bs.rq_pi_on_chip = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_lth = ROCE_RQ_WQECNT_LTH; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen0 = + ROCE_RQ_MTT_PREFETCH_MAXLEN0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen1 = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxwqe = + ROCE_RQ_MTT_PREFETCH_MAXWQE; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_minnum = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_maxnum = + ROCE_RQ_WQE_PREFETCH_MAXNUM; + } +} + +static void roce5_qp_rst2init_set_qpcc(roce_verbs_qp_attr_s *qp_attr, + struct roce5_qp *rqp, + const struct roce5_device *rdev) +{ + qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = rqp->mtt.mtt_paddr; + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->db.dma & (~0x3ULL); /* bits:63-2 */ + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num |= + rqp->mtt.mtt_layers & 0x3; /* bits:1-0 */ + if (rqp->qp_type == IB_QPT_XRC_TGT) { + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->wb_dma; + } + + qp_attr->chip_seg.dw4.bs.sq_rq_mtt_page_size = + rqp->mtt.mtt_page_shift - PAGE_SHIFT_4K; + /* rc_entry_size:2bits */ + qp_attr->chip_seg.dw10.bs.rc_entry_size = + ((u32)ROCE_ILOG2(rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .rdmarc_entry_sz >> + BYTES_TO_16B_SHIFT) & + 0x3); + qp_attr->chip_seg.dw14.bs.qp_rkey_en = 1; + qp_attr->chip_seg.dw10.bs.rc_entry_prefetch_maxnum = 0; +} + +static void roce5_qp_rst2init_set_sqc_sqac(roce_verbs_qp_attr_s *qp_attr, + const struct roce5_qp *rqp, + const struct roce5_pd *pd, + const struct roce5_cq *send_cq) +{ + qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = rqp->pi_on_chip; + qp_attr->chip_seg.dw16.bs.sq_wqebb_size = + ((u32)(rqp->sq.wqe_shift - ROCE_SQ_WQEBB_SIZE_SHIFT)) & + ROCE_SQ_WQEBB_SIZE_MASK; + qp_attr->chip_seg.dw16.bs.sq_pd = ((u32)pd->pdn) & + 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw16.bs.sq_page_size = + ((u32)(rqp->buf.page_shift - PAGE_SHIFT_4K)) & + ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw5.bs.sq_inline_en = (rqp->max_inline_data > 0) ? 1 : + 0; + + qp_attr->chip_seg.dw16.bs.sq_rkey_en = 1; + + qp_attr->chip_seg.dw5.bs.sq_wqecnt_lth = ROCE_SQ_WQECNT_LTH; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_minnum = + ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_maxnum = + ROCE_SQ_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw6.bs.sq_wqe_prefetch_mode = 0; + qp_attr->chip_seg.dw6.bs.sq_mtt_prefetch_maxlen = 0; + + if (rqp->sq.wqebb_cnt != 0) { + qp_attr->chip_seg.dw5.bs.sq_size = + (u32)(ROCE_ILOG2((u32)rqp->sq.wqebb_cnt)) & + ROCE_SQ_SIZE_MASK; + } + + qp_attr->chip_seg.dw17.bs.sqa_cqn = send_cq->cqn & ROCE_SQ_CQN_MASK; + + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_minnum = + ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_maxnum = + ROCE_WQE_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw6.bs.sqa_mtt_prefetch_maxlen = 0; + qp_attr->chip_seg.dw5.bs.sqa_wqe_cache_thd_sel = 0; +} + +static void roce5_qp_rst2init_set_drv(roce_verbs_qp_attr_s *qp_attr, + struct roce5_qp *rqp, + struct roce5_device *rdev) +{ + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce5_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7; + qp_attr->com_info.dw3.bs.dif_en = (u32)rqp->signature_en; + + if (!rqp->ibqp.uobject) { + qp_attr->com_info.dw0.bs.fre = 1; + } + + qp_attr->com_info.dw3.bs.xrc_vld = + (u32)(rqp->qp_type == IB_QPT_XRC_INI); + qp_attr->com_info.dw1.bs.local_qp = rqp->qpn & + 0xffffff; /* local_qp:24bits */ + qp_attr->com_info.dw5.bs.ep = rdev->hw_info.ep_id; + qp_attr->chip_seg.dw4.bs.dsgl = rqp->double_sgl_mode & 0x1; +} + +static void roce5_qp_rst2init_set_common(roce_verbs_qp_attr_s *qp_attr, + const struct roce5_pd *pd, + const struct roce5_qp *rqp, + const struct roce5_cq *recv_cq, + const struct roce5_cq *send_cq) +{ + qp_attr->chip_seg.dw18.bs.ud_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + + if (rqp->shadow != 0) { + qp_attr->com_info.dw11.bs.shadow = 1; + if (recv_cq != NULL) { + qp_attr->com_info.dw11.bs.shadow_vfid = + recv_cq->shadow_vfid; + } + + if (send_cq != NULL) { + qp_attr->com_info.dw11.bs.shadow_cq_vfid = + send_cq->shadow_vfid; + } + } +} + +static void roce5_qp_rst2init_set_xrc_attr(const struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, + const struct roce5_device *rdev, + const struct roce5_cq *recv_cq) +{ + qp_attr->com_info.dw2.bs.xrc_srq_en = 1; + + if ((((u32)attr->qp_access_flags & XRC_CONTAINER_FLAG) != 0) || + (rdev->cfg_info.srq_container_en != 0)) { + qp_attr->chip_seg.dw9.bs.container_en = 1; + qp_attr->com_info.dw3.bs.srq_container = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & + 0xffff; /* xrcd:16bits */ + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw13.bs.srq_state = ROCE_SRQ_INIT_STATE; + } else { + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & + 0xffff; /* xrcd:16bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw14.bs.srq_xrcd = rqp->xrcdn & + 0xffff; /* xrcd:16bits */ + } + if (recv_cq != NULL) { + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & + 0xfffff; /* rq_cqn:20bits */ + } +} + +/* **************************************************************************** + Prototype : roce5_qp_rst2init + Description : parameter set when QP is from RESET to INIT + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2015/8/6 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +void roce5_qp_rst2init(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ + struct roce5_pd *pd = NULL; + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + struct roce5_device *rdev = NULL; + + rdev = to_roce5_dev(rqp->ibqp.device); + pd = roce5_get_pd(rqp); + roce5_get_cqs(rqp, &send_cq, &recv_cq); + + /* CHIP_SEG: QPCC */ + roce5_qp_rst2init_set_qpcc(qp_attr, rqp, rdev); + + /* CHIP_SEG: SQC & SQAC */ + roce5_qp_rst2init_set_sqc_sqac(qp_attr, rqp, pd, send_cq); + + /* RRWC */ + qp_attr->chip_seg.dw4.bs.rrw_mtt_prefetch_maxlen = 0; + + /* RCC */ + qp_attr->chip_seg.dw10.bs.rc_mtt_prefetch_maxlen = + ROCE_QP_MAX_PFETCH_MTT_LAYER; + + /* TIMER_SEG */ + qp_attr->com_info.dw3.bs.tss_timer_num = ROCE_QP_MAX_TIMER_NUM; + + /* DRV_SEG */ + roce5_qp_rst2init_set_drv(qp_attr, rqp, rdev); + + /* UCODE COM_SEG */ + roce5_qp_rst2init_set_common(qp_attr, pd, rqp, recv_cq, send_cq); + + if (rqp->qp_type == IB_QPT_XRC_TGT) { + roce5_qp_rst2init_set_xrc_attr(rqp, attr, qp_attr, rdev, + recv_cq); + } else { + roce5_set_qp_rst2init_attr(rqp, qp_attr, pd, recv_cq); + } + +#ifdef ROCE_BONDING_EN + roce5_bond_rr_set_flow(rdev, rqp, qp_attr); +#endif +} + +/* **************************************************************************** + Prototype : roce5_qp_init2rtr + Description : init2rtr attributes modify + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2015/8/1 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +static void roce5_qp_init2rtr(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ + if (rqp->ibqp.srq || (rqp->qp_type == IB_QPT_XRC_TGT) || + (rqp->qp_type == IB_QPT_XRC_INI) || (rqp->qp_type == IB_QPT_UD) || + (rqp->qp_type == IB_QPT_UC) || (rqp->qp_type == IB_QPT_GSI)) { + qp_attr->com_info.dw3.bs.invalid_credit = 1; + } + + if ((rqp->qp_type == IB_QPT_GSI) || (rqp->qp_type == IB_QPT_UD)) { + qp_attr->com_info.dw3.bs.pmtu = IB_MTU_4096 & + 0x7; /* for UD, mtu = 4096 */ + qp_attr->com_info.dw5.bs.mtu_code = ROCE_MTU_CODE_4K; + } + + qp_attr->com_info.dw2.bs.ack_to = 0xf; /* ack_to:5bits */ + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce5_qp_type((enum ib_qp_type)rqp->qp_type) & + 0x7; /* service_type: 3bit */ +} + +/* **************************************************************************** + Prototype : roce5_qp_rtr2rts + Description : roce5_qp_rtr2rts + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2015/8/2 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +static void roce5_qp_rtr2rts(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ +} + +static void roce5_qp_2err(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ +} + +/* **************************************************************************** + Prototype : roce5_set_abs_field + Description : set abs field to qpc + Input : struct roce5_qp *rqp + struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + enum ib_qp_state cur_state + enum ib_qp_state new_state + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_abs_field(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, roce_verbs_qp_attr_s *qp_attr) +{ + enum ib_qp_state cur_state, new_state; + + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) { + roce5_qp_rst2init(rqp, attr, qp_attr); + } + + if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR)) { + roce5_qp_init2rtr(rqp, attr, qp_attr); + } + + if ((cur_state == IB_QPS_RTR) && (new_state == IB_QPS_RTS)) { + roce5_qp_rtr2rts(rqp, attr, qp_attr); + } + + if (new_state == IB_QPS_ERR) { + roce5_qp_2err(rqp, attr, qp_attr); + } + return; +} + +#ifdef ROCE_LATENCY_PROF_EN +#define ROCE_SET_LATENCY_PROF_EN 5 +static int roce5_latency_prof_en_check(struct roce5_device *rdev, + u32 *latency_prof_en, + u32 *frequency_reduction_ratio) +{ + roce_dfx_set_inbuf_s buf_in = { 0 }; + roce_dfx_set_outbuf_s buf_out = { 0 }; + u32 out_size = sizeof(buf_out); + int ret = 0; + buf_in.show = 1; + buf_in.cmd_type = ROCE_CMD_ROCE_SET; + buf_in.sub_cmd_type = ROCE_SET_LATENCY_PROF_EN; + ret = roce5_adm_dfx_roce_set(rdev, &buf_in, sizeof(buf_in), &buf_out, + &out_size); + if (ret != 0) { + ROCE_ERR("latency prof query l2d err"); + return ret; + } + *latency_prof_en = buf_out.sub_param.latency_prof.latency_prof_en; + *frequency_reduction_ratio = + buf_out.sub_param.latency_prof.frequency_reduction_ratio; + return 0; +} + +static int +roce5_latency_prof_time_get(struct roce5_device *rdev, + struct roce5_modify_qp_ex_resp_data *resp) +{ + struct hinic5_hwdev *hwdev = (struct hinic5_hwdev *)rdev->hwdev; + u32 latency_prof_en = 0; + u32 frequency_reduction_ratio = 1; + u64 time_diff = 0; + int ret = 0; + if (hwdev == NULL) { + ROCE_ERR("latency prof hwdev is null"); + return (-EINVAL); + } + ret = roce5_latency_prof_en_check(rdev, &latency_prof_en, + &frequency_reduction_ratio); + if (ret != 0) { + return ret; + } + if (latency_prof_en == 0) { + resp->latency_prof_en = 0; + resp->non_ptp_time_diff = 0; + return 0; + } + //get clock time and non_ptp_chip time of difference value, convenient at later calculate timestamp + ret = hinic5_get_non_ptp_time_diff(hwdev, &time_diff); + if (ret != 0) { + ROCE_ERR("Failed to read n_ptp time, ret(%d)\n", ret); + return ret; + } + resp->latency_prof_en = 1; + resp->non_ptp_time_diff = time_diff; + resp->frequency_reduction_ratio = frequency_reduction_ratio; + return 0; +} +#endif +static int roce5_record_opt_field(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata, + roce_verbs_cmd_qp_modify_rsp_s *modify_rsp) +{ + int ret = 0; + int rsp_vld = 0; + enum ib_qp_state cur_state, new_state; + struct roce5_modify_qp_ex_resp_data resp = { 0 }; + struct roce5_device *rdev = NULL; + + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + rqp->qp_state = new_state; + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + rqp->sl = attr->ah_attr.sl & 0x7; + } + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) { + rqp->atomic_rd_en = (u8)attr->qp_access_flags; + } + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) { + rqp->port = attr->port_num; + } + + if (roce5_sqp_check(rqp) != 0) { + ret = store_sqp_attrs(rqp, attr, attr_mask); + if (ret != 0) { + ROCE_ERR("Failed to store sqp attribute."); + return ret; + } + } + + rdev = to_roce5_dev(rqp->ibqp.device); + + /* + * TGT QP no has update pi requirement, and db trigger microcode thread and driver no has strict timing disable system, one general in init status trigger microcode, + * but if XRC TGT QP migration 至 rts after again trigger microcode thread, SQ hardware none permission, fetch wqe will exception, because this TGT QP not next db + * but non- TGT QP i.e. make in qp rst status trigger, will go via wqe empty 终结, not will exception + */ + if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT) && + (rqp->pi_on_chip != 0) && (rqp->qp_type != IB_QPT_XRC_TGT)) { + /* avoid PI on chip when PI 残keep, primary dynamic clear to zero */ + ret = roce5_write_sq_db(rdev, rqp, 0, udata); + if (ret != 0) { + ROCE_ERR("Failed to clear PI on chip"); + return -1; + } + } + + if ((udata != NULL) && (udata->outbuf != NULL)) { + if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR) && + (((u32)attr_mask & IB_QP_AV) != 0)) { + resp.cos = rqp->db_cos; + resp.is_cos_valid = 1; + rsp_vld = 1; + } + +#ifdef ROCE_LATENCY_PROF_EN + if (ROCE_IS_DEV_1825V100(rdev) && (cur_state == IB_QPS_RTR) && + (new_state == IB_QPS_RTS)) { + ret = roce5_latency_prof_time_get(rdev, &resp); + if (ret != 0) { + ROCE_ERR("Failed to get time diff, ret(%d).", + ret); + return -1; + } + } +#endif + /* compatible old card not will return ssn value, driver not perform update */ + if (modify_rsp != NULL && + modify_rsp->magic == ROCE_VERBS_MODIFY_RSP_MAGIC) { + resp.is_ssn_valid = modify_rsp->dw1.bs.is_ssn_vld; + resp.ssn = modify_rsp->dw1.bs.ssn; + rqp->sq_wqe_ssn = resp.ssn; + rsp_vld = 1; + } + + if (rsp_vld != 0) { + ret = ib_copy_to_udata( + udata, &resp, + sizeof(struct roce5_modify_qp_ex_resp_data)); + if (ret != 0) { + ROCE_ERR("Failed to copy data to user"); + return -1; + } + } + } + return 0; +} + +/* **************************************************************************** + Prototype : roce5_qp_modify_2rst_cmd + Description : cache out qpc mtt + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2016/6/17 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** */ +int roce5_qp_modify_2rst_cmd(struct roce5_device *rdev, u32 qpn) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_qp_modify2rst_s *qp_moidfy2rst_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_modify2rst_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + qp_moidfy2rst_inbuf = + (roce_uni_cmd_qp_modify2rst_s *)cqm_cmd_inbuf->buf; + qp_moidfy2rst_inbuf->com.index = cpu_to_be32(qpn); + qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_2RST_QP, cqm_cmd_inbuf, + NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send 2RST_QP command, qpn(0x%x), func_id(%d), ret(0x%x)", + qpn, rdev->glb_func_id, ret); + + ret = (-EINVAL); + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + goto err_out; + } + +err_out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_qp_check_rdmarc + Description : roce5_qp_check_rdmarc + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Create function + +**************************************************************************** */ +static int roce5_qp_check_rdmarc(struct roce5_device *rdev, + struct roce5_qp *rqp, int *cost) +{ + int times = 0; + u32 rc_num = 0; + u32 rc_check_index = 0; + void *rdmarc_vaddr = NULL; + dma_addr_t rdmarc_paddr = 0; + + rc_num = + (u32)(((u32)1 << rqp->rdmarc.order) / RDMARC_NUM_PER_CACHELINE); + rdmarc_vaddr = rqp->rdmarc.vaddr; + rdmarc_paddr = rqp->rdmarc.dma_addr; + + times = rdev->try_times; + do { + if ((*((u32 *)rdmarc_vaddr) == + ROCE_RDMARC_DESTROY_CHECK_VALUE) && + (cpu_to_be32(*((u32 *)rdmarc_vaddr + 1)) == rqp->qpn) && + (cpu_to_be64(*((u64 *)rdmarc_vaddr + 1)) == + ((u64)rdmarc_paddr + + ((u64)ROCE_CACHE_LINE_SIZE * (u64)rc_check_index)))) { + rdmarc_vaddr = (void *)((u8 *)rdmarc_vaddr + + ROCE_CACHE_LINE_SIZE); + rc_check_index++; + if (rc_num == rc_check_index) { + break; + } + } else { + ROCE_DELAY(); + times--; + } + } while (times != 0); + + if (rc_num != rc_check_index) { + ROCE_DEV_ERR( + rdev, + "Failed to read rdmarc field back after try %d times, func_id(%d) rc_check_index:%u rc_num:%u", + ((rdev->try_times - times) + 1), rdev->glb_func_id, + rc_check_index, rc_num); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static int roce5_qp_check_state(struct roce5_device *rdev, + const struct roce5_qp *rqp, const u32 *wb_va, + int *cost) +{ + int times = 0; + int read_back_flag = 0; + + times = rdev->try_times; + while ((times--) != 0) { + if (*(u32 *)wb_va == ROCE_QP_DESTROY_CHECK_VALUE_HIGH_4B && + (*((u32 *)wb_va + 1) == + ROCE_QP_DESTROY_CHECK_VALUE_LOW_4B)) { + read_back_flag = 1; + break; + } + ROCE_DELAY(); + } + + if (read_back_flag == 0) { + ROCE_DEV_ERR( + rdev, + "Failed to read QP field back after try %d times,qpn(0x%x), wb_data(0x%x), wb_data+1(0x%x) func_id(%d)", + ((rdev->try_times - times) + 1), rqp->qpn, + *(u32 *)wb_va, *((u32 *)wb_va + 1), rdev->glb_func_id); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static void roce5_qp_fill_cacheout_inbuf(struct roce5_device *rdev, + struct roce5_qp *rqp, + dma_addr_t wb_dma, + cqm_cmd_buf_s *cqm_cmd_inbuf) +{ + roce_uni_cmd_qp_cache_invalid_s *qp_cacheout_inbuf = NULL; + struct rdma_service_cap *rdma_cap = NULL; + u32 page_size = roce5_get_page_size(rdev, true); + + qp_cacheout_inbuf = + (roce_uni_cmd_qp_cache_invalid_s *)(cqm_cmd_inbuf)->buf; + rdma_cap = &rdev->rdma_cap; + + qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + qp_cacheout_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_cacheout_inbuf->qp_cache.sq_buf_len = cpu_to_be32( + ALIGN(((u32)rqp->sq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), + page_size)); + qp_cacheout_inbuf->qp_cache.rq_buf_len = cpu_to_be32( + ALIGN(((u32)rqp->rq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), + page_size)); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_flags = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_num = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + qp_cacheout_inbuf->qp_cache.wb_gpa = cpu_to_be64(wb_dma); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_flags = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_num = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); +} + +static int roce5_qp_send_cache_out_cmd(struct roce5_device *rdev, + struct roce5_qp *rqp, void *wb_va, + dma_addr_t wb_dma, int *cost) +{ + unsigned long end; + int times = 0; + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_cache_invalid_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce5_qp_fill_cacheout_inbuf(rdev, rqp, wb_dma, cqm_cmd_inbuf); + + end = jiffies + + (unsigned long)((unsigned int)rdev->destory_flow_timeout * HZ); + while (time_before(jiffies, end)) { + times++; + ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, + ROCE_CMD_MISC_CACHE_INVLD, + cqm_cmd_inbuf, NULL, + ROCE_CMD_TIME_CLASS_C); + if (ret <= 0) { /* return success or exception happened */ + break; + } + cond_resched(); + }; + + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed MISC_CACHE_INVLD command after try %d times, ret(%d), func_id(%d) qpn(%d)", + times, ret, rdev->glb_func_id, rqp->qpn); + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = times; + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR( + rdev, + "Card not present, return ERROR, func_id(%d)", + rdev->glb_func_id); + ret = -ENODEV; + } + return ret; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = (rdev->try_times - times) + 1; + return 0; +} + +/* **************************************************************************** + Prototype : roce5_qp_cache_out_cmd + Description : cache out qpc mtt + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2016/9/24 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** */ +/* roce5_dealloc_latch_resource() collect use construct temporary change amount rqp to reuse this function. such as newly added for rqp intomember of use, need 排 query whether need adaptation */ +int roce5_qp_cache_out_cmd(struct roce5_device *rdev, struct roce5_qp *rqp) +{ + int ret = 0; + int cost = 0; + void *wb_va = NULL; + dma_addr_t wb_dma = 0; + + wb_va = dma_alloc_coherent(rdev->dev, (size_t)8UL, &wb_dma, GFP_KERNEL); + if (wb_va == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc wb_va, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + ret = roce5_qp_send_cache_out_cmd(rdev, rqp, wb_va, wb_dma, &cost); + if (ret != 0) { + goto err; + } + + if (rqp->rsp_depth > 0) { + ret = roce5_qp_check_rdmarc(rdev, rqp, &cost); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to check qp rdmarc, func_id(%d)", + rdev->glb_func_id); + goto err; + } + } + + ret = roce5_qp_check_state(rdev, rqp, (u32 *)wb_va, &cost); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check qp state, func_id(%d)", + rdev->glb_func_id); + goto err; + } + +err: + dma_free_coherent(rdev->dev, sizeof(u32), wb_va, wb_dma); + + /* At the end of the check, if the failure is caused by a chip link down, release resources on the host side. */ + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + + return ret; +} + +static u8 roce5_get_qp_cmdq_cmd(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask) +{ + static u16 op[ROCE_QP_STATE_NUM][ROCE_QP_STATE_NUM]; + enum ib_qp_state cur_state, new_state; + + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + if ((cur_state > IB_QPS_ERR) || (new_state > IB_QPS_ERR)) { + ROCE_DEV_ERR( + rdev, + "QP state modify invalid, cur_state(%d), new_state(%d), func_id(%d)", + cur_state, new_state, rdev->glb_func_id); + return 0; + } + op[IB_QPS_RESET][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RESET][IB_QPS_INIT] = ROCE_CMD_RST2INIT_QP; + + op[IB_QPS_INIT][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_INIT][IB_QPS_INIT] = ROCE_CMD_INIT2INIT_QP; + op[IB_QPS_INIT][IB_QPS_RTR] = ROCE_CMD_INIT2RTR_QP; + + op[IB_QPS_RTR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTR][IB_QPS_RTS] = ROCE_CMD_RTR2RTS_QP; + + op[IB_QPS_RTS][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTS][IB_QPS_RTS] = ROCE_CMD_RTS2RTS_QP; + op[IB_QPS_RTS][IB_QPS_SQD] = ROCE_CMD_RTS2SQD_QP; + + op[IB_QPS_SQD][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQD][IB_QPS_RTS] = ROCE_CMD_SQD2RTS_QP; + op[IB_QPS_SQD][IB_QPS_SQD] = ROCE_CMD_SQD2SQD_QP; + + op[IB_QPS_SQE][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQE][IB_QPS_RTS] = ROCE_CMD_SQERR2RTS_QP; + + op[IB_QPS_ERR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + + return (u8)op[cur_state][new_state]; +} + +static int roce5_qp_rts2sqd_cmd(struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + int sqd_event = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_qp_modify_rts2sqd_s *qp_rts2sqd_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_modify_rts2sqd_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + sqd_event = ((((u32)attr_mask) & IB_QP_EN_SQD_ASYNC_NOTIFY) != 0) && + (attr->en_sqd_async_notify != 0); + qp_rts2sqd_inbuf = + (roce_uni_cmd_qp_modify_rts2sqd_s *)cqm_cmd_inbuf->buf; + qp_rts2sqd_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_rts2sqd_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + qp_rts2sqd_inbuf->sqd_event_en = cpu_to_be32((u32)(sqd_event != 0)); + + ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_RTS2SQD_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RTS2SQD_QP cmd, func_id(%d), ret(0x%x)", + rdev->glb_func_id, ret); + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +static int roce5_qp_handle_err(struct roce5_device *rdev, + const struct roce5_qp *rqp, u8 cmd, int ret) +{ + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_QP command, command(%u), func_id(%d), ret(%d)", + cmd, rdev->glb_func_id, ret); + + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + return -1; +} + +static int roce5_qp_modify_cmd(struct roce5_device *rdev, struct roce5_qp *rqp, + u8 cmd, roce_modify_qp_param_s *modify_para) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_modify_qpc_s *qp_modify_inbuf = NULL; + roce_verbs_qp_attr_s *qp_attr = &modify_para->qp_attr; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_qpc_s), &cqm_cmd_outbuf, + ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return (-ENOMEM); + } + + qp_modify_inbuf = (roce_uni_cmd_modify_qpc_s *)cqm_cmd_inbuf->buf; + + roce5_transmit_ci_record_addr_ext(rqp, qp_attr); + roce5_qpc_to_be(qp_attr, rqp, (u32 *)&qp_modify_inbuf->qp_attr); + qp_modify_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + qp_modify_inbuf->com.opt = cpu_to_be32(modify_para->optpar); + ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, cmd, cqm_cmd_inbuf, + cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ret = roce5_qp_handle_err(rdev, rqp, cmd, ret); + } else { + roce5_be32_2_le32( + cqm_cmd_outbuf->buf, (u32 *)&modify_para->modify_rsp, + sizeof(roce_verbs_cmd_qp_modify_rsp_s) / sizeof(u32)); + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +void roce5_cq_clean(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq) +{ + spin_lock_irq(&cq->lock); + roce5_cq_clean_process(cq, qpn, srq); + spin_unlock_irq(&cq->lock); +} + +static int roce5_qp_clean_kernel_res(struct roce5_qp *rqp) +{ + int ret = 0; + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + + roce5_get_cqs(rqp, &send_cq, &recv_cq); + + roce5_cq_clean(recv_cq, rqp->qpn, + rqp->ibqp.srq ? to_roce5_srq(rqp->ibqp.srq) : NULL); + if (send_cq != recv_cq) { + roce5_cq_clean(send_cq, rqp->qpn, NULL); + } + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + rqp->sq_next_wqe = 0; + rqp->sq_wqe_ssn = 0; + + *(rqp->db.db_record) = 0; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + if (rqp->has_rq && rqp->qp_buf_info->q_header_vaddr) { + ret = memset_s(rqp->qp_buf_info->q_header_vaddr, + sizeof(u64), 0, sizeof(u64)); + if (ret != 0) { + ROCE_ERR( + "Failed to memset QP(0x%06x) q_header_vaddr(ret:%d)", + rqp->qpn, ret); + return (-ENOMEM); + } + } + } + + return ret; +} + +static int roce5_qp_modify_2rst(struct roce5_device *rdev, struct roce5_qp *rqp) +{ + int ret = 0; + + ret = roce5_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) to RESET, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce5_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "QP(0x%06x) cache invalid failed, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + rqp->qp_state = IB_QPS_RESET; + + if (rqp->ibqp.uobject == NULL) { + ret = roce5_qp_clean_kernel_res(rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Clean kernel QP(0x%06x) failed, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static void roce5_set_stub_field(struct roce5_device *rdev, + roce_verbs_qp_attr_s *qp_attr) +{ + qp_attr->com_info.dw6.bs.force_local = rdev->local_en; +} + +static int roce5_modify_qp_cmd_process(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + roce_modify_qp_param_s *modify_para) +{ + u8 cmd; + int ret; + cmd = roce5_get_qp_cmdq_cmd(rdev, rqp, attr, attr_mask); + if (cmd == 0) { + ROCE_DEV_ERR(rdev, + "Invalid cmd, func_id(%u), qpn(%u), cmd(%u).", + rdev->glb_func_id, rqp->qpn, cmd); + return -EINVAL; + } + if (cmd == ROCE_CMD_RTS2SQD_QP) { + ret = roce5_qp_rts2sqd_cmd(rdev, rqp, attr, attr_mask); + } else { + ret = roce5_qp_modify_cmd(rdev, rqp, cmd, modify_para); + } + return ret; +} + +static int roce5_qp_modify(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + int ret = 0; + roce_user_handle_t hdl; + roce_modify_qp_param_s modify_para; + roce_verbs_qp_attr_s *qp_attr = &modify_para.qp_attr; + + (void)memset_s(&modify_para, sizeof(roce_modify_qp_param_s), 0, + sizeof(roce_modify_qp_param_s)); + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + ret = roce5_alloc_opt_rdmarc(rqp, attr, attr_mask); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdmarc, func(%d).", + rdev->glb_func_id); + return ret; + } + } + + ret = roce5_set_opt_field(rqp, attr, attr_mask, &modify_para); + if (ret != 0) { + goto err_out; + } + + roce5_set_abs_field(rqp, attr, attr_mask, qp_attr); + roce5_set_stub_field(rdev, qp_attr); + + (void)memset_s(&hdl, sizeof(roce_user_handle_t), 0, + sizeof(roce_user_handle_t)); + ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, qp_attr, NULL, NULL, NULL); + ret = roce5_uld_qp_modify((roce_uld_handle)&hdl, attr, attr_mask, + udata); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify uld qp, func(%d), qpn(%d), ret(%d).", + rdev->glb_func_id, rqp->qpn, ret); + goto err_out; + } + + ret = roce5_modify_qp_cmd_process(rdev, rqp, attr, attr_mask, + &modify_para); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to modify qp, func(%d), qpn(%d), ret(%d).", + rdev->glb_func_id, rqp->qpn, ret); + goto err_out; + } + + ret = roce5_record_opt_field(rqp, attr, attr_mask, udata, + &modify_para.modify_rsp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to record_opt_field, func(%d), qpn(%d).", + rdev->glb_func_id, rqp->qpn); + goto err_out; + } + + return 0; + +err_out: + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + roce5_free_opt_rdmarc(rqp); + } + + return ret; +} + +static int roce5_qp_get_path(struct roce5_device *rdev, struct roce5_qp *rqp, + struct rdma_ah_attr *ah_attr) +{ + int ret = 0; + u16 vlan_id; + + if ((ah_attr->ah_flags & IB_AH_GRH) == 0) { + ROCE_DEV_ERR(rdev, "Not supported, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + ret = roce5_resolve_grh(rdev, ah_attr, &vlan_id, NULL); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to find dmac by grh, func_id(%u)", + (u32)rdev->glb_func_id); + return ret; + } + + return 0; +} + +static int +roce5_qp_modify_check_rd_and_mtu(const struct roce5_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask, + const struct rdma_service_cap *rdma_cap) +{ + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + enum ib_qp_state cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + enum ib_qp_state new_state = (enum ib_qp_state)( + ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state); + enum roce5_mtu rmtu = (enum roce5_mtu)attr->path_mtu; + + if (((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) && + (attr->max_rd_atomic > + rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma)) { + ROCE_ERR( + "Qpn(0x%x), max_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)", + rqp->ibqp.qp_num, attr->max_rd_atomic, cur_state, + new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) && + (attr->max_dest_rd_atomic > + rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma)) { + ROCE_ERR( + "Qpn(0x%x), max_dest_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)", + rqp->ibqp.qp_num, attr->max_dest_rd_atomic, cur_state, + new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) && + ((rmtu < ROCE_MTU_256) || + (rmtu > ((ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? ROCE_MTU_8192 : + ROCE_MTU_4096)))) { + ROCE_ERR("Input path MTU(%u) is invalid", rmtu); + return -EINVAL; + } + + return 0; +} + +static int roce5_qp_modify_check(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, + struct rdma_service_cap *rdma_cap) +{ + int ret; + enum ib_qp_state cur_state, new_state; + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ? + attr->qp_state : + cur_state); + +#ifndef ROCE_VBS_EN + if (!roce5_check_qp_modify_ok( + cur_state, new_state, (enum ib_qp_type)(rqp->ibqp.qp_type), + (enum ib_qp_attr_mask)attr_mask, IB_LINK_LAYER_ETHERNET)) { + ROCE_ERR( + "Qpn(0x%x), invalid attribute mask specified for transition %d to %d, qp_type(%d), " + "attr_mask(0x%x)", + rqp->ibqp.qp_num, cur_state, new_state, + rqp->ibqp.qp_type, attr_mask); + return (-EINVAL); + } +#endif + + if (((((u32)attr_mask) & IB_QP_PORT) != 0) && + (attr->port_num != ROCE_DEFAULT_PORT_NUM)) { + ROCE_ERR( + "Qpn(0x%x), invalid port number(%d) specified for transition %d to %d. qp_type(%d)", + rqp->ibqp.qp_num, attr->port_num, cur_state, new_state, + rqp->ibqp.qp_type); + return (-EINVAL); + } + + if ((((u32)attr_mask) & IB_QP_PKEY_INDEX) != 0) { + if (attr->pkey_index > 0) { + ROCE_ERR( + "Qpn(0x%x), invalid pkey index (%d) specified for transition %d to %d. " + "qp_type(%d)", + rqp->ibqp.qp_num, attr->pkey_index, cur_state, + new_state, rqp->ibqp.qp_type); + return (-EINVAL); + } + } + + ret = roce5_qp_modify_check_rd_and_mtu(rqp, attr, attr_mask, rdma_cap); + + return ret; +} + +#define ROCE5_QP_HANDLE 0 +#define ROCE5_QP_CONTINUE 1 +static int roce5_modify_qp_pre(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + int ret; + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + enum ib_qp_state cur_state, new_state; + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + + ret = roce5_qp_modify_check(rqp, attr, attr_mask, &rdev->rdma_cap); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed qp modify check, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (new_state == IB_QPS_RESET) { + if (cur_state != new_state) { + ret = roce5_qp_modify_2rst(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify 2rst, func_id(%d)", + rdev->glb_func_id); + } + } + return ROCE5_QP_HANDLE; + } + + ret = roce5_modify_qp_pre_ext(rdev, rqp, attr, attr_mask, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to modify qp pre ext, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return ROCE5_QP_CONTINUE; +} + +int roce5_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + mutex_lock(&rqp->mutex); + + ret = roce5_modify_qp_pre(ibqp, attr, attr_mask, udata); + if ((ret == ROCE5_QP_HANDLE) || (ret < 0)) { + goto out; + } + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + ret = roce5_qp_get_path(rdev, rqp, &attr->ah_attr); + if (ret != 0) { + goto out; + } + } + + ret = roce5_qp_modify(rdev, rqp, attr, attr_mask, udata); + if (ret != 0) { + goto out; + } + +out: + mutex_unlock(&rqp->mutex); + return ret; +} + +static void roce5_verbs_shadow_set(struct roce5_device *rdev, u32 qpn, + cqm_cmd_buf_s *buf_in) +{ + roce_verbs_cmd_header_s *cmd_hdr = + (roce_verbs_cmd_header_s *)buf_in->buf; + struct roce5_qp *rqp = NULL; + + if ((qpn == ROCE_QP_SMI_QP_NUM) || (qpn == ROCE_QP_GSI_QP_NUM)) { + return; + } + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + return; + } + + if (rqp->shadow == 0 || cmd_hdr == NULL || + cmd_hdr->dw0.bs.cmd_bitmask == VERBS_CMD_TYPE_VROCE_BITMASK) { + goto shadow_out; + } + + cmd_hdr->dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK); + cmd_hdr->dw3.bs.shadow_vfid = cpu_to_be16(rqp->shadow_vfid); + +shadow_out: + roce5_rqp_put(rqp); +} + +int roce5_send_qp_lb_cmd(u32 qpn, struct roce5_device *rdev, u8 cmd, + cqm_cmd_buf_s *buf_in, cqm_cmd_buf_s *buf_out, + u32 timeout) +{ + int ret = 0; + + roce5_verbs_shadow_set(rdev, qpn, buf_in); + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, cmd, buf_in, + buf_out, NULL, timeout, HINIC5_CHANNEL_ROCE); + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c new file mode 100644 index 000000000..fa48a2671 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c @@ -0,0 +1,238 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/* **************************************************************************** + Prototype : roce5_get_recv_wqe + Description : roce5_get_recv_wqe + Input : struct roce5_qp *rqp + int n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +static void *roce5_get_recv_wqe(struct roce5_qp *rqp, u32 n) +{ + return roce5_get_wqe(rqp, rqp->rq.offset + + (n << (unsigned)rqp->rq.wqe_shift)); +} + +static int roce5_post_recv_check_qp(const struct roce5_qp *qp, + const struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr) +{ + if (ROCE_UNLIKELY((qp->qp_type == IB_QPT_XRC_INI) || + (qp->qp_type == IB_QPT_XRC_TGT))) { + ROCE_DEV_ERR( + rdev, + "Can't post WQE when TGT XRC QP, func_id(%d), qp_type(%d)", + rdev->glb_func_id, qp->qp_type); + *bad_wr = wr; + return -EINVAL; + } + + if (ROCE_UNLIKELY(qp->qp_state == IB_QPS_RESET)) { + ROCE_DEV_ERR(rdev, + "Can't post WQE when QP is RST state, func_id(%d)", + rdev->glb_func_id); + *bad_wr = wr; + return -EINVAL; + } + + return 0; +} + +static int roce5_post_recv_check_wr(struct roce5_qp *qp, + const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, u32 wr_num) +{ + if (roce5_wq_overflow(&qp->rq, wr_num, qp->ibqp.recv_cq) != 0) { + ROCE_ERR_ONCE("SQ is full head:%x, tali:%x, wr_num:%d", + (&qp->rq)->head, (&qp->rq)->tail, wr_num); + *bad_wr = wr; + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > qp->rq.max_sge)) { + *bad_wr = wr; + return -EINVAL; + } + + return 0; +} + +static int roce5_post_recv_check_length(struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, int i) +{ + u32 data_len = 0; + + /* single SGEָ signs data len should less than (2G-1)B */ + if (ROCE_UNLIKELY(wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - + 1)) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Sge data len is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)", + i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return -EINVAL; + } + + data_len += wr->sg_list[i].length; + /* all SGEָ signs data len should less than 2GB */ + if (ROCE_UNLIKELY( + data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Data_len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)", + data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} +static int roce5_post_recv_set_data_seg(const struct ib_recv_wr *wr, + struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + struct roce5_qp *qp, u32 index) +{ + int i = 0; + int ret = 0; + struct roce_wqe_data_seg *dseg = NULL; + + dseg = (struct roce_wqe_data_seg *)roce5_get_recv_wqe(qp, index); + for (i = 0; i < wr->num_sge; ++i) { + ret = roce5_post_recv_check_length(rdev, bad_wr, wr, i); + if (ret != 0) { + return ret; + } + + roce5_set_data_seg(dseg, wr->sg_list + i); + dseg++; + } + + if (wr->num_sge != 0) { + dseg--; + dseg->key = + cpu_to_be32((u32)(wr->sg_list[wr->num_sge - 1].lkey | + ROCE_WQE_NEXT_SGE_INVALID)); + } else { + dseg->key = ROCE_WQE_NEXT_SGE_INVALID; + dseg->key = cpu_to_be32(dseg->key); + dseg->dw2.bs.len = 0; + } + + return 0; +} +/* **************************************************************************** + Prototype : roce5_post_recv + Description : roce5_post_recv + Input : struct ib_qp *ibqp + struct ib_recv_wr *wr + struct ib_recv_wr **bad_wr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +int roce5_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct roce5_qp *qp = NULL; + struct roce5_device *rdev = NULL; + unsigned long flags = 0; /* type required by kernel API */ + int ret = 0; + u32 wr_num = 0; + u32 index = 0; + const struct ib_recv_wr *wr_tmp = wr; + + if (ROCE_UNLIKELY(ibqp == NULL || bad_wr == NULL)) { + ROCE_ERR("ibqp(%d) or bad_wr(%d) is NULL.", !!ibqp, !!bad_wr); + return -EINVAL; + } + + qp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + ret = roce5_post_recv_check_qp(qp, rdev, bad_wr, wr_tmp); + if (ret != 0) { + return ret; + } + + spin_lock_irqsave(&qp->rq.lock, flags); + + index = (unsigned int)qp->rq.head & + ((unsigned int)qp->rq.wqebb_cnt - 1); + + for (wr_num = 0; wr_tmp != NULL; ++wr_num, wr_tmp = wr_tmp->next) { + ret = roce5_post_recv_check_wr(qp, bad_wr, wr_tmp, wr_num); + if (ret != 0) { + goto out; + } + + ret = roce5_post_recv_set_data_seg(wr_tmp, rdev, bad_wr, qp, + index); + if (ret != 0) { + goto out; + } + + qp->rq.wrid[index] = wr_tmp->wr_id; + + index = (index + 1) & (qp->rq.wqebb_cnt - 1); + } + +out: + if (ROCE_LIKELY(wr_num != 0)) { + qp->rq.head += (u32)wr_num; + + wmb(); + + /* SQ need head 4B,RQ need tail 4B */ + *(qp->db.db_record + 1) = cpu_to_be32(qp->rq.head & 0xffff); + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return ret; +} + +int roce5_post_recv_ofa(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + return roce5_post_recv(ibqp, (const struct ib_recv_wr *)wr, + (const struct ib_recv_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c new file mode 100644 index 000000000..ad1a8043a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c @@ -0,0 +1,1205 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_post.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_kernel_compat_gen.h" + +#include "securec.h" + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#define GOTO_LOCAL 1 + +#define GOTO_OUT 2 + +#define ROCE_NEED_JUMP 1 + +#define BYTE_MASK 255 + +#define ROCE_CTRL_SEG_SL 2 +/* **************************************************************************** + Prototype : roce5_get_send_wqe + Description : roce5_get_send_wqe + Input : struct roce5_qp *rqp + u32 n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +static void *roce5_get_send_wqe(struct roce5_qp *rqp, u32 n) +{ + u32 wqebb_index = (n & (((u32)rqp->sq.wqebb_cnt) - 1)); + u32 wqebb_pos = wqebb_index << rqp->sq.wqe_shift; + return roce5_get_wqe(rqp, rqp->sq.offset + wqebb_pos); +} + +/* **************************************************************************** + Prototype : roce5_wq_overflow + Description : roce5_wq_overflow + Input : struct roce5_wq *wq + int wr_num + struct ib_cq *ibcq + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +int roce5_wq_overflow(struct roce5_wq *wq, u32 wr_num, struct ib_cq *ibcq) +{ + unsigned int cur = 0; + struct roce5_cq *cq = NULL; + + cur = wq->head - wq->tail; + if (ROCE_LIKELY((cur + wr_num) < (unsigned int)wq->max_post)) { + return 0; + } + + cq = to_roce5_cq(ibcq); + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return ((cur + (unsigned int)wr_num) >= (unsigned int)wq->max_post); +} + +/* **************************************************************************** + Prototype : roce5_ib_opcode + Description : + Input : + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/7/30 + Author : + Modification : Created function + +**************************************************************************** */ +static const u32 roce5_ib_opcode[] = { + [IB_WR_RDMA_WRITE] = ROCE_WQE_OPCODE_RDMA_WRITE, + [IB_WR_RDMA_WRITE_WITH_IMM] = ROCE_WQE_OPCODE_RDMA_WRITE_IMM, + [IB_WR_SEND] = ROCE_WQE_OPCODE_SEND, + [IB_WR_SEND_WITH_IMM] = ROCE_WQE_OPCODE_SEND_IMM, + [IB_WR_RDMA_READ] = ROCE_WQE_OPCODE_RDMA_READ, + [IB_WR_ATOMIC_CMP_AND_SWP] = ROCE_WQE_OPCODE_ATOMIC_CMP_SWP, + [IB_WR_ATOMIC_FETCH_AND_ADD] = ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD, + [IB_WR_SEND_WITH_INV] = ROCE_WQE_OPCODE_SEND_INVAL, + [IB_WR_LOCAL_INV] = ROCE_WQE_OPCODE_LOCAL_INVAL, + [IB_WR_REG_MR] = ROCE_WQE_OPCODE_FRMR, + [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP, + [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD, + [IB_WR_REG_MR_INTEGRITY] = ROCE_WQE_OPCODE_REG_SIG_MR, +#if defined(IB_WR_OPCODE_HAVE_BIND_MW) + [IB_WR_BIND_MW] = ROCE_WQE_OPCODE_BIND_MW, +#endif +}; + +static const roce_opcode_attr_s g_roce5_opcode_attr[ROCE_KERNEL_OPCODE_NUM] = { + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND = 0x0, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND_WITH_INV, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND_WITH_IMM, + { 0 }, //ROCE_WQE_OPCODE_SEND_FT_TEST + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_WRITE = 0x4, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_WRITE_WITH_IMM, + { 0 }, + { 0 }, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_READ = 0x8, + { 0 }, //ROCE_WQE_OPCODE_ATOMIC_WRITE, + { 0 }, //ROCE_WQE_OPCODE_CFLUSH, + { .remote_opcode = + 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_EXT_CMP_AND_SWP, /* support max 64B */ + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_CMP_AND_SWP = 0xc, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_FETCH_AND_ADD, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_AND_SWP, + { .remote_opcode = + 0x1 }, //ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_AND_ADD, + { 0 }, //ROCE_WQE_OPCODE_FAST_REG_PMR, /* 8'h10-Fast Register PMR is not used by userspace */ + { 0 }, //ROCE_WQE_OPCODE_LOCAL_INV = 0x11, + { 0 }, //ROCE_WQE_OPCODE_BIND_MW, + { 0 } //ROCE_WQE_OPCODE_REG_SIG_MR = 0x13, /* 8'h13-Local opreation(Extended for further local opreation) */ +}; + +static void roce5_set_send_seg(struct roce_wqe_send_tsk_seg *snd_tsk_seg, + const struct ib_send_wr *wr) +{ + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + snd_tsk_seg->immdata_invkey = wr->ex.imm_data; + } + + if (wr->opcode == IB_WR_SEND_WITH_INV) { + snd_tsk_seg->immdata_invkey = + cpu_to_be32(wr->ex.invalidate_rkey); + } +} + +static void roce5_set_atomic_seg(struct roce_wqe_atomic_tsk_seg *atomic_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_atomic_wr *atomic = atomic_wr(wr); + + atomic_tsk_seg->key = cpu_to_be32(atomic->rkey); + atomic_tsk_seg->dw2.va = cpu_to_be64(atomic->remote_addr); + + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + atomic_tsk_seg->dw4.swap_add_data = cpu_to_be64(atomic->swap); + atomic_tsk_seg->dw6.cmp_data = cpu_to_be64(atomic->compare_add); + } else if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { + atomic_tsk_seg->dw4.swap_add_data = + cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->dw6.cmp_data = 0; + } else { + /* IB_WR_MASKED_ATOMIC_FETCH_AND_ADD */ + atomic_tsk_seg->dw4.swap_add_data = + cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->dw6.cmp_data = + cpu_to_be64(atomic->compare_add_mask); + } +} + +static __always_inline void +roce5_set_rdma_seg(struct roce_wqe_rdma_tsk_seg *rdma_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_rdma_wr *rdma = rdma_wr(wr); + + rdma_tsk_seg->va = cpu_to_be64(rdma->remote_addr); + rdma_tsk_seg->rkey = cpu_to_be32(rdma->rkey); + + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { + rdma_tsk_seg->imm_data = cpu_to_be32(wr->ex.imm_data); + } +} + +static void +roce5_set_local_inv_seg(struct roce_wqe_local_inv_tsk_seg *inv_tsk_seg, + const struct ib_send_wr *wr) +{ + inv_tsk_seg->inv_key = cpu_to_be32(wr->ex.invalidate_rkey); +} + +static void +roce5_set_ud_seg_cycle1(struct roce_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ud_wr(wr); + struct roce5_ah *rah = to_roce5_ah(ud->ah); + int ret; + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg_cycle1->immdata_invkey = wr->ex.imm_data; + } + + ud_tsk_seg_cycle1->dw2.value = rah->priv_ah.dw0.value; + ud_tsk_seg_cycle1->dw3.value = rah->priv_ah.dw1.value; + ud_tsk_seg_cycle1->dw4.value = rah->priv_ah.dw2.value; + ret = memcpy_s((void *)ud_tsk_seg_cycle1->dgid, ROCE_GID_LEN, + (void *)rah->priv_ah.dgid, ROCE_GID_LEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return; + } + ud_tsk_seg_cycle1->dw9.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg_cycle1->qkey = cpu_to_be32(ud->remote_qkey); +} + +static void +roce5_set_ud_seg_cycle2(struct roce_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ud_wr(wr); + struct roce5_ah *rah = to_roce5_ah(ud->ah); + + ud_tsk_seg_cycle2->dw0.value = rah->priv_ah.dw7.value; + ud_tsk_seg_cycle2->dmac_l32 = rah->priv_ah.dmac_l32; +} + +static void roce5_set_ud_seg(struct roce_wqe_ud_tsk_seg *ud_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ud_wr(wr); + struct roce5_ah *rah = to_roce5_ah(ud->ah); + int ret; + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + } + + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + ret = memcpy_s((void *)ud_tsk_seg->dgid, sizeof(ud_tsk_seg->dgid), + (void *)rah->priv_ah.dgid, sizeof(ud_tsk_seg->dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return; + } + ud_tsk_seg->dw10.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32(ud->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; +} + +void roce5_set_data_seg(struct roce_wqe_data_seg *dseg, struct ib_sge *sge) +{ + dseg->addr = cpu_to_be64(sge->addr); + + dseg->dw2.bs.len = sge->length & 0x7fffffff; + dseg->dw2.bs.rsvd = 0; + + dseg->dw2.value = cpu_to_be32(dseg->dw2.value); + + dseg->key = cpu_to_be32(sge->lkey); +} + +/* **************************************************************************** + Prototype : roce5_set_gsi_seg + Input : struct roce5_qp *rqp + struct ib_send_wr *wr + void *wqe + Output : None + + 1.Date : 2015/8/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_set_gsi_seg(struct roce5_qp *rqp, const struct ib_send_wr *wr, + void *wqe) +{ + struct roce_wqe_ud_tsk_seg *ud_tsk_seg = + (struct roce_wqe_ud_tsk_seg *)wqe; + struct ib_ud_wr *ud_wr = (struct ib_ud_wr *)(void *)wr; + struct roce5_ah *rah = to_roce5_ah(ud_wr->ah); + int ret; + + if (rqp->sqp_attr == NULL) { + ROCE_ERR("Error! sqp attribute is NULL, qpn(%u).", rqp->qpn); + return -EINVAL; + } + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + ud_tsk_seg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + ud_tsk_seg->dw2.value = cpu_to_be32(ud_tsk_seg->dw2.value); + } + + ud_tsk_seg->common.bs.c = 1; + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + ret = memcpy_s((void *)ud_tsk_seg->dgid, sizeof(ud_tsk_seg->dgid), + (void *)rah->priv_ah.dgid, sizeof(ud_tsk_seg->dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return ret; + } + ud_tsk_seg->dw10.value = cpu_to_be32(ud_wr->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32( + ((ud_wr->remote_qkey & 0x80000000) != 0) ? rqp->sqp_attr->qkey : + ud_wr->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; + + return 0; +} + +/* **************************************************************************** + Prototype : write_invalid_wqe + Description : set the next wqe invalidate + Input : struct roce5_qp *qp + u32 index + Output : None + + 1.Date : 2015/9/8 + Author : qianyuanpan + Modification : Created function + +**************************************************************************** */ +static void write_invalid_wqe(struct roce5_qp *qp, u32 index) +{ + u32 invalid_wqe_dw0 = 0; + struct roce5_wqe_ctrl_seg *ctrl_seg = NULL; + + ctrl_seg = (struct roce5_wqe_ctrl_seg *)roce5_get_send_wqe(qp, index); + invalid_wqe_dw0 = ((index & (u32)qp->sq.wqebb_cnt) == 0) ? 0xff000000 : + 0x7f000000; + ctrl_seg->dw0.value = cpu_to_be32(invalid_wqe_dw0); +} + +#define ROCE5_PCIE_ATOMIC_EN 0x1 +static inline bool roce5_is_pcie_atomic(struct roce5_qp *rqp) +{ + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + return (rdev->cfg_info.pcie_atomic_en == ROCE5_PCIE_ATOMIC_EN); +} + +static inline bool roce5_is_extend_atomic(enum ib_wr_opcode opcode) +{ + return ((roce5_ib_opcode[opcode] == + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP) || + (roce5_ib_opcode[opcode] == + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD)); +} + +static inline int roce5_is_vaild_wr_opcode(struct roce5_qp *rqp, + enum ib_wr_opcode opcode) +{ + if (ROCE_UNLIKELY((opcode >= ARRAY_SIZE(roce5_ib_opcode)))) { + return -EINVAL; + } + + if ((roce5_ib_opcode[opcode] == 0) && (opcode != IB_WR_SEND)) { + return -EINVAL; + } + + if (roce5_is_pcie_atomic(rqp) && roce5_is_extend_atomic(opcode)) { + ROCE_ERR( + "The atomic mode is pcie atomic, but extended atomic is not supported."); + return -EINVAL; + } + + return 0; /* Vaild IB opcode */ +} + +/* **************************************************************************** + Prototype : roce5_validate_wr + Description : roce5_validate_wr + Input : struct roce5_qp *rqp + struct ib_send_wr *wr + int wr_num + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_validate_wr(struct roce5_qp *rqp, const struct ib_send_wr *wr, + u32 wr_num) +{ + if (roce5_wq_overflow(&rqp->sq, wr_num, rqp->ibqp.send_cq) != 0) { + ROCE_ERR("SQ is full"); + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > rqp->sq.max_sge)) { + ROCE_ERR( + "Sge num is invalide, wr->num_sge(%d), rqp->sq.max_sge(%d)", + wr->num_sge, rqp->sq.max_sge); + return -EINVAL; + } + + if (ROCE_UNLIKELY(roce5_is_vaild_wr_opcode(rqp, wr->opcode) != 0)) { + ROCE_ERR("wr->opcode(%d)", wr->opcode); + return -EINVAL; + } + + return 0; +} + +static void +roce5_set_rc_wqe_for_optype_send(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_send_seg((struct roce_wqe_send_tsk_seg *)param->wqe, wr); + param->data_len_addr = + &((struct roce_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_send_tsk_seg); +} + +static void +roce5_set_rc_wqe_for_optype_atomic(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_atomic_seg((struct roce_wqe_atomic_tsk_seg *)param->wqe, wr); + param->wqe = + ((u8 *)param->wqe + sizeof(struct roce_wqe_atomic_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_atomic_tsk_seg); +} + +static void +roce5_set_rc_wqe_for_optype_write(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_rdma_seg((struct roce_wqe_rdma_tsk_seg *)param->wqe, wr); + param->data_len_addr = + &((struct roce_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)(sizeof(struct roce_wqe_rdma_tsk_seg)); +} + +static void +roce5_set_rc_wqe_for_optype_local(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce5_set_local_inv_seg((struct roce_wqe_local_inv_tsk_seg *)param->wqe, + wr); + param->wqe_size += (u32)sizeof(struct roce_wqe_local_inv_tsk_seg); + param->inline_flag = 0; + param->ctrl_seg_tmp.dw0.bs.df = 0; +} + +static void roce5_set_wqe_last_len(u8 *wqe) +{ + struct roce_wqe_send_tsk_seg *snd_task = + (struct roce_wqe_send_tsk_seg *)(void *)wqe; + snd_task->dw3.value = 0; + snd_task->dw3.bs.last_ext_len = ROCE_IMM_EXT_LEN; + snd_task->dw3.value = cpu_to_be32(snd_task->dw3.value); +} + +static inline u32 roce5_log2(u32 x) +{ + u32 shift; + + for (shift = 0; (1U << shift) < x; ++shift) { + ; + } + + return shift; +} + +/* RoCE supports FRMR, for at privileged user, can via post send WR of way for at at free status of MR perform fast register. + frmr wqe contain a pbl(page buffer list) table start address, start va, length, page size etc. information. + 预 先 allocate of store space of start address (pa) need write pbl. only supports one level 寻 址, maximum supports 512(count word) PA. */ +static int set_frmr_wqe_section(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + u64 iova; + u32 fbo; + struct ib_reg_wr *frmr_reg_wr = NULL; + struct roce5_mr *roce5_kernel_frmr = NULL; + struct roce_wqe_frmr_tsk_seg *frmr_wqe_task = NULL; + + frmr_reg_wr = (struct ib_reg_wr *)reg_wr((struct ib_send_wr *)wr); + roce5_kernel_frmr = to_roce5_mr(frmr_reg_wr->mr); + + /* base section set */ + param->wqe_size += sizeof(struct roce_wqe_frmr_tsk_seg); + + frmr_wqe_task = + (struct roce_wqe_frmr_tsk_seg *)((void *)param->tsk_com_seg); + param->dseg = + (roce_wqe_data_seg_s *)(((u8 *)param->tsk_com_seg) + + sizeof(struct roce_wqe_frmr_tsk_seg)); + param->ctrl_seg_tmp.dw0.bs.tsl = + sizeof(struct roce_wqe_frmr_tsk_seg) / ROCE_TASK_SEG_ALIGN; + /* fill wqe ctrl head section:cl bit */ + param->ctrl_seg_tmp.dw1.bs.cl = 1; + + /* local invalidate associated section set */ + param->tsk_com_seg->bs.so = 1; + + iova = (u64)(void *)(uintptr_t)(frmr_reg_wr->mr->iova); + fbo = (u32)(iova & (FRMR_PAGE_SIZE - 1)); + frmr_wqe_task->dw2.bs.fbo = fbo & 0x3FFFFF; + frmr_wqe_task->dw2.value = cpu_to_be32(frmr_wqe_task->dw2.value); + + frmr_wqe_task->dw5.va = + (((u32)frmr_reg_wr->access & IB_ZERO_BASED) != 0) ? + 0 : + cpu_to_be64(iova); + + /* fill in access permission */ + frmr_wqe_task->dw3.bs.zbva = + (u32)(((u32)frmr_reg_wr->access & IB_ZERO_BASED) != 0); + frmr_wqe_task->dw3.bs.be = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_MW_BIND) != + 0); /* bind operation enable */ + frmr_wqe_task->dw3.bs.lre = 1; /* this place read enable */ + frmr_wqe_task->dw3.bs.lwe = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_LOCAL_WRITE) != + 0); /* this place write enable */ + frmr_wqe_task->dw3.bs.rre = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_READ) != + 0); /* remote read enable */ + frmr_wqe_task->dw3.bs.rwe = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_WRITE) != + 0); /* remote write enable */ + frmr_wqe_task->dw3.bs.rae = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_ATOMIC) != + 0); /* remote Atomic enable */ + frmr_wqe_task->dw3.bs.block = 1; + frmr_wqe_task->dw3.bs.rsvd = 0; + frmr_wqe_task->dw3.bs.pa_num = (roce5_kernel_frmr->npages == 0) ? + 1 : + roce5_kernel_frmr->npages; + /* MR memory page size. equals to (2^page_size)*4KB */ + frmr_wqe_task->dw3.bs.page_size = + roce5_log2(frmr_reg_wr->mr->page_size) - 12; //12 is 4K page of shift + if (frmr_reg_wr->mr->page_size != PAGE_SIZE) { + ROCE_WARN( + "Frmr page size doesn't match page frame of OS! It may lead to unexpected memory translation."); + } + frmr_wqe_task->dw3.value = cpu_to_be32(frmr_wqe_task->dw3.value); + + frmr_wqe_task->m_key = cpu_to_be32(frmr_reg_wr->key); + frmr_wqe_task->dw7.len = cpu_to_be64(frmr_reg_wr->mr->length); + frmr_wqe_task->dw9.pbl_addr = cpu_to_be64(roce5_kernel_frmr->page_map); + /* set inline and direct wqe handle section */ + frmr_wqe_task->rsvd[0] = 0; + frmr_wqe_task->rsvd[1] = 0; + + param->inline_flag = 0; + return 0; +} + +static int roce5_post_send_rc(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, + const struct ib_send_wr *wr) +{ + int ret = 0; + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND_WITH_INV: + roce5_set_wqe_last_len(param->wqe); + roce5_set_rc_wqe_for_optype_send(wr, param); + break; + case IB_WR_SEND: + roce5_set_rc_wqe_for_optype_send(wr, param); + break; + + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + roce5_set_rc_wqe_for_optype_atomic(wr, param); + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + roce5_set_wqe_last_len(param->wqe); + roce5_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + roce5_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_LOCAL_INV: + roce5_set_rc_wqe_for_optype_local(wr, param); + ret = ROCE_NEED_JUMP; + break; + + case IB_WR_REG_MR: + set_frmr_wqe_section(wr, param); + ret = ROCE_NEED_JUMP; + break; + + default: + ROCE_DEV_ERR(rdev, "Invalid opcode(%d), func_id(%d)", + wr->opcode, rdev->glb_func_id); + break; + } + + return ret; +} + +static int roce5_post_send_uc(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, + const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND: + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + roce5_set_wqe_last_len(param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_send_seg((struct roce_wqe_send_tsk_seg *)param->wqe, + wr); + param->data_len_addr = + &((struct roce_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + + sizeof(struct roce_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_send_tsk_seg); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_WRITE: + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { + roce5_set_wqe_last_len(param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_rdma_seg((struct roce_wqe_rdma_tsk_seg *)param->wqe, + wr); + param->data_len_addr = + &((struct roce_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + + sizeof(struct roce_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_rdma_tsk_seg); + break; + + case IB_WR_LOCAL_INV: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce5_set_local_inv_seg( + (struct roce_wqe_local_inv_tsk_seg *)param->wqe, wr); + param->wqe_size += + (u32)sizeof(struct roce_wqe_local_inv_tsk_seg); + return ROCE_NEED_JUMP; + + default: + ROCE_DEV_ERR(rdev, "Invalid opcode(%d), func_id(%d)", + wr->opcode, rdev->glb_func_id); + break; + } + + return 0; +} + +static int roce5_post_send_ud(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct roce_wqe_ud_tsk_seg *tsk_sg = + (struct roce_wqe_ud_tsk_seg *)param->wqe; + + if (ROCE_UNLIKELY((wr->opcode != IB_WR_SEND) && + (wr->opcode != IB_WR_SEND_WITH_IMM))) { + ROCE_DEV_ERR(rdev, "Wr->opcode(%d), func_id(%d)", wr->opcode, + rdev->glb_func_id); + *bad_wr = wr; + return ROCE_NEED_JUMP; + } + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + tsk_sg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + tsk_sg->dw2.value = cpu_to_be32(tsk_sg->dw2.value); + } + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->data_len_addr = + &((struct roce_wqe_ud_tsk_seg *)param->wqe)->data_len; + + if (param->sq_rmd_size > (1UL << (unsigned int)rqp->sq.wqe_shift)) { + roce5_set_ud_seg((struct roce_wqe_ud_tsk_seg *)(param->wqe), + wr); + param->wqe = + ((u8 *)param->wqe + sizeof(struct roce_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_ud_tsk_seg); + } else { + roce5_set_ud_seg_cycle1( + (struct roce_wqe_ud_tsk_seg_cycle1 *)param->wqe, wr); + param->wqe = rqp->sq_head_addr; + param->wqe_size += + (u32)sizeof(struct roce_wqe_ud_tsk_seg_cycle1); + roce5_set_ud_seg_cycle2( + (struct roce_wqe_ud_tsk_seg_cycle2 *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + + sizeof(struct roce_wqe_ud_tsk_seg_cycle2)); + param->wqe_size += + (u32)sizeof(struct roce_wqe_ud_tsk_seg_cycle2); + } + + return 0; +} + +static void roce5_post_send_set_normal_data_seg(const struct ib_send_wr *wr, + struct roce_wqe_data_seg **dseg, + struct roce5_qp *rqp) +{ + if (wr->num_sge != 0) { + if ((u8 *)(*dseg) == rqp->sq_head_addr) { + *dseg = (struct roce_wqe_data_seg *)(( + void *)(rqp->sq_tail_addr - + sizeof(struct roce_wqe_data_seg))); + } else { + (*dseg)--; + } + (*dseg)->key = (u32)wr->sg_list[wr->num_sge - 1].lkey | + ROCE_WQE_NEXT_SGE_INVALID; + (*dseg)->key = cpu_to_be32((*dseg)->key); + } +} + +static int +roce5_post_send_set_normal_sge(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int i; + + param->data_len = 0; + param->dseg = (struct roce_wqe_data_seg *)param->wqe; + param->wqe_size += + (u32)(wr->num_sge * (int)sizeof(struct roce_wqe_data_seg)); + for (i = 0; i < wr->num_sge; ++i) { + if (param->sq_rmd_size < sizeof(struct roce_wqe_data_seg)) { + param->dseg = (struct roce_wqe_data_seg + *)((void *)rqp->sq_head_addr); + } + + if (ROCE_UNLIKELY(wr->sg_list[i].length > + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz))) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Sge_data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)", + i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + param->data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(param->data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz)) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Data len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)", + param->data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + roce5_set_data_seg(param->dseg, wr->sg_list + i); + + (param->dseg)++; + param->sq_rmd_size = + (u32)(rqp->sq_tail_addr - (u8 *)param->dseg); + } + + return 0; +} + +static void roce5_copy_inline_data(struct roce5_post_send_normal_param *param, + const struct ib_send_wr *wr, + struct roce5_qp *rqp, int i) +{ + int ret = 0; + struct ib_sge *sge = wr->sg_list + i; + + if (param->sq_rmd_size >= wr->sg_list[i].length) { + /* Set inline data */ + ret = memcpy_s(param->wqe, sge->length, + (void *)(uintptr_t)sge->addr, sge->length); + if (ret != 0) { + ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret); + return; + } + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length); + if ((u8 *)param->wqe == rqp->sq_tail_addr) { + param->wqe = rqp->sq_head_addr; + } + } else { + /* Set inline data cycle1 */ + u32 size = param->sq_rmd_size; + ret = memcpy_s(param->wqe, size, (void *)(uintptr_t)sge->addr, + size); + if (ret != 0) { + ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret); + return; + } + param->wqe = rqp->sq_head_addr; + + /* Set inline data cycle2 */ + size = wr->sg_list[i].length - param->sq_rmd_size; + ret = memcpy_s( + param->wqe, size, + (void *)((u8 *)(void *)(uintptr_t)sge->addr + size), + size); + if (ret != 0) { + ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret); + return; + } + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length - + param->sq_rmd_size); + + param->cycle = 1; + } +} + +static int +roce5_post_send_set_data_seg(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int i; + int ret; + + if ((((unsigned int)wr->send_flags & (unsigned int)IB_SEND_INLINE) != + 0) && + (wr->num_sge != 0) && (param->inline_flag != 0)) { + param->data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + param->wqe_size += wr->sg_list[i].length; + param->data_len += wr->sg_list[i].length; + if (param->data_len > rqp->max_inline_data) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Data len is too big, data_len(%d), max_inline_data(%d), func_id(%d)", + param->data_len, rqp->max_inline_data, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + if ((u8 *)param->wqe == rqp->sq_head_addr) { + param->cycle = 1; + } + + roce5_copy_inline_data(param, wr, rqp, i); + + param->sq_rmd_size = + (u32)(rqp->sq_tail_addr - (u8 *)param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.bdsl = + (ALIGN((u32)(param->data_len), ROCE_TASK_SEG_ALIGN) / + ROCE_TASK_SEG_ALIGN) & + 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 1; + param->inline_flag = 1; + } else { + ret = roce5_post_send_set_normal_sge(param, rdev, rqp, wr, + bad_wr); + if (ret != 0) { + return ret; + } + + roce5_post_send_set_normal_data_seg(wr, ¶m->dseg, rqp); + + param->ctrl_seg_tmp.dw0.bs.bdsl = + (u32)((wr->num_sge * + (int)sizeof(struct roce_wqe_data_seg)) / + ROCE_TASK_SEG_ALIGN) & + 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 0; + } + + return 0; +} + +static void roce5_fill_task_data_len(u32 *data_len_addr, u32 data_len) +{ + if (data_len_addr != NULL) { + *data_len_addr = cpu_to_be32(data_len); + } +} + +static int roce5_construct_wqe(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + int ret; + + switch (rqp->qp_type) { + case IB_QPT_RC: + need_goto = roce5_post_send_rc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) { + return GOTO_LOCAL; + } + break; + + case IB_QPT_UC: + need_goto = roce5_post_send_uc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) { + return GOTO_LOCAL; + } + break; + + case IB_QPT_UD: + need_goto = roce5_post_send_ud(param, rdev, rqp, wr, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + return GOTO_OUT; + } + break; + + case IB_QPT_GSI: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + + ret = roce5_set_gsi_seg(rqp, wr, param->wqe); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set gsi seg, ret(%d).", + ret); + return GOTO_OUT; + } + + param->data_len_addr = + &((struct roce_wqe_ud_tsk_seg *)param->wqe)->data_len; + param->wqe = + ((u8 *)param->wqe + sizeof(struct roce_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_ud_tsk_seg); + break; + + default: + ROCE_DEV_ERR(rdev, "Unknown qp type, func_id(%d)", + rdev->glb_func_id); + break; + } + + return 0; +} + +static void roce5_init_ctrl_seg(struct roce5_post_send_normal_param *param, + const struct ib_send_wr *wr, + const struct roce5_qp *rqp) +{ + u32 wqe_opcode = roce5_ib_opcode[wr->opcode] & 0x1f; + param->ctrl_seg_tmp.dw0.bs.owner = + ((param->index & (u32)rqp->sq.wqebb_cnt) != 0) ? 1 : 0; + param->ctrl_seg_tmp.dw0.bs.drvsl = 0; + param->ctrl_seg_tmp.dw0.bs.wf = 0; + param->ctrl_seg_tmp.dw0.bs.cf = 0; + param->ctrl_seg_tmp.dw0.bs.va = 1; + param->ctrl_seg_tmp.dw0.bs.df = 0; /* va:1bits, 0-sge,1-inline */ + param->ctrl_seg_tmp.dw0.bs.cr = + (u32)(!(((((u32)wr->send_flags) & IB_SEND_SIGNALED) | + rqp->sq_signal_bits) == 0)); + param->ctrl_seg_tmp.dw0.bs.difsl = 0; + param->ctrl_seg_tmp.dw0.bs.csl = 0; + param->ctrl_seg_tmp.dw0.bs.ctrlsl = ROCE_CTRL_SEG_SL; + param->ctrl_seg_tmp.dw0.bs.bdsl = 0; + + param->ctrl_seg_tmp.dw0.bs.wqe_ssn = (param->sq_wqe_ssn & 0x3); + param->sq_wqe_ssn += (u8)g_roce5_opcode_attr[wqe_opcode].remote_opcode; +} + +static void roce5_init_task_com_seg(union roce5_wqe_tsk_com_seg **tsk_com_seg, + u8 *wqe, const struct ib_send_wr *wr, + const struct roce5_qp *rqp) +{ + struct roce_wqe_send_tsk_seg *send_task = + (struct roce_wqe_send_tsk_seg *)(void *)wqe; + *tsk_com_seg = (union roce5_wqe_tsk_com_seg *)(void *)wqe; + + send_task->dw3.value = 0; + (*tsk_com_seg)->bs.c = (u32)(!( + (((unsigned)wr->send_flags & (unsigned)IB_SEND_SIGNALED) | + rqp->sq_signal_bits) == 0)); + (*tsk_com_seg)->bs.se = (u32)(!( + ((unsigned)wr->send_flags & (unsigned)IB_SEND_SOLICITED) == 0)); + (*tsk_com_seg)->bs.f = (u32)(!( + ((unsigned)wr->send_flags & (unsigned)IB_SEND_FENCE) == 0)); + (*tsk_com_seg)->bs.op_type = roce5_ib_opcode[wr->opcode] & 0x1f; +} + +static void +roce5_post_send_local_operation(struct roce5_post_send_normal_param *param, + struct roce5_qp *rqp, struct roce5_device *rdev) +{ + param->opcode = param->tsk_com_seg->bs.op_type; + param->tsk_com_seg->value = cpu_to_be32(param->tsk_com_seg->value); + param->ctrl_seg->dw1.value = cpu_to_be32(param->ctrl_seg_tmp.dw1.value); + param->index += (u32)ROCE_DIV_ROUND_UP(param->wqe_size, + 1U << (u32)rqp->sq.wqe_shift); + + write_invalid_wqe(rqp, param->index); + + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + wmb(); + + param->ctrl_seg->dw0.value = cpu_to_be32(param->ctrl_seg_tmp.dw0.value); +} +/* **************************************************************************** + Prototype : roce5_post_send_normal + Description : roce5_post_send_normal + Input : struct roce5_device *rdev + struct roce5_qp *rqp + struct ib_send_wr *wr + struct ib_send_wr **bad_wr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + + 2.Date : 2015/8/8 + Modification : Modify function + +**************************************************************************** */ +static int roce5_post_send_normal(struct roce5_device *rdev, + struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + int ret = 0; + struct roce5_post_send_normal_param param = { 0 }; + const struct ib_send_wr *wr_tmp = wr; + + spin_lock_irqsave(&rqp->sq.lock, param.flags); + + param.index = rqp->sq_next_wqe; + param.sq_wqe_ssn = rqp->sq_wqe_ssn; + + for (param.wr_num = 0; wr_tmp != NULL; + ++(param.wr_num), wr_tmp = wr_tmp->next) { + ret = roce5_validate_wr(rqp, wr_tmp, param.wr_num); + if (ret != 0) { + *bad_wr = wr_tmp; + ROCE_DEV_ERR(rdev, + " Failed to validate wr, func(%d) qpn(%u)", + rdev->glb_func_id, rqp->qpn); + goto out; + } + + param.wqe = (u8 *)roce5_get_send_wqe(rqp, param.index); + param.ctrl_seg = (struct roce5_wqe_ctrl_seg *)param.wqe; + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + rqp->sq.wrid[(rqp->sq.head + param.wr_num) & + (u32)(rqp->sq.wqebb_cnt - 1)] = wr_tmp->wr_id; + + roce5_init_ctrl_seg(¶m, wr_tmp, rqp); + + param.wqe = ((u8 *)param.wqe + sizeof(*(param.ctrl_seg))); + param.wqe_size = (u32)sizeof(*param.ctrl_seg); + roce5_init_task_com_seg(¶m.tsk_com_seg, param.wqe, wr_tmp, + rqp); + + ret = roce5_construct_wqe(¶m, rdev, rqp, wr_tmp, bad_wr); + if (ret == GOTO_LOCAL) { + ret = 0; + goto local; + } else if (ret == GOTO_OUT) { + ret = -EINVAL; + goto out; + } + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + need_goto = roce5_post_send_set_data_seg(¶m, rdev, rqp, + wr_tmp, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + ret = -EINVAL; + goto out; + } + + roce5_fill_task_data_len(param.data_len_addr, param.data_len); + +local: + roce5_post_send_local_operation(¶m, rqp, rdev); + } + +out: + if (ROCE_LIKELY(param.wr_num != 0)) { + rqp->sq.head += param.wr_num; + + wmb(); + + ret = roce5_write_sq_db(rdev, rqp, param.index, NULL); + rqp->sq_next_wqe = param.index; + rqp->sq_wqe_ssn = param.sq_wqe_ssn; + } + + spin_unlock_irqrestore(&rqp->sq.lock, param.flags); + + return ret; +} + +static int roce5_post_send_check_qp_type(const struct roce5_qp *rqp) +{ + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_TGT)) { + ROCE_ERR("Can't post WQE when TGT XRC QP"); + return -EINVAL; + } + + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_INI)) { + ROCE_ERR("not support xrc in kernel space"); + return -EINVAL; + } + + if (ROCE_UNLIKELY((rqp->qp_state == IB_QPS_RESET) || + (rqp->qp_state == IB_QPS_INIT) || + (rqp->qp_state == IB_QPS_RTR))) { + ROCE_ERR("Can't post WQE when QP is RST/INIT/RTR state"); + return -EINVAL; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_post_send + Description : roce5_post_send + Input : struct ib_qp *ibqp + struct ib_send_wr *wr + const struct ib_send_wr **bad_wr + Output : None +**************************************************************************** */ +int roce5_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int ret = 0; + struct roce5_qp *rqp = to_roce5_qp(ibqp); + struct roce5_device *rdev = to_roce5_dev(ibqp->device); + + ret = roce5_post_send_check_qp_type(rqp); + if (ret != 0) { + *bad_wr = wr; + ROCE_DEV_ERR(rdev, "Failed to check qp.ret(%d)", ret); + return ret; + } + + ret = roce5_post_send_normal(rdev, rqp, wr, bad_wr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to post send normal wr, ret(%d)", + ret); + return ret; + } + + return 0; +} + +int roce5_post_send_ofa(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + return roce5_post_send(ibqp, (const struct ib_send_wr *)wr, + (const struct ib_send_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c new file mode 100644 index 000000000..260eca49e --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c @@ -0,0 +1,543 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "hinic5_crm.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_restore.h" +#include "roce_npu_cmd.h" +#include "roce_main_extension.h" +#include "roce_com.h" +#include "securec.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/** + * @brief Get roce qp by qpn. When querying succeeded, you have to call roce5_rqp_put() to free + the cqm_obj_qp. But there is no need to call roce5_rqp_put() if querying failed. + * @return Success: return the roce qp. Failure: return NULL. + */ +struct roce5_qp *roce5_get_rqp_by_qpn(struct roce5_device *rdev, u32 qpn) +{ + cqm_object_s *cqm_obj_qp = NULL; + struct roce5_qp *rqp = NULL; + + cqm_obj_qp = cqm5_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, + false); + if (cqm_obj_qp == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find rqp according to qpn(0x%x), func_id(%d)", + qpn, rdev->glb_func_id); + return NULL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to convert obj to rqp."); + cqm5_object_put(cqm_obj_qp); + } + + return rqp; +} + +static enum ib_mig_state to_ib_mig_state(int roce5_mig_state) +{ + switch (roce5_mig_state) { + case ROCE_QP_PM_ARMED: + return IB_MIG_ARMED; + + case ROCE_QP_PM_REARM: + return IB_MIG_REARM; + + case ROCE_QP_PM_MIGRATED: + return IB_MIG_MIGRATED; + + default: + return (enum ib_mig_state)(-1); + } +} + +static int to_ib_qp_access_flags(const roce_verbs_qp_attr_s *verbs_qp_attr) +{ + unsigned int ib_flags = 0; + u32 tmp = 0; + + /* RRE */ + if (verbs_qp_attr->com_info.dw0.bs.rre != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_READ; + ib_flags = tmp; + } + + /* RWE */ + if (verbs_qp_attr->com_info.dw0.bs.rwe != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_WRITE; + ib_flags = tmp; + } + + /* RAE */ + if (verbs_qp_attr->com_info.dw0.bs.rae != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_ATOMIC; + ib_flags = tmp; + } + + return (int)ib_flags; +} + +/* **************************************************************************** + Prototype : to_ib_ah_attr + Description : to_ib_ah_attr + Input : struct roce5_device *rdev + struct ib_ah_attr *ah_attr(struct rdma_ah_attr *ah_attr) + struct roce5_qp_context *context + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +static void to_ib_ah_attr(struct roce5_device *rdev, + struct rdma_ah_attr *ah_attr, + roce_verbs_qp_attr_s *verbs_qp_attr) +{ + int ret; + + ret = memset_s(ah_attr, sizeof(*ah_attr), 0, sizeof(*ah_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset ah_attr.(ret:%d)", ret); + return; + } + + ah_attr->port_num = verbs_qp_attr->com_info.dw5.bs.port; + + ah_attr->sl = 0x7 - verbs_qp_attr->path_info.dw7.bs.sl; + + ah_attr->static_rate = 0; + + ah_attr->ah_flags = IB_AH_GRH; + + ah_attr->grh.sgid_index = verbs_qp_attr->path_info.dw7.bs.sgid_index; + ah_attr->grh.hop_limit = verbs_qp_attr->path_info.dw7.bs.hoplmt; + ah_attr->grh.traffic_class = verbs_qp_attr->path_info.dw6.bs.tclass; + ah_attr->grh.flow_label = verbs_qp_attr->path_info.dw6.bs.flow_label; + + ret = memcpy_s((void *)ah_attr->grh.dgid.raw, + sizeof(ah_attr->grh.dgid.raw), + (void *)verbs_qp_attr->path_info.dgid, + sizeof(ah_attr->grh.dgid.raw)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy dgid.(ret:%d)", ret); + } +} + +int roce5_uni_cmd_qp_query(struct roce5_device *rdev, u32 qpn, + u32 *verbs_qp_attr, int qp_attr_size) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_qp_query_s *qp_query_inbuf = NULL; + roce_uni_cmd_qp_query_outbuf_s *qp_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_qp_query_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + qp_query_inbuf = (roce_uni_cmd_qp_query_s *)cqm_cmd_inbuf->buf; + qp_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + qp_query_inbuf->com.index = cpu_to_be32(qpn); + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_QUERY_QP, cqm_cmd_inbuf, + cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_B); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to send QUERY_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto free_cqm_buf; + } + + qp_query_outbuf = (roce_uni_cmd_qp_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = (int)memcpy_s((void *)verbs_qp_attr, (size_t)qp_attr_size, + (void *)&qp_query_outbuf->qp_info.qp_attr, + (size_t)qp_attr_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy verbs_qp_attr, ret(%d).", + ret); + ret = -EINVAL; + } +free_cqm_buf: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int roce5_qp_query(struct roce5_device *rdev, u32 qpn, u32 *context, + int qpc_size) +{ + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_qp_query(rdev, qpn, context, qpc_size); + } + return -EINVAL; +} + +void qpc_seg_to_le32(struct roce5_qp_context *be_ctx, + struct roce5_qp_context *le_ctx, u32 srq_vld) +{ + /* DRV Seg */ + int ret; + + ret = memcpy_s((void *)le_ctx->sw_seg.path_seg.dgid, + sizeof(be_ctx->sw_seg.path_seg.dgid), + (void *)be_ctx->sw_seg.path_seg.dgid, + sizeof(be_ctx->sw_seg.path_seg.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return; + } + + /* CHIP Seg */ + le_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa = + be64_to_cpu(be_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa); + le_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num = cpu_to_be64( + be_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num); + le_ctx->chip_seg.rcc.rc_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rcc.rc_curt_sge_va); + le_ctx->chip_seg.sqc.sq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.sqc.sq_curt_sge_va); + le_ctx->chip_seg.sqac.sqa_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.sqac.sqa_curt_sge_va); + + if (srq_vld != 0) { + le_ctx->chip_seg.srqc.srq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.srqc.srq_curt_sge_va); + } else { + le_ctx->chip_seg.rqc.rq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rqc.rq_curt_sge_va); + } + + le_ctx->chip_seg.rrwc.rrw_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va); +} + +void roce5_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size) +{ + u32 *ctx = NULL, *ctx1 = NULL; + u32 i = 0; + + ctx = le_ctx; + ctx1 = (u32 *)context; + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) { + *ctx = be32_to_cpu(*ctx1); + } + + return; +} + +static void roce5_get_ah_attr(const struct roce5_qp *rqp, + struct ib_qp_attr *qp_attr, + struct roce5_device *rdev, + roce_verbs_qp_attr_s *verbs_qp_attr) +{ + if ((rqp->qp_type == IB_QPT_RC) || (rqp->qp_type == IB_QPT_UC)) { + to_ib_ah_attr(rdev, &qp_attr->ah_attr, verbs_qp_attr); + } +} + +static void roce5_query_set_common_attr(struct ib_qp_attr *qp_attr, + const struct ib_qp *ibqp, + struct roce5_qp *rqp, + struct ib_qp_init_attr *qp_init_attr) +{ + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = (u32)rqp->rq.wqebb_cnt; + qp_attr->cap.max_recv_sge = (u32)rqp->rq.max_sge; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = (u32)rqp->sq.max_post; + qp_attr->cap.max_send_sge = (u32)rqp->sq.max_sge; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_attr->cap.max_inline_data = rqp->max_inline_data; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = (enum ib_qp_create_flags)0; + + qp_init_attr->sq_sig_type = (rqp->sq_signal_bits == 1) ? + IB_SIGNAL_ALL_WR : + IB_SIGNAL_REQ_WR; +} + +static void roce5_query_set_attr(struct ib_qp_attr *qp_attr, u8 tmp_qp_state, + struct roce5_qp *rqp, + roce_verbs_qp_attr_s *verbs_qp_attr, + struct roce5_device *rdev) +{ + qp_attr->qp_state = (enum ib_qp_state)rqp->qp_state; + qp_attr->path_mtu = (enum ib_mtu)verbs_qp_attr->com_info.dw3.bs.pmtu; + qp_attr->path_mig_state = + (enum ib_mig_state)to_ib_mig_state(ROCE_QP_PM_MIGRATED); + qp_attr->qkey = verbs_qp_attr->com_info.q_key; + qp_attr->rq_psn = verbs_qp_attr->com_info.dw9.bs.next_rcv_psn; + qp_attr->sq_psn = verbs_qp_attr->com_info.dw8.bs.next_send_psn; + qp_attr->dest_qp_num = verbs_qp_attr->com_info.dw0.bs.dest_qp; + qp_attr->qp_access_flags = to_ib_qp_access_flags(verbs_qp_attr); + + roce5_get_ah_attr(rqp, qp_attr, rdev, verbs_qp_attr); + + qp_attr->pkey_index = 0; + + if (qp_attr->qp_state == IB_QPS_INIT) { + qp_attr->port_num = rqp->port; + } else { + qp_attr->port_num = verbs_qp_attr->com_info.dw5.bs.port; + } + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + + /* SQ Draining FLAG */ + qp_attr->sq_draining = + (tmp_qp_state == + (u8)(enum roce_qp_state)ROCE_QP_STATE_SQ_DRAINING); + + qp_attr->max_rd_atomic = (1 << verbs_qp_attr->com_info.dw1.bs.sra_max); + qp_attr->max_dest_rd_atomic = + (1 << verbs_qp_attr->com_info.dw1.bs.rra_max); + + qp_attr->min_rnr_timer = verbs_qp_attr->com_info.dw2.bs.min_rnr_nak; + qp_attr->timeout = verbs_qp_attr->com_info.dw2.bs.ack_to; + qp_attr->retry_cnt = verbs_qp_attr->com_info.dw1.bs.to_retry_limit; + qp_attr->rnr_retry = verbs_qp_attr->com_info.dw1.bs.rnr_retry_limit; +} + +static int roce5_query_qp_struct_clean(struct ib_qp_attr *qp_attr, + struct ib_qp_init_attr *qp_init_attr, + roce_verbs_qp_attr_s *verbs_qp_attr) +{ + int ret; + + ret = memset_s(qp_attr, sizeof(struct ib_qp_attr), 0, + sizeof(struct ib_qp_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = memset_s(qp_init_attr, sizeof(struct ib_qp_init_attr), 0, + sizeof(struct ib_qp_init_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_init_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = memset_s(verbs_qp_attr, sizeof(roce_verbs_qp_attr_s), 0, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset verbs_qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} + +static void roce_qp_attr_com_info_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + verbs_qp_attr->com_info.dw0.value = + be32_to_cpu(verbs_qp_attr->com_info.dw0.value); + verbs_qp_attr->com_info.dw1.value = + be32_to_cpu(verbs_qp_attr->com_info.dw1.value); + verbs_qp_attr->com_info.dw2.value = + be32_to_cpu(verbs_qp_attr->com_info.dw2.value); + verbs_qp_attr->com_info.dw3.value = + be32_to_cpu(verbs_qp_attr->com_info.dw3.value); + verbs_qp_attr->com_info.q_key = + be32_to_cpu(verbs_qp_attr->com_info.q_key); + verbs_qp_attr->com_info.dw5.value = + be32_to_cpu(verbs_qp_attr->com_info.dw5.value); + verbs_qp_attr->com_info.dw6.value = + be32_to_cpu(verbs_qp_attr->com_info.dw6.value); + verbs_qp_attr->com_info.dw7.value = + be32_to_cpu(verbs_qp_attr->com_info.dw7.value); + verbs_qp_attr->com_info.dw8.value = + be32_to_cpu(verbs_qp_attr->com_info.dw8.value); + verbs_qp_attr->com_info.dw9.value = + be32_to_cpu(verbs_qp_attr->com_info.dw9.value); + verbs_qp_attr->com_info.lsn = be32_to_cpu(verbs_qp_attr->com_info.lsn); + verbs_qp_attr->com_info.dw11.value = + be32_to_cpu(verbs_qp_attr->com_info.dw11.value); + verbs_qp_attr->com_info.rsvd = + be32_to_cpu(verbs_qp_attr->com_info.rsvd); +} + +static void roce_qp_attr_path_info_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + verbs_qp_attr->path_info.dw0.value = + be32_to_cpu(verbs_qp_attr->path_info.dw0.value); + verbs_qp_attr->path_info.dmac_l32 = + be32_to_cpu(verbs_qp_attr->path_info.dmac_l32); + verbs_qp_attr->path_info.dw6.value = + be32_to_cpu(verbs_qp_attr->path_info.dw6.value); + verbs_qp_attr->path_info.dw7.value = + be32_to_cpu(verbs_qp_attr->path_info.dw7.value); +} + +static void roce_qp_attr_chip_seg_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + verbs_qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = + be64_to_cpu(verbs_qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa); + verbs_qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = be64_to_cpu( + verbs_qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num); + verbs_qp_attr->chip_seg.dw4.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw4.value); + verbs_qp_attr->chip_seg.dw5.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw5.value); + verbs_qp_attr->chip_seg.dw6.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw6.value); + verbs_qp_attr->chip_seg.dw7.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw7.value); + verbs_qp_attr->chip_seg.dw8.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw8.value); + verbs_qp_attr->chip_seg.dw9.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw9.value); + verbs_qp_attr->chip_seg.dw10.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw10.value); + verbs_qp_attr->chip_seg.rc_page_gpa_l = + be32_to_cpu(verbs_qp_attr->chip_seg.rc_page_gpa_l); + verbs_qp_attr->chip_seg.dw12.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw12.value); + verbs_qp_attr->chip_seg.dw13.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw13.value); + verbs_qp_attr->chip_seg.dw14.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw14.value); + verbs_qp_attr->chip_seg.dw15.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw15.value); + verbs_qp_attr->chip_seg.dw16.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw16.value); + verbs_qp_attr->chip_seg.dw17.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw17.value); + verbs_qp_attr->chip_seg.dw18.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw18.value); + verbs_qp_attr->chip_seg.dw19.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw19.value); +} + +static void roce5_qp_attr_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + roce_qp_attr_com_info_to_le32(verbs_qp_attr); + roce_qp_attr_path_info_to_le32(verbs_qp_attr); + roce_qp_attr_chip_seg_to_le32(verbs_qp_attr); +} + +/* **************************************************************************** + Prototype : roce5_query_qp + Description : roce5_query_qp + Input : struct ib_qp *ibqp + struct ib_qp_attr *qp_attr + int qp_attr_mask + struct ib_qp_init_attr *qp_init_attr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +int roce5_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) +{ + int ret = 0; + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_qp_attr_s verbs_qp_attr; + u32 srq_vld; + u8 tmp_qp_state = 0; + + if ((ibqp == NULL) || (qp_attr == NULL) || (qp_init_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + ret = roce5_query_qp_struct_clean(qp_attr, qp_init_attr, + &verbs_qp_attr); + if (ret != 0) { + return ret; + } + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state == (u8)(enum ib_qp_state)IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + ret = roce5_qp_query(rdev, rqp->qpn, + (u32 *)(void *)(&verbs_qp_attr), + sizeof(verbs_qp_attr)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to query qp with 25V100 or 23V200, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + mutex_unlock(&rqp->mutex); + return ret; + } + } + + srq_vld = (!(rqp->ibqp.srq)) ? ROCE_QP_NO_SRQ : ROCE_QP_HAS_SRQ; + + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + roce5_qp_attr_to_le32(&verbs_qp_attr); + rqp->qp_state = verbs_qp_attr.com_info.dw6.bs + .state; /* 25V100 or 23V200 qp state */ + } + + tmp_qp_state = rqp->qp_state; + if (rqp->qp_state == + (u8)(enum roce_qp_state)ROCE_QP_STATE_SQ_DRAINING) { + rqp->qp_state = (u8)(enum ib_qp_state)IB_QPS_SQD; + } + + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + /* 25V100 or 23V200 set attr */ + roce5_query_set_attr(qp_attr, tmp_qp_state, rqp, &verbs_qp_attr, + rdev); + } + +done: + roce5_query_set_common_attr(qp_attr, ibqp, rqp, qp_init_attr); + + mutex_unlock(&rqp->mutex); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c new file mode 100644 index 000000000..257e9a8aa --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c @@ -0,0 +1,22 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" + +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev) +{ + struct rdma_comp_priv *comp_private = NULL; + comp_private = (struct rdma_comp_priv *)hinic5_get_service_adapter( + hwdev, SERVICE_T_ROCE); + return comp_private; +} + +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->pd_bitmap); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h new file mode 100644 index 000000000..8a1a8d665 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h @@ -0,0 +1,166 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef RDMA_COMP_H +#define RDMA_COMP_H + +/* x86 */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#include <linux/delay.h> +#include <linux/types.h> +#ifdef ROCE_KNL_DT +#include <linux/compiler_types.h> +#endif + +#include "hinic5_crm.h" +#include "hinic5_cqm.h" +#include "hinic5_rdma.h" + +#include "roce_gid.h" +#include "roce_npu_cmd_defs.h" +#include "roce.h" +#include "roce_compat.h" + +#include "rdma_bitmap.h" +#include "hmm_em_inner.h" +#include "hmm_common.h" + +#ifndef pr_fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": [RDMA]" fmt +#endif + +#define RDMA_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define RDMA_INVALID_GUID 0 + +#define RDMA_SERVICE_TYPE_ROCE SERVICE_T_ROCE + +#define RDMA_BIT_SHIFT_1 1 +#define RDMA_BIT_SHIFT_2 2 +#define RDMA_BIT_SHIFT_4 4 +#define RDMA_BIT_SHIFT_8 8 +#define RDMA_BIT_SHIFT_16 16 + +#define ROCE5_RDMARC_MIN_DEPTH 1 +#define ROCE5_RDMARC_MAX_DEPTH 512 +#define ROCE5_RDMARC_MIN_ENTRY 8 /* min entry aligned required by engine */ +#define ROCE5_RDMARC_MAX_ENTRY 128 /* max entry supported by engine */ +#define ROCE5_RDMARC_EXT_ENTRY 384 //384 /* ext tbl 12K. 12k/32 = 384 */ + +#define ROCE5_RDMARC_GPA_H_SHIFT 40 /* high:24bit, 64-24= 40 */ +#define ROCE5_RDMARC_GPA_L_SHIFT 8 /* bit[39:8] */ +#define ROCE5_RDMARC_RRA_MAX_MASK 0x7 /* rra_max:3bit */ +#define ROCE5_RDMARC_EXT_ORDER_SHIFT 3 + +extern u32 g_mtt_page_size; + +#define RDMA_EM_MIN_ORDER 2 /* rdma rc extend table at least 4 page */ + +#define ROCE_MTT_PAGE_SIZE_4K 4096 +#define ROCE_MTT_PAGE_SIZE_64K (64 * 1024) +#define ROCE_MTT_PAGE_SIZE_2M (2 * 1024 * 1024) +#define ROCE_MTT_PAGE_SIZE_4K_SHIFT 12 +#define ROCE_MTT_PAGE_SIZE_64K_SHIFT 16 +#define ROCE_MTT_PAGE_SIZE_2M_SHIFT 21 + +enum { + ROCE5_RDMA_MTT_PAGE_SIZE_4K = 0, + ROCE5_RDMA_MTT_PAGE_SIZE_64K = 1, + ROCE5_RDMA_MTT_PAGE_SIZE_2M = 2 +}; + +enum { + ROCE5_RDMA_CMD_TIME_OUT_A = 30000, + ROCE5_RDMA_CMD_TIME_OUT_B = 40000, + ROCE5_RDMA_CMD_TIME_OUT_C = 50000 +}; + +/** + * @brief struct rdma_comp_resource + * @details RDMA component resource structure, used to store RDMA component of related resources + */ +struct rdma_comp_resource { + struct mutex mutex; /**< gid_entry mutex used by */ + __be64 node_guid; /**< and ibdev in of node_guid same as */ + struct rdma_gid_entry * + *gid_table; /**< gid_entry in rdma component initialize when allocate memory */ +}; + +/** + * @brief enum rdma_mw_type + * @details define an enum type, used to represent RDMA memory window of type + */ +enum rdma_mw_type { RDMA_MW_TYPE_1 = 1, RDMA_MW_TYPE_2 = 2 }; + +/** + * @brief struct rdma_mw + * @details define a rdma_mw structure, used to represent a RDMA memory window + */ +struct rdma_mw { + struct mpt mpt; + u32 key; /**< mw corresponding to key */ + u32 pdn; /**< mw bind of pdn */ + enum rdma_mw_type type; /**< mw of type,type1,type2 */ + int enabled; /**< mw of status */ +}; + +struct rdma_comp_priv { + struct hmm_comp_priv hmm_priv; + struct rdma_comp_resource rdma_comp_res; /* gid & guid */ + struct hmm_rdma rsvd_lkey; + struct hmm_rdma fixed_mr; + struct rdma_service_cap rdma_cap; + + struct rdma_bitmap pd_bitmap; + struct rdma_bitmap xrcd_bitmap; + + struct hmm_buddy rdmarc_buddy; + struct hmm_em_table rdmarc_em_table; +}; + +/* Only V100 use */ +struct rdma_gid_update_inbuf { + roce_verbs_cmd_com_s com; + __be32 port; + __be32 rsvd; + struct rdma_gid_entry gid_entry; +}; + +struct rdma_gid_clear_inbuf { + __be32 port; + __be32 gid_num; +}; + +typedef struct tag_roce_clear_gid { + roce_verbs_cmd_com_s com; + + u32 port; + u32 gid_num; +} roce_cmd_clear_gid_s; + +/* for llt to set stub */ +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv); +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev); +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, + struct rdma_gid_entry *gid_entry); +int roce5_update_gid_mac(struct roce5_device *rdev, u32 port, int *index, + struct rdma_gid_entry *gid_entry); +int roce5_update_gid(struct roce5_device *rdev, u32 port, u32 update_index, + struct rdma_gid_entry *gid_entry); +int roce5_reset_gid_table(struct roce5_device *rdev, u32 port); +int roce5_get_gid(struct roce5_device *rdev, u32 port, u32 gid_index, + struct rdma_gid_entry *gid); + +#endif //RDMA_COMP_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c new file mode 100644 index 000000000..b7919b409 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c @@ -0,0 +1,489 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "roce.h" +#include "roce_cqm_cmd.h" +#include "roce_npu_cmd.h" +#include "roce_mix.h" +#include "roce_gid.h" +#include "securec.h" +#include "roce_npu_cmd_gid_defs.h" + +#define ROCE_GID_IFID_DW_INDEX \ + 2 /* Interface ID starts at dword index 2 in u32 gid[4] (indices 0-1: subnet_prefix, 2-3: interface_id) */ + +static int roce5_uni_cmd_update_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, + struct rdma_gid_entry *new_gid_entry, + int port, int gid_index) +{ + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_update_gid_s *gid_update_inbuf = NULL; + int ret; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_update_gid_s), NULL, 0); + if (ret != 0) { + ROCE_ERR("Failed to alloc cqm_cmd_inoutbuf, ret(%d)", ret); + return -ENOMEM; + } + + gid_update_inbuf = (roce_uni_cmd_update_gid_s *)cqm_cmd_inbuf->buf; + *(u64 *)(&gid_update_inbuf->gid_attr.gid[0]) = + new_gid_entry->global.subnet_prefix; + *(u64 *)(&gid_update_inbuf->gid_attr.gid[ROCE_GID_IFID_DW_INDEX]) = + new_gid_entry->global.interface_id; + gid_update_inbuf->gid_attr.dw4.value = + cpu_to_be32(new_gid_entry->dw4.value); + gid_update_inbuf->gid_attr.dw5.value = + cpu_to_be32(new_gid_entry->hdr_len_value); + gid_update_inbuf->gid_attr.dw6.bs1.ctrl_val = + new_gid_entry->dw6_h.value; + gid_update_inbuf->gid_attr.dw6.bs1.smac_hi16 = + be16_to_cpu(*((u16 *)(&new_gid_entry->smac[0]))); + gid_update_inbuf->gid_attr.dw6.value = + cpu_to_be32(gid_update_inbuf->gid_attr.dw6.value); + gid_update_inbuf->gid_attr.smac_lo32 = + *((u32 *)(&new_gid_entry->smac[2])); //4=lo32 + + gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index); + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + gid_update_inbuf->com.opt = cpu_to_be32((u32)port); + if (rdev->shadow != 0) { + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK); + gid_update_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rdev->shadow_vfid); + gid_update_inbuf->gid_attr.dw10.shadow.shadow_vf_num = + rdev->shadow_vf_num; + gid_update_inbuf->gid_attr.dw10.value = + cpu_to_be32(gid_update_inbuf->gid_attr.dw10.value); + } + + ROCE_PRINT_GID(rdev, new_gid_entry, "gid_index(0x%x)", + gid_update_inbuf->com.index); + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Send cmd update_gid failed, ret(%d)", ret); + } + + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +static int roce5_cmd_update_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, + struct rdma_gid_entry *new_gid_entry, int port, + int gid_index) +{ + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + struct rdma_gid_update_inbuf *gid_update_inbuf = NULL; + int ret; + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + return roce5_uni_cmd_update_gid(rdev, comp_priv, new_gid_entry, + port, gid_index); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct rdma_gid_update_inbuf), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret(%d)", + ret); + return -ENOMEM; + } + + gid_update_inbuf = (struct rdma_gid_update_inbuf *)cqm_cmd_inbuf->buf; + gid_update_inbuf->port = cpu_to_be32((u32)port); + gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index); + gid_update_inbuf->gid_entry.global.subnet_prefix = + new_gid_entry->global.subnet_prefix; + gid_update_inbuf->gid_entry.global.interface_id = + new_gid_entry->global.interface_id; + gid_update_inbuf->gid_entry.dw4.value = + cpu_to_be32(new_gid_entry->dw4.value); + gid_update_inbuf->gid_entry.dw6_h.value = + cpu_to_be16(new_gid_entry->dw6_h.value); + gid_update_inbuf->gid_entry.hdr_len_value = + cpu_to_be32(new_gid_entry->hdr_len_value); + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + + ret = memcpy_s(&gid_update_inbuf->gid_entry.smac[0], ETH_ALEN, + &new_gid_entry->smac[0], ETH_ALEN); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy_s smac.(ret:%d)", ret); + ret = -ENOMEM; + goto err; + } + + ROCE_PRINT_GID(rdev, new_gid_entry, "gid_index(0x%x) ", + gid_update_inbuf->com.index); + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Send cmd update_gid failed, ret(%d)", ret); + ret = -1; + } + +err: + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, + struct rdma_gid_entry *gid_entry) +{ + struct rdma_gid_entry entry1; + struct rdma_gid_entry entry2; + int ret; + + ret = memcpy_s(&entry1, sizeof(struct rdma_gid_entry), gid_tbl_entry, + sizeof(struct rdma_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy entry1.(ret:%d)", ret); + return -ENOMEM; + } + ret = memcpy_s(&entry2, sizeof(struct rdma_gid_entry), gid_entry, + sizeof(struct rdma_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy entry2.(ret:%d)", ret); + return -ENOMEM; + } + + /* compare ip+vlan, compare after clear smac */ + ret = memset_s((void *)entry1.smac, sizeof(entry1.smac), 0, + sizeof(entry1.smac)); + if (ret != 0) { + ROCE_ERR("Failed to memset entry1 smac.(ret:%d)", ret); + return -ENOMEM; + } + ret = memset_s((void *)entry2.smac, sizeof(entry2.smac), 0, + sizeof(entry2.smac)); + if (ret != 0) { + ROCE_ERR("Failed to memset entry2.smac.(ret:%d)", ret); + return -ENOMEM; + } + return memcmp((void *)&entry1, (void *)&entry2, + sizeof(struct rdma_gid_entry)); +} + +static int roce5_uni_cmd_reset_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, u32 port, + u32 gid_num) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_clear_gid_s *gid_clear_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_clear_gid_s), NULL, 0); + if (ret != 0) { + ROCE_ERR("Failed to alloc cqm_cmd_inoutbuf, ret(%d)", ret); + return -ENOMEM; + } + + gid_clear_inbuf = (roce_uni_cmd_clear_gid_s *)cqm_cmd_inbuf->buf; + gid_clear_inbuf->gid_clear.gid_num = cpu_to_be32(gid_num); + gid_clear_inbuf->com.index = 0; + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + gid_clear_inbuf->com.opt = cpu_to_be32((u32)port); + + if (rdev->shadow != 0) { + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK); + gid_clear_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rdev->shadow_vfid); + gid_clear_inbuf->gid_clear.dw1.shadow.shadow_vf_num = + rdev->shadow_vf_num; + gid_clear_inbuf->gid_clear.dw1.value = + cpu_to_be32(gid_clear_inbuf->gid_clear.dw1.value); + } + + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Send cmd clear_gid failed, ret(%d)", ret); + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +static int roce5_cmd_reset_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, u32 port, + u32 gid_num) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_cmd_clear_gid_s *gid_clear_inbuf = NULL; + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + return roce5_uni_cmd_reset_gid(rdev, comp_priv, port, gid_num); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(comp_priv->hmm_priv.hwdev, + &cqm_cmd_inbuf, + (u16)sizeof(roce_cmd_clear_gid_s), + NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret(%d)", + ret); + return -ENOMEM; + } + + gid_clear_inbuf = (roce_cmd_clear_gid_s *)cqm_cmd_inbuf->buf; + gid_clear_inbuf->port = cpu_to_be32(port); + gid_clear_inbuf->gid_num = cpu_to_be32(gid_num); + gid_clear_inbuf->com.index = 0; + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Send cmd clear_gid failed, ret(%d)", ret); + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +int roce5_update_gid_mac(struct roce5_device *rdev, u32 port, int *index, + struct rdma_gid_entry *gid_entry) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + u32 gid_index = 0; + int update_index = + -1; /* -1 is initvalue, indicating no need to update */ + int ret = 0; + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_ERR("Hwdev or gid_tbl is null"); + return -EINVAL; + } + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + /* port num index from zero */ + if (port >= comp_priv->rdma_cap.num_ports) { + ROCE_ERR("Input port(%d) is invalid", port); + return -EINVAL; + } + /* invalid gid cause of with the head fe80:: of IPV6 */ + if (cpu_to_be64(gid_entry->global.subnet_prefix) == + RDMA_DEFAULT_GID_SUBNET_PREFIX) { + return 0; + } + gid_table = comp_priv->rdma_comp_res.gid_table; + mutex_lock(&comp_priv->rdma_comp_res.mutex); + /* loop all gid, ensure all gid cleared not exist */ + for (gid_index = 1; gid_index < comp_priv->rdma_cap.max_gid_per_port; + gid_index++) { + if (rdma_gid_entry_cmp(&gid_table[port][gid_index], + gid_entry) == 0) { + update_index = (int)gid_index; + break; + } + } + + if (update_index > 0) { + ret = roce5_cmd_update_gid(rdev, comp_priv, gid_entry, + (int)port, update_index); + if (ret != 0) { + ROCE_ERR("Update gid tbl failed, ret(%d)", ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + /* update gid table after CMDQ */ + ret = memcpy_s((void *)&gid_table[port][update_index], + sizeof(*gid_entry), (void *)gid_entry, + sizeof(*gid_entry)); + if (ret != 0) { + ROCE_ERR( + "Failed to memcpy gid_entry to gid_table.(ret:%d)", + ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return (-ENOMEM); + } + + *index = update_index; + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; + } + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; +} + +int roce5_update_gid(struct roce5_device *rdev, u32 port, u32 update_index, + struct rdma_gid_entry *gid_entry) +{ + int ret = 0; + u32 port_num = 0; + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_ERR("Hwdev or gid_tbl is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + + /* port start from zero */ + port_num = comp_priv->rdma_cap.num_ports; + if (port >= port_num) { + ROCE_DEV_ERR(rdev, "Input port(%d) is invalid", port); + return -EINVAL; + } + + mutex_lock(&comp_priv->rdma_comp_res.mutex); + ret = roce5_cmd_update_gid(rdev, comp_priv, gid_entry, (int)port, + (int)update_index); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Rdma_update_gid failed"); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + + ret = memcpy_s((void *)&gid_table[port][update_index], + sizeof(*gid_entry), (void *)gid_entry, + sizeof(*gid_entry)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy gid_entry.(ret:%d)", ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return -ENOMEM; + } + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + + return 0; +} + +int roce5_reset_gid_table(struct roce5_device *rdev, u32 port) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + int ret = 0; + u32 gids_per_port = 0; + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + ROCE_ERR("Gid_table is null"); + return -EINVAL; + } + + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + + if (gids_per_port == 0) { + ROCE_ERR("Gids_per_port(%d) is invalid", gids_per_port); + return -EINVAL; + } + + ret = memset_s(&gid_table[port][0], + gids_per_port * sizeof(struct rdma_gid_entry), 0, + gids_per_port * sizeof(struct rdma_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_table.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_cmd_reset_gid(rdev, comp_priv, port, gids_per_port); + if (ret != 0) { + ROCE_ERR("Reset gid table failed, gid_num(%d), ret(%d)", + gids_per_port, ret); + return ret; + } + + return 0; +} + +int roce5_get_gid(struct roce5_device *rdev, u32 port, u32 gid_index, + struct rdma_gid_entry *gid) +{ + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret; + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + if (gid == NULL) { + ROCE_ERR("Gid is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + if (port < 1 || port > rdma_cap->num_ports) { + ROCE_ERR("Port(%d) invalid", port); + return -EINVAL; + } + + if (gid_index >= rdma_cap->max_gid_per_port) { + ROCE_ERR("gid_index(%d) invalid", gid_index); + return -EINVAL; + } + + ret = memcpy_s( + (void *)gid, sizeof(*gid), + (void *)&comp_priv->rdma_comp_res.gid_table[port - 1][gid_index], + sizeof(*gid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c new file mode 100644 index 000000000..ca5e88aff --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c @@ -0,0 +1,443 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "roce_infra.h" +#include "hinic5_hmm.h" +#include "securec.h" + +#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 12K */ + +static int rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.num_pds; + reserved_bot = comp_priv->rdma_cap.reserved_pds; + + ret = rdma_bitmap_init(&comp_priv->pd_bitmap, num, num - 1, + reserved_bot, 0); + if (ret != 0) { + ROCE_ERR("Can't initialize pd's bitmap, ret(%d)", ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + int i = 0; + int port_num = 0; + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + ROCE_ERR("Gid_table is null"); + return; + } + + port_num = (int)comp_priv->rdma_cap.num_ports; + for (i = 0; i < port_num; i++) { + if (gid_table[i]) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + } + + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; + + return; +} + +static int roce5_get_gid_nums(struct rdma_comp_priv *comp_priv) +{ + struct hinic5_hwdev *dev = comp_priv->hmm_priv.hwdev; + struct service_cap *cap = &(dev->cfg_mgmt->svc_cap); + struct dev_roce_svc_own_cap *roce_cap = + &(cap->rdma_cap.dev_rdma_cap.roce_own_cap); + + /* 23v100 and earlier generations: The configuration file does not contain max gid. */ + return (roce_cap->max_gid != 0 ? roce_cap->max_gid : + comp_priv->rdma_cap.max_gid_per_port); +} + +static int rdma_init_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + u32 i = 0; + u32 port_num = 0; + u32 gids_per_port = 0; + + port_num = comp_priv->rdma_cap.num_ports; + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + + if ((port_num == 0) || (gids_per_port == 0)) { + ROCE_ERR( + "Alloc memory for gid_tbl failed, port_num(%d), gids_per_ports(%d)", + port_num, gids_per_port); + return -EINVAL; + } + + gid_table = (struct rdma_gid_entry **)kzalloc( + port_num * sizeof(struct rdma_gid_entry *), GFP_KERNEL); + if (gid_table == NULL) { + ROCE_ERR( + "Alloc memory for gid_tbl failed, ret(%d), port_num(%d)", + -ENOMEM, port_num); + return -ENOMEM; + } + + comp_priv->rdma_comp_res.gid_table = gid_table; + + for (i = 0; i < port_num; i++) { + gid_table[i] = (struct rdma_gid_entry *)kzalloc( + gids_per_port * sizeof(struct rdma_gid_entry), + GFP_KERNEL); + if (gid_table[i] == NULL) { + ROCE_ERR( + "Alloc memory for gid_tbl[%d] failed, ret(%d), gids_per_ports(%d)", + i, -ENOMEM, gids_per_port); + goto err_out; + } + } + return 0; +err_out: + for (i = 0; i < port_num; i++) { + if (gid_table[i]) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + } + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; + + return -ENOMEM; +} + +static int rdma_init_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.max_xrcds; + reserved_bot = comp_priv->rdma_cap.reserved_xrcds; + + ret = rdma_bitmap_init(&comp_priv->xrcd_bitmap, num, num - 1, + reserved_bot, 0); + if (ret != 0) { + ROCE_ERR("Can't initialize xrcd's bitmap, ret(%d)", ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->xrcd_bitmap); +} + +static int rdma_init_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 i = 0; + u32 max_order = 0; /* rdmarc buddy max priority num */ + u32 qp_num = 0; + u32 rdmarc_per_qp = 0; /* rdmarc num per qp */ + u32 rdmarc_size = 0; + u32 log_rdmarc_per_seg = 0; /* min num of entry, 2^log_rdmarc_per_seg */ + u32 rdmarc_pow_of_two = 0; + + qp_num = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qps; + rdmarc_per_qp = + comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qp_dest_rdma + + ROCE_MAX_RDMA_RC_EXTEND; + rdmarc_size = + comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz; + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + for (i = 1; i < qp_num * rdmarc_per_qp; i <<= 1) { + max_order++; + } + + max_order = (max_order > log_rdmarc_per_seg) ? + (max_order - log_rdmarc_per_seg) : + 0; + + ret = hmm_buddy_init(&comp_priv->rdmarc_buddy, max_order); + if (ret != 0) { + ROCE_ERR("Initialize rdmarc's buddy failed, ret(%d)", -ENOMEM); + return -ENOMEM; + } + rdmarc_pow_of_two = + roce_roundup_pow_of_two((u32)(qp_num * rdmarc_per_qp)); + ret = hmm_em_init_table(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table, rdmarc_size, + rdmarc_pow_of_two, 0, RDMA_EM_MIN_ORDER); + if (ret != 0) { + ROCE_ERR("Initialize rdmarc's em_table failed, ret(%d)", ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); + + return -ENOMEM; +} + +static int roce5_rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + ret = rdma_init_pd_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize pd's table failed, ret(%d)", ret); + return ret; + } + + return 0; +} + +static int roce5_rdma_init_mtt_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + ret = hmm_mtt_init((struct hmm_comp_priv *)(void *)comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize mtt's table failed, ret(%d)", ret); + return ret; + } + + return 0; +} + +static int roce5_rdma_init_bitmap(struct rdma_comp_priv *comp_priv) +{ + int ret; + ret = roce5_rdma_init_pd_table(comp_priv); + if (ret != 0) { + return ret; + } + + ret = roce5_rdma_init_mtt_table(comp_priv); + if (ret != 0) { + rdma_cleanup_pd_table(comp_priv); + return ret; + } + + return 0; +} + +static void roce5_rdma_unit_bitmap(struct rdma_comp_priv *comp_priv) +{ + hmm_mtt_cleanup((struct hmm_comp_priv *)(void *)comp_priv); + rdma_cleanup_pd_table(comp_priv); +} + +static int roce5_rdma_init_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = roce5_rdma_init_bitmap(comp_priv); + if (ret != 0) { + return ret; + } + + ret = rdma_init_gid_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize gid table failed, ret(%d)", ret); + goto err_init_gid_table; + } + + ret = rdma_init_xrcd_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize xrcd's table failed, ret(%d)", ret); + goto err_init_xrcd; + } + + ret = rdma_init_rdmarc_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize rdmarc's table failed, ret(%d)", ret); + goto err_init_rdmarc_table; + } + + ROCE_INFO("Rdma init resource successful."); + return 0; + +err_init_rdmarc_table: + rdma_cleanup_xrcd_table(comp_priv); +err_init_xrcd: + rdma_cleanup_gid_table(comp_priv); +err_init_gid_table: + roce5_rdma_unit_bitmap(comp_priv); + return ret; +} + +static void rdma_cleanup_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table); + + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); + + return; +} + +static void roce5_rdma_unit_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + if (hinic5_support_roce(hwdev, NULL)) { + rdma_cleanup_rdmarc_table(comp_priv); + rdma_cleanup_xrcd_table(comp_priv); + rdma_cleanup_gid_table(comp_priv); + } + + roce5_rdma_unit_bitmap(comp_priv); +} + +void roce5_rdma_cleanup_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return; + } + + if (!hinic5_support_rdma(hwdev, NULL)) { + ROCE_ERR("Not support rdma service"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + roce5_rdma_unit_table(hwdev, comp_priv); + + kfree(comp_priv); + + hinic5_unregister_service_adapter((void *)hwdev, SERVICE_T_ROCE); + + ROCE_INFO("Rdma cleanup resource successful."); + + return; +} + +static int roce5_rdma_init_comp_priv(struct rdma_comp_priv *comp_priv, + void *hwdev, + struct rdma_service_cap *rdma_cap) +{ + int ret; + + mutex_init(&comp_priv->rdma_comp_res.mutex); + comp_priv->hmm_priv.hwdev = hwdev; + + comp_priv->hmm_priv.dev = + (struct device *)(((struct hinic5_hwdev *)hwdev)->dev_hdl); + if (comp_priv->hmm_priv.dev == NULL) { + ROCE_ERR("Failed to init, device is NULL."); + return -EINVAL; + } + + ret = memcpy_s((void *)&comp_priv->rdma_cap, + sizeof(struct rdma_service_cap), (void *)rdma_cap, + sizeof(struct rdma_service_cap)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy rdma_cap.(ret:%d)", ret); + return -ENOMEM; + } + //to adapt hmm struct + comp_priv->hmm_priv.dev_cap.log_mtt = rdma_cap->log_mtt; + comp_priv->hmm_priv.dev_cap.num_mtts = rdma_cap->num_mtts; + comp_priv->hmm_priv.dev_cap.log_mtt_seg = rdma_cap->log_mtt_seg; + comp_priv->hmm_priv.dev_cap.mtt_entry_sz = rdma_cap->mtt_entry_sz; + comp_priv->hmm_priv.dev_cap.mpt_entry_sz = rdma_cap->mpt_entry_sz; + comp_priv->hmm_priv.dev_cap.dmtt_cl_start = rdma_cap->dmtt_cl_start; + comp_priv->hmm_priv.dev_cap.dmtt_cl_end = rdma_cap->dmtt_cl_end; + comp_priv->hmm_priv.dev_cap.dmtt_cl_sz = rdma_cap->dmtt_cl_sz; + + switch (g_mtt_page_size) { + case ROCE5_RDMA_MTT_PAGE_SIZE_4K: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + case ROCE5_RDMA_MTT_PAGE_SIZE_64K: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_64K; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_64K_SHIFT; + break; + case ROCE5_RDMA_MTT_PAGE_SIZE_2M: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_2M; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_2M_SHIFT; + break; + default: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + } + + /* Recalculate the gid num. */ + comp_priv->rdma_cap.max_gid_per_port = + (u32)roce5_get_gid_nums(comp_priv); + + ret = roce5_rdma_init_table(hwdev, comp_priv); + if (ret != 0) { + return ret; + } + + ret = hinic5_register_service_adapter((void *)hwdev, (void *)comp_priv, + SERVICE_T_ROCE); + if (ret != 0) { + roce5_rdma_unit_table(hwdev, comp_priv); + ROCE_ERR("put rdma_comp_res failed, ret(%d)", ret); + return ret; + } + + return 0; +} + +int roce5_rdma_init_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + if (hinic5_support_rdma(hwdev, &rdma_cap) == false) { + ROCE_INFO("Neither ROCE nor IWARP"); + return 0; + } + + comp_priv = (struct rdma_comp_priv *)kzalloc( + sizeof(struct rdma_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) { + ROCE_ERR("Alloc memory for comp_priv failed, ret(%d)", -ENOMEM); + return -ENOMEM; + } + + ret = roce5_rdma_init_comp_priv(comp_priv, hwdev, &rdma_cap); + if (ret != 0) { + kfree(comp_priv); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c new file mode 100644 index 000000000..aa252e2db --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "roce_cqm_cmd.h" +#include "hmm_common.h" +#include "securec.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_main_extension.h" +#include "roce_compat.h" +#include "roce_mr.h" +#include "rdma_comp_mw_mr.h" + +static int rdma_mpt_alloc_rsvd_lkey(struct rdma_comp_priv *comp_priv, + struct mpt *mpt) +{ + u32 mpt_entry_size = 0; + + mpt_entry_size = comp_priv->hmm_priv.dev_cap.mpt_entry_sz; + + /* call cqm interface allocate mpt, and mpt_index and vaddr save in rdma_mpt structure */ + mpt->mpt_object = (void *)cqm5_object_qpc_mpt_create( + comp_priv->hmm_priv.hwdev, RDMA_SERVICE_TYPE_ROCE, + CQM_OBJECT_MPT, mpt_entry_size, mpt, mpt->mpt_index, 0, 0); + if (mpt->mpt_object == NULL) { + ROCE_ERR("Alloc mpt_object failed, err(%d)", -ENOMEM); + return -ENOMEM; + } + + mpt->vaddr = (void *)((cqm_qpc_mpt_s *)mpt->mpt_object)->vaddr; + + return 0; +} + +int roce5_rdma_init_rsvd_lkey(struct roce5_device *rdev) +{ + struct hmm_rdma *mr = NULL; + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (rdev == NULL) { + ROCE_ERR("rdev is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + mr = &comp_priv->rsvd_lkey; + mr->mpt.mpt_index = rdma_cap->reserved_lkey >> MR_KEY_LEFT_SHIFT_OFS; + if (mr->mpt.mpt_index >= rdma_cap->reserved_mrws) { + ROCE_ERR("Cfg err, reserved_lkey(0x%x), reserved_mrws(0x%x)", + rdma_cap->reserved_lkey, rdma_cap->reserved_mrws); + return -EINVAL; + } + /* Alloc MPT */ + ret = rdma_mpt_alloc_rsvd_lkey(comp_priv, &mr->mpt); + if (ret != 0) { + ROCE_ERR("Rdma_mpt_alloc_rsvd_lkey failed, ret(%d)", ret); + return ret; + } + mr->mtt.mtt_layers = 0; + mr->iova = 0; + mr->size = ~0ULL; + mr->key = comp_priv->rdma_cap.reserved_lkey; + mr->type = roce5_mr_type2hmm(RDMA_RSVD_LKEY); + mr->access = RDMA_IB_ACCESS_LOCAL_WRITE; + /* Enable MPT */ + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, mr, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Rdma_enable_mr_mpt failed, ret(%d)", ret); + goto err_out; + } + return 0; +err_out: + hmm_mpt_free(rdev->hwdev, &mr->mpt); + return ret; +} + +void roce5_rdma_free_rsvd_lkey(struct roce5_device *rdev) +{ + struct hmm_rdma *mr = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (rdev == NULL) { + ROCE_ERR("rdev is null"); + return; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + mr = &comp_priv->rsvd_lkey; + ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, mr, + RDMA_SERVICE_TYPE_ROCE, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Disable mpt of mr failed, ret(%d)", ret); + return; + } + + hmm_mpt_free(rdev->hwdev, &mr->mpt); + + return; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h new file mode 100644 index 000000000..1e9c1132d --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h @@ -0,0 +1,12 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef RDMA_COMP_MW_MR_H +#define RDMA_COMP_MW_MR_H + +#include "roce.h" + +int roce5_rdma_init_rsvd_lkey(struct roce5_device *rdev); +void roce5_rdma_free_rsvd_lkey(struct roce5_device *rdev); + +#endif /* RDMA_COMP_MW_MR_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c new file mode 100644 index 000000000..ee3486e97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c @@ -0,0 +1,51 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" + +int roce5_rdma_pd_alloc(void *hwdev, u32 *pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (pdn == NULL)) { + ROCE_ERR("Hwdev or pdn is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + *pdn = rdma_bitmap_alloc(&comp_priv->pd_bitmap); + if (*pdn == RDMA_INVALID_INDEX) { + ROCE_ERR("Can't get valid pdn, err(%d)", -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +void roce5_rdma_pd_free(void *hwdev, u32 pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + rdma_bitmap_free(&comp_priv->pd_bitmap, pdn); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c new file mode 100644 index 000000000..d1265e54e --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c @@ -0,0 +1,140 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "hmm_em_inner.h" +#include "hmm_buddy.h" + +static int roce5_rdma_rdmarc_alloc_check(struct rdma_comp_priv **comp_priv, + void *hwdev, + const struct rdma_rdmarc *rdmarc, + u32 *log_rdmarc_per_seg, u32 num) +{ + if ((hwdev == NULL) || (rdmarc == NULL)) { + ROCE_ERR("Hwdev or rdmarc is null"); + return -EINVAL; + } + + *comp_priv = get_rdma_comp_priv(hwdev); + if (*comp_priv == NULL || (*comp_priv)->hmm_priv.dev == NULL) { + ROCE_ERR("Comp_priv(%d) or PCI device is NULL.", + !!(*comp_priv)); + return -EINVAL; + } + + *log_rdmarc_per_seg = (*comp_priv)->rdma_cap.log_rdmarc_seg; + + if ((num < ROCE5_RDMARC_MIN_DEPTH) || (num > ROCE5_RDMARC_MAX_DEPTH)) { + ROCE_ERR("Num is invalid, ret(%d)", -EINVAL); + return -EINVAL; + } + + return 0; +} + +static void roce5_rdma_rdmarc_order_set(struct rdma_rdmarc *rdmarc, u32 order, + u32 ext_order, u32 log_rdmarc_per_seg) +{ + rdmarc->order = order; + rdmarc->ext_order = ext_order + log_rdmarc_per_seg; + return; +} + +int roce5_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + u32 i = 0; + u32 order = 0; + u32 ext_order = 0; + u32 ext_num = num + ROCE5_RDMARC_EXT_ENTRY; + u32 offset = 0; + void *vaddr = NULL; + u32 log_rdmarc_per_seg = 0; + u32 chip_num = (num < ROCE5_RDMARC_MIN_ENTRY) ? ROCE5_RDMARC_MIN_ENTRY : + num; + ret = roce5_rdma_rdmarc_alloc_check(&comp_priv, hwdev, rdmarc, + &log_rdmarc_per_seg, num); + if (ret != 0) { + return ret; + } + for (i = 1; i < chip_num; i <<= 1) { + order++; + } + for (i = 1; i < ext_num; i <<= 1) { + ext_order++; + } + ext_order = (ext_order > log_rdmarc_per_seg) ? + (ext_order - log_rdmarc_per_seg) : + 0; + roce5_rdma_rdmarc_order_set(rdmarc, order, ext_order, + log_rdmarc_per_seg); + offset = hmm_buddy_alloc(&comp_priv->rdmarc_buddy, ext_order); + if (offset == RDMA_INVALID_INDEX) { + ROCE_ERR("Alloc rdmarc index failed, ret(%d)", -ENOMEM); + return -ENOMEM; + } + rdmarc->offset = offset << log_rdmarc_per_seg; + + ret = hmm_em_table_get_range( + comp_priv->hmm_priv.dev, &comp_priv->rdmarc_em_table, + rdmarc->offset, rdmarc->offset + (1U << rdmarc->ext_order) - 1); + if (ret != 0) { + ROCE_ERR("Alloc rdmarc entry failed, ret(%d)", -ENOMEM); + goto err_table_get; + } + vaddr = hmm_em_table_find(&comp_priv->rdmarc_em_table, rdmarc->offset, + &rdmarc->dma_addr); + if (vaddr == NULL) { + ret = -ENOMEM; + ROCE_ERR("Can't find va and pa of rdmarc entry, ret(%d)", + -ENOMEM); + goto err_rdmarc_find; + } + rdmarc->vaddr = vaddr; + return 0; + +err_rdmarc_find: + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); +err_table_get: + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, ext_order); + return ret; +} + +void roce5_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + u32 order = 0; + u32 offset = 0; + u32 log_rdmarc_per_seg = 0; + + if ((hwdev == NULL) || (rdmarc == NULL)) { + ROCE_ERR("Hwdev or rdmarc is null"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL || comp_priv->hmm_priv.dev == NULL) { + ROCE_ERR("Comp_priv(%d) or comp_priv->hmm_priv.dev is NULL", + !!comp_priv); + return; + } + + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); + + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + + order = rdmarc->ext_order - log_rdmarc_per_seg; + offset = rdmarc->offset >> log_rdmarc_per_seg; + + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, order); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h new file mode 100644 index 000000000..f268757a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h @@ -0,0 +1,78 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_COM_H +#define ROCE_COM_H + +#include "roce_cfg_format.h" + +/* kernel mode basic capability negotiation support macro */ +#define ROCE5_SUPPORT_IB_FEATURE(rdev, feature) \ + ((rdev)->cfg_info.feature_cap.roce_kernel_ability & \ + (ROCE_SUPPORT_IB_##feature)) + +#define ROCE5_SUPPORT_IB_CMD_RESIZE_CQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_RESIZE_CQ_MSG_V2) +#define ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_QUERY_QP_MSG_V2) +#define ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_MODIFY_SRQ_MSG_V2) +#define ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_DESTROY_SRQ_MSG_V2) + +/* kernel mode extended capability negotiation support macro */ +#define ROCE5_SUPPORT_FEATURE(rdev, feature) \ + ((rdev)->cfg_info.feature_cap.roce_kernel_ext_ability & \ + (ROCE_SUPPORT_##feature)) + +#define ROCE5_SUPPORT_CQ_MODIFY_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CQ_MODIFY_MSG_V2) +#define ROCE5_SUPPORT_CQ_CHECK_DATA_STATE_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CQ_CHECK_DATA_STATE_MSG_V2) +#define ROCE5_SUPPORT_CQ_CACHE_OUT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CQ_CACHE_OUT_MSG_V2) +#define ROCE5_SUPPORT_QP_SRQN_LB_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, QP_SRQN_LB_MSG_V2) +#define ROCE5_SUPPORT_SET_PI_ON_CHIP_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SET_PI_ON_CHIP_MSG_V2) +#define ROCE5_SUPPORT_SQ_DB_TYPE_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SQ_DB_TYPE_MSG_V2) +#define ROCE5_SUPPORT_GET_QP_RX_PORT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, GET_QP_RX_PORT_MSG_V2) +#define ROCE5_SUPPORT_MODIFY_UDP_SRC_PORT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, MODIFY_UDP_SRC_PORT_MSG_V2) +#define ROCE5_SUPPORT_GET_QP_UDP_SRC_PORT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, GET_QP_UDP_SRC_PORT_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_MPT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_MPT_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_GID_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_GID_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_PI_CI_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_PI_CI_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_CQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_CQ_MSG_V2) +#define ROCE5_SUPPORT_ALLOC_QPC_MSG_FEATURE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, ALLOC_QPC_MSG_FEATURE) +#define ROCE5_SUPPORT_GET_OQID_MSG_FEATURE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, GET_OQID_MSG_FEATURE) +#define ROCE5_SUPPORT_RETRY_LOAD_CQC_MSG_FEATURE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, RETRY_LOAD_CQC_MSG_FEATURE) +#define ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SET_HASH_VALUE_MSG_V2) +#define ROCE5_SUPPORT_CC_GET_DCQCN_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_DCQCN_ENABLE) +#define ROCE5_SUPPORT_CC_GET_LDCP_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_LDCP_ENABLE) +#define ROCE5_SUPPORT_CC_GET_IPQCN_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_IPQCN_ENABLE) +#define ROCE5_SUPPORT_CC_GET_USER_ALGO_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_USER_ALGO_ENABLE) +#define ROCE5_SUPPORT_CC_PARAM_V1(rdev) ROCE5_SUPPORT_FEATURE(rdev, CC_PARAM_V1) +#define ROCE5_SUPPORT_CC_PARAM_V2(rdev) ROCE5_SUPPORT_FEATURE(rdev, CC_PARAM_V2) +#define ROCE5_SUPPORT_8K_MTU(rdev) ROCE5_SUPPORT_FEATURE(rdev, KERNEL_8K_MTU) +#define ROCE5_SUPPORT_SML_MAC_TABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SML_MAC_TABLE) +#define ROCE5_SUPPORT_SQ_DB_TYPE_21(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SQ_DB_TYPE_21) +#endif /* ROCE_COM_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c new file mode 100644 index 000000000..a15e865ec --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c @@ -0,0 +1,1594 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <rdma/ib_mad.h> +#include <rdma/ib_verbs.h> +#include <linux/pci.h> +#include <linux/types.h> + +#include "securec.h" /* Huawei secure functions; */ + +#include "hinic5_crm.h" +#include "hinic5_srv_nic.h" +#include "hinic5_lld.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_pd.h" +#include "roce_qp.h" +#include "roce_cmd.h" +#include "roce_gid.h" +#include "roce_netdev.h" +#include "roce_main_extension.h" +#include "roce_qp_extension.h" +#include "roce_restore.h" +#include "roce_infra.h" +#include "roce_ctx_format.h" +#include "roce_kernel_compat_gen.h" +#include "roce5_gid_type.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif +#include "roce_mix.h" + +uint8_t roce5_get_db_cos_from_vlan_pri(struct roce5_device *rdev, + uint8_t vlan_pri) +{ + uint8_t db_cos; + int ret; + + ret = hinic5_get_cos_by_pri(rdev->hwdev, vlan_pri, &db_cos); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "ret:%d vlan_pri:%u", ret, vlan_pri); + db_cos = 0; + } + + return db_cos; +} + +struct net_device *roce5_ib_get_netdev(struct ib_device *ibdev, u32 port_num) +{ + struct roce5_device *rdev = NULL; + struct net_device *netdev = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return NULL; + } + + rdev = to_roce5_dev(ibdev); + +#ifdef ROCE_BONDING_EN + netdev = roce5_bond_get_netdev(rdev); + if (netdev != NULL) { + return netdev; + } +#endif + + netdev = rdev->ndev; + + dev_hold(netdev); + + return netdev; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +struct net_device *roce5_ib_get_netdev_compat(struct ib_device *ibdev, + u8 port_num) +{ + return roce5_ib_get_netdev(ibdev, (u32)port_num); +} + +static void roce5_parse_fw_version(struct roce5_device *rdev, u64 *fw_ver) +{ + int ret; + int i = 0; + struct hinic5_fw_version fw_version; + char *fw_str = (char *)fw_version.microcode_ver; + char *fw_temp = NULL; + u64 fw_verion[ROCE_FW_VERSION_LEN] = { 0 }; + + ret = hinic5_get_fw_version(rdev->hwdev, &fw_version, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_WARN("get fw version failed"); + *fw_ver = ROCE_FW_VER; + return; + } + ROCE_INFO("fw ver:%s - %s - %s", fw_version.boot_ver, + fw_version.mgmt_ver, fw_version.microcode_ver); + + while (((fw_temp = strsep(&fw_str, ".")) != NULL) && + (i < ROCE_FW_VERSION_LEN)) { + ret = sscanf_s(fw_temp, "%llu", &fw_verion[i]); + if (ret != 1) { + ROCE_WARN("parse fw version failed ret(%d)", ret); + *fw_ver = ROCE_FW_VER; + return; + } + + i++; + } + + *fw_ver = (((fw_verion[0] & 0xffffffff) + << 32) | // 0 is fw_version array idx, 32 is offset + ((fw_verion[1] & 0xffff) + << 16) | // 1 is fw_version array idx, 16 is offset + ((fw_verion[2] & 0xff) + << 8) | // 2 is fw_version array idx, 8 is offset + (fw_verion[3] & 0xff)); // 3 is fw_version array idx +} + +static void roce5_set_local_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if (((rdma_cap->flags & RDMA_BMME_FLAG_LOCAL_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_REMOTE_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_FAST_REG_WR) != 0)) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_MGT_EXTENSIONS; + } +} + +static void roce5_set_bmme_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if ((rdma_cap->flags & RDMA_BMME_FLAG_TYPE_2_WIN) != 0) { + if ((rdma_cap->flags & RDMA_BMME_FLAG_WIN_TYPE_2B) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2B; + } else { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2A; + } + } +} + +static void roce5_query_device_props_set(struct roce5_device *rdev, + struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if (rdev->lld_dev->dev_type == HINIC5_DEVICE_T_PCI) { + struct pci_dev *pdev = + container_of(rdev->dev, struct pci_dev, dev); + props->vendor_id = pdev->vendor; + props->vendor_part_id = pdev->device; + } + roce5_parse_fw_version(rdev, &props->fw_ver); + props->hw_ver = ROCE_HW_VER; + + /* sys_image_guid equal GID */ + props->sys_image_guid = rdev->ib_dev.node_guid; + + props->max_mr_size = ~0ULL; + props->page_size_cap = rdma_cap->page_size_cap; + props->max_qp = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_qps - + rdma_cap->reserved_qps); + props->max_qp_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes; + /* We should ensure sq's max size(max_sq_sg) as the max_sge */ + props->max_send_sge = (int)rdma_cap->max_sq_sg; + props->max_recv_sge = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg; + props->max_cq = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs - + rdma_cap->reserved_cqs); + props->max_cqe = (int)rdma_cap->max_cqes; + props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - + rdma_cap->reserved_mrws); + props->max_mw = props->max_mr; + props->max_pd = (int)(rdma_cap->num_pds - rdma_cap->reserved_pds); + props->max_qp_rd_atom = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma; + props->max_qp_init_rd_atom = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma; + props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; + props->max_srq = + (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - + rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs); + props->max_srq_wr = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_wqes; + props->max_srq_sge = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_sge; + props->max_fast_reg_page_list_len = rdma_cap->max_frpl_len; + props->local_ca_ack_delay = (u8)rdma_cap->local_ca_ack_delay; + props->atomic_cap = + ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_ATOMIC) != 0) ? + IB_ATOMIC_HCA : + IB_ATOMIC_NONE; + props->masked_atomic_cap = props->atomic_cap; + props->max_pkeys = (u16)rdma_cap->max_pkeys; + props->max_sge_rd = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg; +#if defined(IB_DEV_ATTR_HAVE_MAX_FMR) + props->max_fmr = props->max_map_per_fmr; +#endif +#if defined(IB_DEV_ATTR_HAVE_MAX_MAP_PER_FMR) + props->max_map_per_fmr = (int)rdma_cap->max_fmr_maps; +#endif + props->max_ah = INT_MAX; + return; +} +/* **************************************************************************** + Prototype : roce5_query_device + Description : query device attribute + Input : struct ib_device *ibdev + struct ib_device_attr *props + struct ib_udata *uhw + Output : struct ib_device_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_query_device(struct ib_device *ibdev, struct ib_device_attr *props, + struct ib_udata *uhw) +{ + struct roce5_device *rdev = NULL; + struct rdma_service_cap *rdma_cap = NULL; + int ret; + + if ((ibdev == NULL) || (props == NULL)) { + ROCE_ERR("Ibdev or props is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + ret = memset_s(props, sizeof(*props), 0, sizeof(*props)); + if (ret != 0) { + ROCE_ERR("Failed to memset props.(ret:%d)", ret); + return -ENOMEM; + } + + rdma_cap = &rdev->rdma_cap; + + props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | + IB_DEVICE_RC_RNR_NAK_GEN; + + /* APM */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_APM) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_AUTO_PATH_MIG; + } + + /* rsvd_lKey */ + if ((rdma_cap->flags & RDMA_BMME_FLAG_RESERVED_LKEY) != 0) { +#if defined(HAVE_IB_DEVICE_LOCAL_DMA_LKEY) + props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; +#elif defined(HAVE_IB_KERNEL_CAP_FLAGS) + props->kernel_cap_flags |= IBK_LOCAL_DMA_LKEY; +#endif + } + + roce5_set_local_cap_flag(rdma_cap, props); + + /* support XRC */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_XRC) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_XRC; + } + + /* support MW */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_MEM_WINDOW) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW; + } + + roce5_set_bmme_cap_flag(rdma_cap, props); + + roce5_query_device_props_set(rdev, rdma_cap, props); + return 0; +} + +static void roce5_eth_link_get_speed(struct ib_port_attr *props, + enum mag_cmd_port_speed speed) +{ + switch (speed) { + /* 10G <==> 1X x 10G */ + case PORT_SPEED_10GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 25G <==> 1X x 25G */ + case PORT_SPEED_25GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 40G <==> 4X x 10G */ + case PORT_SPEED_40GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 100G <==> 4X x 25G */ + case PORT_SPEED_100GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 200G <==> 8X x 25G */ + case PORT_SPEED_200GB: + props->active_width = IB_WIDTH_8X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 400G <==> 8X x 50G */ + case PORT_SPEED_400GB: + props->active_width = IB_WIDTH_8X; + props->active_speed = IB_SPEED_HDR; + break; + + default: + props->active_width = 0; + props->active_speed = 0; + break; + } +} + +static void roce5_set_ib_port_attr(struct ib_port_attr *props, + struct roce5_device *rdev) +{ + struct rdma_comp_priv *comp_priv = get_rdma_comp_priv(rdev->hwdev); + + if (comp_priv == NULL) { + ROCE_ERR("comp_priv is NULL"); + return; + } + + props->port_cap_flags = IB_PORT_CM_SUP; + props->gid_tbl_len = (int)comp_priv->rdma_cap.max_gid_per_port; + props->max_msg_sz = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz; + props->pkey_tbl_len = (u16)rdev->rdma_cap.max_pkeys; + props->max_mtu = (enum ib_mtu)((ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? + ROCE_MTU_8192 : + ROCE_MTU_4096); + props->state = IB_PORT_DOWN; + props->phys_state = ROCE_PORT_PHYS_STATE_DISABLED; + props->active_mtu = IB_MTU_256; +} + +static inline int roce5_get_port_speed_by_rdev(struct roce5_device *rdev, + enum mag_cmd_port_speed *speed) +{ + enum mag_cmd_port_speed speed_table[] = { + [ROCE5_4_25G_HCA] = PORT_SPEED_25GB, + [ROCE5_2_25G_HCA] = PORT_SPEED_25GB, + [ROCE5_1_100G_HCA] = PORT_SPEED_100GB, + [ROCE5_2_100G_HCA] = PORT_SPEED_100GB, + [ROCE5_2_200G_HCA] = PORT_SPEED_200GB, + [ROCE5_2_400G_HCA] = PORT_SPEED_400GB, + [ROCE5_4_200G_HCA] = PORT_SPEED_200GB, + }; + int hca_type = rdev->hw_info.hca_type; + + if (hca_type >= ARRAY_SIZE(speed_table) || + speed_table[hca_type] == PORT_SPEED_NOT_SET) { + return -EINVAL; + } + + *speed = speed_table[hca_type]; + return 0; +} + +static void roce5_eth_link_query_port(struct ib_device *ibdev, + struct ib_port_attr *props) +{ + struct roce5_device *rdev = NULL; + struct net_device *netdev = NULL; +#ifdef ROCE_BONDING_EN + struct net_device *upper = NULL; +#endif + enum roce5_mtu rmtu; + enum mag_cmd_port_speed speed = PORT_SPEED_10GB; + int ret = 0; + + rdev = to_roce5_dev(ibdev); + + roce5_set_ib_port_attr(props, rdev); + + ret = roce5_get_port_speed_by_rdev(rdev, &speed); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to get speed, func_id(%d), hca_type(%d)", + rdev->glb_func_id, rdev->hw_info.hca_type); + props->active_width = 0; + props->active_speed = 0; + return; + } + + roce5_eth_link_get_speed(props, speed); + + netdev = roce5_ib_get_netdev(ibdev, ROCE_DEFAULT_PORT_NUM); + if (netdev == NULL) { + return; + } + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + rcu_read_lock(); + upper = netdev_master_upper_dev_get_rcu(netdev); + if (upper != NULL) { + dev_put(netdev); + netdev = upper; + dev_hold(netdev); + } + rcu_read_unlock(); + } +#endif + + if (netif_running(netdev) && netif_carrier_ok(netdev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = ROCE_PORT_PHYS_STATE_LINKUP; + } + + rmtu = (ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? + roce5_get_mtu((int)netdev->mtu) : + (enum roce5_mtu)iboe_get_mtu((int)netdev->mtu); + + dev_put(netdev); + + props->active_mtu = + (enum ib_mtu)ROCE_MIN((enum roce5_mtu)props->max_mtu, rmtu); +} + +/* **************************************************************************** + Prototype : roce5_query_port + Description : query port attribute + Input : struct ib_device *ibdev + u8 port + struct ib_port_attr *props + Output : struct ib_port_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props) +{ + int ret; + if ((ibdev == NULL) || (props == NULL)) { + ROCE_ERR("Ibdev or props is null"); + return -EINVAL; + } + + ret = memset_s(props, sizeof(*props), 0, sizeof(*props)); + if (ret != 0) { + ROCE_ERR("Failed to memset props.(ret:%d)", ret); + return -ENOMEM; + } + + roce5_eth_link_query_port(ibdev, props); + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_query_port_compat(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) +{ + return roce5_query_port(ibdev, (u32)port, props); +} + +/* **************************************************************************** + Prototype : roce5_query_gid + Description : query gid + Input : struct ib_device *ibdev + u8 port + int index + union ib_gid *gid + Output : union ib_gid *gid + + 1.Date : 2015/5/8 + Modification : Created function + + 2.Date : 2015/6/8 + Modification : Modify function + +**************************************************************************** */ +int roce5_query_gid(struct ib_device *ibdev, u32 port, int index, + union ib_gid *gid) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + struct rdma_gid_entry gid_entry; + + if ((ibdev == NULL) || (gid == NULL)) { + ROCE_ERR("Ibdev or gid is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + ret = roce5_get_gid(rdev, port, (u32)index, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = memcpy_s((void *)gid->raw, sizeof(*gid), (void *)gid_entry.raw, + sizeof(*gid)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy gid.(ret:%d)", ret); + return -ENOMEM; + } + + // per per OFED of gid generation method convert GID, only IPv4 scenario need convert + if (gid_entry.dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + // not add of gid returns directly, no need to convert + if ((gid->global.subnet_prefix == 0) && + (gid->global.interface_id == 0)) { + return 0; + } + ipv6_addr_set_v4mapped(*((u32 *)(void *)gid + ROCE_GID_IP_IDX), + (struct in6_addr *)gid); + } + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_query_gid_compat(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid) +{ + return roce5_query_gid(ibdev, (u32)port, index, gid); +} + +/* **************************************************************************** + Prototype : roce5_query_pkey + Description : query pkey + Input : struct ib_device *ibdev + u8 port + u16 index + u16 *pkey + Output : u16 *pkey + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) +{ + struct roce5_device *rdev = NULL; + if ((ibdev == NULL) || (pkey == NULL)) { + ROCE_ERR("Ibdev or pkey is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + if (pkey == NULL) { + ROCE_ERR("Pkey is null"); + return -EINVAL; + } + + *pkey = 0xffff; + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_query_pkey_compat(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey) +{ + return roce5_query_pkey(ibdev, (u32)port, index, pkey); +} + +/* **************************************************************************** + Prototype : roce5_modify_device + Description : modify device attribute + Input : struct ib_device *ibdev + int mask + struct ib_device_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_device(struct ib_device *ibdev, int mask, + struct ib_device_modify *props) +{ + unsigned long flags = 0; /* type required by kernel API */ + struct roce5_device *rdev = NULL; + int ret; + + if ((ibdev == NULL) || (props == NULL)) { + ROCE_ERR("Ibdev or props is null"); + return -EINVAL; + } + + if (((unsigned int)mask & ~IB_DEVICE_MODIFY_NODE_DESC) != 0) { + ROCE_ERR("Not supported to modify node description"); + return -EOPNOTSUPP; + } + + if ((((u32)mask) & IB_DEVICE_MODIFY_NODE_DESC) == 0) { + ROCE_INFO("No need to modify node description"); + return 0; + } + + rdev = to_roce5_dev(ibdev); + + spin_lock_irqsave(&rdev->node_desc_lock, flags); + ret = memcpy_s((void *)ibdev->node_desc, IB_DEVICE_NODE_DESC_MAX, + (void *)props->node_desc, IB_DEVICE_NODE_DESC_MAX); + if (ret != 0) { + ROCE_ERR("Failed to memcpy node_desc.(ret:%d)", ret); + spin_unlock_irqrestore(&rdev->node_desc_lock, flags); + return -ENOMEM; + } + spin_unlock_irqrestore(&rdev->node_desc_lock, flags); + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_modify_port + Description : modify port attribute + Input : struct ib_device *ibdev + u8 port + int mask + struct ib_port_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_port(struct ib_device *ibdev, u32 port, int mask, + struct ib_port_modify *props) +{ + int ret = 0; + struct ib_port_attr attr; + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + ret = memset_s(&attr, sizeof(struct ib_port_attr), 0, + sizeof(struct ib_port_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset attr.(ret:%d)", ret); + return -ENOMEM; + } + + mutex_lock(&rdev->cap_mask_mutex); + + ret = roce5_query_port(ibdev, port, &attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query port, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + mutex_unlock(&rdev->cap_mask_mutex); + + return ret; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_modify_port_compat(struct ib_device *ibdev, u8 port, int mask, + struct ib_port_modify *props) +{ + return roce5_modify_port(ibdev, (u32)port, mask, props); +} + +static int roce5_alloc_ucontext_set(struct roce5_device *rdev, + struct roce5_alloc_ucontext_resp *resp) +{ + struct rdma_service_cap *rdma_cap = NULL; + + rdma_cap = &rdev->rdma_cap; + + resp->num_qps = rdma_cap->dev_rdma_cap.roce_own_cap.max_qps; + resp->num_xsrqs = rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs; + resp->cqe_size = rdma_cap->cqe_size; + resp->wqebb_size = rdma_cap->wqebb_size; + resp->dwqe_size = rdma_cap->direct_wqe_size; + resp->max_msg_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_msg_sz; + resp->max_comp_vector = rdma_cap->num_comp_vectors; + resp->max_inline_size = + rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + + resp->storage_aa_en = 0; + resp->phy_port = rdev->hw_info.phy_port; + resp->srq_container_en = rdev->cfg_info.srq_container_en; + resp->srq_container_mode = rdev->cfg_info.srq_container_mode; + resp->xrc_srq_container_mode = rdev->cfg_info.xrc_srq_container_mode; + resp->warn_th = rdev->cfg_info.warn_th; + resp->func_id = rdev->glb_func_id; + resp->pcie_atomic_en = rdev->cfg_info.pcie_atomic_en; + resp->roce_user_ext_ability = + rdev->cfg_info.feature_cap.roce_user_ext_ability; + resp->roce_uld_ext_ability = + rdev->cfg_info.feature_cap.uld_feature.roce_uld_user_ext_ability; + resp->max_send_sge = (uint8_t)rdev->rdma_cap.max_sq_sg; + resp->max_recv_sge = + (uint8_t)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg; + resp->max_sq_wqe_len = rdev->rdma_cap.max_sq_desc_sz; + resp->huge_page_en = rdev->huge_page_en; + return hinic5_get_chip_node_id(rdev->lld_dev, &(resp->card_id)); +} + +static int roce5_alloc_ucontext_pre_check(struct ib_device *ibdev, + const struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + + if ((ibdev == NULL) || (udata == NULL)) { + ROCE_ERR("Ibdev or udata is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + if (!rdev->ib_active) { + ROCE_DEV_ERR(rdev, "Device is abnormal, func_id(%d)", + rdev->glb_func_id); + return -EAGAIN; + } + + return 0; +} + +static int roce5_alloc_ucontext_return(struct roce5_device *rdev, + struct ib_udata *udata, + struct roce5_ucontext *context, + struct roce5_alloc_ucontext_resp *resp) +{ + int ret; + + resp->db_offset = context->db_dma_addr & ((1 << PAGE_SHIFT) - 1); +#ifdef ROCE_DWQE_EN + resp->dwqe_offset = context->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1); + + if (context->dwqe_dma_addr == 0) { + ROCE_DEV_ERR(rdev, "dwqe_dma_addr = 0"); + resp->dwqe_size = 0; + } +#else + resp->dwqe_size = 0; +#endif + + INIT_LIST_HEAD(&context->db_page_list); + mutex_init(&context->db_page_mutex); + + /* Copy data to user space */ + ret = ib_copy_to_udata(udata, resp, + sizeof(struct roce5_alloc_ucontext_resp)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to copy data to user space, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + return 0; +} + +int roce5_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) +{ + int ret; + + struct roce5_ucontext *context = to_roce5_ucontext(ibucontext); + struct roce5_device *rdev = to_roce5_dev(ibucontext->device); + struct roce5_alloc_ucontext_resp *resp = NULL; + + ret = roce5_alloc_ucontext_pre_check(ibucontext->device, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "pre check err, ret(%d), func_id(%d)", ret, + rdev->glb_func_id); + goto err; + } + + resp = kzalloc(sizeof(struct roce5_alloc_ucontext_resp), GFP_KERNEL); + if (resp == NULL) { + ret = (-ENOMEM); + ROCE_DEV_ERR(rdev, "Failed alloc ucontext, func_id(%d)", + rdev->glb_func_id); + goto err; + } + + ret = roce5_alloc_ucontext_set(rdev, resp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get chip node id, func_id(%d)", + rdev->glb_func_id); + goto err_db; + } + /* Alloc user space context Doorbell and DWQE */ + ret = hinic5_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr, + &context->dwqe_dma_addr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc DB pa, ret(%d), func_id(%d)", ret, + rdev->glb_func_id); + goto err_db; + } + + /* Copy data to user space */ + ret = roce5_alloc_ucontext_return(rdev, udata, context, resp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to return to user, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto err_return; + } + + kfree(resp); +#ifdef __ROCE_DFX__ + rdev->ucontext = context; +#endif + return 0; + +err_return: + hinic5_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, + context->dwqe_dma_addr); +err_db: + kfree(resp); +err: + return ret; +} + +void roce5_dealloc_ucontext(struct ib_ucontext *ibcontext) +{ + struct roce5_ucontext *context = NULL; + struct roce5_device *rdev = NULL; + + if (ibcontext == NULL) { + ROCE_ERR("Ibcontext is null"); + return; + } + + context = to_roce5_ucontext(ibcontext); + rdev = to_roce5_dev(ibcontext->device); + + hinic5_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, + context->dwqe_dma_addr); + + return; +} + +struct ib_ucontext *roce5_alloc_ucontext_ofa(struct ib_device *ibdev, + struct ib_udata *udata) +{ + int ret; + struct roce5_ucontext *context = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return ERR_PTR(-EINVAL); + } + + context = kzalloc(sizeof(struct roce5_ucontext), GFP_KERNEL); + if (context == NULL) { + ROCE_ERR("Failed to alloc ucontext"); + return ERR_PTR(-ENOMEM); + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + context->ibucontext.device = ibdev; + ret = roce5_alloc_ucontext(&context->ibucontext, udata); + if (ret != 0) { + kfree(context); + return ERR_PTR(ret); + } + + return (struct ib_ucontext *)&context->ibucontext; +} + +/* **************************************************************************** + Prototype : roce5_dealloc_ucontext_ofa + Description : dealloc ucontext + Input : struct ib_ucontext *ibcontext + Output : None + + 1.Date : 2023/11/9 + Modification : Created function + +**************************************************************************** */ +int roce5_dealloc_ucontext_ofa(struct ib_ucontext *ibcontext) +{ + struct roce5_ucontext *context = NULL; + + if (ibcontext == NULL) { + ROCE_ERR("Ibcontext is null"); + return -EINVAL; + } + + roce5_dealloc_ucontext(ibcontext); + context = to_roce5_ucontext(ibcontext); + + kfree(context); + return 0; +} + +/* **************************************************************************** + Prototype : roce5_mmap + Description : memory map + Input : struct ib_ucontext *ibcontext + struct vm_area_struct *vma + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) +{ + struct roce5_device *rdev = NULL; + struct roce5_ucontext *ucontext = NULL; + unsigned long db_pfn = 0; + unsigned long dwqe_pfn = 0; + + if ((ibcontext == NULL) || (vma == NULL)) { + ROCE_ERR("Ibcontext or vma is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibcontext->device); + ucontext = to_roce5_ucontext(ibcontext); + db_pfn = ucontext->db_dma_addr >> PAGE_SHIFT; + dwqe_pfn = ucontext->dwqe_dma_addr >> PAGE_SHIFT; + + if ((vma->vm_end - vma->vm_start) != PAGE_SIZE) { + ROCE_DEV_ERR( + rdev, + "(Vm_end(%lu) - vm_start(%lu)) is not equal to PAGE_SIZE, func_id(%d)", + vma->vm_end, vma->vm_start, rdev->glb_func_id); + return -EINVAL; + } + + /* map hw DB to physical page from user */ + if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* construct vm_start~vm_start+PAGE_SIZE page table + * db_pfn is page number + * vm_page_prot means attr + */ + if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to do db io remap, func_id(%d)", + rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } + +#ifdef ROCE_DWQE_EN + if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && + (rdev->rdma_cap.direct_wqe_size != 0)) { // DWQE mmap +#ifdef __aarch64__ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + + if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to do dwqe io remap, func_id(%d)", + rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } +#endif + + ROCE_DEV_ERR(rdev, "Not support, func_id(%d)", rdev->glb_func_id); + return -EINVAL; +} + +enum rdma_link_layer roce5_port_link_layer(struct ib_device *ibdev, + u32 port_num) +{ + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + return IB_LINK_LAYER_UNSPECIFIED; + } + + rdev = to_roce5_dev(ibdev); + + if (port_num != ROCE_DEFAULT_PORT_NUM) { + return IB_LINK_LAYER_UNSPECIFIED; + } + + return IB_LINK_LAYER_ETHERNET; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +enum rdma_link_layer roce5_port_link_layer_compat(struct ib_device *ibdev, + u8 port_num) +{ + return roce5_port_link_layer(ibdev, (u32)port_num); +} + +static void roce5_resolve_cb(int status, struct sockaddr *src_addr, + struct rdma_dev_addr *addr, void *context) +{ + ((struct roce5_resolve_cb_context *)context)->status = status; + complete(&((struct roce5_resolve_cb_context *)context)->comp); +} + +static int roce5_rdma_addr_find_l2_eth_by_grh( + const union ib_gid *sgid, const union ib_gid *dgid, u8 *dmac, + const struct net_device *ndev, int *hoplimit, struct roce5_device *rdev, + const struct ib_gid_attr *sgid_attr) +{ + struct rdma_dev_addr dev_addr; + struct roce5_resolve_cb_context ctx; + union { + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } sgid_addr, dgid_addr; + int ret; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid); + rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid); + + ret = memset_s(&dev_addr, sizeof(dev_addr), 0, sizeof(dev_addr)); + if (ret != 0) { + ROCE_ERR("Failed to memset dev_addr.(ret:%d)", ret); + return -ENOMEM; + } + + if (ndev) { + dev_addr.bound_dev_if = ndev->ifindex; + dev_addr.net = dev_net(ndev); + } else { + dev_addr.net = &init_net; + } + + init_completion(&ctx.comp); +#if defined(RDMA_RESOLVE_IP_HAVE_GID_ATTR) + dev_addr.sgid_attr = sgid_attr; + ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr, + (struct sockaddr *)&dgid_addr, &dev_addr, + RESOLVE_IP_TIME_OUT, roce5_resolve_cb, true, + &ctx); +#else + ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr, + (struct sockaddr *)&dgid_addr, &dev_addr, + RESOLVE_IP_TIME_OUT, roce5_resolve_cb, &ctx); +#endif + if (ret != 0) { + ROCE_ERR("rdma_resolve_ip failed. Igonore the err(%d).", ret); + roce5_resolve_cb(0, (struct sockaddr *)&sgid_addr, &dev_addr, + &ctx); + } + + wait_for_completion(&ctx.comp); + + ret = memcpy_s(dmac, ETH_ALEN, dev_addr.dst_dev_addr, ETH_ALEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac.(ret:%d)", ret); + return -ENOMEM; + } + if (hoplimit) { + *hoplimit = dev_addr.hoplimit; + } + + return 0; +} + +static int roce5_ah_valid_check(struct ib_global_route *grh, u16 *vlan_id, + u8 *dmac) +{ + u8 unicast_gid0[ROCE_GID_LEN] = { 0 }; + u8 unicast_gid1[ROCE_GID_LEN] = { 0 }; + + /* check gid(unicast gid can not be 0 or 1) */ + unicast_gid0[ROCE_GID_HIGHEST_BYTE] = 0; + unicast_gid1[ROCE_GID_HIGHEST_BYTE] = 1; + + if ((ROCE_MEMCMP(grh->dgid.raw, unicast_gid0, sizeof(union ib_gid)) == + 0) || + (ROCE_MEMCMP(grh->dgid.raw, unicast_gid1, sizeof(union ib_gid)) == + 0)) { + ROCE_ERR( + "Invalid unicast dgid, The ROCE_GID_HIGHEST_BYTE of unicast_dgid is: 0x%02x", + *(grh->dgid.raw + ROCE_GID_HIGHEST_BYTE)); + return (-EINVAL); + } + + if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) != 0) { + rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, dmac); + *vlan_id = ROCE_DEFAULT_VLAN_ID; + } + + return 0; +} + +static int roce5_fill_gid_attr(struct roce5_device *rdev, + struct rdma_ah_attr *ah_attr, union ib_gid *sgid, + const struct ib_gid_attr **sgid_attr) +{ + int ret = 0; + + ret = rdma_query_gid(&rdev->ib_dev, ah_attr->port_num, + ah_attr->grh.sgid_index, sgid); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to query gid func_id(%hu),port_num(%d),gid_index(%d),ret(%d)", + rdev->glb_func_id, ah_attr->port_num, + ah_attr->grh.sgid_index, ret); + return ret; + } + + *sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, ah_attr->port_num, + ah_attr->grh.sgid_index); + if (IS_ERR_OR_NULL(*sgid_attr)) { + ret = (int)PTR_ERR(*sgid_attr); + ROCE_PRINT_GID(rdev, sgid->raw, + "Failed to get sgid_attr, func_id(%hu), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + return ret; +} + +static void roce5_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr) +{ + rdma_put_gid_attr(sgid_attr); +} + +static struct net_device *roce5_fill_netdev(struct roce5_device *rdev, + union ib_gid *sgid) +{ + struct net_device *netdev = NULL; + union { + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } socket_addr; + + rdma_gid2ip((struct sockaddr *)&socket_addr, sgid); + + /* find netdev,rdev->ndevis not valid in vlan scenario */ + netdev = ip_dev_find(&init_net, + socket_addr._sockaddr_in.sin_addr.s_addr); + if (netdev) { + dev_put(netdev); + } + + return netdev; +} + +int roce5_resolve_grh(struct roce5_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata) +{ + int ret = 0; + u8 zero_mac[ETH_ALEN] = { 0 }; + u8 *dmac = NULL; + union ib_gid sgid = { 0 }; + const struct ib_gid_attr *sgid_attr = NULL; + struct net_device *netdev = NULL; + + if ((rdev == NULL) || (ah_attr == NULL) || (vlan_id == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + dmac = ah_attr->roce.dmac; + ret = roce5_ah_valid_check(&ah_attr->grh, vlan_id, dmac); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to check grh input, func_id(%d), ret(%d)," + "The grh inputs are: vlan_id(%d), dmac:%02x-%02x-%02x-%02x-%02x-%02x", + rdev->glb_func_id, ret, *vlan_id, + *((u8 *)dmac + MAC_INDEX_0), + *((u8 *)dmac + MAC_INDEX_1), + *((u8 *)dmac + MAC_INDEX_2), + *((u8 *)dmac + MAC_INDEX_3), + *((u8 *)dmac + MAC_INDEX_4), + *((u8 *)dmac + MAC_INDEX_5)); + return ret; + } + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) != 0) { + return 0; + } + + ret = roce5_fill_gid_attr(rdev, ah_attr, &sgid, &sgid_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to fill gid attr, func_id(%hu), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + netdev = roce5_fill_netdev(rdev, &sgid); + + /* reparse dmac avoiding invalid damc from OFED */ + ret = roce5_rdma_addr_find_l2_eth_by_grh( + &sgid, &ah_attr->grh.dgid, dmac, netdev, NULL, rdev, sgid_attr); + if (ret != 0) { + ROCE_PRINT_GID( + rdev, sgid.raw, + "Failed to find dmac by grh, func_id(%hu), ret(%d)", + rdev->glb_func_id, ret); + goto resolve_grh_end; + } + + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) == 0) { + ROCE_DEV_ERR(rdev, "Failed to find valid dmac, func_id(%hu)", + rdev->glb_func_id); + ret = (-EINVAL); + goto resolve_grh_end; + } + + if (sgid_attr != NULL && sgid_attr->ndev != NULL) { + *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev); + } + +resolve_grh_end: + roce5_release_gid_ref_cnt(sgid_attr); + + return ret; +} + +static int roce5_ah_get_vlan_id(struct roce5_device *rdev, struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, u32 *vlan_id) +{ + u32 ret = 0; + struct net_device *ndev = NULL; + rcu_read_lock(); + ndev = rcu_dereference(ah_attr->grh.sgid_attr->ndev); + if (ndev == NULL) { + rcu_read_unlock(); + ROCE_DEV_ERR(rdev, "Net device is NULL, func_id(%hu)", + rdev->glb_func_id); + return -EINVAL; + } + ret = rdma_vlan_dev_vlan_id(ndev); + if (ret != 0xffff) { + *vlan_id = ret; + } + rcu_read_unlock(); + return 0; +} + +static int roce5_create_ib_ah_kernel(struct roce5_device *rdev, + struct ib_pd *pd, struct roce5_ah *rah, + struct rdma_ah_attr *ah_attr) +{ + int ret; + u8 *dmac = ah_attr->roce.dmac; + u32 vlan_id = 0; + + ret = roce5_ah_get_vlan_id(rdev, pd, ah_attr, &vlan_id); + if (ret != 0) { + ROCE_ERR("Failed to get vlan_id (ret:%d)", ret); + return -EFAULT; + } + + if (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) != 0) { + ret = memcpy_s((void *)rah->priv_ah.dgid, + sizeof(rah->priv_ah.dgid), + (void *)ah_attr->grh.dgid.raw, + sizeof(rah->priv_ah.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return -ENOMEM; + } + rah->priv_ah.dw2.bs.flow_label = ah_attr->grh.flow_label & + 0xfffff; + rah->priv_ah.dw1.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f; + rah->priv_ah.dw1.bs.hoplimit = ah_attr->grh.hop_limit; + rah->priv_ah.dw1.bs.tclass = + (u8)(ah_attr->grh.traffic_class | IP_ECN_PERMIT); + } + rah->priv_ah.dw0.bs.pd = to_roce5_pd(pd)->pdn & 0x3ffff; + rah->priv_ah.dw0.bs.wqe_cos = + roce5_get_db_cos_from_vlan_pri(rdev, ah_attr->sl); + rah->priv_ah.dw0.value = cpu_to_be32(rah->priv_ah.dw0.value); + + rah->priv_ah.dw1.bs.port = ah_attr->port_num & 0xf; + rah->priv_ah.dw2.bs.smac_index = + rdev->glb_func_id; /* set global Function ID */ + rah->priv_ah.dw2.value = cpu_to_be32(rah->priv_ah.dw2.value); + + rah->priv_ah.dw1.bs.resv = 0; + rah->priv_ah.dw7.bs.vlan_id = vlan_id & 0xfff; + rah->priv_ah.dw7.bs.vlan_pri = ah_attr->sl & 0x7; + + rah->priv_ah.dw1.value = cpu_to_be32(rah->priv_ah.dw1.value); + + rah->priv_ah.dw7.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | + dmac[1]; + rah->priv_ah.dw7.value = cpu_to_be32(rah->priv_ah.dw7.value); + + ret = memcpy_s((void *)&rah->priv_ah.dmac_l32, + sizeof(rah->priv_ah.dmac_l32), + (void *)&dmac[ROCE_RAH_DMAC_L32_START], + sizeof(rah->priv_ah.dmac_l32)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac_l32.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} + +static int roce5_create_ib_ah_user(struct roce5_device *rdev, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata) +{ + struct roce5_create_ah_resp resp; + roce5_get_cos_inbuf_s dcb_inbuf; + int ret; + + ret = memset_s(ah_attr->roce.dmac, sizeof(ah_attr->roce.dmac), 0, + sizeof(ah_attr->roce.dmac)); + if (ret != 0) { + ROCE_ERR("Failed to memset dmac."); + return (-ENOMEM); + } + + ret = roce5_resolve_grh(rdev, ah_attr, &resp.vlan_id, udata); + if (ret != 0) { + ROCE_ERR("Failed to resolve grh"); + return -EINVAL; + } + + ret = memcpy_s(resp.dmac, ETH_ALEN, ah_attr->roce.dmac, ETH_ALEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy_s ah dmac."); + return (-ENOMEM); + } + + dcb_inbuf.sl = ah_attr->sl; + dcb_inbuf.sgid_index = ah_attr->grh.sgid_index; + dcb_inbuf.port_num = ah_attr->port_num; + dcb_inbuf.traffic_class = ah_attr->grh.traffic_class; + ret = roce5_get_dcb_cfg_cos(rdev, &dcb_inbuf, &resp.cos, IB_QPT_UD); + if (ret != 0) { + ROCE_ERR("Failed to get cos from dcb info, ret:%d", ret); + return ret; + } + + ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (ret != 0) { + ROCE_ERR("Failed to copy data to user"); + return ret; + } + + return 0; +} + +int roce5_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_ah *rah = NULL; + struct roce5_device *rdev = NULL; + struct rdma_ah_attr *ah_attr = NULL; + enum rdma_ah_attr_type ah_type = 0; + + if ((ibah == NULL) || (init_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rah = to_roce5_ah(ibah); + rdev = to_roce5_dev(ibah->device); + ah_attr = init_attr->ah_attr; + ah_type = ah_attr->type; + + if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && + (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) == 0)) { + ROCE_DEV_ERR(rdev, "Invalid ah_type(%d)", ah_type); + return -EINVAL; + } + + if (udata != NULL) { + ret = roce5_create_ib_ah_user(rdev, ah_attr, udata); + } else { + ret = roce5_create_ib_ah_kernel(rdev, ibah->pd, rah, ah_attr); + } + return ret; +} + +struct ib_ah *roce5_create_ah_ofa(struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_ah *rah = NULL; + struct roce5_device *rdev = NULL; + struct rdma_ah_init_attr init_attr; + + if ((pd == NULL) || (ah_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + ret = -EINVAL; + goto out; + } + + rdev = to_roce5_dev(pd->device); + init_attr.ah_attr = ah_attr; + + rah = kzalloc(sizeof(*rah), GFP_ATOMIC); + if (rah == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, "Failed to alloc ah, func_id(%d)", + rdev->glb_func_id); + goto out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + rah->ibah.device = pd->device; + rah->ibah.pd = pd; + ret = roce5_create_ah(&rah->ibah, &init_attr, udata); + if (ret != 0) { + kfree(rah); + } + +out: + return (ret != 0) ? ERR_PTR(ret) : &rah->ibah; +} + +int roce5_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ + struct roce5_ah *ah = NULL; + struct roce5_priv_ah priv_ah; + int ret; + + if ((ibah == NULL) || (ah_attr == NULL)) { + ROCE_ERR("Ibah or ah_attr is null"); + return -EINVAL; + } + + ah = to_roce5_ah(ibah); + ret = memset_s(ah_attr, sizeof *ah_attr, 0, sizeof *ah_attr); + if (ret != 0) { + ROCE_ERR("Failed to memset ah_attr.(ret:%d)", ret); + return -ENOMEM; + } + + priv_ah.dw1.value = be32_to_cpu(ah->priv_ah.dw1.value); + priv_ah.dw2.value = be32_to_cpu(ah->priv_ah.dw2.value); + priv_ah.dw7.value = be32_to_cpu(ah->priv_ah.dw7.value); + + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->sl = priv_ah.dw7.bs.vlan_pri; + ah_attr->port_num = priv_ah.dw1.bs.port; + ah_attr->grh.traffic_class = priv_ah.dw1.bs.tclass; + ah_attr->grh.hop_limit = priv_ah.dw1.bs.hoplimit; + ah_attr->grh.sgid_index = priv_ah.dw1.bs.sgid_index; + ah_attr->grh.flow_label = priv_ah.dw2.bs.flow_label; + + ret = memcpy_s((void *)ah_attr->grh.dgid.raw, sizeof(ah->priv_ah.dgid), + (void *)ah->priv_ah.dgid, sizeof(ah->priv_ah.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return -ENOMEM; + } + return 0; +} + +int roce5_destroy_ah(struct ib_ah *ibah, u32 flags) +{ + return 0; +} + +int roce5_destroy_ah_ofa(struct ib_ah *ibah) +{ + struct roce5_ah *rah = NULL; + + if (ibah == NULL) { + ROCE_ERR("Ibah is null"); + return -EINVAL; + } + + rah = to_roce5_ah(ibah); + + kfree(rah); + + return 0; +} + +int roce5_port_immutable(struct ib_device *ibdev, u32 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + + if ((ibdev == NULL) || (immutable == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + immutable->core_cap_flags = + RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; // only rocev2 + +#if defined(IB_QUERY_PORT_HAVE_U8_PORT) + err = ib_query_port(ibdev, (u8)port_num, &attr); +#elif defined(IB_QUERY_PORT_HAVE_U32_PORT) + err = ib_query_port(ibdev, port_num, &attr); +#endif + if (err != 0) { + ROCE_ERR("dev(%s) query ib port(%d) failed, err(%d)", + ibdev->name, port_num, err); + return err; + } + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_port_immutable_compat(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + return roce5_port_immutable(ibdev, (u32)port_num, immutable); +} + +#define ROCE_DEFAULT_COS 3 + +int roce5_get_dcb_cfg_cos(struct roce5_device *rdev, + roce5_get_cos_inbuf_s *inbuf, u8 *cos, + enum ib_qp_type qp_type) +{ + int ret; + u8 pri; + struct rdma_gid_entry gid; + struct hinic5_dcb_state dcb = { 0 }; + +#if !defined(UVERBS_IOCTL_HAVE_DECLARE_UVERBS_WRITE_EX) + /* Note: low version of ofed in modify qp when unused udate->outbuf, cause cannot from kernel mode get db cos, therefore stub uses a fixed value. */ + *cos = ROCE_DEFAULT_COS; + goto cos_success; +#endif + + ret = roce5_get_gid(rdev, inbuf->port_num, inbuf->sgid_index, &gid); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init gid info, ret(%d).", ret); + return -EINVAL; + } + + ret = hinic5_get_dcb_state(rdev->hwdev, &dcb); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "hinic5_get_dcb_state failed, ret(%d).", + ret); + return -EINVAL; + } + + *cos = dcb.default_cos; + gid.dw6_h.value = cpu_to_le16(gid.dw6_h.value); + if ((dcb.trust == ROCE5_DCB_PCP) && + (gid.dw6_h.bs.tag == ROCE_GID_VLAN_INVALID)) { + // pcp cfg & no vlan should use default cos + goto cos_success; + } + + pri = (dcb.trust == ROCE5_DCB_PCP) ? + inbuf->sl : + (inbuf->traffic_class >> ROCE5_DSCP_IDX); + ret = hinic5_get_cos_by_pri(rdev->hwdev, pri, cos); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "get cos failed, ret(%d), pri(%u), dcb_on(%u), trust(%u).", + ret, pri, dcb.dcb_on, dcb.trust); + return -EINVAL; + } + +cos_success: + roce5_cos_update_ext(rdev, cos, qp_type); + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h new file mode 100644 index 000000000..2fb0eb2b9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h @@ -0,0 +1,269 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MIX_H +#define ROCE_MIX_H + +#include <net/arp.h> +#include <net/neighbour.h> +#include <net/route.h> +#include <net/addrconf.h> +#include <net/ip6_route.h> +#include <linux/mutex.h> +#include <linux/inetdevice.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <linux/module.h> + +#include <rdma/ib_addr.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_pack.h> + +#include "roce.h" + +#define ROCE_HW_VER 0ULL +#define ROCE_FW_VER_0 1ULL +#define ROCE_FW_VER_1 4ULL +#define ROCE_FW_VER_2 0ULL +#define ROCE_FW_VER_3 0ULL +#define ROCE_FW_VER \ + ((ROCE_FW_VER_0 << 32) | (ROCE_FW_VER_1 << 16) | \ + (ROCE_FW_VER_2 << 8) | ROCE_FW_VER_3) + +#define RESOLVE_IP_TIME_OUT 1000 +#define ROCE_GID_HIGHEST_BYTE 15 +#define ROCE_FW_VERSION_LEN 4 +#define ROCE_DEFAULT_VLAN_ID 0xFFFF +#define ROCE_RAH_TC_SHIFT 2 +#define ROCE_RAH_DMAC_H16_SHIFT 8 +#define ROCE_RAH_DMAC_L32_START 2 +#define ROCE_UDP_SRC_PORT_H8_SHIFT 8 + +#define ROCE5_DCB_PCP 0 +#define ROCE5_DCB_DSCP 1 + +#define ROCE5_DSCP_IDX 2 //dscp begin from bit 2 in traffic_class + +#define ROCE_GID_IP_IDX 3 + +#define MAC_INDEX_0 0 +#define MAC_INDEX_1 1 +#define MAC_INDEX_2 2 +#define MAC_INDEX_3 3 +#define MAC_INDEX_4 4 +#define MAC_INDEX_5 5 + +#define IP_INDEX_0 0 +#define IP_INDEX_1 1 +#define IP_INDEX_2 2 +#define IP_INDEX_3 3 +#define IP_INDEX_4 4 +#define IP_INDEX_5 5 +#define IP_INDEX_6 6 +#define IP_INDEX_7 7 + +#define ROCE_MTU_256_INT 256 +#define ROCE_MTU_512_INT 512 +#define ROCE_MTU_1024_INT 1024 +#define ROCE_MTU_2048_INT 2048 +#define ROCE_MTU_4096_INT 4096 +#define ROCE_MTU_8192_INT 8192 + +enum { + USR_MMAP_DB_OFFSET = 0, + USR_MMAP_DWQE_OFFSET, + USR_MMAP_DFX_OFFSET, + USR_MMAP_DBAR3_OFFSET +}; + +enum roce_port_state_e { + ROCE_PORT_PHYS_STATE_NO_CHANGE = 0, + ROCE_PORT_PHYS_STATE_SLEEP, + ROCE_PORT_PHYS_STATE_POLLING, + ROCE_PORT_PHYS_STATE_DISABLED = 3, + ROCE_PORT_PHYS_STATE_PORTCONFTRAIN, + ROCE_PORT_PHYS_STATE_LINKUP = 5, + ROCE_PORT_PHYS_STATE_LINKERRRECOVER, + ROCE_PORT_PHYS_STATE_PHYTEST +}; + +enum ROCE_MBOX_CMD { ROCE_MBOX_CMD_SEND_MAIL_BOX, ROCE_MBOX_CMD_MAX }; + +enum roce5_mtu { + ROCE_MTU_256 = IB_MTU_256, + ROCE_MTU_512 = IB_MTU_512, + ROCE_MTU_1024 = IB_MTU_1024, + ROCE_MTU_2048 = IB_MTU_2048, + ROCE_MTU_4096 = IB_MTU_4096, + ROCE_MTU_8192, +}; + +static inline int roce5_mtu_enum_to_int(enum roce5_mtu mtu) +{ + switch (mtu) { + case ROCE_MTU_256: + return ROCE_MTU_256_INT; + case ROCE_MTU_512: + return ROCE_MTU_512_INT; + case ROCE_MTU_1024: + return ROCE_MTU_1024_INT; + case ROCE_MTU_2048: + return ROCE_MTU_2048_INT; + case ROCE_MTU_4096: + return ROCE_MTU_4096_INT; + case ROCE_MTU_8192: + return ROCE_MTU_8192_INT; + default: + return -1; + } +} + +static inline enum roce5_mtu roce5_get_mtu(int mtu) +{ + int local_mtu = mtu - (IB_GRH_BYTES + IB_UDP_BYTES + IB_BTH_BYTES + + IB_EXT_XRC_BYTES + IB_EXT_ATOMICETH_BYTES + + IB_ICRC_BYTES); + + if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_8192)) + return ROCE_MTU_8192; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_4096)) + return ROCE_MTU_4096; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_2048)) + return ROCE_MTU_2048; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_1024)) + return ROCE_MTU_1024; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_512)) + return ROCE_MTU_512; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_256)) + return ROCE_MTU_256; + else + return 0; +} + +struct roce5_priv_ah { + /* DW0 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; + u32 wqe_cos : 3; + u32 fl : 1; /* forcelock flag, it is used in multicast communication */ +#else + u32 f1 : 1; + u32 wqe_cos : 3; + u32 stat_rate : 4; + u32 rsvd0 : 6; + u32 pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 hoplimit : 8; + u32 sgid_index : 7; + u32 resv : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tclass : 8; +#else + u32 tclass : 8; + u32 rsvd1 : 4; + u32 port : 4; + u32 resv : 1; + u32 sgid_index : 7; + u32 hoplimit : 8; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 flow_label : 20; + u32 smac_index : 10; + u32 rsvd : 2; +#else + u32 rsvd : 2; + u32 smac_index : 10; + u32 flow_label : 20; +#endif + } bs; + u32 value; + } dw2; + + /* DW3~6 */ + u8 dgid[16]; + + /* DW7 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 rsvd : 1; + u32 vlan_pri : 3; +#else + u32 vlan_pri : 3; + u32 rsvd : 1; + u32 vlan_id : 12; + u32 dmac_h16 : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + u32 dmac_l32; +}; + +struct roce5_ah { + struct ib_ah ibah; + struct roce5_priv_ah priv_ah; +}; + +union gid_addr { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; +}; + +struct mailbox_header { + u32 dest_host_id; + u32 dest_global_function_id; +}; + +struct roce5_mail_box_buffer { + struct mailbox_header header; + u32 data[0]; +}; + +struct roce5_resolve_cb_context { + struct completion comp; + int status; +}; + +typedef struct roce5_get_cos_inbuf { + u8 sl; + u8 sgid_index; + u8 port_num; + u8 traffic_class; +} roce5_get_cos_inbuf_s; + +int roce5_resolve_grh(struct roce5_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata); +int roce5_get_dcb_cfg_cos(struct roce5_device *rdev, + roce5_get_cos_inbuf_s *inbuf, u8 *cos, + enum ib_qp_type qp_type); +uint8_t roce5_get_db_cos_from_vlan_pri(struct roce5_device *rdev, + uint8_t vlan_pri); + +#endif /* ROCE_MIX_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c new file mode 100644 index 000000000..92a3ada97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c @@ -0,0 +1,138 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_pd.h" + +int roce5_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + int ret; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if (CHECK_2LAYER_IS_NULL(ibpd, device)) { + ROCE_ERR( + "Invalid NULL parameter, ibpd or ibpd->device is NULL."); + ret = -EINVAL; + goto err_out; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + ret = roce5_rdma_pd_alloc(rdev->hwdev, &pd->pdn); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc pdn, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto err_out; + } + + if (udata != NULL) { + if (udata->outbuf == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid NULL parameter, udata->outbuf is NULL."); + ret = -EINVAL; + goto err_copy_to_user; + } + + if (ib_copy_to_udata(udata, &pd->pdn, sizeof(u32)) != 0) { + ret = -EFAULT; + ROCE_DEV_ERR( + rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + goto err_copy_to_user; + } + } + + return 0; + +err_copy_to_user: + roce5_rdma_pd_free(rdev->hwdev, pd->pdn); +err_out: + return ret; +} + +struct ib_pd *roce5_alloc_pd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibdev is null"); + goto err_out; + } + + rdev = to_roce5_dev(ibdev); + + pd = (struct roce5_pd *)kmalloc(sizeof *pd, GFP_KERNEL); + if (pd == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, + "Failed to alloc memory for roce pd, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + pd->ibpd.device = ibdev; + ret = roce5_alloc_pd(&pd->ibpd, udata); + if (ret != 0) { + goto err_alloc_pdn; + } + + return &pd->ibpd; + +err_alloc_pdn: + kfree(pd); + +err_out: + return (struct ib_pd *)ERR_PTR((long)ret); +} + +int roce5_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return -EINVAL; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + roce5_rdma_pd_free(rdev->hwdev, pd->pdn); + + return 0; +} + +int roce5_dealloc_pd_ofa(struct ib_pd *ibpd) +{ + int ret; + struct roce5_pd *pd = NULL; + + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return -EINVAL; + } + + pd = to_roce5_pd(ibpd); + ret = roce5_dealloc_pd(ibpd, NULL); + if (ret == 0) { + kfree(pd); + } + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h new file mode 100644 index 000000000..e40c5a476 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_PD_H +#define ROCE_PD_H + +#include <rdma/ib_verbs.h> + +struct roce5_pd { + struct ib_pd ibpd; + u32 pdn; +}; + +#endif //ROCE_PD_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h new file mode 100644 index 000000000..1886af73f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h @@ -0,0 +1,191 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SRQ_H +#define ROCE_SRQ_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_pd.h" +#include "roce_db.h" +#include "roce_compat.h" +#include "roce_wqe_format.h" + +/* definition SRQ SGE maximum max and minimum value */ +#define ROCE_SRQ_CONT_MAX_SGE 16 +#define ROCE_SRQ_MAX_SGE 15 +#define ROCE_SRQ_MID_SGE 7 +#define ROCE_SRQ_MIN_SGE 3 + +#define ROCE_SRQN_INVLD CQM_INDEX_INVALID +#define ROCE_SRQ_CONTAINER_LWM_MASK 0xFFFF +#define ROCE_SRQ_CONTAINER_WARTH_MASK 0xF + +#define MAX_SUPPORT_CONTAINER_MODE 3 +#define DWORD_LEN 32 +#define XRC_CQN_FIRST_LEN 10 +#define XRC_CQN_SECOND_LEN 3 +#define SRQ_GPA_SIG_LEN 3 +#define SRQ_WQEBB_SIZE_CAL_SECTTOR \ + 4 /* (2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 */ +#define RDMA_PREFETCH_WQE_MAX 7 +#define RDMA_PREFETCH_MTT_LEN_MAX 3 +#define ROCE_WQE_BB_SIZE_MIN 64 + +#define LAST_SGE_NO_PRESENT \ + 0x80000000UL /* Lbit:0 - The next SGE is present in the list \ + * Lbit:1 - The last SGE, no SGE is present in the list */ +#define NORMAL_FMT_AND_NEXT_SGE_PRESENT \ + 0x3FFFFFFFUL /* Ebit:b0 - Normal format, without extension. \ + * Ebit:b1 - The pointer inside of SGE points to the next SGL. \ + * "Length" and "Key" fields are */ +#define NORMAL_FMT_AND_LAST_SGE_NO_PRESENT 0xBFFFFFFFUL + +/** + * CHIP container_mode: + * chip mode: 0 -> Not container + * chip mode: 1 -> container_size: 2 + * chip mode: 2 -> container_size: 4 + * chip mode: 3 -> container_size: 8 + * chip mode: 4 -> container_size: 16 + */ +enum roce5_chip_srq_mode { + ROCE_CHIP_SRQ_MODE_N = 0, + ROCE_CHIP_SRQ_MODE_1, + ROCE_CHIP_SRQ_MODE_2, + ROCE_CHIP_SRQ_MODE_3, + ROCE_CHIP_SRQ_MODE_4 +}; + +enum srq_state { + ROCE_SRQ_STATE_INVALID = 0x0, + ROCE_SRQ_STATE_ERR = 0x1, + ROCE_SRQ_STATE_VALID = 0xf, + ROCE_SRQ_STATE_MEM_INIT = 0xa +}; + +#define ROCE_SRQ_STATE_CHECK_VALUE 0x0 + +struct roce5_wqe_srq_next_seg { + u16 reserved1; + __be16 pcnt; /* indicate the pi */ + u8 signature; + u8 reserved2; + __be16 next_wqe_index; + u32 reserved3[2]; +}; + +struct roce5_wqe_container_srq_next_seg { + u32 next_gpa_h; + + struct { + u32 rsvd : 11; + u32 next_gpa_vd : 1; + u32 next_gpa_h : 20; /* indicate the pi */ + } dw1; + + struct { + u32 next_idx : 16; + u32 rsvd : 16; + } dw2; + + struct { + u32 rsvd2 : 30; + u32 link_flag : 1; + u32 rsvd : 1; + } dw3; + + struct { + u32 osd_next_idx : 16; + u32 osd_cur_idx : 16; + } dw4; +}; + +#define ROCE_SRQ_SGE_LAST 1 +#define ROCE_SRQ_SGE_NLAST 0 + +#define ROCE_SRQ_SGE_LKEY_NOEXT 0 +#define ROCE_SRQ_SGE_LKEY_EXT 1 + +struct roce5_srq { + struct ib_srq ibsrq; /* ibsrq */ + cqm_queue_s *cqm_srq; + + u32 srqn; + int max_depth; + int max_gs; + int wqe_shift; + + cqm_buf_s *buf; + struct roce5_db db; + u64 *wrid; + spinlock_t lock; + int head; + int tail; + u16 wqe_ctr; + u8 xrc_en; + u8 rsvd; + struct ib_umem *umem; + struct mtt mtt; + struct mutex mutex; + u32 rqe_cnt_th; + u8 container_flag; + u8 container_size; + u8 container_mode; + u8 container_warn_th; + + int buf_sz; +}; + +static inline u8 roce5_srq_get_max_sge(struct roce5_srq *rsrq) +{ + if (rsrq->container_flag != 0) { + return ROCE_SRQ_CONT_MAX_SGE; + } + return ROCE_SRQ_MAX_SGE; +} + +static inline int roce5_srq_init_wqe_size(struct roce5_srq *rsrq) +{ + int desc_size = 0; + + if (rsrq->container_flag != 0) { + desc_size = rsrq->max_gs * sizeof(struct roce_wqe_srq_data_seg); + } else { + desc_size = sizeof(struct roce5_wqe_srq_next_seg) + + rsrq->max_gs * sizeof(struct roce_wqe_srq_data_seg); + } + desc_size = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)desc_size)); + desc_size = ROCE_MAX(ROCE_WQE_BB_SIZE_MIN, desc_size); + + rsrq->wqe_shift = (int)ROCE_ILOG2((unsigned)desc_size); + return desc_size; +} + +void roce5_free_srq_wqe(struct roce5_srq *srq, int wqe_index); +void roce5_srq_async_event(struct roce5_device *rdev, struct roce5_srq *srq, + int type); +void *roce5_srq_get_wqe(struct roce5_srq *srq, int n); +u8 roce5_get_container_sz(u32 container_mode); +u32 roce5_srq_max_avail_wr_set(struct roce5_srq *rsrq); +int roce5_create_srq_common(struct roce5_device *rdev, struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, u32 index); +struct roce5_srq *roce5_get_rsrq_by_srqn(struct roce5_device *rdev, u32 srqn); +static inline void roce5_rsrq_put(struct roce5_srq *rsrq) +{ + if (rsrq == NULL) { + return; + } + cqm5_object_put(&rsrq->cqm_srq->object); +} + +#endif //ROCE_SRQ_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c new file mode 100644 index 000000000..0a3beb338 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c @@ -0,0 +1,116 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_aeq_format.h" +#include "roce_srq_extension.h" +#include "roce_restore.h" + +static void *roce5_srq_buf_offset(struct tag_cqm_buf *buf, int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce5_srq_get_wqe(struct roce5_srq *srq, int n) +{ + return roce5_srq_buf_offset(srq->buf, + (int)((u32)n << (unsigned)srq->wqe_shift)); +} + +void roce5_srq_async_event(struct roce5_device *rdev, struct roce5_srq *srq, + int type) +{ + struct ib_srq *ibsrq = &srq->ibsrq; + struct ib_event event; + int ret; + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset event.(ret:%d)", + ret); + } + if (ibsrq->event_handler) { + event.device = ibsrq->device; + event.element.srq = ibsrq; + switch (type) { + case ROCE_EVENT_TYPE_SRQ_LIMIT: + event.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + event.event = IB_EVENT_SRQ_ERR; + break; + + default: + ROCE_DEV_WARN_LIMIT( + rdev, + "unexpected event type(%d) on SRQ(%06x), func_id(%d)", + type, srq->srqn, rdev->glb_func_id); + return; + } + + ibsrq->event_handler(&event, ibsrq->srq_context); + } +} + +/* **************************************************************************** + Prototype : roce5_free_srq_wqe + Description : roce5_free_srq_wqe + Input : struct roce5_srq *srq + int wqe_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_free_srq_wqe(struct roce5_srq *srq, int wqe_index) +{ + struct roce5_wqe_srq_next_seg *next = NULL; + + spin_lock(&srq->lock); + + next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(srq, + srq->tail); + next->next_wqe_index = (u16)wqe_index; + srq->tail = wqe_index; + + spin_unlock(&srq->lock); +} + +/** + * @brief Get roce srq by srqn. When querying succeeded, you have to call roce5_rsrq_put() to free + the cqm_obj_srq. But there is no need to call roce5_rsrq_put() if querying failed. + * @return Success: return the roce srq. Failure: return NULL. + */ +struct roce5_srq *roce5_get_rsrq_by_srqn(struct roce5_device *rdev, u32 srqn) +{ + cqm_object_s *cqm_obj_srq = NULL; + struct roce5_srq *rsrq = NULL; + + cqm_obj_srq = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find srq according to srqn(%u), func_id(%d).", + srqn, rdev->glb_func_id); + return NULL; + } + + rsrq = cqmobj_to_roce5_srq(cqm_obj_srq); + if (rsrq == NULL) { + ROCE_DEV_ERR(rdev, "Failed to convert obj to rsrq."); + cqm5_object_put(cqm_obj_srq); + } + + return rsrq; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c new file mode 100644 index 000000000..47c86b2fa --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c @@ -0,0 +1,712 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/kernel.h> + +#include "comm_defs.h" +#include "roce.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_restore.h" +#include "roce_npu_cmd_srq_defs.h" +#include "roce_npu_cmd_defs.h" + +#include "roce_srq_extension.h" +#include "roce_main_extension.h" +#include "hinic5_hmm.h" +#include "roce_npu_cmd.h" +#include "securec.h" +#include "roce_os_compat.h" + +/* **************************************************************************** + Prototype : roce5_srq_sw2hw + Description : roce5_srq_sw2hw + Input : struct roce5_device *rdev + struct roce5_srq *rsrq + u32 pdn + u32 cqn + u16 xrcdn + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + + 1.Date : 2015/7/27 + Modification : modify function +**************************************************************************** */ +static void roce5_srq_sw2hw_set(roce_verbs_srq_attr_s *srq_attr, + struct roce5_srq *rsrq, u32 pdn, u32 cqn, + u16 xrcdn, int page_shift) +{ + srq_attr->dw0.bs.size = (u32)(ROCE_ILOG2((unsigned)rsrq->max_depth)); + srq_attr->dw0.bs.page_size = (u32)page_shift; + srq_attr->dw0.bs.wqebb_size = + (u32)rsrq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->dw0.bs.mtt_page_size = + rsrq->mtt.mtt_page_shift - PAGE_SHIFT_4K; + srq_attr->dw0.bs.xrcd = xrcdn; + + srq_attr->dw1.bs.dma_attr_idx = 0; + srq_attr->dw1.bs.so_ro = 1; + srq_attr->dw1.bs.state = ROCE_SRQ_STATE_VALID; + + srq_attr->dw3.bs.container = rsrq->container_flag; + srq_attr->dw3.bs.lth_pre_en = + (rsrq->container_flag == 0); //rdma engine prefetch + srq_attr->dw3.bs.pcnt_on_chip = 0; + srq_attr->dw3.bs.rkey_en = 1; + srq_attr->dw3.bs.pd = pdn; + + srq_attr->srqn = cpu_to_be32(rsrq->srqn); + srq_attr->xrc_cqn = cpu_to_be32(cqn); + + if (rsrq->container_flag == 0) { //normal srq + srq_attr->dw1.bs.ep = 0; + srq_attr->dw1.bs.cos = 0; + + srq_attr->dw2.value = 0; + srq_attr->dw2.bs.wqe_prefetch_max_num = + RDMA_PREFETCH_WQE_MAX; //rdma engine wqe prefetch max num + srq_attr->dw2.bs.wqecnt_lth = 0xe; + srq_attr->dw2.bs.mtt_prefetch_maxlen = + 0; //rdma engine mtt prefetch max len + + srq_attr->dw3.bs.lth_pre_en = 1; //rdma engine prefetch + srq_attr->dw3.bs.lth_gap = 0; + } else { + srq_attr->cont.dw2.bs.head_idx = 0; + srq_attr->cont.dw2.bs.warn_th = + 0; //Do NOT init warn_th until user arm srq + srq_attr->cont.dw2.bs.cont_size = + (MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode) & + 0x3; + srq_attr->cont.dw2.value = + cpu_to_be32(srq_attr->cont.dw2.value); + srq_attr->record_gpa_at_hop_num = + cpu_to_be64((rsrq->mtt.mtt_layers & 0x3)); + } + + srq_attr->dw0.value = cpu_to_be32(srq_attr->dw0.value); + srq_attr->dw1.value = cpu_to_be32(srq_attr->dw1.value); + srq_attr->dw2.value = cpu_to_be32(srq_attr->dw2.value); + srq_attr->dw3.value = cpu_to_be32(srq_attr->dw3.value); + + srq_attr->l0mtt_gpa = rsrq->mtt.mtt_paddr; + + srq_attr->l0mtt_gpa = cpu_to_be64(srq_attr->l0mtt_gpa); + + srq_attr->record_gpa_at_hop_num = + cpu_to_be64(rsrq->db.dma | (rsrq->mtt.mtt_layers & 0x3)); + + return; +} + +static int roce5_srq_sw2hw(struct roce5_device *rdev, struct roce5_srq *rsrq, + u32 pdn, u32 cqn, u16 xrcdn, int page_shift) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_verbs_srq_attr_s *srq_attr = NULL; + roce_uni_cmd_create_srq_s *srq_sw2hw_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_create_srq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_sw2hw_inbuf = (roce_uni_cmd_create_srq_s *)cqm_cmd_inbuf->buf; + srq_sw2hw_inbuf->com.index = cpu_to_be32((u32)rsrq->srqn); + srq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + srq_attr = &srq_sw2hw_inbuf->srq_attr; + + roce5_srq_sw2hw_set(srq_attr, rsrq, pdn, cqn, xrcdn, page_shift); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_SW2HW_SRQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send SW2HW_SRQ command, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + + /* update dev status after cmdq timeout or exception, notify PCIe reset */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static int roce5_create_srq_check(const struct ib_pd *ibpd, + const struct ib_srq_init_attr *init_attr, + const struct roce5_srq *rsrq) +{ + struct roce5_device *rdev = NULL; + + if ((ibpd == NULL) || (init_attr == NULL)) { + ROCE_ERR("Ibpd or init_attr is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibpd->device); + if ((init_attr->attr.max_wr > + (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes) || + (init_attr->attr.max_sge > + ((u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge + + rsrq->container_flag))) { + ROCE_DEV_ERR( + rdev, + "Invalid input parms, max_wr(%d), max_srq_wqes(%d), max_sge(%d), max_srq_sge(%d)", + init_attr->attr.max_wr, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes, + init_attr->attr.max_sge, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge); + return -EINVAL; + } + + return 0; +} + +static int roce5_create_srq_write_mtt_and_db(struct roce5_device *rdev, + struct roce5_srq *rsrq, + struct roce5_create_srq_cmd *ucmd, + struct roce5_pd *pd) +{ + int ret = 0; + + ret = roce5_umem_write_mtt(rdev, &rsrq->mtt, rsrq->umem, 1); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = roce5_db_map_user(to_roce5_ucontext(pd->ibpd.uobject->context), + ucmd->db_addr, &rsrq->db, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to map db to user space, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return ret; +} + +static int roce5_create_srq_get_umem(struct roce5_create_srq_cmd *ucmd, + struct ib_udata *udata, + struct roce5_srq *rsrq, + struct roce5_device *rdev, + struct roce5_pd *pd) +{ + int ret = 0; + + if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0) { + ret = -EFAULT; + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + rsrq->umem = roce5_umem_get(&rdev->ib_dev, pd->ibpd.uobject->context, + ucmd->buf_addr, (size_t)rsrq->buf_sz, 0, 0); + if (IS_ERR(rsrq->umem)) { + ROCE_DEV_ERR( + rdev, + "Failed to get ib_umem, func_id(%d) rsrq->buf_sz:%u", + rdev->glb_func_id, rsrq->buf_sz); + ret = (int)PTR_ERR(rsrq->umem); + return ret; + } + + return ret; +} + +static int roce5_create_user_srq_update(struct roce5_device *rdev, + struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + int ret = 0; + + cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce5_cq(init_attr->ext.cq)->cqn : + 0); + xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce5_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; + + /* set SRQC */ + ret = roce5_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, + (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to handle srq_sw2hw, func_id(%d)", + rdev->glb_func_id); + goto err_unmap; + } + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + + if (ib_copy_to_udata(udata, &rsrq->srqn, sizeof(u32)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + goto err_unmap; + } + + return 0; + +err_unmap: + roce5_db_unmap_user(to_roce5_ucontext(pd->ibpd.uobject->context), + &rsrq->db, 0); + return ret; +} + +u32 roce5_srq_max_avail_wr_set(struct roce5_srq *rsrq) +{ + u32 link_wqe_count = 0; + + if (rsrq->container_flag != 0) { + link_wqe_count = (u32)rsrq->max_depth / rsrq->container_size; + return ((u32)rsrq->max_depth - link_wqe_count) - + (rsrq->container_size - 1); + } else { + return (u32)rsrq->max_depth - 1; + } +} + +/* **************************************************************************** + Prototype : roce5_create_user_srq + Description : roce5_create_user_srq + Input : struct roce5_device *rdev + struct roce5_srq *rsrq + struct roce5_pd *pd + struct ib_srq_init_attr *init_attr + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_user_srq(struct roce5_device *rdev, + struct roce5_srq *rsrq, struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, u32 index) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + struct roce5_create_srq_cmd ucmd = { 0 }; + + /* dispatch SRQN and SRQC */ + rsrq->cqm_srq = cqm5_object_rdma_queue_create(rdev->hwdev, + SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SRQ, 0, + rsrq, false, index, 0, 0); + if (rsrq->cqm_srq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue by cqm_object, func_id(%d) index(%d)", + rdev->glb_func_id, index); + return (-ENOMEM); + } + + rsrq->srqn = rsrq->cqm_srq->index; + + ret = roce5_create_srq_get_umem(&ucmd, udata, rsrq, rdev, pd); + if (ret != 0) { + goto err_free_cqm_srq; + } + + page_shift = + roce5_umem_get_buf_page_shift(rdev, rsrq->umem, ucmd.buf_addr); + npages = roce5_umem_get_mtt_npages(rdev, rsrq->umem, (u32)page_shift); + + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &rsrq->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, " Failed to alloc rdma_mtt, func_id(%d)", + rdev->glb_func_id); + goto err_release_umem; + } + + ret = roce5_create_srq_write_mtt_and_db(rdev, rsrq, &ucmd, pd); + if (ret != 0) { + goto err_free_mtt; + } + + ret = roce5_create_user_srq_update(rdev, rsrq, pd, init_attr, udata, + page_shift); + if (ret != 0) { + goto err_free_mtt; + } + + init_attr->attr.max_wr = roce5_srq_max_avail_wr_set(rsrq); + + return 0; + +err_free_mtt: + hmm_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + +err_release_umem: + ib_umem_release(rsrq->umem); + +err_free_cqm_srq: + cqm5_object_delete(&rsrq->cqm_srq->object); + + return ret; +} + +static void roce5_get_cqn_xrcdn(u32 *cqn, u16 *xrcdn, + struct ib_srq_init_attr *init_attr, + struct roce5_device *rdev) +{ + *cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce5_cq(init_attr->ext.cq)->cqn : + 0); + *xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce5_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; +} + +static int roce5_create_kernel_srq_update(struct roce5_device *rdev, + struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + int ret; + ret = roce5_buf_write_mtt(rdev, &rsrq->mtt, rsrq->buf); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + rsrq->wrid = (u64 *)kmalloc( + (size_t)((u32)rsrq->max_depth * sizeof(u64)), GFP_KERNEL); + if (rsrq->wrid == NULL) { + rsrq->wrid = (u64 *)vzalloc( + (size_t)((u32)rsrq->max_depth * sizeof(u64))); + if (rsrq->wrid == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to alloc srq wrid, func_id(%d)", + rdev->glb_func_id); + ret = -ENOMEM; + goto err_free_mtt; + } + } + + roce5_get_cqn_xrcdn(&cqn, &xrcdn, init_attr, rdev); + + ret = roce5_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, + (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to handle srq sw2hw, func_id(%d)", + rdev->glb_func_id); + goto err_free_wrid; + } + + return 0; + +err_free_wrid: + kvfree(rsrq->wrid); + rsrq->wrid = NULL; + +err_free_mtt: + hmm_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_create_kernel_srq + Description : roce5_create_kernel_srq + Input : struct roce5_device *rdev + struct roce5_srq *rsrq + struct roce5_pd *pd + struct ib_srq_init_attr *init_attr + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_kernel_srq(struct roce5_device *rdev, + struct roce5_srq *rsrq, struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + u32 index) +{ + int i = 0; + int ret = 0; + int page_shift = 0; + struct roce5_wqe_srq_next_seg *next = NULL; + struct roce_wqe_srq_data_seg *scatter = NULL; + + /* alloc queue Buf/ Soft DB/SRQN/SRQC */ + rsrq->cqm_srq = cqm5_object_rdma_queue_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_SRQ, + (u32)rsrq->buf_sz, rsrq, true, index, 0, 0); + if (rsrq->cqm_srq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue by cqm_object, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + rsrq->srqn = rsrq->cqm_srq->index; + rsrq->buf = &rsrq->cqm_srq->q_room_buf_1; + + /* set Soft DB */ + rsrq->db.db_record = (__be32 *)(void *)(&rsrq->cqm_srq->q_header_vaddr + ->doorbell_record); + rsrq->db.dma = rsrq->cqm_srq->q_header_paddr; + *rsrq->db.db_record = 0; + + rsrq->head = 0; + rsrq->tail = rsrq->max_depth - 1; + rsrq->wqe_ctr = 0; + + for (i = 0; i < rsrq->max_depth; ++i) { + next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(rsrq, + i); + next->next_wqe_index = cpu_to_be16( + (u16)((u32)(i + 1) & (u32)(rsrq->max_depth - 1))); + + scatter = (struct roce_wqe_srq_data_seg *)(next + 1); + scatter->dw2.length = 0; + + /* first SGE = last SGE */ + scatter->dw3.lkey = LAST_SGE_NO_PRESENT; + scatter->dw3.lkey = cpu_to_be32(scatter->dw3.lkey); + } + + page_shift = (int)ROCE_ILOG2(rsrq->buf->buf_size); + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, rsrq->buf->buf_number, (u32)page_shift, + &rsrq->mtt, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdma_mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_cqm_srq; + } + + ret = roce5_create_kernel_srq_update(rdev, rsrq, pd, init_attr, + page_shift); + if (ret != 0) { + goto err_free_cqm_srq; + } + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + init_attr->attr.max_wr = (u32)rsrq->max_depth - 1; + return 0; + +err_free_cqm_srq: + cqm5_object_delete(&rsrq->cqm_srq->object); + + return ret; +} + +static void roce5_set_srq_depth(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq) +{ + u32 link_wqe_count = 0; + u32 remain_wqe_count = 0; + + /* + * +---+---+---+---+---+---+---+---+ + * |wqe|wqe|wqe|wqe|wqe|wqe|wqe|nil| + * +---+---+---+---+---+---+---+---+ + * ^ + * | + * full condition:head==tail==nil + */ + if (rsrq->container_flag != 0) { + link_wqe_count = + init_attr->attr.max_wr / (rsrq->container_size - 1u); + remain_wqe_count = + init_attr->attr.max_wr % (rsrq->container_size - 1u); + link_wqe_count += + ((remain_wqe_count == 0) ? + 0 : + (rsrq->container_size - remain_wqe_count)); + rsrq->max_depth = (int)(init_attr->attr.max_wr + + link_wqe_count + rsrq->container_size); + } else { + rsrq->max_depth = (int)(init_attr->attr.max_wr + 1); + } + rsrq->max_depth = + (int)((ROCE_ROUNDUP_POW_OF_TWO((u32)rsrq->max_depth)) & + 0xffffffff); +} + +static void roce5_rsrq_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, struct roce5_device *rdev) +{ + int buf_size = 0; + int desc_size = 0; + + roce5_srq_container_init(init_attr, rsrq, rdev); + + mutex_init(&rsrq->mutex); + spin_lock_init(&rsrq->lock); + + roce5_set_srq_depth(init_attr, rsrq); + + rsrq->max_gs = (init_attr->attr.max_sge <= ROCE_SRQ_MIN_SGE) ? + ROCE_SRQ_MIN_SGE : + ((init_attr->attr.max_sge <= ROCE_SRQ_MID_SGE) ? + ROCE_SRQ_MID_SGE : + roce5_srq_get_max_sge(rsrq)); + init_attr->attr.max_sge = (u32)rsrq->max_gs; + + /* 0 <= max_gs <= 3, desc_size = 64 + * 4 <= max_gs <= 7, desc_size = 128 + */ + desc_size = roce5_srq_init_wqe_size(rsrq); + buf_size = rsrq->max_depth * desc_size; + + /* WQEBB align 1 PAGE */ + if (buf_size < (int)PAGE_SIZE) { + rsrq->max_depth = + (int)((u64)PAGE_SIZE >> (unsigned int)rsrq->wqe_shift); + } + + /* align with PAGE SIZE */ + buf_size = (int)ROCE_ALIGN((u32)buf_size, PAGE_SIZE); + rsrq->buf_sz = buf_size; + + ROCE_DEV_INFO( + rdev, + "RoCE srq mode: container_EN(%d), container_mode(%d), XRC_EN(%d), MAX_DEPTH(%d), MAX_SGE(%d), desc_size(%d)", + rsrq->container_flag, rsrq->container_mode, rsrq->xrc_en, + rsrq->max_depth, rsrq->max_gs, desc_size); +} + +int roce5_create_srq_common(struct roce5_device *rdev, struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, u32 index) +{ + int ret; + roce5_rsrq_init(init_attr, rsrq, rdev); + + ret = roce5_create_srq_check(&pd->ibpd, init_attr, rsrq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check srq info"); + return ret; + } + + if (rsrq->ibsrq.uobject != NULL) { + ret = roce5_create_user_srq(rdev, rsrq, pd, init_attr, udata, + index); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to create user srq, func_id(%d), index(%d)", + rdev->glb_func_id, index); + } + } else { + ret = roce5_create_kernel_srq(rdev, rsrq, pd, init_attr, index); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create kernel srq, func_id(%d)", + rdev->glb_func_id); + } + } + + ROCE_DEV_INFO( + rdev, + "SRQ create:srqn:%d,container_en_mode:%d_%d,xrc_en:%d,max_depth:%d", + rsrq->srqn, rsrq->container_flag, rsrq->container_mode, + rsrq->xrc_en, rsrq->max_depth); + + return ret; +} + +int roce5_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + struct roce5_srq *rsrq = NULL; + + if ((ibsrq == NULL) || (init_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + pd = to_roce5_pd(ibsrq->pd); + rdev = to_roce5_dev(ibsrq->device); + rsrq = to_roce5_srq(ibsrq); + + return roce5_create_srq_common(rdev, rsrq, pd, init_attr, udata, + ROCE_SRQN_INVLD); +} + +struct ib_srq *roce5_create_srq_ofa(struct ib_pd *ibpd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_srq *rsrq = NULL; + + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return ERR_PTR(-EINVAL); + } + + rsrq = (struct roce5_srq *)kzalloc(sizeof(struct roce5_srq), + GFP_KERNEL); + if (rsrq == NULL) { + ROCE_ERR("Failed to alloc srq"); + return ERR_PTR(-ENOMEM); + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + rsrq->ibsrq.pd = ibpd; + rsrq->ibsrq.device = ibpd->device; +#if defined(IB_SRQ_HAVE_UOBJECT) + rsrq->ibsrq.uobject = + (udata ? ibpd->uobject : + NULL); /* udata not NULL means user srq create */ +#endif + ret = roce5_create_srq(&rsrq->ibsrq, init_attr, udata); + if (ret != 0) { + kfree(rsrq); + return ERR_PTR(ret); + } + + return &rsrq->ibsrq; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c new file mode 100644 index 000000000..887a857ad --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c @@ -0,0 +1,635 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "comm_defs.h" +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_mix.h" +#include "roce_cqm_cmd.h" +#include "roce_restore.h" +#include "hinic5_hmm.h" +#include "roce_main_extension.h" +#include "roce_srq_extension.h" +#include "roce_npu_cmd.h" +#include "roce_com.h" +#include "securec.h" +#include "roce_os_compat.h" + +/* and above next is V100 compatible data structure */ +typedef struct tag_roce_cmd_srq_arm { + roce_verbs_cmd_com_s com; + union { + u32 limitwater; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lwm : 16; + u32 warth : 4; + u32 th_up_en : 1; + u32 cont_en : 1; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 cont_en : 1; + u32 th_up_en : 1; + u32 warth : 4; + u32 lwm : 16; +#endif + } bs; + }; +} roce_cmd_srq_arm_s; + +typedef struct tag_roce_cmd_srq_hw2sw { + roce_verbs_cmd_com_s com; + roce_xq_mtt_info_s mtt_info; + u32 srq_buf_len; + u32 wqe_cache_line_start; + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; +} roce_cmd_srq_hw2sw_s; + +static u8 roce5_cal_srq_container_num(u32 avail_wr, struct roce5_srq *srq) +{ + u32 container_num = 0; + u32 srqe_num = 0; + + /* cal container_num by avail_wr from user */ + srqe_num = srq->container_size - 1; + + /* explain the formula: round up by srqe_num */ + container_num = (avail_wr + srqe_num - 1) / srqe_num; + return roce5_calculate_cont_th(container_num); +} + +static void roce5_srq_arm_config_cqm_cmd(cqm_cmd_buf_s *cqm_cmd_inbuf, + struct roce5_srq *srq, u32 srq_limit, + u8 *warth) +{ + roce_uni_cmd_srq_arm_s *srq_arm_inbuf = NULL; + + srq_arm_inbuf = (roce_uni_cmd_srq_arm_s *)cqm_cmd_inbuf->buf; + srq_arm_inbuf->com.index = cpu_to_be32((u32)(srq->srqn)); + srq_arm_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); + if (srq->container_flag != 0) { + srq_arm_inbuf->srq_arm.bs.cont_en = srq->container_flag; + srq_arm_inbuf->srq_arm.bs.th_up_en = + 1; //Valid for setting up container warn_threshold + *warth = roce5_cal_srq_container_num(srq_limit, srq); + srq_arm_inbuf->srq_arm.bs.warth = *warth & + ROCE_SRQ_CONTAINER_WARTH_MASK; + srq_arm_inbuf->srq_arm.limitwater = + cpu_to_be32(srq_arm_inbuf->srq_arm.limitwater); + } else { + srq_arm_inbuf->srq_arm.bs.lwm = srq_limit; + srq_arm_inbuf->srq_arm.limitwater = + cpu_to_be32(srq_arm_inbuf->srq_arm.limitwater); + } +} + +/* **************************************************************************** + Prototype : roce5_srq_arm + Description : roce5_srq_arm + Input : struct roce5_device *rdev + struct roce5_srq *srq + u32 srq_limit + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_srq_arm(struct roce5_device *rdev, struct roce5_srq *srq, + u32 srq_limit) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + uint8_t warth = 0; + uint16_t inbuf_size = 0; + + if (ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) != 0) { + inbuf_size = sizeof(roce_uni_cmd_srq_arm_s); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + inbuf_size, NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + if (ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) != 0) { + roce5_srq_arm_config_cqm_cmd(cqm_cmd_inbuf, srq, srq_limit, + &warth); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_ARM_SRQ, + cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "send ARM_SRQ command Failed, ret(%d), func_id(%d), warth(%d)", + ret, rdev->glb_func_id, warth); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_modify_srq + Description : roce5_modify_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *attr + enum ib_srq_attr_mask attr_mask + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + struct roce5_srq *srq = NULL; + struct roce5_modify_srq_cmd ucmd = { 0 }; + u32 max_avail_wr = 0; + + if ((ibsrq == NULL) || (attr == NULL)) { + ROCE_ERR("Ibsrq or attr is null"); + return (-EINVAL); + } + + if (((unsigned int)attr_mask & IB_SRQ_MAX_WR) != 0) { + ROCE_ERR("Not support resizing SRQs"); + return (-EINVAL); + } + + rdev = to_roce5_dev(ibsrq->device); + srq = to_roce5_srq(ibsrq); + + if ((udata != NULL) && + (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0)) { + ROCE_DEV_ERR( + rdev, + "SRQN:(%d), Failed to copy from user space, func_id(%d)", + srq->srqn, rdev->glb_func_id); + return (-EFAULT); + } + + if (((unsigned int)attr_mask & IB_SRQ_LIMIT) != 0) { + /* calculate max_wr */ + max_avail_wr = roce5_srq_max_avail_wr_set(srq); + if (attr->srq_limit > max_avail_wr) { + ROCE_DEV_ERR( + rdev, + "srq_limit > max_wr, func_id(%d), max_avail_wr(%d)", + rdev->glb_func_id, max_avail_wr); + return (-EINVAL); + } + + mutex_lock(&srq->mutex); + ret = roce5_srq_arm(rdev, srq, attr->srq_limit); + mutex_unlock(&srq->mutex); + + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "[Failed to handle srq arm, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + } + + return 0; +} + +static int roce5_cmd_srq_query(struct roce5_device *rdev, struct roce5_srq *srq, + u32 *srq_limit) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_srq_query_s *srq_query_inbuf = NULL; + roce_uni_cmd_srq_query_outbuf_s *srq_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_srq_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_srq_query_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_query_inbuf = (roce_uni_cmd_srq_query_s *)cqm_cmd_inbuf->buf; + srq_query_outbuf = + (roce_uni_cmd_srq_query_outbuf_s *)cqm_cmd_outbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_SRQ, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_A, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send QUERY_SRQ command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto err_cmd; + } + + *srq_limit = be32_to_cpu(srq_query_outbuf->com.srq_limit); + + if (srq->container_flag != 0) { + /* parse srqc to get warn_th */ + u32 warn_th = be32_to_cpu(srq_query_outbuf->com.warn_th); + *srq_limit = (warn_th == 0) ? 0 : + ((u32)((srq->container_size - 1) * + (1U << warn_th))); + } + +err_cmd: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_query_srq + Description : roce5_query_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *srq_attr + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + int ret = 0; + u32 limit_water = 0; + struct roce5_device *rdev = NULL; + struct roce5_srq *rsrq = NULL; + + if ((ibsrq == NULL) || (srq_attr == NULL)) { + ROCE_ERR("Ibsrq or srq_attr is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibsrq->device); + rsrq = to_roce5_srq(ibsrq); + + ret = roce5_cmd_srq_query(rdev, rsrq, &limit_water); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query srq, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + srq_attr->srq_limit = limit_water; + srq_attr->max_wr = roce5_srq_max_avail_wr_set(rsrq); + srq_attr->max_sge = (u32)rsrq->max_gs; + + return 0; +} + +static void roce5_srq_hw2sw_config_cqm_cmd(struct roce5_device *rdev, + cqm_cmd_buf_s *cqm_cmd_inbuf, + struct rdma_service_cap *rdma_cap, + struct roce5_srq *srq) +{ + roce_uni_cmd_srq_hw2sw_s *srq_hw2sw_inbuf = NULL; + + srq_hw2sw_inbuf = (roce_uni_cmd_srq_hw2sw_s *)cqm_cmd_inbuf->buf; + srq_hw2sw_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + srq_hw2sw_inbuf->srq_cache.srq_buf_len = cpu_to_be32((u32)srq->buf_sz); + + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_flags = 0; + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_num = 0; + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); +} + +/* **************************************************************************** + Prototype : roce5_srq_hw2sw + Description : roce5_srq_hw2sw + Input : struct roce5_device *rdev + struct roce5_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_srq_hw2sw(struct roce5_device *rdev, struct roce5_srq *srq) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + uint16_t inbuf_size = 0; + + if (ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) != 0) { + inbuf_size = sizeof(roce_uni_cmd_srq_hw2sw_s); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + inbuf_size, NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + if (ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) != 0) { + roce5_srq_hw2sw_config_cqm_cmd(rdev, cqm_cmd_inbuf, + &rdev->rdma_cap, srq); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_HW2SW_SRQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to send HW2SW_SRQ command, func_id(%d)", + rdev->glb_func_id); + + ret = -1; + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + + goto err_out; + } + +err_out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void roce5_free_srq(struct ib_srq *ibsrq, struct roce5_device *rdev, + struct roce5_srq *srq, struct ib_udata *udata) +{ + hmm_mtt_free(rdev->hwdev, &srq->mtt, SERVICE_T_ROCE); + + if (ibsrq->uobject) { + struct roce5_ucontext *ucontext = roce5_queue_to_ucontext( + (void *)ibsrq, udata, ROCE_Q_TYPE_SRQ); + roce5_db_unmap_user(ucontext, &srq->db, 0); + ib_umem_release(srq->umem); + } else { + kfree(srq->wrid); + } + + cqm5_object_delete(&srq->cqm_srq->object); +} + +int roce5_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + int ret = 0; + int times = 0; + struct roce5_device *rdev = NULL; + struct roce5_srq *rsrq = NULL; + + if (ibsrq == NULL) { + ROCE_ERR("Ibsrq is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibsrq->device); + rsrq = to_roce5_srq(ibsrq); + times = rdev->try_times; + + ret = roce5_srq_hw2sw(rdev, rsrq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to handle srq hw2sw, func_id(%d), ret:%d", + rdev->glb_func_id, ret); + return ret; + } + + roce5_free_srq(ibsrq, rdev, rsrq, udata); + return 0; +} + +int roce5_destroy_srq_ofa(struct ib_srq *ibsrq) +{ + int ret = roce5_destroy_srq(ibsrq, NULL); + if (ret == 0) { + kfree(ibsrq); + } + + return ret; +} + +static int roce5_check_sge_length(const struct ib_recv_wr *wr, + struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + struct roce_wqe_srq_data_seg *scat) +{ + int i = 0; + int ret = 0; + u32 data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + if (ROCE_UNLIKELY( + wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - + 1)) { + ret = -EINVAL; + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Sge data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)", + i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + return ret; + } + + data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz)) { + ret = -EINVAL; + *bad_wr = wr; + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + + ROCE_DEV_ERR( + rdev, + "Data length is over range, data_len(0x%x), max_msg_sz(0x%x), func_id(%d)", + data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + return ret; + } + + scat[i].dw2.length = cpu_to_be32(wr->sg_list[i].length); + scat[i].dw3.lkey = + cpu_to_be32((u32)(wr->sg_list[i].lkey & + NORMAL_FMT_AND_NEXT_SGE_PRESENT)); + scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); + } + + if (ROCE_UNLIKELY(i == 0)) { + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + scat[0].dw2.length = 0; + } else { + scat[i - 1].dw3.lkey = cpu_to_be32( + (u32)((wr->sg_list[i - 1].lkey | LAST_SGE_NO_PRESENT) & + NORMAL_FMT_AND_LAST_SGE_NO_PRESENT)); + } + + return ret; +} + +static int roce5_post_srq_recv_precheck(struct ib_srq *ibsrq, + const struct ib_recv_wr **bad_wr) +{ + if (ibsrq == NULL) { + ROCE_ERR("Ibsrq is null"); + return -EINVAL; + } + + if (bad_wr == NULL) { + ROCE_ERR("Bad_wr is null"); + return -EINVAL; + } + + return 0; +} + +int roce5_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + int ret = 0; + int nreq = 0; + unsigned long flags = 0; /* type required by kernel API */ + struct roce5_srq *srq = NULL; + struct roce5_device *rdev = NULL; + struct roce5_wqe_srq_next_seg *next = NULL; + struct roce_wqe_srq_data_seg *scat = NULL; + const struct ib_recv_wr *wr_tmp = wr; + + ret = roce5_post_srq_recv_precheck(ibsrq, bad_wr); + if (ret != 0) { + return -EINVAL; + } + + srq = to_roce5_srq(ibsrq); + rdev = to_roce5_dev(ibsrq->device); + + spin_lock_irqsave(&srq->lock, flags); + + for (nreq = 0; wr_tmp; ++nreq, wr_tmp = wr_tmp->next) { + if (ROCE_UNLIKELY((wr_tmp->num_sge > srq->max_gs) || + (srq->head == srq->tail))) { + ret = (wr_tmp->num_sge > srq->max_gs) ? (-EINVAL) : + (-ENOMEM); + *bad_wr = wr_tmp; + ROCE_DEV_ERR( + rdev, + "Num_sge(%d) > max_gs(%d) or srq is empty(head(%d) == tail(%d)), func_id(%d)", + wr_tmp->num_sge, srq->max_gs, rdev->glb_func_id, + srq->head, srq->tail); + break; + } + + next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe( + srq, srq->head); + scat = (struct roce_wqe_srq_data_seg *)(next + 1); + + ret = roce5_check_sge_length(wr_tmp, rdev, bad_wr, scat); + if (ret != 0) { + goto sge_len_err; + } + + next->pcnt = (u16)cpu_to_be16(srq->wqe_ctr + (u16)nreq); + srq->wrid[srq->head] = wr_tmp->wr_id; + srq->head = be16_to_cpu(next->next_wqe_index); + } + +sge_len_err: + if (ROCE_UNLIKELY(nreq != 0)) { + srq->wqe_ctr += (u16)nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *srq->db.db_record = cpu_to_be32(srq->wqe_ctr); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return ret; +} + +int roce5_post_srq_recv_ofa(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + return roce5_post_srq_recv(ibsrq, (const struct ib_recv_wr *)wr, + (const struct ib_recv_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h new file mode 100644 index 000000000..12d501b4c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h @@ -0,0 +1,36 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SRQ_FORMAT_H +#define ROCE_SRQ_FORMAT_H + +#include <linux/types.h> + +typedef struct roce5_srq_context { + /* DW0 */ + union { + struct { + u32 size : 5; + u32 page_size : 4; + u32 wqebb_size : 3; + u32 rsvd : 2; + u32 pdn : 18; + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { + u32 rsvd : 24; + u32 dma_attr_idx : 6; + u32 so_ro : 2; + } bs_c; + u32 value; + } dw1; + + u32 rsvd[14]; +} roce5_srq_context_s; + +#endif /* ROCE_SRQ_FORMAT_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c new file mode 100644 index 000000000..e66a99f39 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c @@ -0,0 +1,729 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "comm_defs.h" +#include "roce_uld_kernel_api.h" +#include "roce_bond.h" +#include "roce_uld_inner.h" +#include "roce_mpu_cmd.h" +#include "roce_cqm_cmd.h" + +#define PLUGIN __attribute__((__used__, __weak__, optimize("O0"))) +#define ROCE_ULD_PRINT_BUF_SIZE 256 + +PLUGIN int roce5_uld_reg_user_mr(roce_uld_handle handle) +{ + return -1; +} + +PLUGIN int roce5_uld_dereg_user_mr(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_adm_dfx_uld_extend(roce_uld_handle handle, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + return 0; +} + +PLUGIN int roce5_uld_qp_modify(roce_uld_handle handle, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + return 0; +} + +PLUGIN void roce5_uld_rdev_remove(roce_uld_handle handle) +{ + return; +} + +PLUGIN int roce5_uld_rdev_add(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_uld_qp_destroy(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_uld_qp_create(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_uld_cq_create(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN void roce5_uld_cq_destroy(roce_uld_handle handle) +{ + return; +} + +PLUGIN int32_t roce5_uld_user2kernel_msg_handle(roce_uld_handle handle, + void *buf_in, u16 buf_in_len, + void *buf_out, u16 buf_out_len, + u16 *buf_out_size) +{ + return 0; +} + +PLUGIN int roce5_uld_service_init(void) +{ + return 0; +} + +PLUGIN void roce5_uld_service_exit(void) +{ + return; +} + +u64 roce5_mr_virt_addr_get(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return 0; + } + + return hdl->virt_addr; +} + +int roce5_mr_virt_addr_set(roce_uld_handle handle, u64 virt_addr) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + hdl->virt_addr = virt_addr; + + return 0; +} + +int roce5_mr_user_data_get(roce_uld_handle handle, u8 *user_data, + u32 user_data_len) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + u32 cpy_len; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + if (user_data == NULL) { + ROCE_ERR("user_data is NULL"); + return -EINVAL; + } + + cpy_len = (user_data_len > hdl->mr->user_data_len) ? + hdl->mr->user_data_len : + user_data_len; + (void)memcpy_s(user_data, user_data_len, hdl->mr->rdmamr.user_data, + cpy_len); + + return cpy_len; +} + +int roce5_mpt_alloc_and_init(roce_uld_handle handle, int mode, u8 *user_data, + u32 user_data_len, u32 mpt_xid_mode) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_mr *mr = NULL; + struct roce5_device *rdev = NULL; + int ret = 0; + u32 npages = 0; + u32 page_shift = PAGE_SHIFT; + u32 mpt_xid_create_mode; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + mr = hdl->mr; + rdev = hdl->rdev; + if (rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return -EINVAL; + } + + if (user_data_len > HMM_USER_DATA_MAX_BYTE) { + ROCE_ERR( + "user_data_len(%u) is invalid, max user_data_len is %u", + user_data_len, HMM_USER_DATA_MAX_BYTE); + return -EINVAL; + } + + if (mode == ROCE_ULD_MPT_NORMAL) { + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + npages = (u32)ib_umem_num_pages(mr->umem); +#if defined(UMEM_HAVE_PAGE_SHIFT) + page_shift = (u32)(mr->umem->page_shift); +#endif + } + + mpt_xid_create_mode = CQM_DYNAMIC_XID_MOD( + CQM_XID_SEARCH_ALL, CQM_XID_LOW_BIT_0_0_1, mpt_xid_mode); + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift, + mpt_xid_create_mode); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc mpt and mtt, func_id(%d), npages(%u), page_shift(%u)", + hinic5_global_func_id(rdev->hwdev), npages, page_shift); + goto err_alloc_tpt; + } + + roce5_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, hdl->pdn, hdl->virt_addr, + hdl->length, hdl->access); + + if (mode == ROCE_ULD_MPT_NORMAL) { + ret = roce5_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + hinic5_global_func_id(rdev->hwdev)); + goto err_write_mtt; + } + } + + if (user_data != NULL) { + (void)memcpy_s(hdl->mr->rdmamr.user_data, user_data_len, + user_data, user_data_len); + hdl->mr->user_data_len = user_data_len; + } else { + hdl->mr->user_data_len = 0; + } + + return 0; + +err_write_mtt: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + return ret; +} + +void roce5_mpt_destroy(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return; + } + + if (hdl->mr == NULL) { + ROCE_ERR("mr is NULL"); + return; + } + + rdev = hdl->rdev; + if (rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return; + } + + if (hdl->mr->rdmamr.enabled != RDMA_MPT_DISABLED) { + roce5_free_tpt(rdev, &hdl->mr->rdmamr); + } + + return; +} + +int roce5_mpt_enable(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + int ret; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + rdev = hdl->rdev; + if (rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return -EINVAL; + } + + /* here hmm will userdata send to npu */ + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &hdl->mr->rdmamr, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of user mr, func_id(%d)", + hinic5_global_func_id(rdev->hwdev)); + } + + return ret; +} + +int roce5_uld_get_func_id(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -1; + } + + if (hdl->rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return -1; + } + + return (int)hdl->rdev->glb_func_id; +} + +int roce5_uld_send_msg_to_mgmt(roce_uld_handle handle, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((cmd < ROCE_MPU_CMD_ULP_START) || (cmd >= ROCE_MPU_CMD_MAX)) { + ROCE_ERR("cmd is invalid"); + return -1; + } + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rdev == NULL) || (buf_in == NULL) || (buf_out == NULL) || + (out_size == NULL)) { + ROCE_ERR( + "handle is invalid, or buf_in(%d), buf_out(%d), out_size(%d) is invalid", + (buf_in == NULL), (buf_out == NULL), + (out_size == NULL)); + return -1; + } + + return hinic5_msg_to_mgmt_sync(hdl->rdev->hwdev, HINIC5_MOD_ROCE, cmd, + buf_in, in_size, buf_out, out_size, 0, + HINIC5_CHANNEL_ROCE); +} + +int roce5_uld_get_qp_info(roce_uld_handle handle, roce_uld_qp_info *qp_info) +{ + struct roce5_qp *rqp = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rqp == NULL) || (qp_info == NULL)) { + ROCE_ERR("param invalid"); + return -1; + } + + rqp = (struct roce5_qp *)hdl->rqp; + qp_info->qpn = rqp->qpn; + qp_info->qp_type = rqp->qp_type; + if (hdl->qp_attr != NULL) { + qp_info->dgid = (u8 *)&hdl->qp_attr->path_info.dgid[0]; + } + return 0; +} + +int roce5_uld_modify_qp_tx_bond_hash(roce_uld_handle handle, u32 tx_bond_hash) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->qp_attr == NULL)) { + ROCE_ERR("param invalid"); + return -1; + } + + hdl->qp_attr->path_info.dw0.bs.bond_tx_hash_value = tx_bond_hash & 0xf; + return 0; +} + +u8 *roce5_get_rdev_uld_def(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + return (u8 *)&rdev->uld_def.self_def[0]; +} + +void *roce5_uld_get_rdev_hwdev(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + void *hwdev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + hwdev = hdl->rdev->hwdev; + if (hwdev == NULL) { + ROCE_ERR("handle is invalid, hwdev is NULL"); + return NULL; + } + return hwdev; +} + +struct ib_device *roce5_uld_get_rdev_ibdev(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + struct ib_device *ib_dev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + ib_dev = &(hdl->rdev->ib_dev); + return ib_dev; +} + +struct hinic5_lld_dev *roce5_uld_get_rdev_lld_dev(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + lld_dev = hdl->rdev->lld_dev; + if (lld_dev == NULL) { + ROCE_ERR("handle is invalid, lld_dev is NULL"); + return NULL; + } + return lld_dev; +} + +#ifdef ROCE_BONDING_EN +int roce5_uld_get_rdev_bond_dev_slave_cnt(roce_uld_handle handle, + u32 *slave_cnt, uint8_t *bond_en) +{ + struct roce5_device *rdev = NULL; + struct roce5_bond_device *bond_dev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (slave_cnt == NULL) || (bond_en == NULL)) { + return -EINVAL; + } + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return -EINVAL; + } + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + *slave_cnt = 0; + *bond_en = 0; + return 0; + } + *slave_cnt = hdl->rdev->bond_dev->slave_cnt; + *bond_en = 1; + return 0; +} +#endif + +u8 *roce5_get_rqp_uld_def(roce_uld_handle handle) +{ + struct roce5_qp *rqp = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rqp = (struct roce5_qp *)hdl->rqp; + if ((rqp == NULL) || (rqp->rqp_magic != ROCE_RQP_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rqp == NULL) ? ROCE_MAGIC_NULL : rqp->rqp_magic); + return NULL; + } + return (u8 *)&rqp->uld_def.uld_def[0]; +} + +u8 *roce5_get_rcq_uld_def(roce_uld_handle handle) +{ + struct roce5_cq *rcq = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rcq = (struct roce5_cq *)hdl->rcq; + if ((rcq == NULL) || (rcq->rcq_magic != ROCE_RCQ_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rcq == NULL) ? ROCE_MAGIC_NULL : rcq->rcq_magic); + return NULL; + } + return (u8 *)&rcq->uld_def.uld_def[0]; +} + +int roce5_uld_cos_set(roce_uld_handle handle, struct ib_qp_attr *attr, u8 type, + u8 value) +{ + u8 cos; + roce5_get_cos_inbuf_s inbuf = { 0 }; + struct roce5_qp *rqp = NULL; + roce_verbs_qp_attr_s *qp_attr = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + int ret; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rqp == NULL) || (hdl->qp_attr == NULL) || + (hdl->rdev == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + rqp = hdl->rqp; + qp_attr = hdl->qp_attr; + + if (type == ROCE5_COS_SET_TYPE_CLASS) { + inbuf.sl = attr->ah_attr.sl; + inbuf.sgid_index = attr->ah_attr.grh.sgid_index; + inbuf.port_num = attr->ah_attr.port_num; + inbuf.traffic_class = value; + + ret = roce5_get_dcb_cfg_cos(hdl->rdev, &inbuf, &cos, + rqp->qp_type); + if (ret != 0) { + ROCE_ERR("get dcb cfg cos failed"); + return ret; + } + + /* update tclass and cos */ + qp_attr->path_info.dw6.bs.tclass = value; + qp_attr->com_info.dw5.bs.cos = cos; + rqp->db_cos = cos; + } else if (type == ROCE5_COS_SET_TYPE_COS) { + if (value > 7) { /* cos range is 0-7 */ + ROCE_ERR("invalid cos(%u)", value); + return -EINVAL; + } + qp_attr->com_info.dw5.bs.cos = value; + rqp->db_cos = value; + } else { + ROCE_ERR("set type(%u) invalid", type); + return -EINVAL; + } + + return 0; +} + +int roce5_support_uld_kernel_feature(roce_uld_handle handle, u64 feature) +{ + roce_user_handle_t *user_handle = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + + if ((!user_handle) || + (user_handle->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + ROCE_ERR("Invalid RoCE handle\n"); + return -EINVAL; + } + + rdev = user_handle->rdev; + + if (rdev == NULL) { + ROCE_ERR("rdev is NULL in handle\n"); + return -EINVAL; + } + + //return capability bit AND result + return (rdev)->cfg_info.feature_cap.uld_feature + .roce_uld_kernel_ext_ability & + feature; +} + +int roce5_uld_set_mpt_xid_mode(roce_uld_handle handle, u8 mode) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rdev == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + rdev = hdl->rdev; + rdev->mpt_xid_mode = mode; + + return 0; +} + +int roce5_uld_get_dgid(roce_uld_handle handle, u8 **dgid) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->qp_attr == NULL) || (dgid == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + *dgid = hdl->qp_attr->path_info.dgid; + return 0; +} + +int roce5_uld_set_bond_tx_hash_value(roce_uld_handle handle, + u32 bond_tx_hash_value) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->qp_attr == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + hdl->qp_attr->path_info.dw0.bs.bond_tx_hash_value = bond_tx_hash_value; + return 0; +} + +int roce5_uld_get_mr_uld_self_def(roce_uld_handle handle, u8 **self_def, + u32 *self_def_len) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->mr == NULL) || (self_def == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *self_def = &hdl->mr->uld_def.self_def[0]; + if (self_def_len != NULL) { + *self_def_len = ROCE_MR_ULD_DEF_LEN; + } + return 0; +} + +int roce5_uld_get_rdmamr(roce_uld_handle handle, struct hmm_rdma **rdmamr_ptr) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->mr == NULL) || (rdmamr_ptr == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *rdmamr_ptr = &(hdl->mr->rdmamr); + return 0; +} + +int roce5_uld_set_qp_attr_ack_to(roce_uld_handle handle, u32 ack_to) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->qp_attr == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + hdl->qp_attr->com_info.dw2.bs.ack_to = ack_to; + return 0; +} + +int roce5_uld_get_qp_state(roce_uld_handle handle, u8 *out_qp_state) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->rqp == NULL) || (out_qp_state == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *out_qp_state = hdl->rqp->qp_state; + return 0; +} + +int roce5_uld_get_config_num_ports(roce_uld_handle handle, + int *out_cfg_num_ports) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->rdev == NULL) || + (out_cfg_num_ports == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *out_cfg_num_ports = hdl->rdev->hw_info.config_num_ports; + return 0; +} + +int roce5_uld_get_handle_length(roce_uld_handle handle, u64 *length) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (length == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *length = hdl->length; + return 0; +} + +void roce5_uld_print_kernel_err_impl(roce_uld_handle handle, const char *func, + int line, const char *fmt, ...) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + va_list args; + char buf[ROCE_ULD_PRINT_BUF_SIZE]; + + va_start(args, fmt); + if (vsnprintf(buf, sizeof(buf), fmt, args) < 0) { + va_end(args); + return; + } + va_end(args); + + if ((hdl != NULL) && (hdl->magic == ROCE_ULD_HANDLE_MAGIC_NUM) && + (hdl->rdev != NULL)) { + ROCE_DEV_ERR(hdl->rdev, "[ULP][%s,%d]%s", func, line, buf); + } else { + ROCE_ERR("[ULP][%s,%d]%s", func, line, buf); + } +} + +int roce5_uld_qp_get_port(roce_uld_handle handle, u8 *port) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->rqp == NULL) || (port == NULL) || + (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *port = hdl->rqp->port; + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h new file mode 100644 index 000000000..e61160c2f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef ROCE_ULD_INNER_H +#define ROCE_ULD_INNER_H + +#include "roce.h" +#include "roce_mr.h" +#include "roce_qp.h" +#include "roce_cq.h" +#include "hmm_common.h" +#include "roce_npu_cmd_mr_defs.h" +#include "roce_npu_cmd_qp_defs.h" + +#define ROCE_ULD_HANDLE_MAGIC_NUM 0x524F4345 /* "ROCE" string of ascii code */ +#define ROCE_MAGIC_NULL 0x12345678 /* used to distinguish null pointer and unassigned */ + +typedef struct { + u32 magic; /* magic number, determine handle whether valid */ + struct roce5_device *rdev; + struct roce5_mr *mr; + roce_verbs_mr_attr_s *mr_attr; + u64 virt_addr; + u64 length; + u32 pdn; + int access; + + struct roce5_qp *rqp; + roce_verbs_qp_attr_s *qp_attr; + + struct roce5_cq *rcq; + int cqe; + + struct ib_udata *udata; + struct mtt * + mtt; /* hmm module of mtt management structure, shadow scenario page_shift and paddr mandatory, mtt_layers constraint is 0 */ + dma_addr_t *db_dma_handle; /* software db of dma address */ + + u8 *shadow; + u32 *shadow_vfid; + u16 *shadow_vf_num; +} roce_user_handle_t; + +#define ROCE_CONSTUCT_ULD_HDL_RQP(hdl, roce_dev, roce_qp, roce_qp_attr, \ + roce_udata, roce_mtt, roce_db_dma_handle) \ + do { \ + (hdl)->magic = ROCE_ULD_HANDLE_MAGIC_NUM; \ + (hdl)->rdev = (roce_dev); \ + (hdl)->rqp = (roce_qp); \ + (hdl)->qp_attr = (roce_qp_attr); \ + (hdl)->udata = (roce_udata); \ + (hdl)->mtt = (roce_mtt); \ + (hdl)->db_dma_handle = (roce_db_dma_handle); \ + } while (0) + +#define ROCE_CONSTUCT_ULD_HDL_RCQ(hdl, roce_dev, roce_cq, roce_udata, \ + roce_mtt, roce_db_dma_handle, roce_cqe) \ + do { \ + (hdl)->magic = ROCE_ULD_HANDLE_MAGIC_NUM; \ + (hdl)->rdev = (roce_dev); \ + (hdl)->rcq = (roce_cq); \ + (hdl)->udata = (roce_udata); \ + (hdl)->mtt = (roce_mtt); \ + (hdl)->db_dma_handle = (roce_db_dma_handle); \ + (hdl)->cqe = (roce_cqe); \ + } while (0) + +#define ROCE_CONSTUCT_ULD_HDL_SHADOW(hdl, roce_shadow, roce_shadow_vfid, \ + roce_shadow_vf_num) \ + do { \ + (hdl)->shadow = (roce_shadow); \ + (hdl)->shadow_vfid = (roce_shadow_vfid); \ + (hdl)->shadow_vf_num = (roce_shadow_vf_num); \ + } while (0) + +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c new file mode 100644 index 000000000..7530a655b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c @@ -0,0 +1,169 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_xrc.h" + +static int roce5_rdma_xrcd_alloc(void *hwdev, u32 *xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (xrcdn == NULL)) { + ROCE_ERR("Hwdev or xrcdn is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + *xrcdn = rdma_bitmap_alloc(&comp_priv->xrcd_bitmap); + if (*xrcdn == RDMA_INVALID_INDEX) { + ROCE_ERR("Can't get valid xrcdn, err(%d)", -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +static void roce5_rdma_xrcd_free(void *hwdev, u32 xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + rdma_bitmap_free(&comp_priv->xrcd_bitmap, xrcdn); +} + +static int roce5_init_xrcd(struct ib_device *ibdev, struct roce5_device *rdev, + struct roce5_xrcd *xrcd) +{ + int ret = 0; + struct ib_cq_init_attr cq_attr = { 0 }; + + ret = roce5_rdma_xrcd_alloc(rdev->hwdev, &xrcd->xrcdn); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma xrcd, ret(%d)", ret); + goto err_alloc_xrcdn; + } + + xrcd->pd = ib_alloc_pd(ibdev, 0); + + if (IS_ERR(xrcd->pd)) { + ret = (int)PTR_ERR(xrcd->pd); + ROCE_DEV_ERR(rdev, "Failed to alloc pd, ret(%d)", ret); + goto err_alloc_pd; + } + + cq_attr.cqe = 1; + xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr); + if (IS_ERR(xrcd->cq)) { + ret = (int)PTR_ERR(xrcd->cq); + ROCE_DEV_ERR(rdev, "Failed to create cq, ret(%d)", ret); + goto err_create_cq; + } + return 0; + +err_create_cq: + ib_dealloc_pd(xrcd->pd); + +err_alloc_pd: + roce5_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + +err_alloc_xrcdn: + return ret; +} + +int roce5_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + struct roce5_xrcd *xrcd = NULL; + + if (ibxrcd == NULL) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rdev = to_roce5_dev(ibxrcd->device); + xrcd = to_roce5_xrcd(ibxrcd); + + return roce5_init_xrcd(ibxrcd->device, rdev, xrcd); +} + +struct ib_xrcd *roce5_alloc_xrcd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret; + struct roce5_device *rdev = NULL; + struct roce5_xrcd *xrcd = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return ERR_PTR(-EINVAL); + } + + rdev = to_roce5_dev(ibdev); + + xrcd = (struct roce5_xrcd *)kzalloc(sizeof(*xrcd), GFP_KERNEL); + if (xrcd == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc memory for xrcd"); + return ERR_PTR(-ENOMEM); + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + xrcd->ibxrcd.device = ibdev; + + ret = roce5_alloc_xrcd(&xrcd->ibxrcd, NULL); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init xrcd, ret(%d)", ret); + kfree(xrcd); + return ERR_PTR(ret); + } + return &xrcd->ibxrcd; +} + +int roce5_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + struct roce5_xrcd *xrcd = NULL; + if (ibxrcd == NULL) { + ROCE_ERR("Ibxrcd is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibxrcd->device); + xrcd = to_roce5_xrcd(ibxrcd); + ib_destroy_cq(xrcd->cq); + + ib_dealloc_pd(xrcd->pd); + roce5_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + return 0; +} + +int roce5_dealloc_xrcd_ofa(struct ib_xrcd *ibxrcd) +{ + struct roce5_xrcd *xrcd = to_roce5_xrcd(ibxrcd); + int ret = roce5_dealloc_xrcd(ibxrcd, NULL); + if (ret == 0) { + kfree(xrcd); + } + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h new file mode 100644 index 000000000..1a4e603ab --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_XRC_H +#define ROCE_XRC_H + +#include <rdma/ib_verbs.h> + +struct roce5_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; + struct ib_pd *pd; + struct ib_cq *cq; +}; + +#endif //ROCE_XRC_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c new file mode 100644 index 000000000..d67dd0844 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c @@ -0,0 +1,162 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +#include <rdma/ib_user_verbs.h> +#include <linux/errno.h> +#include <linux/init.h> + +#include "roce.h" +#include "hinic5_lld.h" +#include "roce_infra.h" +#include "roce_card.h" + +static struct roce5_cards g_roce5_cards[MAX_DPU_CARD_NUM]; + +void roce5_card_init(u16 idx, u64 card_id) +{ + g_roce5_cards[idx].card_status = RUNNING; + g_roce5_cards[idx].card_id = card_id; + INIT_LIST_HEAD(&g_roce5_cards[idx].rdev_list); + mutex_init(&g_roce5_cards[idx].card_mutex); +} + +void roce5_card_deinit(u16 idx) +{ + g_roce5_cards[idx].card_status = INVALID; + g_roce5_cards[idx].card_id = 0; + mutex_destroy(&g_roce5_cards[idx].card_mutex); +} + +int roce5_g_rdev_add(struct roce5_device *rdev) +{ + struct roce5_rdev_list_node *rdev_node = NULL; + u64 card_id; + u16 i; + + if (hinic5_get_chip_node_id(rdev->lld_dev, &card_id) != 0) { + ROCE_ERR("Failed to get chip node id, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + rdev_node = kzalloc(sizeof(struct roce5_rdev_list_node), GFP_KERNEL); + if (rdev_node == NULL) { + ROCE_ERR("Failed to alloc memory."); + return -ENOMEM; + } + rdev_node->rdev = rdev; + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == RUNNING && + g_roce5_cards[i].card_id == card_id) { + break; + } + } + + if (i >= MAX_DPU_CARD_NUM) { + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == INVALID) { + roce5_card_init(i, card_id); + break; + } + } + + if (i >= MAX_DPU_CARD_NUM) { + ROCE_ERR("Exceeded the MAX_DPU_CARD_NUM:0x%x", + MAX_DPU_CARD_NUM); + kfree(rdev_node); + return -ENOMEM; + } + } + + mutex_lock(&g_roce5_cards[i].card_mutex); + list_add_tail(&rdev_node->node, &g_roce5_cards[i].rdev_list); + mutex_unlock(&g_roce5_cards[i].card_mutex); + return 0; +} + +void roce5_g_rdev_remove(struct roce5_device *rdev) +{ + struct roce5_rdev_list_node *rdev_node = NULL, *rdev_next = NULL; + u64 card_id; + int i; + + if (hinic5_get_chip_node_id(rdev->lld_dev, &card_id) != 0) { + ROCE_ERR("Failed to get chip node id, func_id(%d)", + rdev->glb_func_id); + return; + } + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == RUNNING && + g_roce5_cards[i].card_id == card_id) { + break; + } + } + + if (i < MAX_DPU_CARD_NUM) { + mutex_lock(&g_roce5_cards[i].card_mutex); + list_for_each_entry_safe(rdev_node, rdev_next, + &g_roce5_cards[i].rdev_list, node) { + if (rdev_node->rdev == rdev) { + list_del(&rdev_node->node); + kfree(rdev_node); + break; + } + } + mutex_unlock(&g_roce5_cards[i].card_mutex); + /* deinit card if rdev list is empty */ + if (list_empty(&g_roce5_cards[i].rdev_list)) { + roce5_card_deinit(i); + } + } +} + +struct roce5_device *roce5_get_rdev_by_card_func_id(uint64_t card_id, + uint16_t func_id) +{ + struct roce5_rdev_list_node *rdev_node = NULL; + int i; + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == RUNNING && + g_roce5_cards[i].card_id == card_id) { + break; + } + } + + if (i < MAX_DPU_CARD_NUM) { + list_for_each_entry(rdev_node, &g_roce5_cards[i].rdev_list, + node) { + if (rdev_node->rdev->glb_func_id == func_id) { + return rdev_node->rdev; + } + } + } + return NULL; +} + +struct roce5_device *roce5_get_rdev_by_name(const char *dev_name) +{ + struct roce5_rdev_list_node *rdev_node = NULL; + int i; + + if (dev_name == NULL) { + return NULL; + } + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == INVALID) { + continue; + } + + list_for_each_entry(rdev_node, &g_roce5_cards[i].rdev_list, + node) { + if (strncmp(rdev_node->rdev->ib_dev.name, dev_name, + IB_DEVICE_NAME_MAX) == 0) { + return rdev_node->rdev; + } + } + } + return NULL; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h new file mode 100644 index 000000000..5ceabdd1f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h @@ -0,0 +1,27 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +#ifndef ROCE_CARD_H +#define ROCE_CARD_H + +#include <linux/slab.h> +#include <linux/list.h> + +#include "roce.h" + +#define MAX_DPU_CARD_NUM \ + 1024 //current SDK PCIe on current cannot identifies VM on and bare metal on is same one 张 card, temporary add specification supports start multi VM +struct roce5_rdev_list_node { + struct list_head node; + struct roce5_device *rdev; +}; + +typedef enum roce5_card_status { INVALID = 0, RUNNING } roce5_card_status_e; + +struct roce5_cards { + u64 card_id; + roce5_card_status_e card_status; + struct mutex card_mutex; + struct list_head rdev_list; +}; +#endif /* ROCE_CARD_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c new file mode 100644 index 000000000..2ad39f97f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c @@ -0,0 +1,1270 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/errno.h> +#include <linux/if_ether.h> +#include <linux/moduleparam.h> +#include <linux/types.h> + +#include "rdma_comp.h" +#include "roce5_gid_type.h" +#include "roce_compat.h" +#include "roce_com.h" +#include "roce_cqm_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_ctx_format.h" +#include "roce_infra.h" +#include "roce_main_extension.h" +#include "roce_mix.h" +#include "roce_npu_cmd.h" +#include "roce_npu_cmd_ext_defs.h" +#include "roce_srq_extension.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "securec.h" +#include "roce_cmd.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#define OVS_MPU_CMD_QOS_SET_VM_POLICING 30 +#define OVS_MPU_CMD_VPORT_SET 13 +#define OVS_MPU_VPORT_SET_CMD_ATTR 2 + +uint8_t roce5_srq_mode_chip_adapt(uint8_t cfg_mode) +{ + switch (cfg_mode) { + case ROCE_CHIP_SRQ_MODE_1: + return ROCE_SRQ_MODE_3; + case ROCE_CHIP_SRQ_MODE_2: + return ROCE_SRQ_MODE_2; + case ROCE_CHIP_SRQ_MODE_3: + return ROCE_SRQ_MODE_1; + case ROCE_CHIP_SRQ_MODE_4: + return ROCE_SRQ_MODE_0; + default: + return ROCE_SRQ_MODE_3; + } +} + +int roce5_add_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id) +{ + void *hwdev = rdev->hwdev; + u16 er_fwd_id = rdev->glb_func_id; + roce_cfg_mac_cmd_s add_mac_entry_info; + u16 out_size = (u16)sizeof(add_mac_entry_info); + int ret; + + if (ROCE5_SUPPORT_SML_MAC_TABLE(rdev) == 0) { + return 0; + } + + if (rdev->hca_vf_mode != 0) { + ROCE_INFO("No need to add mac table entry."); + return 0; + } + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_ERR("Hwdev or mac_addr is null"); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_ERR("Invalid vlan_id(%u)", vlan_id); + return -EINVAL; + } + + ret = memset_s(&add_mac_entry_info, sizeof(add_mac_entry_info), 0, + sizeof(add_mac_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset add_mac_entry_info.(ret:%d)", ret); + return -ENOMEM; + } + add_mac_entry_info.func_id = func_id; + add_mac_entry_info.vlan_id = vlan_id; + add_mac_entry_info.er_fwd_id = er_fwd_id; + ret = memcpy_s(add_mac_entry_info.mac, ETH_ALEN, mac_addr, ETH_ALEN); + if (ret != 0) { + ROCE_ERR("memory copy failed. ret%d", ret); + return -ENOMEM; + } + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_MAC, + &add_mac_entry_info, + sizeof(add_mac_entry_info), + &add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (add_mac_entry_info.head.status != 0)) { + ROCE_ERR( + "Failed to add mac entry, err(%d), status(0x%x), out size(0x%x)", + ret, add_mac_entry_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce5_del_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id) +{ + void *hwdev = rdev->hwdev; + u16 er_fwd_id = rdev->glb_func_id; + roce_cfg_mac_cmd_s del_mac_tbl_info; + u16 out_size = (u16)sizeof(del_mac_tbl_info); + int ret; + + if (ROCE5_SUPPORT_SML_MAC_TABLE(rdev) == 0) { + return 0; + } + + if (rdev->hca_vf_mode != 0) { + ROCE_DEV_INFO(rdev, "No need to delete mac table entry."); + return 0; + } + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_ERR("Hwdev or mac_addr is null"); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_ERR("Invalid vlan_id(%u)", vlan_id); + return -EINVAL; + } + + ret = memset_s(&del_mac_tbl_info, sizeof(del_mac_tbl_info), 0, + sizeof(del_mac_tbl_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset del_mac_tbl_info.(ret:%d)", ret); + return -ENOMEM; + } + del_mac_tbl_info.func_id = func_id; + del_mac_tbl_info.vlan_id = vlan_id; + del_mac_tbl_info.er_fwd_id = er_fwd_id; + ret = memcpy_s(del_mac_tbl_info.mac, mac_addr_len, mac_addr, + mac_addr_len); + if (ret != 0) { + ROCE_ERR("Failed to memcpy_s dmac.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_MAC, + &del_mac_tbl_info, + sizeof(del_mac_tbl_info), + &del_mac_tbl_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (del_mac_tbl_info.head.status != 0)) { + ROCE_ERR( + "Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)", + ret, del_mac_tbl_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +void roce5_add_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], u32 vlan_id, + u16 func_id, u8 er_id, u16 src_func_id) +{ + void *hwdev = rdev->hwdev; + roce_cfg_ipsu_mac_cmd_s ipsu_add_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_add_mac_entry_info); + int ret; + + if (rdev->hca_vf_mode != 0) { + ROCE_INFO("No need to add ipsu tbl mac entry"); + return; + } + +#ifdef ROCE_BONDING_EN + if (!roce5_get_bond_ipsurx_en()) { + ROCE_ERR("No need to add ipsu tbl mac entry"); + return; + } +#endif + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_ERR("Hwdev or mac_addr is null"); + return; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_ERR("Invalid vlan_id(%u)", vlan_id); + } + + ret = memset_s(&ipsu_add_mac_entry_info, + sizeof(ipsu_add_mac_entry_info), 0, + sizeof(ipsu_add_mac_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_add_mac_entry_info.(ret:%d)", + ret); + return; + } + ipsu_add_mac_entry_info.func_id = func_id; + ipsu_add_mac_entry_info.er_id = er_id; + ipsu_add_mac_entry_info.src_func_id = src_func_id; + ipsu_add_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_add_mac_entry_info.vni_en = 0; + ret = memcpy_s(ipsu_add_mac_entry_info.mac, ETH_ALEN, mac_addr, + ETH_ALEN); + if (ret != 0) { + ROCE_ERR( + "Failed to memcpy ipsu_add_mac_entry_info.mac (ret:%d)", + ret); + return; + } + + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_IPSU_MAC, + &ipsu_add_mac_entry_info, + sizeof(ipsu_add_mac_entry_info), + &ipsu_add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_add_mac_entry_info.head.status != 0) && + (ipsu_add_mac_entry_info.head.status != ERR_EXIST))) { + ROCE_ERR( + "Failed to add mac entry to IPSU, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_add_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_add_mac_entry_info.head.status == ERR_EXIST) { + ROCE_INFO("This mac entry has been added to IPSU"); + } + + return; +} + +static int32_t roce5_del_mac_entry_precheck(struct roce5_device *rdev, + void *hwdev, uint8_t *mac_addr, + uint32_t vlan_id) +{ + if (rdev->hca_vf_mode != 0) { + ROCE_DEV_INFO(rdev, "No need to del ipsu tbl mac entry."); + return -EINVAL; + } + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_DEV_ERR(rdev, "hwdev or mac_addr is null."); + return -EINVAL; + } + +#ifdef ROCE_BONDING_EN + if (!roce5_get_bond_ipsurx_en()) { + ROCE_DEV_INFO(rdev, "No need to del ipsu tbl mac entry."); + return -EINVAL; + } +#endif + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_DEV_ERR(rdev, "Invalid vlan_id(%d).", vlan_id); + return -EINVAL; + } + + return 0; +} + +void roce5_del_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], size_t eth_alen, + u32 vlan_id, u8 er_id, u16 src_func_id) +{ + void *hwdev = rdev->hwdev; + u16 func_id = rdev->glb_func_id; + roce_cfg_ipsu_mac_cmd_s ipsu_del_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_del_mac_entry_info); + int ret; + + ret = roce5_del_mac_entry_precheck(rdev, hwdev, mac_addr, vlan_id); + if (ret != 0) { + return; + } + + ret = memset_s(&ipsu_del_mac_entry_info, + sizeof(ipsu_del_mac_entry_info), 0, + sizeof(ipsu_del_mac_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_del_mac_entry_info.(ret:%d)", + ret); + return; + } + ipsu_del_mac_entry_info.func_id = func_id; + ipsu_del_mac_entry_info.er_id = er_id; + ipsu_del_mac_entry_info.src_func_id = src_func_id; + ipsu_del_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_del_mac_entry_info.vni_en = 0; + ret = memcpy_s(ipsu_del_mac_entry_info.mac, eth_alen, mac_addr, + eth_alen); + if (ret != 0) { + ROCE_ERR("Failed to memcpy_s dmac.(ret:%d)", ret); + return; + } + + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_IPSU_MAC, + &ipsu_del_mac_entry_info, + sizeof(ipsu_del_mac_entry_info), + &ipsu_del_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_mac_entry_info.head.status != 0) && + (ipsu_del_mac_entry_info.head.status != ERR_NOT_FOUND))) { + ROCE_ERR( + "Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_del_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_del_mac_entry_info.head.status == ERR_NOT_FOUND) { + ROCE_INFO("This mac entry of ipsu has been deleted"); + } + + return; +} + +int roce5_add_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info) +{ + roce_cfg_ipsu_ip_cmd_s ipsu_add_ip_entry_info; + u16 out_size = sizeof(ipsu_add_ip_entry_info); + union roce_addr_info *ip = NULL; + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL) || (addr_info == NULL)) { + ROCE_ERR("rdev, hwdev or addr_info is null"); + return -EINVAL; + } + + ip = &addr_info->ip_addr; + + ret = memset_s(&ipsu_add_ip_entry_info, sizeof(ipsu_add_ip_entry_info), + 0, sizeof(ipsu_add_ip_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_add_ip_entry_info.(ret:%d)", + ret); + return -ENOMEM; + } + + ipsu_add_ip_entry_info.func_id = rdev->glb_func_id; + ipsu_add_ip_entry_info.ip_ver = addr_info->ip_ver; + ipsu_add_ip_entry_info.port_id = hinic5_physical_port_id(rdev->hwdev); + ret = memcpy_s(&ipsu_add_ip_entry_info.ip, sizeof(union roce_addr_info), + ip, sizeof(union roce_addr_info)); + if (ret) { + ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret); + return -ENOMEM; + } + + ROCE_INFO("funcid=%u,ipver=%u,portid=%u,ip=0x%x-%x-%x-%x", + ipsu_add_ip_entry_info.func_id, ipsu_add_ip_entry_info.ip_ver, + ipsu_add_ip_entry_info.port_id, ip->dw[0x0], ip->dw[0x1], + ip->dw[0x2], ip->dw[0x3]); + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_ADD_IPSU_IP, + &ipsu_add_ip_entry_info, + sizeof(ipsu_add_ip_entry_info), + &ipsu_add_ip_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_add_ip_entry_info.head.status != 0) && + (ipsu_add_ip_entry_info.head.status != ERR_EXIST))) { + ROCE_ERR( + "Failed to add ip tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_add_ip_entry_info.head.status, out_size); + return -EINVAL; + } + + if (ipsu_add_ip_entry_info.head.status == ERR_EXIST) { + ROCE_INFO("This ip tbl entry has been added to IPSU"); + } + + return 0; +} + +int roce5_del_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info) +{ + roce_cfg_ipsu_ip_cmd_s ipsu_del_ip_entry_info; + u16 out_size = sizeof(ipsu_del_ip_entry_info); + union roce_addr_info *ip = NULL; + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL) || (addr_info == NULL)) { + ROCE_ERR("rdev, hwdev or addr_info is null"); + return -EINVAL; + } + + ip = &addr_info->ip_addr; + + ret = memset_s(&ipsu_del_ip_entry_info, sizeof(ipsu_del_ip_entry_info), + 0, sizeof(ipsu_del_ip_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_add_ip_entry_info.(ret:%d)", + ret); + return -ENOMEM; + } + + ipsu_del_ip_entry_info.func_id = rdev->glb_func_id; + ipsu_del_ip_entry_info.ip_ver = addr_info->ip_ver; + ipsu_del_ip_entry_info.port_id = hinic5_physical_port_id(rdev->hwdev); + ret = memcpy_s(&ipsu_del_ip_entry_info.ip, sizeof(union roce_addr_info), + ip, sizeof(union roce_addr_info)); + if (ret) { + ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret); + return -ENOMEM; + } + + ROCE_INFO("funcid=%u,ipver=%u,portid=%u,ip=0x%x-%x-%x-%x", + ipsu_del_ip_entry_info.func_id, ipsu_del_ip_entry_info.ip_ver, + ipsu_del_ip_entry_info.port_id, ip->dw[0x0], ip->dw[0x1], + ip->dw[0x2], ip->dw[0x3]); + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_IPSU_IP, + &ipsu_del_ip_entry_info, + sizeof(ipsu_del_ip_entry_info), + &ipsu_del_ip_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_ip_entry_info.head.status != 0) && + (ipsu_del_ip_entry_info.head.status != ERR_NOT_FOUND))) { + ROCE_ERR( + "Failed to add ip tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_del_ip_entry_info.head.status, out_size); + return -EINVAL; + } + + if (ipsu_del_ip_entry_info.head.status == ERR_NOT_FOUND) { + ROCE_INFO("This ip tbl entry of ipsu has been deleted"); + } + + return 0; +} + +int roce5_modify_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, u8 type) +{ + struct roce_cmd_mpu_addr_info addr_info = { 0 }; + union roce_addr_info *ip = NULL; + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_ERR("rdev, hwdev or gid_entry is null"); + return -EINVAL; + } + + ip = (union roce_addr_info *)&gid_entry->raw; + + if (rdev->hca_vf_mode == 0) { + ROCE_INFO("No need to modify ipsu tbl ip entry"); + return 0; + } + + if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) { + ROCE_ERR("gid_type:%d is unsupported", + gid_entry->dw6_h.bs.gid_type); + return -EINVAL; + } + + ret = memcpy_s(&addr_info.ip_addr, sizeof(union roce_addr_info), ip, + sizeof(union roce_addr_info)); + if (ret) { + ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret); + return -ENOMEM; + } + + if (gid_entry->dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + addr_info.ip_addr.dw[0x0] = 0x0; + addr_info.ip_addr.dw[0x1] = 0x0; + addr_info.ip_addr.dw[0x2] = 0x0; + addr_info.ip_ver = 0; + } else { + addr_info.ip_ver = 1; + } + + if (type == ROCE_MPU_CMD_ADD_IPSU_IP) { + return roce5_add_ipsu_tbl_ip_entry(rdev, &addr_info); + } + + return roce5_del_ipsu_tbl_ip_entry(rdev, &addr_info); +} + +static int32_t +roce5_add_ipsu_tcam_tbl_entry_precheck(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_DEV_ERR(rdev, "hwdev or gid_entry is null."); + return -EINVAL; + } + + if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) { + ROCE_DEV_ERR(rdev, "gid_type:%d is unsupported.", + gid_entry->dw6_h.bs.gid_type); + return -EINVAL; + } + + return 0; +} + +static int32_t roce5_ipsu_tcam_tbl_set_entry( + struct roce5_device *rdev, struct rdma_gid_entry *gid_entry, + uint32_t gid_index, roce_cfg_ipsu_tcam_cmd_s *ipsu_add_tcam_entry_info) +{ + union roce_addr_info *ip = (union roce_addr_info *)&gid_entry->raw; + int32_t i; + int32_t ret; + + ipsu_add_tcam_entry_info->func_id = rdev->glb_func_id; + ipsu_add_tcam_entry_info->ip_ver = gid_entry->dw6_h.bs.gid_type; + ipsu_add_tcam_entry_info->gid_idx = gid_index; + ret = memcpy_s(&ipsu_add_tcam_entry_info->ip, + sizeof(union roce_addr_info), ip, + sizeof(union roce_addr_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy ip info.(ret:%d).", ret); + return -EINVAL; + } + + for (i = 0; i < 0x4; i++) { + ipsu_add_tcam_entry_info->ip.dw[i] = + htonl(ipsu_add_tcam_entry_info->ip.dw[i]); + } + + if (gid_entry->dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + ipsu_add_tcam_entry_info->ip.dw[0x0] = 0x0; + ipsu_add_tcam_entry_info->ip.dw[0x1] = 0x0; + ipsu_add_tcam_entry_info->ip.dw[0x2] = 0x0; + } + + return 0; +} + +void roce5_add_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, + u32 gid_index) +{ + roce_cfg_ipsu_tcam_cmd_s ipsu_add_tcam_entry_info; + u16 out_size = sizeof(roce_cfg_ipsu_tcam_cmd_s); + int ret; + + ret = roce5_add_ipsu_tcam_tbl_entry_precheck(rdev, gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to precheck params(ret:%d).", ret); + return; + } + + ret = memset_s(&ipsu_add_tcam_entry_info, + sizeof(ipsu_add_tcam_entry_info), 0, + sizeof(ipsu_add_tcam_entry_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to memset ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ret = roce5_ipsu_tcam_tbl_set_entry(rdev, gid_entry, gid_index, + &ipsu_add_tcam_entry_info); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ROCE_DEV_INFO(rdev, "funcid=%u,ipver=%u,gid_idx=%u,ip=0x%x-%x-%x-%x.", + ipsu_add_tcam_entry_info.func_id, + ipsu_add_tcam_entry_info.ip_ver, + ipsu_add_tcam_entry_info.gid_idx, + ipsu_add_tcam_entry_info.ip.dw[0x0], + ipsu_add_tcam_entry_info.ip.dw[0x1], + ipsu_add_tcam_entry_info.ip.dw[0x2], + ipsu_add_tcam_entry_info.ip.dw[0x3]); + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_ADD_IPSU_TCAM, + &ipsu_add_tcam_entry_info, + sizeof(ipsu_add_tcam_entry_info), + &ipsu_add_tcam_entry_info, &out_size); + if (ipsu_add_tcam_entry_info.head.status == ERR_FULL) { + ROCE_DEV_INFO(rdev, "TCAM Table is full."); + } else if ((ret == 0) && (out_size != 0) && + (ipsu_add_tcam_entry_info.head.status == ERR_EXIST)) { + ROCE_DEV_INFO(rdev, "This tcam entry has been added to IPSU."); + } else if ((ret != 0) || (out_size == 0) || + ((ipsu_add_tcam_entry_info.head.status != 0))) { + ROCE_DEV_ERR( + rdev, + "Failed to add tcam tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x).", + ret, ipsu_add_tcam_entry_info.head.status, out_size); + return; + } + + return; +} + +static int32_t +roce5_del_ipsu_tcam_tbl_entry_precheck(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_DEV_ERR(rdev, "hwdev or gid_entry is null."); + return -EINVAL; + } + + if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) { + ROCE_DEV_ERR(rdev, "gid_type:%d is unsupported", + gid_entry->dw6_h.bs.gid_type); + return -EINVAL; + } + + return 0; +} + +void roce5_del_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + roce_cfg_ipsu_tcam_cmd_s ipsu_del_tcam_entry_info; + u16 out_size = sizeof(roce_cfg_ipsu_tcam_cmd_s); + int ret; + + ret = roce5_del_ipsu_tcam_tbl_entry_precheck(rdev, gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to precheck params(ret:%d).", ret); + return; + } + + ret = memset_s(&ipsu_del_tcam_entry_info, + sizeof(ipsu_del_tcam_entry_info), 0, + sizeof(ipsu_del_tcam_entry_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to memset ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ret = roce5_ipsu_tcam_tbl_set_entry(rdev, gid_entry, 0, + &ipsu_del_tcam_entry_info); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ROCE_DEV_INFO(rdev, "funcid=%u,ipver=%u,gid_idx=%u,ip=0x%x-%x-%x-%x", + ipsu_del_tcam_entry_info.func_id, + ipsu_del_tcam_entry_info.ip_ver, + ipsu_del_tcam_entry_info.gid_idx, + ipsu_del_tcam_entry_info.ip.dw[0x0], + ipsu_del_tcam_entry_info.ip.dw[0x1], + ipsu_del_tcam_entry_info.ip.dw[0x2], + ipsu_del_tcam_entry_info.ip.dw[0x3]); + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_IPSU_TCAM, + &ipsu_del_tcam_entry_info, + sizeof(ipsu_del_tcam_entry_info), + &ipsu_del_tcam_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_tcam_entry_info.head.status != 0) && + (ipsu_del_tcam_entry_info.head.status != ERR_NOT_FOUND))) { + ROCE_DEV_ERR( + rdev, + "Failed to del tcam entry, err(%d), status(0x%x), out size(0x%x).", + ret, ipsu_del_tcam_entry_info.head.status, out_size); + return; + } + + if (ipsu_del_tcam_entry_info.head.status == ERR_NOT_FOUND) { + ROCE_DEV_INFO(rdev, + "This tcam entry of ipsu has been deleted."); + } + return; +} + +int roce5_set_sysfs_cfg_param(void *hwdev, u16 func_id, u32 *param, + u32 param_len, u16 type) +{ + roce_cc_cfg_param_cmd_s cfg_param; + u16 out_size = (u16)sizeof(cfg_param); + int ret; + int param_idx = 0; + int param_dw_num; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + param_dw_num = (int)(param_len / sizeof(u32)); + if (param_dw_num > ROCE_CC_CFG_PARAM_MAX_DW_NUM) { + ROCE_ERR("cfg_param len :%d dw_num:%d exceed max, type:0x%x", + param_len, param_dw_num, type); + return -EINVAL; + } + + ret = memset_s(&cfg_param, sizeof(cfg_param), 0, sizeof(cfg_param)); + if (ret != 0) { + ROCE_ERR("Failed to memset cfg_param.(ret:%d)", ret); + return -ENOMEM; + } + + cfg_param.func_id = func_id; + for (; param_idx < param_dw_num; param_idx++) { + cfg_param.param[param_idx] = param[param_idx]; + } + + ret = roce5_msg_to_mgmt_sync(hwdev, type, &cfg_param, sizeof(cfg_param), + &cfg_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_param.head.status != 0)) { + ROCE_ERR( + "Failed to cfg param, err(%d), status(0x%x), out size(0x%x), type(0x%x)", + ret, cfg_param.head.status, out_size, type); + + return -EINVAL; + } + return 0; +} + +int roce5_set_func_tbl_func_state(void *hwdev, u8 func_state, u16 func_id) +{ + roce_set_func_state_cmd_s set_func_state_info; + u16 out_size = (u16)sizeof(set_func_state_info); + int ret; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&set_func_state_info, sizeof(set_func_state_info), 0, + sizeof(set_func_state_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset set_func_state_info.(ret:%d)", ret); + return (-ENOMEM); + } + set_func_state_info.func_id = func_id; + set_func_state_info.func_en = func_state; + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_FUNC_STATE, + &set_func_state_info, + sizeof(set_func_state_info), + &set_func_state_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (set_func_state_info.head.status != 0)) { + ROCE_ERR( + "Failed to set func state(%d), err(%d), status(0x%x), out size(0x%x), func_id(%d)", + func_state, ret, set_func_state_info.head.status, + out_size, func_id); + return (-EINVAL); + } + + return 0; +} + +int roce5_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value) +{ + roce_get_func_table_cmd_s get_func_table; + roce_get_func_table_rsp_s get_func_table_rsp; + u16 out_size = (u16)sizeof(get_func_table_rsp); + int ret; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&get_func_table, sizeof(get_func_table), 0, + sizeof(get_func_table)); + if (ret != 0) { + ROCE_ERR("Failed to memset get_func_table.(ret:%d)", ret); + return (-ENOMEM); + } + ret = memset_s(&get_func_table_rsp, out_size, 0, out_size); + if (ret != 0) { + ROCE_ERR("Failed to memset get_func_table_rsp.(ret:%d)", ret); + return (-ENOMEM); + } + get_func_table.func_id = func_id; + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_FUNC_TABLE, + &get_func_table, sizeof(get_func_table), + &get_func_table_rsp, &out_size); + if ((ret != 0) || (out_size == 0) || + (get_func_table_rsp.head.status != 0)) { + ROCE_ERR( + "Failed to get func table, err(%d), status(0x%x), out size(0x%x), func_id(%d)", + ret, get_func_table_rsp.head.status, out_size, func_id); + return (-EINVAL); + } + + *func_tbl_value = get_func_table_rsp.func_tbl_value; + + return 0; +} + +int roce5_set_func_tbl_cpu_endian(struct roce5_device *rdev) +{ + roce_set_cpu_endian_cmd_s set_cpu_endian_info; + u16 out_size = (u16)sizeof(set_cpu_endian_info); + int ret; + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + ret = memset_s(&set_cpu_endian_info, sizeof(set_cpu_endian_info), 0, + sizeof(set_cpu_endian_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to memset set_cpu_endian_info.(ret:%d).", + ret); + return -ENOMEM; + } + set_cpu_endian_info.func_id = rdev->glb_func_id; + set_cpu_endian_info.cpu_endian = rdev->hw_info.cpu_endian; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_SET_CPU_ENDIAN, + &set_cpu_endian_info, + sizeof(set_cpu_endian_info), + &set_cpu_endian_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (set_cpu_endian_info.head.status != 0)) { + ROCE_DEV_ERR( + rdev, + "Failed to set func state, err(%d), status(0x%x), out size(0x%x).", + ret, set_cpu_endian_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +static void roce5_init_cfg_info_parse_resp(struct roce5_device *rdev, + roce_get_cfg_info_resp_s *resp) +{ + rdev->cfg_info.scence_id = resp->scence_id; + rdev->cfg_info.lb_en = resp->lb_en; + rdev->cfg_info.lb_mode = resp->lb_mode; + rdev->cfg_info.srq_container_en = + (resp->container_mode != ROCE_CHIP_SRQ_MODE_N); + rdev->cfg_info.srq_container_mode = + roce5_srq_mode_chip_adapt(resp->container_mode); + rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3; + rdev->cfg_info.warn_th = 0; + + rdev->cfg_info.fake_en = resp->fake_en; + rdev->cfg_info.pf_start_bit = resp->pf_start_bit; + rdev->cfg_info.pf_end_bit = resp->pf_end_bit; + rdev->cfg_info.page_bit = resp->page_bit; + + rdev->cfg_info.port_num = resp->port_num; + rdev->cfg_info.host_num = resp->host_num; + rdev->cfg_info.pcie_atomic_en = resp->pcie_atomic_en; + + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes = resp->max_wqe; + rdev->rdma_cap.max_cqes = resp->max_cqe; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes = + resp->max_srq_wqe; + rdev->rdma_cap.max_sq_sg = resp->max_sq_sge; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg = resp->max_rq_sge; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge = + resp->max_srq_sge; + rdev->rdma_cap.max_sq_desc_sz = resp->max_sq_wqe_len; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_desc_sz = + resp->max_rq_wqe_len; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz = + resp->max_sq_inline; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_pd = resp->max_pd; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_xrcd = resp->max_xrcd; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_gid = resp->max_gid; + + rdev->hca_vf_mode = resp->ip2func; + + roce5_cfg_info_update_ext(rdev, resp); +} + +int roce5_get_feature(struct roce5_device *rdev, + roce_get_chip_ability_s *roce_feature) +{ + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_verbs_cmd_header_s *cmd_hdr = NULL; + roce_get_chip_ability_s *resp_buf = NULL; + int ret = 0; + + ret = memset_s(roce_feature, sizeof(*roce_feature), 0, + sizeof(*roce_feature)); + if (ret != 0) { + ROCE_ERR("Failed to memset roce_feature, ret(%d).", ret); + return (-ENOMEM); + } + + // allocate input and output buffer + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + sizeof(roce_verbs_cmd_header_s), + &cqm_cmd_outbuf, + sizeof(roce_get_chip_ability_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to allocate cqm buffers, ret = %d.", + ret); + return ret; + } + + // fill input command + cmd_hdr = (roce_verbs_cmd_header_s *)cqm_cmd_inbuf->buf; + cmd_hdr->dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_COMM_BITMASK); + + // send command and wait response + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_GET_CHIP_ABILITY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_C, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send feature cmd, ret = %d.", + ret); + goto free_buf; + } + + // get response data + resp_buf = (roce_get_chip_ability_s *)cqm_cmd_outbuf->buf; + roce_feature->roce_kernel_ability = + be64_to_cpu(resp_buf->roce_kernel_ability); + roce_feature->roce_kernel_ext_ability = + be64_to_cpu(resp_buf->roce_kernel_ext_ability); + roce_feature->roce_user_ext_ability = + be64_to_cpu(resp_buf->roce_user_ext_ability); + + // get on layer service supported attribute + roce_feature->uld_feature.roce_uld_kernel_ext_ability = + be64_to_cpu(resp_buf->uld_feature.roce_uld_kernel_ext_ability); + roce_feature->uld_feature.roce_uld_user_ext_ability = + be64_to_cpu(resp_buf->uld_feature.roce_uld_user_ext_ability); +free_buf: + // free buffer + if (cqm_cmd_inbuf || cqm_cmd_outbuf) { + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + } + return ret; +} + +int roce5_uni_cmd_init_cfg_info(struct roce5_device *rdev) +{ + roce_get_cfg_info_cmd_s get_cfg_info; + roce_get_cfg_info_resp_s resp; + u16 out_size = (u16)sizeof(resp); + int ret = 0; + + ret = memset_s(&get_cfg_info, sizeof(get_cfg_info), 0, + sizeof(get_cfg_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset get_cfg_info, ret(%d).", ret); + return (-ENOMEM); + } + ret = memset_s(&resp, sizeof(resp), 0, sizeof(resp)); + if (ret != 0) { + ROCE_ERR("Failed to memset resp, ret(%d).", ret); + return (-ENOMEM); + } + + get_cfg_info.func_id = rdev->glb_func_id; + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO, + &get_cfg_info, sizeof(get_cfg_info), &resp, + &out_size); + if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) { + ROCE_DEV_ERR(rdev, + "Failed, err(%d), status(0x%x), out size(0x%x)", + ret, get_cfg_info.head.status, out_size); + return (-EINVAL); + } + + roce5_init_cfg_info_parse_resp(rdev, &resp); + + ROCE_DEV_INFO( + rdev, + "init, func_id(%d), lb_en(%d), lb_mode(%d), srq_mode(%d), scence_id(%d), pcie_atomic_en(%d).", + rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode, + rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id, + rdev->cfg_info.pcie_atomic_en); + ROCE_DEV_INFO( + rdev, + "max_wqes(0x%x), max_cqes(0x%x), max_srq_wqes(0x%x), max_sq_sg(0x%x), max_rq_sg(0x%x), " + "max_srq_sge(0x%x), max_sq_desc_sz(0x%x), max_rq_desc_sz(0x%x), max_sq_inline_data_sz(0x%x).", + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + rdev->rdma_cap.max_cqes, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes, + rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge, + rdev->rdma_cap.max_sq_desc_sz, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_desc_sz, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz); + + return 0; +} + +int roce5_init_cfg_info(struct roce5_device *rdev) +{ + roce_get_cfg_info_cmd_s get_cfg_info; + roce_get_cfg_info_comp_resp_s resp; + u16 out_size = (u16)sizeof(resp); + int ret = 0; + + if (rdev == NULL || rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + return roce5_uni_cmd_init_cfg_info(rdev); + } + + (void)memset_s(&get_cfg_info, sizeof(get_cfg_info), 0, + sizeof(get_cfg_info)); + (void)memset_s(&resp, sizeof(resp), 0, sizeof(resp)); + + get_cfg_info.func_id = rdev->glb_func_id; + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO, + &get_cfg_info, sizeof(get_cfg_info), &resp, + &out_size); + if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) { + ROCE_DEV_ERR(rdev, + "Failed, err(%d), status(0x%x), out size(0x%x)", + ret, get_cfg_info.head.status, out_size); + return (-EINVAL); + } + + rdev->cfg_info.scence_id = resp.scence_id; + rdev->cfg_info.lb_en = resp.lb_en; + rdev->cfg_info.lb_mode = resp.lb_mode; + rdev->cfg_info.srq_container_en = + (resp.container_mode != ROCE_CHIP_SRQ_MODE_N); + rdev->cfg_info.srq_container_mode = + roce5_srq_mode_chip_adapt(resp.container_mode); + rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3; + rdev->cfg_info.warn_th = 0; + + rdev->cfg_info.fake_en = resp.fake_en; + rdev->cfg_info.pf_start_bit = resp.pf_start_bit; + rdev->cfg_info.pf_end_bit = resp.pf_end_bit; + rdev->cfg_info.page_bit = resp.page_bit; + + rdev->cfg_info.port_num = resp.port_num; + rdev->cfg_info.host_num = resp.host_num; + rdev->cfg_info.pcie_atomic_en = resp.pcie_atomic_en; + + rdev->hca_vf_mode = 0; // V100 fixed to mac mode + + ROCE_DEV_INFO( + rdev, + "init, func_id(%d), lb_en(%d), lb_mode(%d), srq_mode(%d), scence_id(%d), pcie_atomic_en(%d).", + rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode, + rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id, + rdev->cfg_info.pcie_atomic_en); + return 0; +} + +int roce5_del_func_res(struct roce5_device *rdev) +{ + roce_set_func_state_cmd_s del_func_res_info; + u16 out_size = (u16)sizeof(del_func_res_info); + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + ROCE_ERR("rdev/Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&del_func_res_info, sizeof(del_func_res_info), 0, + sizeof(del_func_res_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset del_func_res_info.(ret:%d)", ret); + return (-ENOMEM); + } + del_func_res_info.func_id = rdev->glb_func_id; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_FUNC_RES, + &del_func_res_info, + sizeof(del_func_res_info), + &del_func_res_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (del_func_res_info.head.status != 0)) { + ROCE_ERR( + "Failed to set func state, err(%d), status(0x%x), out size(0x%x)", + ret, del_func_res_info.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce5_set_bw_ctrl_state(struct roce5_device *rdev, u8 cmd, + struct roce5_bw_ctrl_inbuf *inbuf) +{ + int ret = 0; + roce_cc_cfg_bw_ctrl_cmd_s set_bw_ctrl; + u16 out_size = (u16)sizeof(set_bw_ctrl); + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&set_bw_ctrl, sizeof(set_bw_ctrl), 0, + sizeof(set_bw_ctrl)); + if (ret != 0) { + ROCE_ERR("Memset failed when set_bw_ctrl cmd"); + return (-ENOMEM); + } + + set_bw_ctrl.func_id = rdev->glb_func_id; + set_bw_ctrl.cmd = cmd; + set_bw_ctrl.cir = inbuf->ctrl_param.cir; + set_bw_ctrl.pir = inbuf->ctrl_param.pir; + set_bw_ctrl.cnp = inbuf->ctrl_param.cnp; + ROCE_INFO("func_id:%u, cmd:%u, cir:%u, pir:%u, cnp:%u", + rdev->glb_func_id, set_bw_ctrl.cmd, set_bw_ctrl.cir, + set_bw_ctrl.pir, set_bw_ctrl.cnp); + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_SET_BW_CTRL, + &set_bw_ctrl, sizeof(set_bw_ctrl), + &set_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (set_bw_ctrl.head.status != 0)) { + ROCE_ERR( + "Failed to set set_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)", + ret, set_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce5_query_bw_ctrl_state(struct roce5_device *rdev, + struct roce5_bw_ctrl_param *bw_ctrl_param) +{ + int ret = 0; + roce_cc_cfg_bw_ctrl_cmd_s query_bw_ctrl; + u16 out_size = (u16)sizeof(query_bw_ctrl); + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&query_bw_ctrl, sizeof(query_bw_ctrl), 0, + sizeof(query_bw_ctrl)); + if (ret != 0) { + ROCE_ERR("Memset failed when set_bw_ctrl cmd"); + return (-ENOMEM); + } + + query_bw_ctrl.func_id = rdev->glb_func_id; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_QUERY_BW_CTRL, + &query_bw_ctrl, sizeof(query_bw_ctrl), + &query_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (query_bw_ctrl.head.status != 0)) { + ROCE_ERR( + "Failed to set query_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)", + ret, query_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + ret = memcpy_s((void *)bw_ctrl_param, + sizeof(struct roce5_bw_ctrl_param), + (void *)((u8 *)(&query_bw_ctrl) + ROCE_CMD_DEFAULT_SIZE), + sizeof(struct roce5_bw_ctrl_param)); + if (ret != 0) { + ROCE_ERR("memory copy failed. ret%d", ret); + return -ENOMEM; + } + + ROCE_INFO("query_bw_ctrl_state:func_id:%u, cir:%u, pir:%u, cnp:%u", + rdev->glb_func_id, bw_ctrl_param->cir, bw_ctrl_param->pir, + bw_ctrl_param->cnp); + + return 0; +} + +#ifdef ROCE_CHIP_TEST +int roce5_set_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf) +{ + int ret = 0; + roce_cc_cfg_vnic_waterline_cmd_s set_vnic_waterline; + u16 out_size = (u16)sizeof(set_vnic_waterline); + + (void)outbuf; + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&set_vnic_waterline, sizeof(set_vnic_waterline), 0, + sizeof(set_vnic_waterline)); + if (ret != 0) { + ROCE_ERR("Memset failed when set_vnic_waterline cmd"); + return (-ENOMEM); + } + + set_vnic_waterline.func_id = rdev->glb_func_id; + set_vnic_waterline.ip_ver = 0; + set_vnic_waterline.param.red_waterline = + inbuf->waterline_param.red_waterline; + set_vnic_waterline.param.yellow_waterline = + inbuf->waterline_param.yellow_waterline; + set_vnic_waterline.param.ip_info.ipv4.val = + inbuf->waterline_param.ip_val; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, + ROCE_MPU_CMD_CC_SET_VNIC_WATERLINE, + &set_vnic_waterline, + sizeof(set_vnic_waterline), + &set_vnic_waterline, &out_size); + if ((ret != 0) || (out_size == 0) || + (set_vnic_waterline.head.status != 0)) { + ROCE_ERR( + "Failed to set set_vnic_waterline cmd, err(%d), status(0x%x), out size(0x%x)", + ret, set_vnic_waterline.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce5_get_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf) +{ + int ret = 0; + roce_cc_cfg_vnic_waterline_cmd_s get_vnic_waterline; + u16 out_size = (u16)sizeof(get_vnic_waterline); + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&get_vnic_waterline, sizeof(get_vnic_waterline), 0, + sizeof(get_vnic_waterline)); + if (ret != 0) { + ROCE_ERR("Memset failed when get_vnic_waterline cmd"); + return (-ENOMEM); + } + + get_vnic_waterline.func_id = rdev->glb_func_id; + get_vnic_waterline.ip_ver = 0; + get_vnic_waterline.param.ip_info.ipv4.val = + inbuf->waterline_param.ip_val; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, + ROCE_MPU_CMD_CC_GET_VNIC_WATERLINE, + &get_vnic_waterline, + sizeof(get_vnic_waterline), + &get_vnic_waterline, &out_size); + if ((ret != 0) || (out_size == 0) || + (get_vnic_waterline.head.status != 0)) { + ROCE_ERR( + "Failed to set get_vnic_waterline cmd, err(%d), status(0x%x), out size(0x%x)", + ret, get_vnic_waterline.head.status, out_size); + return (-EINVAL); + } + + outbuf->waterline_param.red_waterline = + get_vnic_waterline.param.red_waterline; + outbuf->waterline_param.yellow_waterline = + get_vnic_waterline.param.yellow_waterline; + + return 0; +} +#endif /* ROCE_CHIP_TEST */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h new file mode 100644 index 000000000..5fd787a67 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h @@ -0,0 +1,82 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CMD_H +#define ROCE_CMD_H + +#include "comm_defs.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" + +#include "roce_mpu_cmd_defs.h" +#include "roce_cfg_format.h" + +#define ROCE_VLAN_ID_MASK 0x7FFF +#define ROCE_VLAN_N_VID 4096 + +#define ROCE_CMD_DEFAULT_SIZE 12 + +#define ERR_EXIST 6 +#define ERR_FULL 12 +#define ERR_NOT_FOUND 13 + +#define roce5_msg_to_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, out_size) \ + hinic5_msg_to_mgmt_sync(hwdev, HINIC5_MOD_ROCE, cmd, buf_in, in_size, \ + buf_out, out_size, 0, HINIC5_CHANNEL_ROCE) +#define roce5_msg_to_ovs_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, \ + out_size) \ + hinic5_msg_to_mgmt_sync(hwdev, HINIC5_MOD_OVS, cmd, buf_in, in_size, \ + buf_out, out_size, 90000, HINIC5_CHANNEL_OVS) + +int roce5_add_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id); +int roce5_del_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id); +typedef int (*roce5_modify_mac_tbl)(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], size_t eth_alen, + u32 vlan_id, u16 func_id); +void roce5_add_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], u32 vlan_id, + u16 func_id, u8 er_id, u16 src_func_id); +void roce5_del_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], size_t eth_alen, + u32 vlan_id, u8 er_id, u16 src_func_id); +int roce5_add_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info); +int roce5_del_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info); +int roce5_modify_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, u8 type); +void roce5_add_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, + u32 gid_index); +void roce5_del_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry); +typedef void (*roce5_modify_ipsu_tbl_mac)(void *hwdev, u8 mac_addr[ETH_ALEN], + u32 vlan_id, u16 func_id, u8 er_id); + +int roce5_set_sysfs_cfg_param(void *hwdev, u16 func_id, u32 *param, + u32 param_len, u16 type); +int roce5_set_bw_ctrl_state(struct roce5_device *rdev, u8 cmd, + struct roce5_bw_ctrl_inbuf *inbuf); +int roce5_query_bw_ctrl_state(struct roce5_device *rdev, + struct roce5_bw_ctrl_param *bw_ctrl_param); +#ifdef ROCE_CHIP_TEST +int roce5_set_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf); +int roce5_get_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf); +#endif /* ROCE_CHIP_TEST */ +int roce5_set_func_tbl_func_state(void *hwdev, u8 func_state, u16 func_id); +int roce5_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value); +int roce5_set_func_tbl_cpu_endian(struct roce5_device *rdev); +int roce5_get_feature(struct roce5_device *rdev, + roce_get_chip_ability_s *roce_feature); +int roce5_init_cfg_info(struct roce5_device *rdev); +int roce5_del_func_res(struct roce5_device *rdev); + +#endif /* ROCE_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c new file mode 100644 index 000000000..11bd63d04 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c @@ -0,0 +1,86 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/kernel.h> + +#include "roce.h" +#include "roce_os_compat.h" + +int roce5_db_map_user(struct roce5_ucontext *context, unsigned long virt, + struct roce5_db *db, u8 shadow_db) +{ + int ret = 0; + struct roce5_db_page *db_page = NULL; + + if ((context == NULL) || (db == NULL)) { + return -EINVAL; + } + if (shadow_db != 0) { + return ret; + } + mutex_lock(&context->db_page_mutex); + + list_for_each_entry(db_page, &context->db_page_list, list) { + if (db_page->user_virt == (virt & PAGE_MASK)) { + goto found; + } + } + + db_page = (struct roce5_db_page *)kmalloc(sizeof(*db_page), GFP_KERNEL); + if (db_page == NULL) { + ret = -ENOMEM; + ROCE_ERR("Failed to alloc DB page"); + goto out; + } + + db_page->user_virt = (virt & PAGE_MASK); + db_page->refcnt = 0; + db_page->umem = roce5_umem_get(context->ibucontext.device, + &context->ibucontext, virt & PAGE_MASK, + PAGE_SIZE, 0, 0); + if (IS_ERR(db_page->umem)) { + ret = (int)PTR_ERR(db_page->umem); + ROCE_ERR("Failed to get ib_umem ret:%d", ret); + kfree(db_page); + goto out; + } + + list_add(&db_page->list, &context->db_page_list); + +found: +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + db->dma = sg_dma_address(db_page->umem->sgt_append.sgt.sgl) + + (virt & ~PAGE_MASK); +#else + db->dma = sg_dma_address(db_page->umem->sg_head.sgl) + + (virt & ~PAGE_MASK); +#endif + db->user_page = db_page; + ++db_page->refcnt; + +out: + mutex_unlock(&context->db_page_mutex); + + return ret; +} + +void roce5_db_unmap_user(struct roce5_ucontext *context, struct roce5_db *db, + u8 shadow_db) +{ + if ((context == NULL) || (db == NULL)) { + return; + } + if (shadow_db != 0) { + return; + } + mutex_lock(&context->db_page_mutex); + + if ((--db->user_page->refcnt) == 0) { + list_del(&db->user_page->list); + ib_umem_release(db->user_page->umem); + kfree(db->user_page); + } + + mutex_unlock(&context->db_page_mutex); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h new file mode 100644 index 000000000..7232e9edc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_DB_H +#define ROCE_DB_H + +#include <rdma/ib_umem.h> +#include <linux/types.h> +#include <linux/list.h> + +struct roce5_db_page { + struct list_head list; + struct ib_umem *umem; + unsigned long user_virt; + int refcnt; +}; + +struct roce5_db { + __be32 *db_record; + dma_addr_t dma; + struct roce5_db_page *user_page; +}; + +static inline void roce5_write64(u32 val[2], void __iomem *dest) +{ + __raw_writeq(*(u64 *)(void *)val, dest); +} + +#endif //ROCE_DB_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c b/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c new file mode 100644 index 000000000..7d589afe1 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c @@ -0,0 +1,1701 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <net/ipv6.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_user_verbs.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> +#include <linux/security.h> +#include <linux/kref.h> +#include <linux/list.h> +#include <linux/netdevice.h> +#include <linux/sizes.h> + +#include "hinic5_crm.h" +#include "hinic5_hw.h" +#include "hinic5_mt.h" +#include "hinic5_srv_nic.h" +#include "hinic5_rdma.h" +#include "hinic5_id_tbl.h" + +#include "roce_dfx.h" +#include "roce_mr.h" +#include "roce_main_extension.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_netlink.h" +#include "roce_cmd.h" +#include "roce_qp.h" +#include "roce_infra.h" +#include "rdma_comp_mw_mr.h" +#include "roce_xrc.h" +#include "roce_com.h" +#include "roce_kernel_compat_gen.h" +#include "roce_netdev.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifdef ENABLE_HYPER_ROCE +#include "hyper_roce_tool.h" +#endif + +#include "securec.h" + +MODULE_AUTHOR(HIROCE5_DRV_AUTHOR); +MODULE_DESCRIPTION(HIROCE5_DRV_DESC); +MODULE_VERSION(HIROCE5_DRV_VERSION); +MODULE_LICENSE("GPL"); + +static int g_loop_times = 50000; +module_param(g_loop_times, int, S_IRUGO); +MODULE_PARM_DESC(g_loop_times, + "default: 50000. The number must be greater than 0."); + +static u8 g_destory_flow_timeout = 10; +module_param(g_destory_flow_timeout, byte, S_IRUGO); +MODULE_PARM_DESC(g_destory_flow_timeout, "default: 10"); + +static bool g_local_en = false; // IP mode because IP in this place need switch control whether advancein this place switch +module_param(g_local_en, bool, S_IRUGO); +MODULE_PARM_DESC(g_local_en, "default false"); + +static bool g_full_qp_spec = true; // calculate standard card scenario, supports single cq create full specification qp +module_param(g_full_qp_spec, bool, S_IRUGO); +MODULE_PARM_DESC( + g_full_qp_spec, + "Supports creating a maximum number of QPs through a single CQ. default true."); + +static bool g_huge_page_en = 0; +module_param(g_huge_page_en, bool, S_IRUGO); +MODULE_PARM_DESC( + g_huge_page_en, + "default: 0. value: 0 or 1, set huge page en, and users need to reserve huge pages in advance."); + +#define ROCE_BOND_SLAVE_BIT_MASK 0xf + +/* **************************************************************************** + Prototype : get_cpu_endian + Description : Acquire CPU's enianness + Return Value : 0: little endian; 1: big endian + + 1.Date : 2016/6/23 + Modification : Created function + +**************************************************************************** */ +static int get_cpu_endian(void) +{ + int cpu_mode = 0; + union { + unsigned int i; + unsigned char s[4]; + } c; + c.i = 0x12345678; + + if (c.s[0] == 0x12) { + ROCE_INFO("CPU is be"); + cpu_mode = 1; + } else { + ROCE_INFO("CPU is le"); + cpu_mode = 0; + } + + return cpu_mode; +} + +static int roce5_alloc_hw_resource(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_rdma_init_rsvd_lkey(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to init rsvd lkey, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + /* Clear gid table before register for a new IB device */ + ret = roce5_reset_gid_table(rdev, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to reset gid_table, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_gid_reset; + } + + return 0; + +err_gid_reset: + roce5_rdma_free_rsvd_lkey(rdev); + + return ret; +} + +static void roce5_dealloc_hw_resource(struct roce5_device *rdev) +{ + roce5_rdma_free_rsvd_lkey(rdev); +} + +/* 1825 v100 esl sub did */ +#define HI_1825_V100_SUB_DEV_ID_2X100 0x1825 +#define HI_1825_V100_SUB_DEV_ID_COMPUTE_2X200 0x40B1 +#define HI_1825_V100_SUB_DEV_ID_COMPUTE_DY_YH 0x30C4 +#define HI_1825_V100_SUB_DEV_ID_COMPUTE_UBX 0x30C8 + +static int roce5_init_ver_attr_pci(struct hinic5_lld_dev *lld_dev, + struct roce5_device *rdev) +{ + struct pci_dev *pdev = container_of(lld_dev->dev, struct pci_dev, dev); + + if ((pdev->device == HINIC5_DEV_ID_25V1_PF) || + (pdev->device == HINIC5_DEV_ID_25V1_VF)) { + rdev->device_type = ROCE_DEV_TYPE_HI1825_V100; + return 0; + } + + switch (pdev->subsystem_device) { + case HI_1825_V100_SUB_DEV_ID_2X100: + case HI_1825_V100_SUB_DEV_ID_COMPUTE_2X200: + case HI_1825_V100_SUB_DEV_ID_COMPUTE_DY_YH: + case HI_1825_V100_SUB_DEV_ID_COMPUTE_UBX: + rdev->device_type = ROCE_DEV_TYPE_HI1825_V100; + break; + default: + ROCE_DEV_ERR(rdev, "Unknown subsystem id(%u).", + pdev->subsystem_device); + rdev->device_type = ROCE_DEV_TYPE_UNKNOWN; + return -ENXIO; + } + + return 0; +} + +static int roce5_init_ver_attr(struct hinic5_lld_dev *lld_dev, + struct roce5_device *rdev) +{ + int ret; + + switch (lld_dev->dev_type) { + case HINIC5_DEVICE_T_PCI: + ret = roce5_init_ver_attr_pci(lld_dev, rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to init PCI version attributes, ret(%d).", + ret); + return ret; + } + break; + + case HINIC5_DEVICE_T_UB: + /* + * The 'struct ub_dev' contains a 'module' member that can be used to distinguish the version of the card + * under the UBC. However, this member is only available in the most recent version of 'struct ub_dev', and + * the current development environment does not include the updated structure definition, making it + * inaccessible at present. Here, the fixed version(Hi1825V100) is returned. + */ + rdev->device_type = ROCE_DEV_TYPE_HI1825_V100; + break; + + default: + ROCE_DEV_ERR(rdev, "Unknown low level device type(%u).", + lld_dev->dev_type); + rdev->device_type = ROCE_DEV_TYPE_UNKNOWN; + return -ENXIO; + } + + return 0; +} + +static int roce5_get_perf_cq_step(struct roce5_device *rdev) +{ + int perf_step = 0; + + if (ROCE_IS_DEV_1825V100(rdev)) { + perf_step = ROCE_1825V100_STEP; + } else { + ROCE_DEV_INFO( + rdev, + "not support high performance cq, device type(%u), func_id(%u).", + rdev->device_type, rdev->glb_func_id); + } + return perf_step; +} + +static int roce5_get_perf_cq_cnt(struct roce5_device *rdev) +{ + if (rdev->cqn_ctrl.rsvd_cq_step == 0) { + return 0; + } + /* When you insmod the driver, CQ0 is consumed by default. */ + return (int)((rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs - + 1) / + rdev->cqn_ctrl.rsvd_cq_step) + + 1; +} + +void roce5_init_perf_cq_attr(struct roce5_device *rdev) +{ + /* Calculate the step size and number of high-performance CQs */ + rdev->cqn_ctrl.rsvd_cq_step = roce5_get_perf_cq_step(rdev); + rdev->cqn_ctrl.high_perf_cq_cnt = roce5_get_perf_cq_cnt(rdev); +#ifdef ROCE_VBS_EN + // VBS RoCE scenario disable high performance cq + rdev->cqn_ctrl.high_perf_cq_cnt = 0; +#endif + rdev->cqn_ctrl.alloc_cq_cnt = 0; +} + +static struct roce5_device * +roce5_rdev_alloc(struct hinic5_lld_dev *lld_dev, void **uld_dev, + const struct rdma_service_cap *rdma_cap) +{ + struct roce5_device *rdev = NULL; + int ret; + + rdev = (struct roce5_device *)roce5_rdev_alloc_ext(); + if (rdev == NULL) { + ROCE_ERR("Failed to alloc rdev"); + return NULL; + } + + *uld_dev = (void *)rdev; + rdev->lld_dev = lld_dev; + rdev->hwdev = lld_dev->hwdev; + rdev->dev = lld_dev->dev; + dma_set_max_seg_size(rdev->dev, SZ_2G); + ret = roce5_init_ver_attr(lld_dev, rdev); + if (ret != 0) { + ROCE_ERR( + "Failed to init version attribute of the device, ret(%d)", + ret); + goto alloc_failed; + } + + mutex_init(&rdev->qp_cnt.cur_qps_mutex); + mutex_init(&rdev->cqn_ctrl.cq_ctrl_mutex); + + rdev->hwdev_hdl = ((struct hinic5_hwdev *)(rdev->hwdev))->dev_hdl; + if (memcpy_s((void *)&rdev->rdma_cap, sizeof(*rdma_cap), + (void *)rdma_cap, sizeof(*rdma_cap)) != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy rdma_cap."); + goto alloc_failed; + } + + rdev->ndev = hinic5_get_netdev_by_lld(rdev->lld_dev); + if (rdev->ndev == NULL) { + ROCE_DEV_ERR(rdev, "roce add failed, netdev is null."); + goto alloc_failed; + } + + roce5_init_perf_cq_attr(rdev); + rdev->rdev_magic = ROCE_DEVICE_MAGIC; + return rdev; + +alloc_failed: + ib_dealloc_device(&rdev->ib_dev); + return NULL; +} + +static int roce5_board_info_get(struct roce5_device *rdev) +{ + int ret = 0; + + ret = hinic5_get_board_info(rdev->hwdev, &rdev->board_info, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to get board info"); + return ret; + } + + ROCE_DEV_INFO( + rdev, + "Get board info success, board_type:0x%x, port_num:0x%x, pf_num:0x%x, vf_total_num:0x%x, " + "work_mode:0x%x, service_mode:0x%x, port_speed:0x%x.", + rdev->board_info.board_type, rdev->board_info.port_num, + rdev->board_info.pf_num, rdev->board_info.vf_total_num, + rdev->board_info.work_mode, rdev->board_info.service_mode, + rdev->board_info.port_speed); + + ret = roce5_board_cfg_check(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check board cfg info"); + return ret; + } + + return 0; +} + +static void roce5_fix_ibdev_name(struct roce5_device *rdev) +{ + int ret; + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + ret = strcpy_s(rdev->ib_dev.name, IB_DEVICE_NAME_MAX, + "hrn5_bond_%d"); + if (ret != EOK) { + ROCE_DEV_ERR(rdev, "Strcpy_s error"); + } + return; + } +#endif + ret = strcpy_s(rdev->ib_dev.name, IB_DEVICE_NAME_MAX, "hrn5_%d"); + if (ret != EOK) { + ROCE_DEV_ERR(rdev, "Strcpy_s error"); + } +} + +static int roce5_register_template(struct roce5_device *rdev) +{ + service_register_template_s svc_template; + int ret; + + ret = memset_s(&svc_template, sizeof(svc_template), 0, + sizeof(svc_template)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memset svc_template.(ret:%d)", + ret); + return -ENOMEM; + } + + svc_template.service_type = SERVICE_T_ROCE; + svc_template.scq_ctx_size = rdev->rdma_cap.cqc_entry_sz; + svc_template.srq_ctx_size = + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.srqc_entry_sz; + svc_template.service_handle = rdev; + svc_template.embedded_cq_ceq_callback = NULL; + svc_template.no_cq_ceq_callback = NULL; + + svc_template.shared_cq_ceq_callback = roce5_cq_completion; + svc_template.aeq_level_callback = roce5_async_event_level; + svc_template.aeq_callback = roce5_async_event; + + ret = cqm5_service_register(rdev->hwdev, &svc_template); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to register cqm_service, func(%d).", + rdev->glb_func_id); + return ret; + } + + return 0; +} + +/* Alloc CEQs and alloc CEQN */ +static int roce5_alloc_ceq(struct roce5_device *rdev) +{ + int ret; + + ret = hinic5_alloc_ceqs(rdev->hwdev, SERVICE_T_ROCE, + rdev->ib_dev.num_comp_vectors, rdev->ceqn, + &rdev->ceq_num); + if (ret < 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqes, num_comp_vectors(%d), func_id(%d), ret(%d)", + rdev->ib_dev.num_comp_vectors, rdev->glb_func_id, ret); + return ret; + } + + return 0; +} + +static void roce5_free_ceq(struct roce5_device *rdev) +{ + int i; + + for (i = 0; i < rdev->ceq_num; ++i) { + hinic5_free_ceq(rdev->hwdev, SERVICE_T_ROCE, rdev->ceqn[i]); + } +} + +static void roce5_init_hw_info(struct roce5_device *rdev) +{ + rdev->hw_info.config_num_ports = (int)rdev->rdma_cap.num_ports; + rdev->hw_info.ep_id = hinic5_ep_id(rdev->hwdev); + rdev->hw_info.phy_port = 1; + rdev->hw_info.is_vf = roce5_func_is_vf(rdev->hwdev); + rdev->hw_info.cpu_endian = (u8)get_cpu_endian(); +} + +static int roce5_init_dev_info(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_set_func_tbl_cpu_endian(rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set func_tbl cpu_endian, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_init; + } + + /* Set function table to ENABLE */ + ret = roce5_set_func_tbl_func_state(rdev->hwdev, ROCE_FUNC_ENABLE, + rdev->glb_func_id); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set func_tbl, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_init; + } + + return 0; + +err_init: + return ret; +} +#ifdef __ROCE_DFX__ +static void device_list_lock_init(struct roce5_device_res *device_res) +{ + mutex_init(&device_res->mr_list.node_lock); + INIT_LIST_HEAD(&device_res->mr_list.node); +} + +static void device_list_lock_deinit(struct roce5_device_res *device_res) +{ + struct roce5_mr *mr = NULL; + struct roce5_mr *tmp_mr = NULL; + + mutex_lock(&device_res->mr_list.node_lock); + list_for_each_entry_safe(mr, tmp_mr, &device_res->mr_list.node, node) { + list_del(&(mr->node)); + } + mutex_unlock(&device_res->mr_list.node_lock); + + mutex_destroy(&(device_res->mr_list.node_lock)); +} + +static int roce5_dfx_device_init(struct roce5_device *rdev) +{ + rdev->device_res = kzalloc(sizeof(struct roce5_device_res), GFP_KERNEL); + if (rdev->device_res == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc device resource."); + return -ENOMEM; + } + + device_list_lock_init(rdev->device_res); + + return 0; +} + +static void roce5_dfx_device_deinit(struct roce5_device *rdev) +{ + if (rdev->device_res != NULL) { + device_list_lock_deinit(rdev->device_res); + kfree(rdev->device_res); + rdev->device_res = NULL; + } +} +#endif // __ROCE_DFX__ + +static void roce5_deinit_dev_info(struct roce5_device *rdev) +{ + (void)roce5_set_func_tbl_func_state(rdev->hwdev, ROCE_FUNC_DISABLE, + rdev->glb_func_id); +} + +static int roce5_ib_register_device(struct roce5_device *rdev) +{ +#if defined(IB_REG_DEV_HAVE_PORT_CALLBACK) + roce5_fix_ibdev_name(rdev); + return ib_register_device(&rdev->ib_dev, NULL); +#elif defined(IB_REG_DEV_HAVE_NAME) && defined(IB_REG_DEV_HAVE_DEVICE) +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + return ib_register_device(&rdev->ib_dev, "hrn5_bond_%d", + rdev->dev); + } +#endif + + return ib_register_device(&rdev->ib_dev, "hrn5_%d", rdev->dev); +#endif +} + +static int roce5_dev_register(struct roce5_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce5_ib_register_device(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to reg ibdev, func_id(%d)", + rdev->glb_func_id); + goto err_reg_dev; + } + + ret = memcpy_s(uld_dev_name, ROCE_ULD_DEV_NAME_LEN, rdev->ib_dev.name, + ROCE_ULD_DEV_NAME_LEN); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy uld_dev_name.(ret:%d)", + ret); + goto err_memcpy_uld; + } + + return 0; + +err_memcpy_uld: + ib_unregister_device(&rdev->ib_dev); +err_reg_dev: + return ret; +} + +static void roce5_dev_unregister(struct roce5_device *rdev) +{ + ib_unregister_device(&rdev->ib_dev); +} + +static int roce5_handle_register_event(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_add_real_device_mac(rdev, rdev->ndev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to add real device mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (netif_running(rdev->ndev) && + test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) { + roce5_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ACTIVE); + } + + return 0; +} + +static int roce5_init_dev(struct roce5_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce5_init_dev_info(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init dev info, func_id(%d)", + rdev->glb_func_id); + goto err_init_dev_info; + } + + ret = roce5_alloc_hw_resource(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc hw res, func_id(%d)", + rdev->glb_func_id); + goto err_alloc_hw_res; + } + + ret = roce5_dev_register(rdev, uld_dev_name); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init register, func_id(%d)", + rdev->glb_func_id); + goto err_init_register; + } + + ret = roce5_g_rdev_add(rdev); + if (ret != 0) { + ROCE_ERR("Failed to add rdev to g_rdev"); + goto err_add_rdev; + } + + ret = roce5_get_feature(rdev, &rdev->cfg_info.feature_cap); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get feature, ret=%d", ret); + goto err_get_feature_or_init_sysfs; + } + + ret = roce5_init_sysfs(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init sysfs, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + goto err_get_feature_or_init_sysfs; + } + rdev->ib_active = true; + + ret = roce5_handle_register_event(rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to handle register event, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + goto err_register_event; + } + + ROCE_DEV_INFO(rdev, "Add init all ok, func_id(%d), dev_name(%s)", + rdev->glb_func_id, rdev->ib_dev.name); + + return 0; + +err_register_event: + roce5_remove_sysfs(rdev); +err_get_feature_or_init_sysfs: + roce5_g_rdev_remove(rdev); +err_add_rdev: + roce5_dev_unregister(rdev); +err_init_register: + roce5_dealloc_hw_resource(rdev); +err_alloc_hw_res: + roce5_deinit_dev_info(rdev); +err_init_dev_info: + return ret; +} + +static void roce5_rdev_qp_lb_init(struct roce5_device *rdev) +{ + int i = 0; + mutex_init(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + for (; i < ROCE_LB_QPN_MAX; i++) { + rdev->qp_lb_scqn_ctrl.qp[i] = 0; + } +} + +static int roce5_add_rdev_init(struct roce5_device *rdev) +{ + int ret = 0; + + ret = roce5_board_info_get(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get board info"); + return ret; + } + +#ifdef ROCE_BONDING_EN + ret = roce5_bond_attach(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to attach bond device."); + return ret; + } + + atomic_set(&rdev->next_port, 0); +#endif + + rdev->gid_dev = (struct net_device **)kzalloc( + rdev->rdma_cap.max_gid_per_port * sizeof(struct net_device *), + GFP_KERNEL); + if (rdev->gid_dev == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc gid dev memory"); + return -ENOMEM; + } + + ret = roce5_rdma_init_resource(rdev->hwdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init rdma resources."); + goto err_init_resource; + } + + rdev->glb_func_id = hinic5_global_func_id(rdev->hwdev); + roce5_rdev_qp_lb_init(rdev); + + roce5_init_hw_info(rdev); + + ret = roce5_init_cfg_info(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get roce cfg, func_id(%d)", + rdev->glb_func_id); + goto err_init_cfg; + } + + ROCE_DEV_INFO(rdev, "Initializing."); + return 0; + +err_init_cfg: + roce5_rdma_cleanup_resource(rdev->hwdev); + +err_init_resource: + if (rdev->gid_dev != NULL) { + kfree(rdev->gid_dev); + } + return ret; +} + +static void roce5_add_rdev_unit(struct roce5_device *rdev) +{ + roce5_rdma_cleanup_resource(rdev->hwdev); + if (rdev->gid_dev != NULL) { + kfree(rdev->gid_dev); + } +} + +static __be64 rdma_gen_node_guid(u8 *dev_mac) +{ + u8 guid[8]; + u8 mac_addr[6]; + __be64 node_guid = 0; + int ret; + + if (dev_mac == NULL) { + ROCE_ERR("Dev_mac is null"); + return RDMA_INVALID_GUID; + } + + ret = memcpy_s((void *)&mac_addr[0], sizeof(mac_addr), (void *)dev_mac, + sizeof(mac_addr)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mac_addr.(ret:%d)", ret); + return ENOMEM; + } + + guid[0] = mac_addr[0] ^ + DEV_ADDR_FIRST_BYTE_VAL_MASK; // 0 is mac & guid array idx + guid[1] = mac_addr[1]; // 1 is mac & guid array idx + guid[2] = mac_addr[2]; // 2 is mac & guid array idx + guid[3] = 0xff; // 3 is guid array idx + guid[4] = 0xfe; // 4 is guid array idx + guid[5] = mac_addr[3]; // 5 is guid array idx, 3 is mac array idx + guid[6] = mac_addr[4]; // 6 is guid array idx, 4 is mac array idx + guid[7] = mac_addr[5]; // 7 is guid array idx, 5 is mac array idx + + /* node_guid is calculated by guid. */ + node_guid = + ((u64)guid[0] << 56) | // 0 is guid array idx, 56 is guid offset + ((u64)guid[1] << 48) | // 1 is guid array idx, 48 is guid offset + ((u64)guid[2] << 40) | // 2 is guid array idx, 40 is guid offset + ((u64)guid[3] << 32) | // 3 is guid array idx, 32 is guid offset + ((u64)guid[4] << 24) | // 4 is guid array idx, 24 is guid offset + ((u64)guid[5] << 16) | // 5 is guid array idx, 16 is guid offset + ((u64)guid[6] << 8) | // 6 is guid array idx, 8 is guid offset + (u64)guid[7]; // 7 is guid array idx + + return (__be64)cpu_to_be64(node_guid); +} + +static __be64 roce5_rdma_init_guid(void *hwdev, struct net_device *netdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (netdev == NULL)) { + ROCE_ERR("Hwdev or netdev is null"); + return ~0ULL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return ~0ULL; + } + + comp_priv->rdma_comp_res.node_guid = + rdma_gen_node_guid((u8 *)netdev->dev_addr); + + return comp_priv->rdma_comp_res.node_guid; +} + +#if defined(HAVE_IB_DEVICE_OPS) +static const struct ib_device_ops dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HRN, + .uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION, + + .modify_qp = roce5_modify_qp, + .query_qp = roce5_query_qp, + .destroy_qp = roce5_destroy_qp, + .post_send = roce5_post_send, + .post_recv = roce5_post_recv, + .create_cq = roce5_create_cq, + .modify_cq = roce5_modify_cq, + .resize_cq = roce5_resize_cq, + .destroy_cq = roce5_destroy_cq, + .poll_cq = roce5_poll_cq, + .req_notify_cq = roce5_arm_cq, + .create_srq = roce5_create_srq, + .modify_srq = roce5_modify_srq, + .query_srq = roce5_query_srq, + .destroy_srq = roce5_destroy_srq, + .post_srq_recv = roce5_post_srq_recv, + .map_mr_sg = roce5_map_kernel_frmr_sg, + .get_dma_mr = roce5_get_dma_mr, + .reg_user_mr = roce5_reg_user_mr, + .dereg_mr = roce5_dereg_mr, + .alloc_mr = roce5_alloc_mr, + .alloc_mw = roce5_alloc_mw, + .dealloc_mw = roce5_dealloc_mw, + .query_device = roce5_query_device, + .add_gid = roce5_ib_add_gid, + .del_gid = roce5_ib_del_gid, + .modify_device = roce5_modify_device, + .alloc_ucontext = roce5_alloc_ucontext, + .dealloc_ucontext = roce5_dealloc_ucontext, + .mmap = roce5_mmap, + .alloc_pd = roce5_alloc_pd, + .dealloc_pd = roce5_dealloc_pd, + .create_ah = roce5_create_ah, + .query_ah = roce5_query_ah, + .destroy_ah = roce5_destroy_ah, + .alloc_xrcd = roce5_alloc_xrcd, + .dealloc_xrcd = roce5_dealloc_xrcd, + +#if defined(ROCE_CREATE_QP_INT) + .create_qp = roce5_create_qp, +#elif defined(ROCE_CREATE_QP_IB_QP) + .create_qp = roce5_create_qp_compat, +#endif + +#if defined(HAVE_ALL_ROCE_FUNCS_U32_PORT) + .query_port = roce5_query_port, +#elif defined(HAVE_ALL_ROCE_FUNCS_U8_PORT) + .query_port = roce5_query_port_compat, +#endif + +#if defined(GET_LINK_LAYER_HAVE_U32_PORT) + .get_link_layer = roce5_port_link_layer, +#elif defined(GET_LINK_LAYER_HAVE_U8_PORT) + .get_link_layer = roce5_port_link_layer_compat, +#endif + +#if defined(QUERY_GID_HAVE_U32_PORT) + .query_gid = roce5_query_gid, +#elif defined(QUERY_GID_HAVE_U8_PORT) + .query_gid = roce5_query_gid_compat, +#endif + +#if defined(QUERY_PKEY_HAVE_U32_PORT) + .query_pkey = roce5_query_pkey, +#elif defined(QUERY_PKEY_HAVE_U8_PORT) + .query_pkey = roce5_query_pkey_compat, +#endif + +#if defined(MODIFY_PORT_HAVE_U32_PORT) + .modify_port = roce5_modify_port, +#elif defined(MODIFY_PORT_HAVE_U8_PORT) + .modify_port = roce5_modify_port_compat, +#endif + +#if defined(GET_PORT_IMMUTABLE_HAVE_U32_PORT) + .get_port_immutable = roce5_port_immutable, +#elif defined(GET_PORT_IMMUTABLE_HAVE_U8_PORT) + .get_port_immutable = roce5_port_immutable_compat, +#endif + +#if defined(GET_NETDEV_HAVE_U32_PORT) + .get_netdev = roce5_ib_get_netdev, +#elif defined(GET_NETDEV_HAVE_U8_PORT) + .get_netdev = roce5_ib_get_netdev_compat, +#endif + +#if defined(IB_DEVICE_OPS_HAVE_CREATE_USER_AH) + .create_user_ah = roce5_create_ah, +#endif + + INIT_RDMA_OBJ_SIZE(ib_ah, roce5_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, roce5_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_pd, roce5_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_srq, roce5_srq, ibsrq), + INIT_RDMA_OBJ_SIZE(ib_ucontext, roce5_ucontext, ibucontext), + INIT_RDMA_OBJ_SIZE(ib_xrcd, roce5_xrcd, ibxrcd), + INIT_RDMA_OBJ_SIZE(ib_mw, roce5_mw, ibmw), +#if defined(IB_DEVICE_OPS_HAVE_IB_QP_OBJ) + INIT_RDMA_OBJ_SIZE(ib_qp, roce5_qp, ibqp), +#endif +}; + +static void roce5_add_init(struct roce5_device *rdev) +{ + struct ib_device *ib_dev = &rdev->ib_dev; + + ib_dev->local_dma_lkey = rdev->rdma_cap.reserved_lkey; + ib_dev->phys_port_cnt = (u8)rdev->rdma_cap.num_ports; + ib_dev->num_comp_vectors = + (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ? + (int)rdev->rdma_cap.num_comp_vectors : + MAX_CEQ_NEED; + ib_dev->node_type = RDMA_NODE_IB_CA; + ib_dev->node_guid = roce5_rdma_init_guid(rdev->hwdev, rdev->ndev); + ib_dev->dma_device = rdev->dev; + ib_dev->dev.parent = ib_dev->dma_device; + (void)strcpy_s(ib_dev->node_desc, IB_DEVICE_NODE_DESC_MAX, "hrn5"); + + rdev->ib_dev.uverbs_cmd_mask = ROCE_UVERBS_CMD_BASE_MASK; +#if defined(IB_DEV_HAVE_UVERBS_EX_CMD_MASK) + rdev->ib_dev.uverbs_ex_cmd_mask = ROCE_UVERBS_EXT_CMD_MASK; +#endif + + ib_set_device_ops(ib_dev, &dev_ops); + roce5_init_dev_ext_handlers(rdev); +} + +#else +static void roce5_init_dev_qp_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.create_qp = roce5_create_qp_compat; + rdev->ib_dev.modify_qp = roce5_modify_qp; + rdev->ib_dev.query_qp = roce5_query_qp; + rdev->ib_dev.destroy_qp = roce5_destroy_qp_ofa; + rdev->ib_dev.post_send = roce5_post_send; + rdev->ib_dev.post_recv = roce5_post_recv; +} + +static void roce5_init_dev_cq_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.create_cq = roce5_create_cq_ofa; + rdev->ib_dev.modify_cq = roce5_modify_cq; + rdev->ib_dev.resize_cq = roce5_resize_cq; + rdev->ib_dev.destroy_cq = roce5_destroy_cq_ofa; + rdev->ib_dev.poll_cq = roce5_poll_cq; + rdev->ib_dev.req_notify_cq = roce5_arm_cq; +} + +static void roce5_init_dev_srq_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.create_srq = roce5_create_srq_ofa; + rdev->ib_dev.modify_srq = roce5_modify_srq; + rdev->ib_dev.query_srq = roce5_query_srq; + rdev->ib_dev.destroy_srq = roce5_destroy_srq_ofa; + rdev->ib_dev.post_srq_recv = roce5_post_srq_recv; +} + +static void roce5_init_dev_mr_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.map_mr_sg = roce5_map_kernel_frmr_sg; + + rdev->ib_dev.get_dma_mr = roce5_get_dma_mr; + rdev->ib_dev.reg_user_mr = roce5_reg_user_mr; + rdev->ib_dev.dereg_mr = roce5_dereg_mr_ofa; + rdev->ib_dev.alloc_mr = roce5_alloc_mr; + + rdev->ib_dev.alloc_mw = roce5_alloc_mw_ofa; + rdev->ib_dev.dealloc_mw = roce5_dealloc_mw_ofa; +} + +static void roce5_init_dev_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.query_device = roce5_query_device; + rdev->ib_dev.query_port = roce5_query_port_compat; + rdev->ib_dev.get_link_layer = roce5_port_link_layer_compat; + rdev->ib_dev.query_gid = roce5_query_gid_compat; + rdev->ib_dev.add_gid = roce5_ib_add_gid; + rdev->ib_dev.del_gid = roce5_ib_del_gid; + rdev->ib_dev.query_pkey = roce5_query_pkey_compat; + rdev->ib_dev.modify_device = roce5_modify_device; + rdev->ib_dev.modify_port = roce5_modify_port_compat; + + rdev->ib_dev.alloc_ucontext = roce5_alloc_ucontext_ofa; + rdev->ib_dev.dealloc_ucontext = roce5_dealloc_ucontext_ofa; + rdev->ib_dev.mmap = roce5_mmap; + + rdev->ib_dev.alloc_pd = roce5_alloc_pd_ofa; + rdev->ib_dev.dealloc_pd = roce5_dealloc_pd_ofa; + + rdev->ib_dev.create_ah = roce5_create_ah_ofa; + rdev->ib_dev.query_ah = roce5_query_ah; + rdev->ib_dev.destroy_ah = roce5_destroy_ah_ofa; + + roce5_init_dev_qp_handlers(rdev); + roce5_init_dev_cq_handlers(rdev); + roce5_init_dev_srq_handlers(rdev); + roce5_init_dev_mr_handlers(rdev); + + rdev->ib_dev.alloc_xrcd = roce5_alloc_xrcd_ofa; + rdev->ib_dev.dealloc_xrcd = roce5_dealloc_xrcd_ofa; + + rdev->ib_dev.get_port_immutable = roce5_port_immutable_compat; + rdev->ib_dev.get_netdev = roce5_ib_get_netdev_compat; +} + +static void roce5_add_init(struct roce5_device *rdev) +{ + int ret; + + rdev->ib_dev.owner = THIS_MODULE; + rdev->ib_dev.node_type = RDMA_NODE_IB_CA; + rdev->ib_dev.local_dma_lkey = rdev->rdma_cap.reserved_lkey; + rdev->ib_dev.phys_port_cnt = (u8)rdev->rdma_cap.num_ports; + rdev->ib_dev.num_comp_vectors = + (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ? + (int)rdev->rdma_cap.num_comp_vectors : + MAX_CEQ_NEED; + rdev->ib_dev.dma_device = rdev->dev; + rdev->ib_dev.dev.parent = rdev->ib_dev.dma_device; + rdev->ib_dev.uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION; + rdev->ib_dev.node_guid = roce5_rdma_init_guid(rdev->hwdev, rdev->ndev); + ret = memcpy_s((void *)rdev->ib_dev.node_desc, ROCE_NODE_DESC_LEN, + "hrn5", ROCE_NODE_DESC_LEN); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy node_desc.(ret:%d)", ret); + return; + } + + rdev->ib_dev.uverbs_cmd_mask = roce5_get_uverbs_cmd_mask(); + rdev->ib_dev.uverbs_ex_cmd_mask = ROCE_UVERBS_EXT_CMD_MASK; + + roce5_init_dev_handlers(rdev); + roce5_init_dev_ext_handlers(rdev); + return; +} +#endif + +static void roce5_mod_param_parse(struct roce5_device *rdev) +{ + if (g_loop_times <= 0) { + ROCE_DEV_WARN( + rdev, + "g_loop_times has been amended to 1, because original value(%d) is smaller than " + "or equal to 0.", + g_loop_times); + g_loop_times = 1; + } + rdev->try_times = g_loop_times; + rdev->destory_flow_timeout = g_destory_flow_timeout; + rdev->local_en = g_local_en; + rdev->full_qp_spec = g_full_qp_spec; + rdev->huge_page_en = g_huge_page_en; +} + +static void roce5_init_mac_vlan(struct roce5_device *rdev) +{ + INIT_LIST_HEAD(&rdev->mac_vlan_list_head); + mutex_init(&rdev->mac_vlan_mutex); +} + +static int roce5_rdev_init(struct roce5_device *rdev) +{ + int ret; + + ret = hinic5_stateful_init(rdev->hwdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init stateful resource"); + return ret; + } + + roce5_init_mac_vlan(rdev); + + ret = roce5_add_rdev_init(rdev); + if (ret != 0) { + goto err; + } + + spin_lock_init(&rdev->node_desc_lock); + rwlock_init(&rdev->rwlock_func_tbl_bond); + mutex_init(&rdev->cap_mask_mutex); + spin_lock_init(&rdev->reset_flow_resource_lock); + INIT_LIST_HEAD(&rdev->qp_list); + roce5_mod_param_parse(rdev); + + roce5_fix_ibdev_name(rdev); + + return 0; +err: + roce5_clean_vlan_device_mac(rdev); + hinic5_stateful_deinit(rdev->hwdev); + return ret; +} + +static void roce5_rdev_unit(struct roce5_device *rdev) +{ + /* FLR by MPU when hotplug, don't need deinit anymore */ + hinic5_stateful_deinit(rdev->hwdev); + + roce5_add_rdev_unit(rdev); + + mutex_destroy(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); +} + +static int roce5_add_do_init(struct roce5_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce5_rdev_init(rdev); + if (ret != 0) { + goto err_init_rdev; + } + + ret = roce5_register_template(rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to register cqm_service, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + goto err_cqm_register; + } + + ret = hinic5_alloc_db_addr(rdev->hwdev, &rdev->kernel_db_map, NULL); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc db or dwqe, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_alloc_db; + } +#ifdef __ROCE_DFX__ + ret = roce5_dfx_device_init(rdev); + if (ret != 0) { + ROCE_ERR("Failed to init dfx device"); + goto err_init_dfx; + } +#endif + roce5_add_init(rdev); + + ret = roce5_alloc_ceq(rdev); + if (ret < 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc ceqs, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_alloc_ceq; + } + + ret = roce5_init_dev(rdev, uld_dev_name); + if (ret != 0) { + goto err_init_dev; + } + + return 0; + +err_init_dev: + roce5_free_ceq(rdev); + +err_alloc_ceq: +#ifdef __ROCE_DFX__ + roce5_dfx_device_deinit(rdev); + +err_init_dfx: +#endif + hinic5_free_db_addr(rdev->hwdev, rdev->kernel_db_map, NULL); + +err_alloc_db: + cqm5_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + +err_cqm_register: + roce5_rdev_unit(rdev); + +err_init_rdev: + return ret; +} + +static int roce5_add_check(const struct hinic5_lld_dev *lld_dev) +{ + int ret = 0; + + if (lld_dev == NULL || lld_dev->hwdev == NULL) { + ROCE_ERR("Lld_dev is null"); + return (-EINVAL); + } + + ret = roce5_add_check_ext(lld_dev); + return ret; +} + +static int roce5_stateful_init(struct hinic5_lld_dev *lld_dev) +{ + int ret = 0; + struct hinic5_lld_dev *ppf_lld_dev = NULL; + + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + return 0; + } + + ppf_lld_dev = hinic5_get_ppf_lld_dev(lld_dev); + if (ppf_lld_dev == NULL) { + ROCE_ERR("Failed to get ppf lld dev, func id(%u).", + hinic5_global_func_id(lld_dev->hwdev)); + return -EINVAL; + } + + ret = hinic5_stateful_init(ppf_lld_dev->hwdev); + hinic5_lld_dev_put(ppf_lld_dev); + + return ret; +} + +static void roce5_stateful_deinit(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_lld_dev *ppf_lld_dev = NULL; + + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + return; + } + + ppf_lld_dev = hinic5_get_ppf_lld_dev(lld_dev); + if (ppf_lld_dev == NULL) { + ROCE_ERR("Failed to get ppf lld dev, func id(%u).", + hinic5_global_func_id(lld_dev->hwdev)); + return; + } + + hinic5_stateful_deinit(ppf_lld_dev->hwdev); + + hinic5_lld_dev_put(ppf_lld_dev); +} + +static void roce5_do_remove(struct roce5_device *rdev, u16 glb_func_id, + const char *dev_name) +{ +#ifdef __ROCE_DFX__ + roce5_dfx_device_deinit(rdev); +#endif + + roce5_g_rdev_remove(rdev); + + roce5_remove_sysfs(rdev); + + ib_unregister_device(&rdev->ib_dev); + + roce5_dealloc_hw_resource(rdev); + + roce5_deinit_dev_info(rdev); + +#ifdef ROCE_BONDING_EN + roce5_set_bond_ipsurx_en(true); +#endif + + roce5_clean_vlan_device_mac(rdev); + roce5_clean_real_device_mac(rdev); + + roce5_free_ceq(rdev); + + hinic5_free_db_addr(rdev->hwdev, rdev->kernel_db_map, NULL); + + cqm5_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + + roce5_del_func_res(rdev); + ROCE_INFO("Function level resource clear ok, func_id(%d), name(%s)", + glb_func_id, dev_name); + + roce5_rdev_unit(rdev); +} + +static void roce5_remove(struct hinic5_lld_dev *lld_dev, void *uld_dev) +{ + struct roce5_device *rdev = NULL; + char *device_name = NULL; + u16 glb_func_id; + roce_user_handle_t hdl = { 0 }; + + if ((uld_dev == NULL) || (lld_dev == NULL)) { + ROCE_ERR("input param is null"); + return; + } + + if (hinic5_support_roce(lld_dev->hwdev, NULL) == false) { + ROCE_WARN("Not support RoCE."); + return; + } + + rdev = (struct roce5_device *)uld_dev; + rdev->ib_active = false; + device_name = rdev->ib_dev.name; + glb_func_id = rdev->glb_func_id; + + ROCE_DEV_INFO(rdev, "Remove start, func_id(%d), name(%s)", + rdev->glb_func_id, device_name); + + hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + hdl.rdev = rdev; + roce5_uld_rdev_remove((roce_uld_handle)&hdl); + + roce5_do_remove(rdev, glb_func_id, device_name); + +#ifdef ROCE_BONDING_EN + roce5_bond_detach(rdev); +#endif + + ib_dealloc_device(&rdev->ib_dev); + + roce5_stateful_deinit(lld_dev); + + ROCE_INFO("Remove end, func_id(%d), name(%s)", glb_func_id, + device_name); +} + +/* **************************************************************************** + Prototype : roce5_add + Description : roce5_add + Input : struct hinic_lld_dev *lld_dev + void **uld_dev + char *uld_dev_name + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_add(struct hinic5_lld_dev *lld_dev, void **uld_dev, + char *uld_dev_name) +{ + struct roce5_device *rdev = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + roce_user_handle_t hdl = { 0 }; + + ret = roce5_add_check(lld_dev); + if (ret != 0) { + return ret; + } + + /* return 0(false) if the hwdev don't support ROCE, make sure it probe success */ + if (hinic5_support_roce(lld_dev->hwdev, &rdma_cap) == false) { + ROCE_WARN("Not support RoCE, func: %d", + hinic5_global_func_id(lld_dev->hwdev)); + // this pf no has enable RoCE, but is not affect RoCE driver load, no need return exception value + ret = 0; + goto err_check; + } + + ret = roce5_stateful_init(lld_dev); + if (ret != 0) { + ROCE_ERR("Failed to add ref to ppf."); + goto err_check; + } + + rdev = roce5_rdev_alloc(lld_dev, uld_dev, &rdma_cap); + if (rdev == NULL) { + ROCE_ERR("Failed to alloc rdev, func: %d", + hinic5_global_func_id(lld_dev->hwdev)); + ret = -EINVAL; + goto err_alloc; + } + + ret = roce5_add_do_init(rdev, uld_dev_name); + if (ret != 0) { + ROCE_ERR("Failed to init rdev."); + goto err_do_init; + } + + hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + hdl.rdev = rdev; + ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rdev->shadow, &rdev->shadow_vfid, + &rdev->shadow_vf_num); + ret = roce5_uld_rdev_add((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_ERR("Failed to add uld rdev."); + roce5_do_remove(rdev, rdev->glb_func_id, uld_dev_name); + goto err_do_init; + } + + ROCE_DEV_INFO(rdev, "Device(0x%p) init success.", rdev); + return 0; + +err_do_init: + ib_dealloc_device(&rdev->ib_dev); +err_alloc: + roce5_stateful_deinit(lld_dev); +err_check: + *uld_dev = NULL; + return ret; +} + +static int roce5_set_nic_event(struct roce5_device *rdev, + const struct hinic5_event_info *event) +{ + switch (event->type) { + case EVENT_NIC_LINK_UP: + if (test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) != 0) { + return -1; + } + ROCE_DEV_INFO(rdev, "Report NIC LINK_UP, func_id(%d)", + rdev->glb_func_id); + return IB_EVENT_PORT_ACTIVE; + + case EVENT_NIC_LINK_DOWN: + if (test_and_clear_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) { + return -1; + } + ROCE_DEV_INFO(rdev, "RoCE report NIC LINK_DOWN, func_id(%d)", + rdev->glb_func_id); + return IB_EVENT_PORT_ERR; + + case EVENT_NIC_DCB_STATE_CHANGE: + ROCE_DEV_INFO( + rdev, + "DCB state change no longer requires RoCE involvement, func_id(%d)", + rdev->glb_func_id); + return -1; + + default: + ROCE_DEV_ERR(rdev, + "Nic event unsupported, func_id(%d), event(%d)", + rdev->glb_func_id, event->type); + return -1; + } +} + +static int roce5_set_ib_event(struct roce5_device *rdev, + const struct hinic5_event_info *event) +{ + switch (event->service) { + case EVENT_SRV_NIC: + return roce5_set_nic_event(rdev, event); + + case EVENT_SRV_COMM: + return roce5_set_comm_event(rdev, event); + + default: + ROCE_DEV_ERR( + rdev, + "func_id(%d) event from svc(%d) is not supported, event(%d)", + rdev->glb_func_id, event->service, event->type); + return -1; + } +} + +void roce5_event(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct hinic5_event_info *event) +{ + struct ib_event ibevent; + struct roce5_device *rdev = NULL; + int type, ret = 0; + + if ((lld_dev == NULL) || (uld_dev == NULL)) { + ROCE_ERR("Input params is null"); + goto out; + } + roce5_lock_rdev(); + + ret = roce5_get_rdev_by_uld(lld_dev, uld_dev, &rdev, event); + if ((ret != 0) || (rdev == NULL)) { + ROCE_ERR("find rdev failed"); + goto err_unlock; + } +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + roce5_handle_bonded_port_state_event(rdev); + goto err_unlock; + } +#endif + type = roce5_set_ib_event(rdev, event); + if (type == -1) { + goto err_unlock; + } + + ibevent.event = (enum ib_event_type)type; + ibevent.device = &rdev->ib_dev; + ibevent.element.port_num = ROCE_DEFAULT_PORT_NUM; + + if (rdev->ib_active) { + ROCE_DEV_INFO(rdev, "report event: %d.", type); + ib_dispatch_event(&ibevent); + } + +err_unlock: + roce5_unlock_rdev(); + +out: + return; +} + +int roce5_adm_dfx_get_dev_type(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct roce5_dfx_common_inbuf *inbuf = + (struct roce5_dfx_common_inbuf *)buf_in; + union roce5_dfx_common_outbuf *outbuf = + (union roce5_dfx_common_outbuf *)buf_out; + + struct roce5_device *find_rdev = + roce5_get_rdev_by_name(inbuf->dev_name); + + if (in_size < sizeof(struct roce5_dfx_common_inbuf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, + (int)sizeof(struct roce5_dfx_common_inbuf)); + return -ENOMEM; + } + if (find_rdev == NULL) { + ROCE_ERR("Failed to find rdev by device name(%s).", + inbuf->dev_name); + return -EINVAL; + } + + outbuf->dev_type = find_rdev->device_type; + *out_size = (u32)sizeof(union roce5_dfx_common_outbuf); + return 0; +} + +static roce5_adm_cmd_proc_t g_roce5_adm_cmd_proc[] = { +#ifdef __ROCE_DFX__ + { ROCE_CMD_GET_QPC_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_QPC_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_CQC_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_CQC_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_SRQC_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_SRQC_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_MPT_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_MPT_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_GID_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_QPC_CQC_PI_CI, roce5_adm_dfx_query }, + { ROCE_CMD_GET_QP_COUNT, roce5_adm_dfx_query }, + { ROCE_CMD_GET_DEV_ALGO, roce5_adm_dfx_query }, + { ROCE_CMD_GET_DEV_TYPE, roce5_adm_dfx_get_dev_type }, +#ifdef ENABLE_DEBUG + { ROCE_CMD_DFX_ATTACK, roce5_dfx_attack_func }, +#endif +#ifdef ROCE_CHIP_TEST + { ROCE_CMD_SET_VNIC_WATERLINE, roce5_adm_dfx_vnic_wl_ctrl }, + { ROCE_CMD_GET_VNIC_WATERLINE, roce5_adm_dfx_vnic_wl_ctrl }, +#endif /* ROCE_CHIP_TEST */ + { ROCE_CMD_ROCE_SET, roce5_adm_dfx_roce_set }, + { ROCE_CMD_ROCE_SET_DEBUG, roce5_adm_dfx_roce_set_debug }, +#endif /* __ROCE_DFX__ */ + { ROCE_CMD_ENABLE_BW_CTRL, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_DISABLE_BW_CTRL, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_CHANGE_BW_CTRL_PARAM, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_QUERY_BW_CTRL_PARAM, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_SET_LDCP_PARAM, roce5_adm_dfx_set_ldcp_param }, + { ROCE_CMD_PORT_TRAFFIC, roce5_adm_dfx_port_traffic }, + { ROCE_CMD_ULD_IOCTL_EXTEND, + (roce5_adm_func_t)roce5_adm_dfx_uld_extend }, +#ifdef ENABLE_HYPER_ROCE + { ROCE_CMD_HYPER, roce5_hyper_tool_dispatch }, +#endif +}; + +static int roce5_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct roce5_device *rdev = (struct roce5_device *)uld_dev; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *)g_roce5_adm_cmd_proc; + int elem_count = + sizeof(g_roce5_adm_cmd_proc) / sizeof(g_roce5_adm_cmd_proc[0]); + roce5_adm_func_t roce5_adm_func = NULL; + + /* check whether is root user */ + if (!capable(CAP_SYS_ADMIN)) { + ROCE_ERR("Invalid permission"); + return -EPERM; /* if not is root, return error */ + } + + if (cmd == GET_DRV_VERSION) { + return roce5_get_drv_version(buf_out, out_size); + } + + if (uld_dev == NULL || buf_in == NULL || buf_out == NULL || + out_size == NULL) { + ROCE_ERR("Input params is null"); + return -EINVAL; + } + + if (hinic5_support_roce(rdev->hwdev, NULL) == false) { + ROCE_ERR("Not support RoCE."); + return -EINVAL; + } + + if (cmd == GET_DRV_VERSION) { + return roce5_get_drv_version(buf_out, out_size); + } + + roce5_adm_func = (roce5_adm_func_t)roce5_adm_generic_find_cmd_func( + generic_array, elem_count, cmd); + if (roce5_adm_func == NULL) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", cmd); + return -EINVAL; + } + if (cmd == ROCE_CMD_ULD_IOCTL_EXTEND) { + roce_user_handle_t hdl = { 0 }; + hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + hdl.rdev = rdev; + return roce5_adm_dfx_uld_extend(&hdl, buf_in, in_size, buf_out, + out_size); + } else { + return roce5_adm_func(rdev, buf_in, in_size, buf_out, out_size); + } +} + +struct hinic5_uld_info roce5_info = { + .probe = roce5_add, + .remove = roce5_remove, + .suspend = NULL, + .resume = NULL, + .event = roce5_event, + .ioctl = roce5_ioctl, +}; + +struct hinic5_uld_info *roce5_info_get(void) +{ + return &roce5_info; +} + +/* **************************************************************************** + Prototype : roce5_service_init + Description : + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** */ +static int __init roce5_service_init(void) +{ + int ret; + + roce5_service_init_pre(); + + ret = hinic5_register_uld(SERVICE_T_ROCE, roce5_info_get()); + if (ret != 0) { + ROCE_ERR("Failed to register uld. ret(%d)", ret); + return ret; + } + + ret = roce5_netlink_init(); + if (ret != 0) { + ROCE_ERR("Failed to init netlink, ret(%d)", ret); + hinic5_unregister_uld(SERVICE_T_ROCE); + return ret; + } + +#ifdef ROCE_BONDING_EN + ret = roce5_bond_init(); + if (ret != 0) { + ROCE_ERR("Failed to init RoCE Bond. ret(%d).", ret); + return ret; + } +#endif + + roce5_uld_service_init(); + + ret = roce5_register_netdev_notifier(); + if (ret != 0) { + ROCE_ERR("Failed to register netdev notifier. ret(%d)", ret); + return ret; + } + + ROCE_INFO("Register roce service successful"); + return 0; +} + +/* **************************************************************************** + Prototype : roce5_service_exit + Description : roce service disable + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** */ +static void __exit roce5_service_exit(void) +{ + roce5_unregister_netdev_notifier(); + + roce5_uld_service_exit(); + +#ifdef ROCE_BONDING_EN + roce5_bond_exit(); +#endif + + roce5_netlink_deinit(); + + hinic5_unregister_uld(SERVICE_T_ROCE); + + ROCE_INFO("Unregister roce service successful"); +} + +module_init(roce5_service_init); +module_exit(roce5_service_exit); diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c new file mode 100644 index 000000000..99d3b3afa --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c @@ -0,0 +1,1082 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_netdev.h" +#include "roce_main_extension.h" +#include "roce_ctx_format.h" +#include "roce_gid.h" +#include "roce_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_restore.h" +#include "rdma_comp.h" +#include "roce_os_compat.h" +#include "roce5_gid_type.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif +#include "hinic5_srv_nic.h" + +static struct notifier_block g_roce5_netdev_notifier; + +/* **************************************************************************** + Prototype : roce5_fill_gid_type + Description : fill gid type + Input : struct rdma_gid_entry *gid_entry + Output : None + + 1.Date : 2016/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_fill_gid_type(struct rdma_gid_entry *gid_entry, + enum roce5_gid_type new_gid_type) +{ + static u32 index_roce5_gid_mapping[32][4] = { + { 0x120E1436, 3452, 224529784, 0x180E252A }, + { 0x120E004A, 3472, 224660876, 0x6C0E403E }, + { 0x120E0042, 3464, 224660836, 0x840E4036 }, + { 0x120E0042, 3464, 224398688, 0xC00E0036 }, + { 0x1212143A, 3456, 224791932, 0x1812252E }, + { 0x1212004E, 3476, 224923024, 0x6C124042 }, + { 0x12120046, 3468, 224922984, 0x8412403A }, + { 0x12120046, 3468, 224660836, 0xC012003A }, + { 0x1216143E, 3460, 225054080, 0x18162532 }, + { 0x12160052, 3480, 225185172, 0x6C164046 }, + { 0x1216004A, 3472, 225185132, 0x8416403E }, + { 0x1216004A, 3472, 224922984, 0xC016003E }, + { 0x10301458, 3484, 226626968, 0x1830254C }, + { 0x1030006C, 3504, 226758060, 0x6C304060 }, + { 0x10300064, 3496, 226758020, 0x84304058 }, + { 0x10300064, 3496, 226495872, 0xC0300058 }, + { 0x10401468, 838864300, 227675560, 0x1840255C }, + { 0x1040007C, 1174408640, 227806652, 0x6C404070 }, + { 0x10400074, 1040190904, 227806612, 0x84404068 }, + { 0x10400074, 1040190904, 227544464, 0xC0400068 }, + { 0x1044146C, 905973168, 227937708, 0x18442560 }, + { 0x10440080, 1241517508, 228068800, 0x6C444074 }, + { 0x10440078, 1107299772, 228068760, 0x8444406C }, + { 0x10440078, 1107299772, 227806612, 0xC044006C }, + { 0x10481470, 973082036, 228199856, 0x18482564 }, + { 0x10480084, 1308626376, 228330948, 0x6C484078 }, + { 0x1048007C, 1174408640, 228330908, 0x84484070 }, + { 0x1048007C, 1174408640, 228068760, 0xC0480070 }, + { 0x1262148A, 1040190928, 230034892, 0x1862257E }, + { 0x1262009E, 1375735268, 230165984, 0x6C624092 }, + { 0x12620096, 1241517532, 230165944, 0x8462408A }, + { 0x12620096, 1241517532, 229903796, 0xC062008A }, + }; + + u8 i = 0; + gid_entry->dw6_h.bs.gid_type = new_gid_type; + gid_entry->dw6_h.bs.gid_update = (new_gid_type == ROCE_IPv4_ROCEv2_GID); + + i = ROCE_GID_MAP_TBL_IDX_GET(gid_entry->dw6_h.bs.tag, (u8)new_gid_type); + gid_entry->hdr_len_value = index_roce5_gid_mapping[i][0]; + if (new_gid_type == ROCE_IPv4_ROCEv2_GID) { + *((u32 *)(void *)gid_entry + 1) = + cpu_to_be32(index_roce5_gid_mapping[i][1]); + *((u32 *)(void *)gid_entry + ROCE_GID_MAP_TBL_IDX2) = + cpu_to_be32( + index_roce5_gid_mapping[i] + [ROCE_GID_MAP_TBL_IDX2]); + } +} + +static void roce5_fill_gid_smac(struct net_device *net_dev, + struct rdma_gid_entry *gid_entry) +{ + int ret; + ret = memcpy_s((void *)gid_entry->smac, sizeof(gid_entry->smac), + (void *)net_dev->dev_addr, sizeof(gid_entry->smac)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy smac.(ret:%d)", ret); + } +} + +/* + 2024-04-05 add rdev parameters, because is in roce driver load, via roce5_ib_add_gid to next call, roce5_fill_gid_entry_ext + via net_dev get to rdev is empty, function in advance return, cannot execute expected of tunnel assignment logical +*/ +static void roce5_fill_gid_vlan(const struct roce5_device *rdev, + struct net_device *net_dev, + struct rdma_gid_entry *gid_entry) +{ + if (rdma_vlan_dev_vlan_id(net_dev) != 0xffff) { + gid_entry->dw4.bs.cvlan = rdma_vlan_dev_vlan_id(net_dev) & + 0xfff; + gid_entry->dw6_h.bs.tag = ROCE_GID_VLAN_VALID; + } + + roce5_fill_gid_entry_ext(rdev, gid_entry); +} + +/* **************************************************************************** + Prototype : roce5_dispatch_event + Description : roce5_dispatch_event + Input : struct roce5_device *rdev + int port_num + enum ib_event_type type + Output : None + + 1.Date : 2016/3/29 + Modification : Created function + +**************************************************************************** */ +static void roce5_dispatch_event(struct roce5_device *rdev, int port_num, + enum ib_event_type type) +{ + struct ib_event event; + int ret; + + if (rdev == NULL) { + ROCE_ERR("Rdev is null"); + return; + } + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memset event.(ret:%d)", ret); + } + + event.device = &rdev->ib_dev; + event.element.port_num = (u8)port_num; + event.event = type; + + ROCE_DEV_INFO(rdev, "driver dispatch event: %d.", type); + ib_dispatch_event(&event); +} + +static int roce5_del_old_gid_entry(struct roce5_device *rdev) +{ + struct rdma_gid_entry old_gid_entry; + int ret = 0; + int index = 0; + + ret = memset_s(&old_gid_entry, sizeof(old_gid_entry), 0, + sizeof(old_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + ret = roce5_get_gid(rdev, 1, index, &old_gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &old_gid_entry, + ROCE_MPU_CMD_DEL_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to del ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_del_ipsu_tcam_tbl_entry(rdev, &old_gid_entry); + } + return ret; +} + +static int roce5_netdev_event_ipv4_dev_scan(struct roce5_device *rdev, + struct net_device *event_netdev, + struct rdma_gid_entry *gid_entry) +{ + struct in_device *in_dev = NULL; + int ret = 0; + int index = 0; + + ret = roce5_del_old_gid_entry(rdev); + if (ret != 0) { + ROCE_ERR("Failed to del gid_entry.(ret:%d)", ret); + return ret; + } + + in_dev = in_dev_get(event_netdev); + if (in_dev) { + for_ifa(in_dev) + { + ipv6_addr_set_v4mapped( + ifa->ifa_address, + (struct in6_addr *)gid_entry->raw); + + roce5_fill_gid_vlan(rdev, event_netdev, gid_entry); + roce5_fill_gid_smac(event_netdev, gid_entry); + + /* RoCE V2 */ + if (rdma_protocol_roce_udp_encap( + &rdev->ib_dev, ROCE_DEFAULT_PORT_NUM)) { + roce5_fill_gid_type(gid_entry, + ROCE_IPv4_ROCEv2_GID); + ret = roce5_update_gid_mac(rdev, 0, &index, + gid_entry); + } + + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update type(%u) gid(IPv4) when process netdev's event, func_id(%d)", + gid_entry->dw6_h.bs.gid_type, + rdev->glb_func_id); + in_dev_put(in_dev); + return ret; + } + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry( + rdev, gid_entry, + ROCE_MPU_CMD_ADD_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to add ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tcam_tbl_entry(rdev, gid_entry, + index); + } + roce5_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, + IB_EVENT_GID_CHANGE); + } + endfor_ifa(in_dev); + in_dev_put(in_dev); + } + + return 0; +} + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) +struct sin6_list { + struct list_head list; + struct sockaddr_in6 sin6; +}; + +static int roce5_gid_fill(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, + struct net_device *event_netdev, + struct sin6_list *sin6_iter) +{ + int ret = 0; + int index; + + ret = roce5_del_old_gid_entry(rdev); + if (ret != 0) { + ROCE_ERR("Failed to del gid_entry.(ret:%d)", ret); + return ret; + } + + if (cpu_to_be64(gid_entry->global.subnet_prefix) == + ROCE_DEFAULT_GID_SUBNET_PREFIX) { + return ret; + } + + roce5_fill_gid_vlan(rdev, event_netdev, gid_entry); + roce5_fill_gid_smac(event_netdev, gid_entry); + + /* RoCE V2 */ + if (rdma_protocol_roce_udp_encap(&rdev->ib_dev, + ROCE_DEFAULT_PORT_NUM)) { + roce5_fill_gid_type(gid_entry, ROCE_IPv6_ROCEv2_GID); + ret = roce5_update_gid_mac(rdev, 0, &index, gid_entry); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update V2 gid(IPv6), func_id(%d)", + rdev->glb_func_id); + list_del(&sin6_iter->list); + kfree(sin6_iter); + return ret; + } + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry( + rdev, gid_entry, ROCE_MPU_CMD_ADD_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to add ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tcam_tbl_entry(rdev, gid_entry, index); + } + } + + roce5_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, IB_EVENT_GID_CHANGE); + + return ret; +} + +static int roce5_netdev_event_ipv6_dev_scan(struct roce5_device *rdev, + struct net_device *event_netdev, + struct rdma_gid_entry *gid_entry) +{ + struct sin6_list *sin6_temp = NULL; + struct sin6_list *sin6_iter = NULL; + struct inet6_dev *in6_dev = NULL; + struct inet6_ifaddr *ifp = NULL; + struct sin6_list *entry = NULL; + int ret; + + LIST_HEAD(sin6_init_list); + + in6_dev = in6_dev_get(event_netdev); + if (in6_dev == NULL) { + return 0; + } + + read_lock_bh(&in6_dev->lock); + list_for_each_entry(ifp, &in6_dev->addr_list, if_list) { + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (entry == NULL) { + continue; + } + + entry->sin6.sin6_family = AF_INET6; + entry->sin6.sin6_addr = ifp->addr; + list_add_tail(&entry->list, &sin6_init_list); + } + + read_unlock_bh(&in6_dev->lock); + in6_dev_put(in6_dev); + + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + if (memcpy_s((void *)gid_entry->raw, sizeof(union ib_gid), + (void *)&sin6_iter->sin6.sin6_addr, + sizeof(union ib_gid)) != 0) { + ROCE_WARN("Failed to do memcpy_s"); + } + + ret = roce5_gid_fill(rdev, gid_entry, event_netdev, sin6_iter); + if (ret != 0) { + goto err_free; + } + + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + + return 0; + +err_free: + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + return ret; +} +#endif + +static int roce5_netdev_event_ip_dev_scan(struct roce5_device *rdev, + struct net_device *event_netdev) +{ + struct rdma_gid_entry gid_entry; + int ret; + + ret = memset_s(&gid_entry, sizeof(gid_entry), 0, sizeof(gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + ret = roce5_netdev_event_ipv4_dev_scan(rdev, event_netdev, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to scan IPv4, func_id(%d)", + rdev->glb_func_id); + return ret; + } + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) + ret = roce5_netdev_event_ipv6_dev_scan(rdev, event_netdev, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to scan IPv6, func_id(%d)", + rdev->glb_func_id); + return ret; + } +#endif + + return 0; +} + +static struct roce5_vlan_dev_list * +roce5_find_vlan_device_list(const struct list_head *mac_list, + const struct net_device *netdev) +{ + struct roce5_vlan_dev_list *vlan_dev_list = NULL; + + list_for_each_entry(vlan_dev_list, mac_list, list) { + if (vlan_dev_list->net_dev == netdev) { + return vlan_dev_list; + } + } + + return NULL; +} + +static int roce5_add_vlan_device(struct roce5_device *rdev, + struct net_device *netdev) +{ + struct roce5_vlan_dev_list *new_list = NULL; + int ret; + + new_list = (struct roce5_vlan_dev_list *)kzalloc(sizeof(*new_list), + GFP_KERNEL); + if (new_list == NULL) { + ROCE_DEV_ERR(rdev, + "Malloc mac_vlan list_head is NULL, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + new_list->ref_cnt = 1; + new_list->net_dev = netdev; + new_list->vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID( + ((u32)rdma_vlan_dev_vlan_id(netdev))); + ret = memcpy_s(new_list->mac, ROCE_MAC_ADDR_LEN, netdev->dev_addr, + ROCE_MAC_ADDR_LEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mac.(ret:%d)", ret); + return -ENOMEM; + } + INIT_LIST_HEAD(&new_list->list); + list_add_tail(&new_list->list, &rdev->mac_vlan_list_head); + + return 0; +} + +static void roce5_del_vlan_device(struct roce5_device *rdev, + struct roce5_vlan_dev_list *old_list) +{ + list_del(&old_list->list); + kfree(old_list); +} + +int roce5_add_vlan_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + int ret = 0; + u32 vlan_id = 0; + + vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID( + ((u32)rdma_vlan_dev_vlan_id(netdev))); + ROCE_DEV_INFO( + rdev, + "enter roce5_add_vlan_device_mac, vlan_id(0x%x), func_id(%d).", + vlan_id, rdev->glb_func_id); + + ret = roce5_add_mac_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, + ETH_ALEN, vlan_id, rdev->glb_func_id); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set vlan device mac, vlan_id(0x%x), func_id(%d)", + vlan_id, rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, vlan_id, + rdev->glb_func_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + ret = roce5_add_bond_vlan_slave_mac( + rdev, (u8 *)netdev->dev_addr, (u16)vlan_id); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set bond vlan slave mac, vlan_id(0x%x), func_id(%d)", + vlan_id, rdev->glb_func_id); + goto err_add_bond_vlan_slave_mac; + } + } +#endif + + return ret; + +#ifdef ROCE_BONDING_EN +err_add_bond_vlan_slave_mac: + roce5_del_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, ETH_ALEN, + vlan_id, hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + + (void)hinic5_del_mac(rdev->hwdev, (u8 *)netdev->dev_addr, (u16)vlan_id, + rdev->glb_func_id, HINIC5_CHANNEL_ROCE); + + return ret; +#endif +} + +void roce5_del_vlan_device_mac(struct roce5_device *rdev, + struct roce5_vlan_dev_list *old_list) +{ +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + roce5_del_bond_vlan_slave_mac(rdev, old_list->mac, + (u16)old_list->vlan_id); + } +#endif + + roce5_del_ipsu_tbl_mac_entry(rdev, old_list->mac, ETH_ALEN, + old_list->vlan_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + + (void)roce5_del_mac_tbl_mac_entry(rdev, old_list->mac, ETH_ALEN, + old_list->vlan_id, rdev->glb_func_id); +} + +static int roce5_update_vlan_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + struct roce5_vlan_dev_list *old_list = NULL; + int ret; + + mutex_lock(&rdev->mac_vlan_mutex); + old_list = + roce5_find_vlan_device_list(&rdev->mac_vlan_list_head, netdev); + if (old_list == NULL) { + mutex_unlock(&rdev->mac_vlan_mutex); + return 0; + } + + if (ROCE_MEMCMP(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN) != + 0) { + roce5_del_vlan_device_mac(rdev, old_list); + + ret = roce5_add_vlan_device_mac(rdev, netdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to add mac_vlan, func_id(%d)", + rdev->glb_func_id); + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + + ret = memcpy_s(old_list->mac, ROCE_MAC_ADDR_LEN, + netdev->dev_addr, ROCE_MAC_ADDR_LEN); + if (ret != 0) { + roce5_del_vlan_device_mac(rdev, old_list); + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + } + + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +int roce5_add_real_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + int ret; + + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, 0, + rdev->glb_func_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + ret = roce5_add_bond_real_slave_mac(rdev, + (u8 *)netdev->dev_addr); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to add bond read device ipsu mac, func_id(%d)", + rdev->glb_func_id); + roce5_del_ipsu_tbl_mac_entry( + rdev, (u8 *)netdev->dev_addr, ETH_ALEN, 0, + hinic5_er_id(rdev->hwdev), rdev->glb_func_id); + return ret; + } + } +#endif + + ret = memcpy_s(rdev->mac, sizeof(rdev->mac), netdev->dev_addr, + sizeof(rdev->mac)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mac.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} + +void roce5_del_real_device_mac(struct roce5_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + roce5_del_bond_real_slave_mac(rdev); + } +#endif + + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); +} + +static int roce5_update_real_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + int ret; + + roce5_del_real_device_mac(rdev); + + ret = roce5_add_real_device_mac(rdev, netdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to add real device ipsu mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return 0; +} + +void roce5_clean_vlan_device_mac(struct roce5_device *rdev) +{ + struct roce5_vlan_dev_list *pos = NULL; + struct roce5_vlan_dev_list *tmp = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + list_for_each_entry_safe(pos, tmp, &rdev->mac_vlan_list_head, list) { +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + (void)roce5_del_bond_vlan_slave_mac(rdev, pos->mac, + (u16)pos->vlan_id); + } +#endif + roce5_del_ipsu_tbl_mac_entry(rdev, pos->mac, ETH_ALEN, + pos->vlan_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + + (void)roce5_del_mac_tbl_mac_entry(rdev, pos->mac, ETH_ALEN, + pos->vlan_id, + rdev->glb_func_id); + + list_del(&pos->list); + kfree(pos); + } + mutex_unlock(&rdev->mac_vlan_mutex); +} + +void roce5_clean_real_device_mac(struct roce5_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + (void)roce5_del_bond_real_slave_mac(rdev); + } +#endif + + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); +} + +void roce5_ifconfig_up_down_event_report(struct roce5_device *rdev, + u8 net_event) +{ + struct ib_event event = {}; + + event.device = &rdev->ib_dev; + event.event = net_event; + event.element.port_num = ROCE_DEFAULT_PORT_NUM; + + ib_dispatch_event(&event); +} + +static int roce5_netdev_event_raw_dev(unsigned long event, + struct roce5_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_CHANGEADDR) { + ret = roce5_update_real_device_mac(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update real device mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = roce5_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to scan ip_dev, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + if ((event == NETDEV_DOWN) || (event == NETDEV_UP)) { + roce5_handle_bonded_port_state_event(rdev); + } + return 0; + } +#endif + if (event == NETDEV_DOWN) { + if (test_and_clear_bit(ROCE5_PORT_EVENT, &rdev->status) != 0) { + roce5_ifconfig_up_down_event_report(rdev, + IB_EVENT_PORT_ERR); + } + } + + if (event == NETDEV_UP) { + if (test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) { + roce5_ifconfig_up_down_event_report( + rdev, IB_EVENT_PORT_ACTIVE); + } + } + + return 0; +} + +static int roce5_netdev_event_vlan_dev(unsigned long event, + struct roce5_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_CHANGEADDR) { + ret = roce5_update_vlan_device_mac(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update vlan device mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = roce5_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to scan vlan device ip list, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static int roce5_netdev_event(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct net_device *event_netdev = NULL; + struct roce5_device *rdev = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + struct net_device *real_dev = NULL; + int ret; + + if (nb == NULL || ptr == NULL) { + goto err_out; + } + + event_netdev = netdev_notifier_info_to_dev(ptr); + if (event_netdev->type != ARPHRD_ETHER) { + goto err_out; + } + + real_dev = (rdma_vlan_dev_real_dev(event_netdev) != 0) ? + rdma_vlan_dev_real_dev(event_netdev) : + event_netdev; + + lld_dev = hinic5_get_lld_dev_by_netdev(real_dev); + if (lld_dev == NULL) { + goto err_out; + } + + /* + * hinic5_get_uld_dev internal will call uld5_dev_hold add device reference count + * via uld5_dev_put free reference count + */ + rdev = (struct roce5_device *)hinic5_get_uld_dev(lld_dev, + SERVICE_T_ROCE); + + /* + * passed in of rdev is empty value when,roce5_bond_event_cfg_rdev will found lld corresponding to bond device rdev, call uld5_dev_hold add device reference count + * passed in of rdev non-null value when, reserved rdev original value, not will call uld5_dev_hold add device reference count + * via uld5_dev_put free reference count + */ + ret = roce5_bond_event_cfg_rdev(lld_dev, (void *)rdev, &rdev, true); + if (ret != 0 || rdev == NULL) { + goto err_out; + } + + /* not need extra of roce5_is_port_of_net_dev check */ + if (real_dev == event_netdev) { + ret = roce5_netdev_event_raw_dev(event, rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to deal with raw device, func_id(%d)", + rdev->glb_func_id); + goto err_put; + } + } else { + ret = roce5_netdev_event_vlan_dev(event, rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to deal with vlan device, func_id(%d)", + rdev->glb_func_id); + goto err_put; + } + } + +err_put: + uld5_dev_put(rdev->lld_dev, SERVICE_T_ROCE); +err_out: + return NOTIFY_DONE; +} + +int roce5_register_netdev_notifier(void) +{ + int ret; + + g_roce5_netdev_notifier.notifier_call = roce5_netdev_event; + ret = register_netdevice_notifier(&g_roce5_netdev_notifier); + if (ret != 0) { + ROCE_ERR("Failed to register netdev notifier, ret(%d)", ret); + g_roce5_netdev_notifier.notifier_call = NULL; + } + + return ret; +} + +void roce5_unregister_netdev_notifier(void) +{ + if (g_roce5_netdev_notifier.notifier_call != NULL) { + unregister_netdevice_notifier(&g_roce5_netdev_notifier); + g_roce5_netdev_notifier.notifier_call = NULL; + } +} + +static int roce5_set_gid_entry(const struct ib_gid_attr *attr, + struct rdma_gid_entry *gid_entry) +{ + int ret = 0; + enum rdma_network_type network_type; + static enum roce5_gid_type gid_type_map[] = { + // map rdma_network_type to roce5_gid_type + [RDMA_NETWORK_IB] = ROCE_ROCEv1_GID, + [RDMA_NETWORK_ROCE_V1] = ROCE_ROCEv1_GID, + [RDMA_NETWORK_IPV4] = ROCE_IPv4_ROCEv2_GID, + [RDMA_NETWORK_IPV6] = ROCE_IPv6_ROCEv2_GID, + }; + + ret = memset_s(gid_entry, sizeof(*gid_entry), 0, sizeof(*gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + network_type = rdma_gid_attr_network_type(attr); + ret = memcpy_s((void *)gid_entry->raw, sizeof(union ib_gid), + (void *)(&attr->gid), sizeof(union ib_gid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + if (network_type == RDMA_NETWORK_ROCE_V1 || + network_type == RDMA_NETWORK_IB) { + ROCE_ERR("network type(%d) is unsupported.", network_type); + return -EINVAL; + } + + roce5_fill_gid_vlan(to_roce5_dev(attr->device), attr->ndev, gid_entry); + roce5_fill_gid_type(gid_entry, gid_type_map[network_type]); + roce5_fill_gid_smac(attr->ndev, gid_entry); + + return 0; +} + +/* add vlan_mac list or ref_cnt + 1 + * add mac_vlan when new list add + */ +static int roce5_notify_vlan(struct roce5_device *rdev, + const struct ib_gid_attr *attr) +{ + int ret; + struct roce5_vlan_dev_list *old_list = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + ROCE_DEV_INFO(rdev, "Add vlan: func_id(%d), netdev(%s).", + rdev->glb_func_id, attr->ndev->name); + old_list = roce5_find_vlan_device_list(&rdev->mac_vlan_list_head, + attr->ndev); + if (old_list != NULL) { + old_list->ref_cnt++; + ret = 0; + goto out; + } + + ret = roce5_add_vlan_device(rdev, attr->ndev); + if (ret != 0) { + goto out; + } + + ret = roce5_add_vlan_device_mac(rdev, attr->ndev); + if (ret != 0) { + old_list = roce5_find_vlan_device_list( + &rdev->mac_vlan_list_head, attr->ndev); + if (old_list) { + roce5_del_vlan_device(rdev, old_list); + } + } + +out: + mutex_unlock(&rdev->mac_vlan_mutex); + + return ret; +} + +int roce5_ib_add_gid_common(const struct ib_gid_attr *attr) +{ + int ret; + struct rdma_gid_entry gid_entry; + struct roce5_device *rdev = NULL; + unsigned int index; + + // the input gid pointer is not needed to check on Euler OS + if (attr == NULL) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rdev = to_roce5_dev(attr->device); + index = attr->index; + + ret = roce5_set_gid_entry(attr, &gid_entry); + if (ret != 0) { + ROCE_ERR("Failed to set gid entry."); + return ret; + } + + ret = roce5_update_gid(rdev, 0, index, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to update gid. ret(%d),func_id(%d)", + ret, rdev->glb_func_id); + return ret; + } + + rdev->gid_dev[index] = attr->ndev; + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &gid_entry, + ROCE_MPU_CMD_ADD_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to add ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tcam_tbl_entry(rdev, &gid_entry, index); + } + + if (rdma_vlan_dev_real_dev(attr->ndev)) { + ret = roce5_notify_vlan(rdev, attr); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to nofify vlan, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + } + + return 0; +} + +int roce5_ib_add_gid(const struct ib_gid_attr *attr, + __always_unused void **context) +{ + return roce5_ib_add_gid_common(attr); +} + +int roce5_ib_del_gid(const struct ib_gid_attr *attr, + __always_unused void **context) +{ + struct roce5_vlan_dev_list *old_list = NULL; + struct roce5_device *rdev = NULL; + struct rdma_gid_entry gid_entry; + u32 index = 0; + int ret; + + if ((attr == NULL) || (attr->device == NULL)) { + ROCE_ERR("Attr or attr->device is null"); + return (-EINVAL); + } + + rdev = to_roce5_dev(attr->device); + index = attr->index; + if (index >= rdev->rdma_cap.max_gid_per_port) { + ROCE_DEV_ERR(rdev, "Invalid gid index(%u), func_id(%d)", index, + rdev->glb_func_id); + return (-EINVAL); + } + + /* del vlan_mac list or ref_cnt - 1 */ + /* del mac_vlan when ref_cnt = 0 */ + if (rdev->ndev != rdev->gid_dev[index]) { + mutex_lock(&rdev->mac_vlan_mutex); + old_list = roce5_find_vlan_device_list( + &rdev->mac_vlan_list_head, rdev->gid_dev[index]); + if (old_list) { + old_list->ref_cnt--; + if (old_list->ref_cnt == 0) { + roce5_del_vlan_device_mac(rdev, old_list); + + roce5_del_vlan_device(rdev, old_list); + } + } + mutex_unlock(&rdev->mac_vlan_mutex); + } + + ret = memset_s(&gid_entry, sizeof(gid_entry), 0, sizeof(gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + ret = roce5_get_gid(rdev, 1, index, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d), index(%u)", + rdev->glb_func_id, index); + return ret; + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &gid_entry, + ROCE_MPU_CMD_DEL_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to del ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_del_ipsu_tcam_tbl_entry(rdev, &gid_entry); + } + + /* + * delete gid no longer send cmd to ucode which write 0s to target entry, + * preventing PPE from building malformed packets. */ + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h new file mode 100644 index 000000000..f77f75dbc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h @@ -0,0 +1,70 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_NETDEV_H +#define ROCE_NETDEV_H + +#include <net/ipv6.h> +#include <net/bonding.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include <rdma/ib_addr.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_mix.h" +#include "roce_cmd.h" + +#ifdef ROCE_BONDING_EN +#include "hinic5_srv_nic.h" +#endif + +#include "securec.h" + +#define ROCE_GID_SET_VLAN_32BIT_VLAID(vlan_id) \ + ((vlan_id) | (((u32)1) << 31)) /* the 31 bit is vlan eable bit */ +#define ROCE_MAC_ADDR_LEN 6 + +/* ipv4 header len. unit:4B */ +#define IPV4_HDR_LEN 5 + +#define IP_ECN_PERMIT 0x2 //ip header tclass bit[0:2] + +#define ROCE_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define ROCE_GID_MAP_TBL_IDX_GET(tag, new_gid_type) \ + ((u8)(((tag) << 2) | (new_gid_type))) + +#define ROCE_GID_MAP_TBL_IDX2 2 + +enum roce5_gid_vlan_type { ROCE_GID_VLAN_INVALID = 0, ROCE_GID_VLAN_VALID }; + +enum roce5_gid_tunnel_type { + ROCE_GID_TUNNEL_INVALID = 0, + ROCE_GID_TUNNEL_VALID +}; + +struct roce5_vlan_dev_list { + u8 mac[ETH_ALEN]; + u32 vlan_id; + u32 ref_cnt; + struct net_device *net_dev; + struct list_head list; +}; + +#if !defined(INETDEV_HAVE_FOR_IFA) +#define for_ifa(in_dev) \ + { \ + struct in_ifaddr *ifa; \ + for (ifa = (in_dev)->ifa_list; ifa; ifa = ifa->ifa_next) +#define endfor_ifa(in_dev) } +#endif + +int roce5_add_real_device_mac(struct roce5_device *rdev, + struct net_device *netdev); + +#endif /* ROCE_NETDEV_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c new file mode 100644 index 000000000..6a16c7f29 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c @@ -0,0 +1,413 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. + + +#include <linux/genetlink.h> +#include <linux/netlink.h> +#include <linux/rtnetlink.h> +#include <linux/skbuff.h> +#include <linux/socket.h> +#include <net/genetlink.h> +#include <net/net_namespace.h> +#include <net/netlink.h> + +#include "roce.h" +#include "hinic5_crm.h" +#include "hinic5_lld.h" +#include "roce_drv_msg_intf.h" +#include "roce_infra.h" +#include "roce_kernel_compat_gen.h" +#include "roce_netlink_process.h" +#include "securec.h" +#include "udk_usrnl.h" +#ifdef ENABLE_HYPER_ROCE +#include "hyper_roce_netlink.h" +#endif +#include "roce_netlink.h" + +static int roce5_netlink_cmd_dispatch(struct sk_buff *skb, + struct genl_info *nl_info); + +static DEFINE_RWLOCK(g_cmd_process_rwlock); /* lock for cmd_cbs */ +static roce5_netlink_cmd_excute_callback cmd_cbs[USRNL_ROCE_CMD_NUM]; + +static int roce5_netlink_register_cb(enum roce_usrnl_cmd cmd, + roce5_netlink_cmd_excute_callback cb) +{ + int ret = 0; + + if (cmd >= USRNL_ROCE_CMD_NUM) { + ROCE_ERR("Invalid command(%u)", cmd); + return -EINVAL; + } + + write_lock(&g_cmd_process_rwlock); + + if (cmd_cbs[cmd] != NULL) { + ROCE_ERR( + "Repeated command setting, the callback of the command(%u) already exists.", + cmd); + ret = -EINVAL; + goto out; + } + + cmd_cbs[cmd] = cb; + +out: + write_unlock(&g_cmd_process_rwlock); + return ret; +} + +static void roce5_netlink_unregister_cb(enum roce_usrnl_cmd cmd) +{ + if (cmd >= USRNL_ROCE_CMD_NUM) { + ROCE_ERR("Invalid command(%u)", cmd); + return; + } + + write_lock(&g_cmd_process_rwlock); + cmd_cbs[cmd] = NULL; + write_unlock(&g_cmd_process_rwlock); +} + +int roce5_netlink_register_cbs(roce5_netlink_cmd_process cmd_procs[], + u8 cmd_num) +{ + int ret, i; + + for (i = 0; i < cmd_num; ++i) { + ret = roce5_netlink_register_cb(cmd_procs[i].cmd, + cmd_procs[i].callback); + if (ret != 0) { + ROCE_ERR("Failed to set callback in netlink, ret(%d)", + ret); + goto err_out; + } + } + + return 0; + +err_out: + for (--i; i >= 0; --i) { + roce5_netlink_unregister_cb(cmd_procs[i].cmd); + } + return ret; +} + +static roce5_netlink_cmd_process g_cmd_process[] = { +#ifdef ROCE_BONDING_EN + { USRNL_ROCE_CMD_QUERY_QP_TX_PORT, roce5_genl_query_qp_tx_port }, + { USRNL_ROCE_CMD_SET_QP_TX_PORT, roce5_genl_set_qp_tx_port }, + { USRNL_ROCE_CMD_QUERY_QP_RX_PORT, roce5_genl_query_qp_rx_port }, + { USRNL_ROCE_CMD_QUERY_BOND_PORT_INFO, + roce5_genl_query_bond_port_info }, + { USRNL_ROCE_CMD_SET_QP_UDP_SRC_PORT, roce5_genl_set_qp_udp_src_port }, + { USRNL_ROCE_CMD_QUERY_QP_UDP_SRC_PORT, + roce5_genl_query_qp_udp_src_port }, +#endif + { USRNL_ROCE_CMD_QUERY_NEXT_WQE_IDX, roce5_genl_query_next_wqe_idx }, + { USRNL_ROCE_CMD_QUERY_QPC, roce5_genl_query_qpc }, + { USRNL_ROCE_CMD_QUERY_CQC, roce5_genl_query_cqc }, + { USRNL_ROCE_CMD_QP_HASH_VALUE_PROCESS, + roce5_genl_qp_hash_value_process }, + { USRNL_ROCE_CMD_ULD, + roce5_netlink_uld_process }, /* usr kernel of uld interface */ +}; + +static int roce5_netlink_init_cbs(void) +{ + int ret; + + ret = roce5_netlink_register_cbs( + g_cmd_process, + sizeof(g_cmd_process) / sizeof(roce5_netlink_cmd_process)); + if (ret != 0) { + ROCE_ERR("Failed to register netlink callbacks, ret(%d).", ret); + return ret; + } + +#ifdef ENABLE_HYPER_ROCE + ret = roce5_hyper_netlink_init(); + if (ret != 0) { + ROCE_ERR("Failed to init hyper roce in netlink, ret(%d).", ret); + return ret; + } +#endif + + return 0; +} + +#if defined(NLA_POLICY_HAVE_MIN_MAX) +static const struct nla_policy roce5_policy[USRNL_GENL_ATTR_MAX + 1] = { + [USRNL_GENL_ATTR] = { .type = NLA_BINARY, + .len = CMD_BUF_LEN_MAX, + .min = 0, + .max = CMD_BUF_LEN_MAX }, +}; +#else +static const struct nla_policy roce5_policy[USRNL_GENL_ATTR_MAX + 1] = { + [USRNL_GENL_ATTR] = { .type = NLA_BINARY, .len = CMD_BUF_LEN_MAX }, +}; +#endif + +static const struct genl_ops roce5_genl_ops[] = { { + .cmd = USRNL_DEV_DRV_CMD_EXECUTE, +#ifdef HAVE_GENL_OPS_FIELD_VALIDATE + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, +#endif + .flags = GENL_ADMIN_PERM, + .policy = roce5_policy, + .doit = roce5_netlink_cmd_dispatch, +} }; + +static struct genl_family roce5_genl_family = { + .hdrsize = 0, + .name = ROCE_GENL_FAMILY_NAME, + .version = 1, + .maxattr = USRNL_GENL_ATTR_MAX, + .netnsok = true, + .parallel_ops = true, + .ops = roce5_genl_ops, + .n_ops = ARRAY_SIZE(roce5_genl_ops), +}; + +static void roce5_drv_msg_hdr_push(struct sk_buff *skb, + usrnl_drv_msg_hdr_s *in_msg_hdr, + uint16_t out_msg_data_len) +{ + usrnl_drv_msg_hdr_s *out_msg_hdr = NULL; + + out_msg_hdr = skb_push(skb, ROCE_DRV_MSG_HDRLEN); + out_msg_hdr->cmd = in_msg_hdr->cmd; + out_msg_hdr->msg_len = ROCE_DRV_MSG_HDRLEN + out_msg_data_len; + out_msg_hdr->module = in_msg_hdr->module; + out_msg_hdr->src_func_idx = in_msg_hdr->src_func_idx; + out_msg_hdr->err_code = 0; +} + +static void roce5_nlahdr_push(struct sk_buff *skb, uint16_t type) +{ + struct nlattr *nlahdr = NULL; + uint16_t attrlen = (uint16_t)skb->len; + uint32_t padlen = (uint32_t)nla_padlen(attrlen); + + nlahdr = (struct nlattr *)skb_push(skb, NLA_HDRLEN); + nlahdr->nla_len = (uint16_t)nla_attr_size(attrlen); + nlahdr->nla_type = type; + + if ((skb->end - skb->tail) >= padlen) { + (void)skb_put(skb, padlen); + } +} + +static struct nlmsghdr *nlmsg_push(struct sk_buff *skb, uint32_t portid, + uint32_t seq, uint16_t type, uint32_t len, + uint16_t flags) +{ + struct nlmsghdr *nlh = NULL; + uint32_t size = (uint32_t)nlmsg_msg_size(len); + + nlh = (struct nlmsghdr *)skb_push(skb, NLMSG_ALIGN(size)); + if (nlh != NULL) { + nlh->nlmsg_type = type; + nlh->nlmsg_len = skb->len; + nlh->nlmsg_flags = flags; + nlh->nlmsg_pid = portid; + nlh->nlmsg_seq = seq; + if (!__builtin_constant_p(size) || + NLMSG_ALIGN(size) - size != 0) { + (void)memset_s((char *)nlmsg_data(nlh) + len, + NLMSG_ALIGN(size) - size, 0, + NLMSG_ALIGN(size) - size); + } + } + + return nlh; +} + +static inline struct nlmsghdr *roce5_nlmsg_push(struct sk_buff *skb, + uint32_t portid, uint32_t seq, + uint16_t type, uint32_t payload, + uint16_t flags) +{ + if (unlikely(skb_headroom(skb) < + (unsigned int)nlmsg_total_size(payload))) { + return NULL; + } + + return nlmsg_push(skb, portid, seq, type, payload, flags); +} + +static void *roce5_genlmsg_push(struct sk_buff *skb, uint32_t portid, + uint32_t seq, const struct genl_family *family, + uint16_t flags, uint8_t cmd) +{ + struct nlmsghdr *nlh = NULL; + struct genlmsghdr *hdr = NULL; + + nlh = roce5_nlmsg_push(skb, portid, seq, (int)family->id, + (int)(GENL_HDRLEN + family->hdrsize), flags); + if (nlh == NULL) { + return NULL; + } + + hdr = nlmsg_data(nlh); + hdr->cmd = cmd; + hdr->version = (uint8_t)family->version; + hdr->reserved = 0; + + return (void *)nlh; +} + +static inline roce5_netlink_cmd_excute_callback +roce5_netlink_get_callback(uint16_t cmd) +{ + roce5_netlink_cmd_excute_callback cb = NULL; + + if (cmd >= USRNL_ROCE_CMD_NUM) { + ROCE_ERR("Invalid command(%u).", cmd); + return NULL; + } + + read_lock(&g_cmd_process_rwlock); + cb = cmd_cbs[cmd]; + read_unlock(&g_cmd_process_rwlock); + + return cb; +} + +static int32_t +roce5_netlink_buf_handler(struct nlattr *cmd_attr, + roce5_netlink_cmd_excute_callback callback, + struct roce5_device *rdev, struct genl_info *nl_info, + struct sk_buff *reply) +{ + int32_t ret = 0; + uint16_t buf_in_len = 0; + uint16_t max_buf_out_len = 0; + uint16_t buf_out_len = 0; + usrnl_drv_msg_hdr_s *in_msg_hdr = NULL; + + in_msg_hdr = (usrnl_drv_msg_hdr_s *)nla_data(cmd_attr); + if (in_msg_hdr->msg_len < ROCE_DRV_MSG_HDRLEN) { + ROCE_DEV_ERR(rdev, "Invalid parameter: msg_len(%d).", + in_msg_hdr->msg_len); + return -EINVAL; + } + + buf_in_len = in_msg_hdr->msg_len - ROCE_DRV_MSG_HDRLEN; + max_buf_out_len = + NLMSG_DEFAULT_SIZE - + (NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN + ROCE_DRV_MSG_HDRLEN); + + if (buf_in_len == 0) { + ret = callback(rdev, NULL, 0, reply->data, max_buf_out_len, + &buf_out_len); + } else { + ret = callback(rdev, nla_data(cmd_attr) + ROCE_DRV_MSG_HDRLEN, + buf_in_len, reply->data, max_buf_out_len, + &buf_out_len); + } + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Do netlink %u cmd failed, errcode:%d", + nl_info->genlhdr->cmd, ret); + nlmsg_free(reply); + return -EIO; + } + + if (buf_out_len != 0) { + skb_put(reply, buf_out_len); + } + + roce5_drv_msg_hdr_push(reply, in_msg_hdr, buf_out_len); + + return 0; +} + +static int roce5_netlink_cmd_dispatch(struct sk_buff *skb, + struct genl_info *nl_info) +{ + int32_t ret = 0; + struct sk_buff *reply = NULL; + struct nlattr *cmd_attr = NULL; + roce5_drv_msg_hdr_s *in_msg_hdr = NULL; + roce5_netlink_cmd_excute_callback callback = NULL; + struct roce5_device *rdev = NULL; + + cmd_attr = nl_info->attrs[USRNL_GENL_ATTR]; + if (cmd_attr == NULL) { + ROCE_ERR("Do netlink cmd from portid:%u failed for NULL attr", + nl_info->snd_portid); + return -EINVAL; + } else if (nla_len(cmd_attr) < ROCE_DRV_MSG_HDRLEN) { + ROCE_ERR("Invalid parameter: nla_len(%d).", nla_len(cmd_attr)); + return -EINVAL; + } + + in_msg_hdr = (roce5_drv_msg_hdr_s *)nla_data(cmd_attr); + + rdev = roce5_get_rdev_by_card_func_id( + in_msg_hdr->card_id, in_msg_hdr->common_hdr.src_func_idx); + if (rdev == NULL) { + ROCE_ERR("Invalid param: src_func_idx: %u, card_id: %llu", + in_msg_hdr->common_hdr.src_func_idx, + in_msg_hdr->card_id); + return -EINVAL; + } + + callback = roce5_netlink_get_callback(in_msg_hdr->common_hdr.cmd); + if (callback == NULL) { + ROCE_DEV_ERR(rdev, "Unable to handle cmd: %u", + in_msg_hdr->common_hdr.cmd); + return -EINVAL; + } + + reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (reply == NULL) { + ROCE_DEV_ERR(rdev, "Alloc reply buf for netlink cmd failed"); + return -ENOMEM; + } + + skb_reserve(reply, NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN + + ROCE_DRV_MSG_HDRLEN); + + ret = roce5_netlink_buf_handler(cmd_attr, callback, rdev, nl_info, + reply); + if (ret != 0) { + return ret; + } + + roce5_nlahdr_push(reply, USRNL_GENL_ATTR); + roce5_genlmsg_push(reply, nl_info->snd_portid, nl_info->snd_seq, + &roce5_genl_family, 0, nl_info->genlhdr->cmd); + + genlmsg_reply(reply, nl_info); + + return 0; +} + +int32_t roce5_netlink_init(void) +{ + int ret; + + ret = roce5_netlink_init_cbs(); + if (ret != 0) { + ROCE_ERR("Failed to init netlink callbacks, ret(%d).", ret); + return ret; + } + + ret = genl_register_family(&roce5_genl_family); + if (ret != 0) { + ROCE_ERR("Failed to register netlink, ret(%d).", ret); + return ret; + } + + ROCE_INFO("netlink init successfully."); + return 0; +} + +void roce5_netlink_deinit(void) +{ + genl_unregister_family(&roce5_genl_family); + ROCE_INFO("netlink deinit successfully."); +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h new file mode 100644 index 000000000..9c4cfff02 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h @@ -0,0 +1,53 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. + + +#ifndef ROCE5_NETLINK_H +#define ROCE5_NETLINK_H + +#include <linux/types.h> +#include "roce_drv_msg_intf.h" +#include "roce.h" + +#define ROCE_GENL_FAMILY_NAME "roce5_genl" +#define ROCE_DRV_MSG_HDRLEN (sizeof(roce5_drv_msg_hdr_s)) +#define CMD_BUF_LEN_MAX 2048 + +typedef int32_t (*roce5_netlink_cmd_excute_callback)(struct roce5_device *rdev, + void *in, uint16_t in_len, + void *out, + uint16_t out_len, + uint16_t *out_size); + +enum roce_usrnl_cmd { + USRNL_ROCE_CMD_QUERY_QP_TX_PORT, + USRNL_ROCE_CMD_SET_QP_TX_PORT, + USRNL_ROCE_CMD_QUERY_QP_RX_PORT, + USRNL_ROCE_CMD_QUERY_BOND_PORT_INFO, + USRNL_ROCE_CMD_SET_QP_UDP_SRC_PORT, + USRNL_ROCE_CMD_QUERY_QP_UDP_SRC_PORT, + USRNL_ROCE_CMD_QUERY_NEXT_WQE_IDX, + USRNL_ROCE_CMD_QUERY_QPC, + USRNL_ROCE_CMD_QUERY_CQC, + USRNL_ROCE_CMD_QP_HASH_VALUE_PROCESS, + USRNL_ROCE_CMD_ULD, /* uld use */ + + /* Command number for HyperRoCE */ + USRNL_ROCE_HYPER_ROCE_START = 16, + USRNL_ROCE_CMD_HYPER_QP_SET = USRNL_ROCE_HYPER_ROCE_START, + USRNL_ROCE_CMD_MODIFY_QP_EXTEND, + USRNL_ROCE_HYPER_ROCE_END = 31, + USRNL_ROCE_CMD_NUM +}; + +typedef struct { + enum roce_usrnl_cmd cmd; + roce5_netlink_cmd_excute_callback callback; +} roce5_netlink_cmd_process; + +int32_t roce5_netlink_init(void); +void roce5_netlink_deinit(void); +int roce5_netlink_register_cbs(roce5_netlink_cmd_process cmd_procs[], + u8 cmd_num); + +#endif /* ROCE5_NETLINK_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c new file mode 100644 index 000000000..8b7445ab0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c @@ -0,0 +1,832 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/types.h> +#include <linux/semaphore.h> + +#include "hinic5_cqm.h" +#include "hinic5_mt.h" +#include "roce_cqm_cmd.h" +#include "roce_cmd.h" +#include "roce_com.h" +#include "roce_dfx.h" +#include "roce_main_extension.h" +#include "roce_mix.h" +#include "roce_npu_cmd.h" +#include "roce_npu_cmd_defs.h" +#include "roce_qp.h" +#include "roce_restore.h" +#include "roce_srq.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_netlink_process.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +static int32_t roce5_set_hash_value(struct roce5_device *rdev, uint32_t qpn, + uint32_t hash_type, uint32_t new_hash_value) +{ + int32_t ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + void *modify_hash_value = NULL; + size_t buf_size = 0; + + if (ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) != 0) { + buf_size = sizeof(roce_uni_cmd_modify_hash_s); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)buf_size, NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + modify_hash_value = cqm_cmd_inbuf->buf; + + if (ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) != 0) { + roce_uni_cmd_modify_hash_s *cmd = + (roce_uni_cmd_modify_hash_s *)modify_hash_value; + cmd->com.index = cpu_to_be32(qpn); + cmd->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + cmd->info.type = cpu_to_be32(hash_type); + cmd->info.hash_value = cpu_to_be32(new_hash_value); + } + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_HASH_VALUE_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d), qpn(%u), type(%u), hash_value(%u), ret=%d", + rdev->glb_func_id, qpn, hash_type, new_hash_value, ret); + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int32_t roce5_get_hash_value(struct roce5_device *rdev, uint32_t qpn, + uint32_t hash_type, uint32_t *hash_value) +{ + int32_t ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_get_hash_s *get_hash_value = NULL; + roce_verbs_qp_hash_info_s *get_hash_value_out = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_get_hash_s), &cqm_cmd_outbuf, + (uint16_t)sizeof(roce_verbs_qp_hash_info_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_hash_value = (roce_uni_cmd_get_hash_s *)cqm_cmd_inbuf->buf; + get_hash_value->com.index = cpu_to_be32(qpn); + get_hash_value->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + get_hash_value->info.type = cpu_to_be32(hash_type); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_HASH_VALUE_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, + ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send ROCE_CMD_GET_HASH_VALUE_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto out; + } + + get_hash_value_out = (roce_verbs_qp_hash_info_s *)cqm_cmd_outbuf->buf; + *hash_value = be32_to_cpu(get_hash_value_out->hash_value); + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +#ifdef ROCE_BONDING_EN +int32_t roce5_genl_query_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + uint32_t slave_cnt = 0; + struct roce5_qp *rqp = NULL; + uint32_t func_tbl_value = 0; + struct roce5_qp_in *msg_in = NULL; + struct roce5_port_out *msg_out = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_port_out) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + *out_size = sizeof(struct roce5_port_out); + msg_out = (struct roce5_port_out *)out; + + if (!roce5_bond_is_active(rdev)) { + msg_out->port = 0; + return 0; + } + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + read_lock(&rdev->rwlock_func_tbl_bond); + func_tbl_value = rdev->func_tbl_bond; + slave_cnt = func_tbl_value & 0xfU; + if (slave_cnt != 0) { + msg_out->port = (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - + ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & + ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + } + read_unlock(&rdev->rwlock_func_tbl_bond); + + if (slave_cnt == 0) { + ROCE_ERR("slave_cnt is 0"); + ret = -EINVAL; + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int32_t roce5_set_new_tx_hash_value(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct roce5_qp_port_in *msg_in) +{ + int32_t ret = 0; + uint32_t func_tbl_value = 0; + uint32_t slave_cnt = 0; + uint32_t target_hash_port = 0; + uint32_t old_hash_port = 0; + uint32_t new_tx_hash_value; + + read_lock(&rdev->rwlock_func_tbl_bond); + func_tbl_value = rdev->func_tbl_bond; + read_unlock(&rdev->rwlock_func_tbl_bond); + + slave_cnt = func_tbl_value & 0xfU; + if (slave_cnt == 0) { + ROCE_DEV_ERR(rdev, "slave_cnt is 0."); + return -EINVAL; + } + target_hash_port = msg_in->port & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + old_hash_port = + (func_tbl_value >> (ROCE_BOND_FWD_ID_TBL_ALL_BITS - + ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & + ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + if (target_hash_port == old_hash_port) { + return 0; + } + + new_tx_hash_value = + rqp->tx_hash_value + + (((slave_cnt - old_hash_port) + target_hash_port) % slave_cnt); + ret = roce5_set_hash_value(rdev, rqp->qpn, ROCE_VERBS_QP_HASH_TYPE_BOND, + new_tx_hash_value); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) Bond hash value, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + rqp->tx_hash_value = new_tx_hash_value; + + return 0; +} + +int32_t roce5_genl_set_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_qp_port_in *msg_in = NULL; + struct roce5_qp *rqp = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_qp_port_in) || out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_port_in *)in; + + if (!roce5_bond_is_active(rdev)) { + return 0; + } + + /* check qp exist */ + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + ret = roce5_set_new_tx_hash_value(rdev, rqp, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set new tx hash value, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_uni_cmd_get_qp_rx_port(struct roce5_device *rdev, uint32_t qpn, + uint32_t *rx_port) +{ + int32_t ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_get_port_info_s *get_rx_port_inbuf = NULL; + roce_uni_cmd_get_port_info_outbuf_s *get_rx_port_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_s), &cqm_cmd_outbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_rx_port_inbuf = (roce_uni_cmd_get_port_info_s *)cqm_cmd_inbuf->buf; + get_rx_port_inbuf->com.index = cpu_to_be32(qpn); + get_rx_port_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_RX_PORT_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, + ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send ROCE_CMD_GET_RX_PORT_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto out; + } + get_rx_port_outbuf = + (roce_uni_cmd_get_port_info_outbuf_s *)cqm_cmd_outbuf->buf; + *rx_port = be32_to_cpu(get_rx_port_outbuf->value.rx_port); +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static int32_t roce5_get_qp_rx_port(struct roce5_device *rdev, uint32_t qpn, + uint32_t *rx_port) +{ + if (ROCE5_SUPPORT_GET_QP_RX_PORT_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_get_qp_rx_port(rdev, qpn, rx_port); + } + return -EINVAL; +} + +int32_t roce5_genl_query_qp_rx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + uint32_t rx_port = 0; + struct roce5_qp *rqp = NULL; + struct roce5_qp_in *msg_in = NULL; + struct roce5_port_out *msg_out = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_port_out) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + if (rqp->qp_type != IB_QPT_RC) { + ROCE_DEV_ERR(rdev, "not support qp type(%d), only support RC.", + rqp->qp_type); + ret = -EINVAL; + goto err_out; + } + + *out_size = sizeof(struct roce5_port_out); + msg_out = (struct roce5_port_out *)out; + + if (!roce5_bond_is_active(rdev)) { + msg_out->port = 0; + ret = 0; + goto err_out; + } + + ret = roce5_get_qp_rx_port(rdev, msg_in->qpn, &rx_port); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get QP(0x%06x) rx port", + msg_in->qpn); + goto err_out; + } + + msg_out->port = rx_port; + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +int32_t roce5_genl_query_bond_port_info(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + uint32_t func_tbl_value = 0; + struct roce5_bond_device *bond_dev = NULL; + struct roce5_bond_port_info_out *msg_out = NULL; + uint32_t alive_port_num; + uint32_t i; + + if (out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_bond_port_info_out) || + out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, out:%d, out_len:%u, out_size:%d", + !!out, out_len, !!out_size); + return -EINVAL; + } + + *out_size = sizeof(struct roce5_bond_port_info_out); + msg_out = (struct roce5_bond_port_info_out *)out; + + if (!roce5_bond_is_active(rdev)) { + return 0; + } + + bond_dev = rdev->bond_dev; + + /* Read func_tbl_bond and derive alive port info under lock to prevent data race */ + read_lock(&rdev->rwlock_func_tbl_bond); + func_tbl_value = rdev->func_tbl_bond; + alive_port_num = func_tbl_value & 0xfU; + for (i = 0; i < alive_port_num; i++) { + msg_out->alive_port[i] = + (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - + ((i + 1) * ROCE_BOND_FWD_ID_TBL_PER_BITS))) & + ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + } + read_unlock(&rdev->rwlock_func_tbl_bond); + msg_out->alive_port_num = + (int32_t)alive_port_num; /* netlink protocol field */ + + /* Read slave info under slave_lock (separate from rwlock to avoid sleeping in atomic context) */ + msg_out->original_port_num = + (int32_t)bond_dev->slave_cnt; /* netlink protocol field */ + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + msg_out->original_port[i] = bond_dev->slaves[i].func_id; + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +static int32_t roce5_uni_cmd_modify_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, + uint32_t new_udp_src_port) +{ + int32_t ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_set_udp_src_port_s *modify_udp_src_port = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_set_udp_src_port_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + modify_udp_src_port = + (roce_uni_cmd_set_udp_src_port_s *)cqm_cmd_inbuf->buf; + modify_udp_src_port->com.index = cpu_to_be32(qpn); + modify_udp_src_port->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + modify_udp_src_port->info.udp_src_port = cpu_to_be32(new_udp_src_port); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + } + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int32_t roce5_modify_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, + uint32_t new_udp_src_port) +{ + if (ROCE5_SUPPORT_MODIFY_UDP_SRC_PORT_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_modify_udp_src_port(rdev, qpn, + new_udp_src_port); + } + return -EINVAL; +} + +int32_t roce5_genl_set_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_qp *rqp = NULL; + struct roce5_qp_udp_src_port_in *msg_in = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_qp_udp_src_port_in) || + out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_udp_src_port_in *)in; + + /* check qp exist */ + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + ret = roce5_modify_udp_src_port(rdev, rqp->qpn, msg_in->udp_src_port); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) hash value, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + ret = -EIO; + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int32_t roce5_uni_cmd_get_qp_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, + uint32_t *udp_src_port) +{ + int32_t ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_get_port_info_s *get_udp_src_port = NULL; + roce_uni_cmd_get_port_info_outbuf_s *get_udp_src_port_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s), + &cqm_cmd_outbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_udp_src_port = (roce_uni_cmd_get_port_info_s *)cqm_cmd_inbuf->buf; + get_udp_src_port->com.index = cpu_to_be32(qpn); + get_udp_src_port->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, + ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send ROCE_CMD_GET_UDP_SRC_PORT_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto out; + } + + get_udp_src_port_outbuf = + (roce_uni_cmd_get_port_info_outbuf_s *)cqm_cmd_outbuf->buf; + *udp_src_port = + be32_to_cpu(get_udp_src_port_outbuf->value.udp_src_port); +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int32_t roce5_get_qp_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, uint32_t *udp_src_port) +{ + if (ROCE5_SUPPORT_GET_QP_UDP_SRC_PORT_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_get_qp_udp_src_port(rdev, qpn, + udp_src_port); + } + return -EINVAL; +} + +int32_t roce5_genl_query_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + uint32_t udp_src_port = 0; + struct roce5_qp_in *msg_in = NULL; + struct roce5_port_out *msg_out = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_port_out) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + + ret = roce5_get_qp_udp_src_port(rdev, msg_in->qpn, &udp_src_port); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get QP(0x%06x) udp src port", + msg_in->qpn); + return ret; + } + + *out_size = sizeof(struct roce5_port_out); + msg_out = (struct roce5_port_out *)out; + + msg_out->port = udp_src_port; + + return 0; +} +#endif + +int32_t roce5_genl_query_next_wqe_idx(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size) +{ + int32_t ret = 0; + int32_t next_wqe_idx = 0; + uint32_t container_mode = 0; + uint32_t container_size = 0; + struct roce5_next_wqe_idx_in *msg_in = NULL; + struct roce5_next_wqe_idx_out *msg_out = NULL; + roce_verbs_srq_attr_s *srq_attr = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_next_wqe_idx_in) || out == NULL || + out_len == 0 || out_len < sizeof(struct roce5_next_wqe_idx_out) || + out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + srq_attr = kmalloc(sizeof(roce_verbs_srq_attr_s), GFP_KERNEL); + if (srq_attr == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to allocate memory for dfx_query_outbuf"); + return -ENOMEM; + } + + msg_in = (struct roce5_next_wqe_idx_in *)in; + msg_out = (struct roce5_next_wqe_idx_out *)out; + ret = roce5_dfx_cmd_query_srq(rdev, msg_in->srqn, srq_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to do roce5_dfx_cmd_query_srq, ret(%d)", + ret); + ret = -EINVAL; + goto free_outbuf; + } + + srq_attr->dw2.value = be32_to_cpu(srq_attr->dw2.value); + srq_attr->dw3.value = be32_to_cpu(srq_attr->dw3.value); + srq_attr->cont.dw2.value = be32_to_cpu(srq_attr->cont.dw2.value); + if (srq_attr->dw3.bs.container == 0) { + next_wqe_idx = (int32_t)(srq_attr->dw2.bs.next_wqe_idx); + } else { + container_mode = MAX_SUPPORT_CONTAINER_MODE - + srq_attr->cont.dw2.bs.cont_size; + container_size = roce5_get_container_sz(container_mode); + next_wqe_idx = (int32_t)(container_size * + srq_attr->cont.dw2.bs.head_idx); + } + + msg_out->next_wqe_idx = next_wqe_idx; + *out_size = sizeof(struct roce5_next_wqe_idx_out); + ret = 0; + +free_outbuf: + kfree(srq_attr); + return ret; +} + +int32_t roce5_genl_query_qpc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_qp_in *msg_in = NULL; + roce_verbs_qp_attr_s *verbs_qp_attr = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(roce_verbs_qp_attr_s) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + verbs_qp_attr = (roce_verbs_qp_attr_s *)out; + + ret = roce5_qp_query(rdev, msg_in->qpn, (u32 *)(void *)verbs_qp_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query qpc, ret(%0x)", ret); + return ret; + } + + *out_size = sizeof(roce_verbs_qp_attr_s); + return 0; +} + +int32_t roce5_genl_query_cqc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_cq_in *msg_in = NULL; + roce_verbs_cq_attr_s *verbs_cq_attr = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_cq_in) || + out == NULL || out_len == 0 || + out_len < sizeof(roce_verbs_cq_attr_s) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_cq_in *)in; + verbs_cq_attr = (roce_verbs_cq_attr_s *)out; + + ret = roce5_dfx_query_cq(rdev, msg_in->cqn, + (u32 *)(void *)verbs_cq_attr, + sizeof(roce_verbs_cq_attr_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query cqc, ret(%0x)", ret); + return ret; + } + + *out_size = sizeof(roce_verbs_cq_attr_s); + return 0; +} + +int32_t roce5_genl_qp_hash_value_process(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size) +{ + uint32_t hash_value = 0; + struct roce5_qp_hash_value_in *msg_in = NULL; + struct roce5_qp_hash_value_out *msg_out = NULL; + int32_t ret; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_qp_hash_value_in)) { + ROCE_DEV_ERR(rdev, "Invalid input param, in:%d, in_len:%u", + !!in, in_len); + return -EINVAL; + } + + msg_in = (struct roce5_qp_hash_value_in *)in; + + if (msg_in->type == ROCE_QP_HASH_VALUE_GET) { + if (out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_qp_hash_value_out) || + out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid output param, out:%d, out_len:%u, out_size:%d", + !!out, out_len, !!out_size); + return -EINVAL; + } + ret = roce5_get_hash_value(rdev, msg_in->qpn, + ROCE_VERBS_QP_HASH_TYPE_UBC, + &hash_value); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to get QP(0x%06x) hash value, func_id(%d)", + msg_in->qpn, rdev->glb_func_id); + return ret; + } + + *out_size = sizeof(struct roce5_qp_hash_value_out); + msg_out = (struct roce5_qp_hash_value_out *)out; + msg_out->hash_value = hash_value; + } else { + ret = roce5_set_hash_value(rdev, msg_in->qpn, + ROCE_VERBS_QP_HASH_TYPE_UBC, + msg_in->hash_value); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set QP(0x%06x) hash value, func_id(%d)", + msg_in->qpn, rdev->glb_func_id); + } + } + + return ret; +} + +int32_t roce5_netlink_uld_process(struct roce5_device *rdev, void *buf_in, + u16 buf_in_len, void *buf_out, + u16 buf_out_len, u16 *buf_out_size) +{ + int32_t ret = 0; + roce_user_handle_t handle; + + handle.rdev = rdev; + handle.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + + ret = roce5_uld_user2kernel_msg_handle((void *)&handle, buf_in, + buf_in_len, buf_out, buf_out_len, + buf_out_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "ULD process err, func_id(%d)", + rdev->glb_func_id); + } + return ret; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h new file mode 100644 index 000000000..d09bcf733 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h @@ -0,0 +1,133 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_NETLINK_PROCESS_H +#define ROCE_NETLINK_PROCESS_H + +#include "roce.h" +#include "roce_compat.h" + +typedef enum { + ROCE_QP_HASH_VALUE_SET, + ROCE_QP_HASH_VALUE_GET, +} ROCE_QP_HASH_VALUE_PROC_TYPE; + +#ifdef ROCE_BONDING_EN +/* and above next data structure is adaptation V100 use */ +struct roce5_get_rx_port { + __be32 rx_port; +}; + +struct roce5_get_udp_src_port { + __be32 udp_src_port; +}; + +typedef struct tag_roce_get_qp_rx_port { + roce_verbs_cmd_com_s com; +} roce_get_qp_rx_port_s; + +typedef struct tag_roce_modify_hash_value { + roce_verbs_cmd_com_s com; + u32 hash_value; +} roce_modify_hash_value_s; + +typedef struct tag_roce_modify_udp_src_port { + roce_verbs_cmd_com_s com; + u32 udp_src_port; +} roce_modify_udp_src_port_s; + +typedef struct roce_get_qp_udp_src_port { + roce_verbs_cmd_com_s com; +} roce_get_qp_udp_src_port_s; + +struct roce5_qp_port_in { + uint32_t qpn; + uint32_t port; +}; + +struct roce5_port_out { + uint32_t port; +}; + +struct roce5_bond_port_info_out { + int32_t original_port_num; + uint8_t original_port[8]; + int32_t rsvd1; + + int32_t alive_port_num; + uint8_t alive_port[8]; + int32_t rsvd2; +}; + +struct roce5_qp_udp_src_port_in { + uint32_t qpn; + uint32_t udp_src_port; +}; + +#endif + +struct roce5_qp_in { + uint32_t qpn; +}; + +struct roce5_cq_in { + uint32_t cqn; +}; + +struct roce5_next_wqe_idx_in { + uint32_t srqn; +}; + +struct roce5_next_wqe_idx_out { + int32_t next_wqe_idx; +}; + +struct roce5_qp_hash_value_in { + uint32_t type; + uint32_t qpn; + uint32_t hash_value; +}; + +struct roce5_qp_hash_value_out { + uint32_t hash_value; +}; + +#ifdef ROCE_BONDING_EN + +int32_t roce5_genl_query_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_set_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size); +int32_t roce5_genl_query_qp_rx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_query_bond_port_info(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_set_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_query_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); + +#endif + +int32_t roce5_genl_query_next_wqe_idx(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size); +int32_t roce5_genl_query_qpc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size); +int32_t roce5_genl_query_cqc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size); +int32_t roce5_genl_qp_hash_value_process(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size); +int32_t roce5_netlink_uld_process(struct roce5_device *rdev, void *buf_in, + u16 buf_in_len, void *buf_out, + u16 buf_out_len, u16 *buf_out_size); +#endif /* ROCE_NETLINK_PROCESS_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c new file mode 100644 index 000000000..69549fbe3 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c @@ -0,0 +1,74 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "securec.h" + +#include "roce_cqm_cmd.h" + +void roce5_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf) +{ + if (cqm_cmd_outbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_outbuf); + } + + if (cqm_cmd_inbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_inbuf); + } +} + +int roce5_cqm_cmd_zalloc_inoutbuf(void *ex_handle, + cqm_cmd_buf_s **cqm_cmd_inbuf, u16 inbuf_size, + cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size) +{ + int ret; + + if (cqm_cmd_inbuf != NULL) { + *cqm_cmd_inbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_inbuf == NULL) { + ret = -ENOMEM; + goto inbuf_err; + } + (*cqm_cmd_inbuf)->size = inbuf_size; + + ret = memset_s((*cqm_cmd_inbuf)->buf, inbuf_size, 0, + inbuf_size); + if (ret != 0) { + ret = -EINVAL; + goto inbuf_err; + } + } + + if (cqm_cmd_outbuf != NULL) { + *cqm_cmd_outbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_outbuf == NULL) { + ret = -ENOMEM; + goto outbuf_err; + } + (*cqm_cmd_outbuf)->size = outbuf_size; + + ret = memset_s((*cqm_cmd_outbuf)->buf, outbuf_size, 0, + outbuf_size); + if (ret != 0) { + ret = -EINVAL; + goto outbuf_err; + } + } + + return 0; +outbuf_err: + if (cqm_cmd_outbuf != NULL) { + roce5_cqm5_cmd_free_inoutbuf(ex_handle, NULL, *cqm_cmd_outbuf); + *cqm_cmd_outbuf = NULL; + } +inbuf_err: + if (cqm_cmd_inbuf != NULL) { + roce5_cqm5_cmd_free_inoutbuf(ex_handle, *cqm_cmd_inbuf, NULL); + *cqm_cmd_inbuf = NULL; + } + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h new file mode 100644 index 000000000..0130fa320 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h @@ -0,0 +1,21 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQM_CMD_H +#define ROCE_CQM_CMD_H +#ifdef ROCE_KNL_DT +#include <linux/compiler_types.h> +#endif +#include "hinic5_cqm.h" + +#include "roce.h" + +void roce5_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf); +int roce5_cqm_cmd_zalloc_inoutbuf(void *ex_handle, + cqm_cmd_buf_s **cqm_cmd_inbuf, u16 inbuf_size, + cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size); + +#endif //ROCE_CQM_CMD_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c new file mode 100644 index 000000000..3000c6b97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c @@ -0,0 +1,240 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + +#include "roce.h" +#include "roce_sysfs.h" +#include "roce_cc_format.h" + +#include "roce_cmd.h" +#include "roce_multipath_sysfs.h" + +#define MAX_STRING_FORMAT_LEN 64 +#define to_roce5_ecn_multipath_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_multipath_ctx, ecn_multipath_root) + +static void roce_ecn_multipath_sysfs_release(struct kobject *kobj) +{ +} + +static ssize_t roce5_show_slow_path_psn_threshold(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + (u32)ecn_multipath_ctx->slow_path_psn_threshold); +} + +static ssize_t roce5_store_slow_path_psn_threshold(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, + size_t count) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + u32 old_slow_path_psn_threshold = 0; + int slow_path_psn_threshold = 0; + int ret; + char remaining; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + ret = sscanf_s(buf, "%d %c", &slow_path_psn_threshold, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, slow_path_psn_threshold should be unsigned int\n"); + return -EIO; + } + + if (slow_path_psn_threshold > + ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX) { + ROCE_ERR( + "Invalid input, slow_path_psn_threshold(%d) over max slow_path_psn_threshold(%u), slow_path_psn_threshold", + slow_path_psn_threshold, + (u32)ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX); + return -EIO; + } + + if (ecn_multipath_ctx->slow_path_psn_threshold != + (u32)slow_path_psn_threshold) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_slow_path_psn_threshold = + ecn_multipath_ctx->slow_path_psn_threshold; + ecn_multipath_ctx->slow_path_psn_threshold = + (u32)slow_path_psn_threshold; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update slow_path_psn_threshold(%d), ret(%d).", + slow_path_psn_threshold, ret); + ecn_multipath_ctx->slow_path_psn_threshold = + old_slow_path_psn_threshold; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce5_show_rtt_req_event_mode(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_multipath_ctx->rtt_req_event_mode); +} + +static ssize_t roce5_store_rtt_req_event_mode(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 old_rtt_req_event_mode = 0; + u32 rtt_req_event_mode = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + char remaining; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + ret = sscanf_s(buf, "%u %c", &rtt_req_event_mode, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_req_event_mode should be unsigned int ret(%d), rtt_req_event_mode(%u)\n", + ret, rtt_req_event_mode); + return -EIO; + } + + if ((rtt_req_event_mode & ROCE_MULTIPATH_RTT_EN_MASK) != 0) { + ROCE_ERR( + "Invalid input, rtt_req_event_mode (%u)only 0 and 1 supported\n", + rtt_req_event_mode); + return -EIO; + } + + if (ecn_multipath_ctx->rtt_req_event_mode != (u32)rtt_req_event_mode) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rtt_req_event_mode = ecn_multipath_ctx->rtt_req_event_mode; + ecn_multipath_ctx->rtt_req_event_mode = (u32)rtt_req_event_mode; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update rtt_req_event_mode(%d), ret(%d).", + rtt_req_event_mode, ret); + ecn_multipath_ctx->rtt_req_event_mode = + old_rtt_req_event_mode; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_ATTR_RW(slow_path_psn_threshold, roce5_show_slow_path_psn_threshold, + roce5_store_slow_path_psn_threshold); +ROCE_ATTR_RW(rtt_req_event_mode, roce5_show_rtt_req_event_mode, + roce5_store_rtt_req_event_mode); + +static struct attribute *ecn_multipath_ctx_attrs[] = { + ROCE_ATTR_PTR(rtt_req_event_mode), + ROCE_ATTR_PTR(slow_path_psn_threshold), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_multipath_ctx_attr_grp = { + .attrs = ecn_multipath_ctx_attrs, +}; + +static const struct attribute_group *ecn_multipath_ctx_groups[] = { + &ecn_multipath_ctx_attr_grp, + NULL, +}; +#endif + +static struct kobj_type roce_ecn_multipath_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_multipath_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_multipath_ctx_groups, +#else + .default_attrs = ecn_multipath_ctx_attrs, +#endif +}; + +int roce5_init_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx) +{ + int ret = 0; + + if (rdev->device_type != ROCE_DEV_TYPE_HI1825_V100) { + return 0; + } + if (ecn_multipath_ctx == NULL) { + pr_err_ratelimited( + "[ROCE, ERR] %s: ecn_multipath_ctx is null\n", + __func__); + return -EINVAL; + } + + ecn_multipath_ctx->rtt_req_event_mode = + ROCE_MULTIPATH_DEFAULT_RTT_EVENT_MODE; + ecn_multipath_ctx->slow_path_psn_threshold = + ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD; + + ret = kobject_init_and_add(&ecn_multipath_ctx->ecn_multipath_root, + &roce_ecn_multipath_ktype, kobj, + "roce5_multipath"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_ultipath.(ret:%d)\n", + ret); + kobject_put(&ecn_multipath_ctx->ecn_multipath_root); + return ret; + } + + return 0; +} + +void roce5_remove_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx) +{ + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + if (ecn_multipath_ctx == NULL) { + ROCE_ERR("ecn_multipath_ctx is null\n"); + return; + } + + kobject_put(&ecn_multipath_ctx->ecn_multipath_root); + } + + return; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h new file mode 100644 index 000000000..2a26de8e2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifndef ROCE_MULTIPATH_SYSFS_H +#define ROCE_MULTIPATH_SYSFS_H + +#include <linux/types.h> +#include <linux/stat.h> +#include <linux/kobject.h> + +struct roce5_device; + +#define ROCE_MULTIPATH_DEFAULT_RTT_EVENT_MODE 0 +#define ROCE_MULTIPATH_RTT_EN_MASK 0xfffffffe +#define ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD 1 +#define ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX 255 + +struct roce5_ecn_multipath_ctx { + struct kobject ecn_multipath_root; + u32 slow_path_psn_threshold; + u32 rtt_req_event_mode; +}; +int roce5_init_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx); +void roce5_remove_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx); +#endif /* ROCE_MULTIPATH_SYSFS_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c new file mode 100644 index 000000000..869a97571 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c @@ -0,0 +1,2157 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/stat.h> + +#include "roce.h" +#include "ccp_algo_format.h" +#include "hinic5_hw.h" +#include "roce_cc_format.h" +#include "roce_cmd.h" +#include "roce_com.h" +#include "roce_dfx.h" +#include "roce_infra.h" +#include "roce_kernel_compat_gen.h" +#include "roce_mix.h" +#include "roce_mpu_cmd.h" +#include "roce_multipath_sysfs.h" +#include "roce_npu_cmd_srq_defs.h" +#include "roce_qp.h" +#include "roce_ucc_sysfs.h" +#include "securec.h" +#include "roce_sysfs.h" + +#define MAX_STRING_FORMAT_LEN 64 + +/* generic macro template: generate show/store function, used for rp_ctx parameters (with range check) */ +#define ROCE5_PARAM_SHOW_STORE_RP(_name, _member, _min_macro, _max_macro) \ + static ssize_t roce5_show_##_name( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct roce5_ecn_rp_ctx *ecn_rp_ctx = \ + to_roce5_ecn_rp_ctx(kobj); \ + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \ + (int)ecn_rp_ctx->_member); \ + } \ + static ssize_t roce5_store_##_name(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + int ret, _val; \ + u32 _old; \ + struct roce5_ecn_ctx *ecn_ctx = \ + to_roce5_ecn_ctx(kobj->parent); \ + struct roce5_ecn_rp_ctx *ecn_rp_ctx = \ + to_roce5_ecn_rp_ctx(kobj); \ + struct roce5_device *rdev = \ + container_of(ecn_ctx, struct roce5_device, ecn_ctx); \ + ret = sscanf_s(buf, "%d", &_val); \ + if (ret != 1) \ + return -EIO; \ + if (ROCE5_PARAM_CHECK_BOUND( \ + _val, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._min_macro, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._max_macro)) \ + return -EIO; \ + if (ecn_rp_ctx->_member != (u32)_val) { \ + mutex_lock(&ecn_ctx->ecn_mutex); \ + _old = ecn_rp_ctx->_member; \ + ecn_rp_ctx->_member = (u32)_val; \ + ret = roce5_update_ecn_param(ecn_ctx); \ + if (ret != 0) { \ + ecn_rp_ctx->_member = _old; \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + return -EIO; \ + } \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + } \ + return (ssize_t)count; \ + } + +/* generic macro template: generate show/store function, used for np_ctx parameters (with range check) */ +#define ROCE5_PARAM_SHOW_STORE_NP(_name, _member, _min_macro, _max_macro) \ + static ssize_t roce5_show_##_name( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct roce5_ecn_np_ctx *ecn_np_ctx = \ + to_roce5_ecn_np_ctx(kobj); \ + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \ + (int)ecn_np_ctx->_member); \ + } \ + static ssize_t roce5_store_##_name(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + int ret, _val; \ + u32 _old; \ + struct roce5_ecn_ctx *ecn_ctx = \ + to_roce5_ecn_ctx(kobj->parent); \ + struct roce5_ecn_np_ctx *ecn_np_ctx = \ + to_roce5_ecn_np_ctx(kobj); \ + struct roce5_device *rdev = \ + container_of(ecn_ctx, struct roce5_device, ecn_ctx); \ + ret = sscanf_s(buf, "%d", &_val); \ + if (ret != 1) \ + return -EIO; \ + if (ROCE5_PARAM_CHECK_BOUND( \ + _val, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._min_macro, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._max_macro)) \ + return -EIO; \ + if (ecn_np_ctx->_member != (u32)_val) { \ + mutex_lock(&ecn_ctx->ecn_mutex); \ + _old = ecn_np_ctx->_member; \ + ecn_np_ctx->_member = (u32)_val; \ + ret = roce5_update_ecn_param(ecn_ctx); \ + if (ret != 0) { \ + ecn_np_ctx->_member = _old; \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + return -EIO; \ + } \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + } \ + return (ssize_t)count; \ + } + +/* generic macro template: generate show/store function, used for ldcp_ctx parameters (with range check) */ +#define ROCE5_PARAM_SHOW_STORE_LDCP(_name, _member, _min_macro, _max_macro) \ + static ssize_t roce5_show_##_name( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct roce5_ecn_ldcp_ctx *ecn_ldcp_ctx = \ + to_roce5_ecn_ldcp_ctx(kobj); \ + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \ + (int)ecn_ldcp_ctx->_member); \ + } \ + static ssize_t roce5_store_##_name(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + int ret, _val; \ + u32 _old; \ + struct roce5_ecn_ctx *ecn_ctx = \ + to_roce5_ecn_ctx(kobj->parent); \ + struct roce5_ecn_ldcp_ctx *ecn_ldcp_ctx = \ + to_roce5_ecn_ldcp_ctx(kobj); \ + struct roce5_device *rdev = \ + container_of(ecn_ctx, struct roce5_device, ecn_ctx); \ + ret = sscanf_s(buf, "%d", &_val); \ + if (ret != 1) { \ + ROCE_ERR("Failed to get " #_member ", ret(%d).", ret); \ + return -EIO; \ + } \ + if (ROCE5_PARAM_CHECK_BOUND( \ + _val, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._min_macro, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._max_macro)) { \ + ROCE_ERR(#_member "(%u) param is out bound.", _val); \ + return -EIO; \ + } \ + if (ecn_ldcp_ctx->_member != (u32)_val) { \ + mutex_lock(&ecn_ctx->ecn_mutex); \ + _old = ecn_ldcp_ctx->_member; \ + ecn_ldcp_ctx->_member = (u32)_val; \ + ret = roce5_update_ecn_param(ecn_ctx); \ + if (ret != 0) { \ + ecn_ldcp_ctx->_member = _old; \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + return -EIO; \ + } \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + } \ + return (ssize_t)count; \ + } + +const struct roce_cc_ecn_param g_roce_cc_ecn_param[] = { + { 0 }, // invalid + { .default_ip_enable_en = 1, + .default_bypass = 0, + .default_min_cnp_period = 16, + .default_alpha_dec_period = 160, + .default_rate_dec_period = 32, + .default_rate_inc_period = 480, + .default_factor_gita = 7, + .default_initial_alpha = 1023, + .default_rate_inc_ai = 2, + .default_rate_inc_hai = 8, + .default_rate_first_set = 1024, + .default_rate_target_clamp = 1, + .default_cnp_prio_enable = 0, + .default_cnp_prio = 0, + .default_token_period = 16, + .default_min_rate = 1, + .default_ecn_cnp_enable = 1, + .default_flow_ctrl_unit = 0, + .default_rtt_rsp_prio = 0, + .default_rtt_rsp_prio_enable = 0, + .default_rtt_version = 0, + .max_rtt_version = 0, + .min_bypass = 0, + .max_bypass = 0, + .min_rate_inc_hai = 1, + .max_rate_inc_hai = 255, + .min_rate_inc_ai = 1, + .max_rate_inc_ai = 63, + .min_initial_alpha = 1, + .max_initial_alpha = 1023, + .min_rate_inc_period = 1, + .max_rate_inc_period = 1023, + .min_alpha_dec_period = 1, + .max_alpha_dec_period = 1023, + .min_token_period = 4, + .max_token_period = 255, + .min_flow_min_rate = 1, + .max_flow_min_rate = 63, + .min_rate_dec_period = 1, + .max_rate_dec_period = 255, + .min_min_cnp_period = 1, + .max_min_cnp_period = 255, + .min_factor_gita = 1, + .max_factor_gita = 15, + .min_rate_first_set = 128, + .max_rate_first_set = 8191, + .default_ldcp_wnd_min = 5, + .default_ldcp_wnd_default = 1024, + .default_ldcp_alpha = 2, + .default_ldcp_beta = 12, + .default_ldcp_gamma = 9, + .default_ldcp_eta = 3, + .min_ldcp_wnd_min = 1, + .max_ldcp_wnd_min = 0x8, + .min_ldcp_wnd_default = 1024, + .max_ldcp_wnd_default = 32768, + .min_ldcp_alpha = 0, + .max_ldcp_alpha = 0xf, + .min_ldcp_beta = 0, + .max_ldcp_beta = 0xf, + .min_ldcp_gamma = 0, + .max_ldcp_gamma = 0xf, + .min_ldcp_eta = 0, + .max_ldcp_eta = 0xf }, // v1 + { .default_ip_enable_en = 1, + .default_bypass = 0, + .default_min_cnp_period = 16, + .default_alpha_dec_period = 160, + .default_rate_dec_period = 48, + .default_rate_inc_period = 480, + .default_factor_gita = 7, + .default_initial_alpha = 1023, + .default_rate_inc_ai = 2, + .default_rate_inc_hai = 12, + .default_rate_first_set = 1024, + .default_rate_target_clamp = 1, + .default_cnp_prio_enable = 0, + .default_cnp_prio = 0, + .default_token_period = 16, + .default_min_rate = 1, + .default_ecn_cnp_enable = 1, + .default_flow_ctrl_unit = 0, + .default_rtt_rsp_prio = 0, + .default_rtt_rsp_prio_enable = 0, + .default_rtt_version = 0, + .max_rtt_version = 0, + .min_bypass = 0, + .max_bypass = 2, + .min_rate_inc_hai = 1, + .max_rate_inc_hai = 255, + .min_rate_inc_ai = 1, + .max_rate_inc_ai = 255, + .min_initial_alpha = 1, + .max_initial_alpha = 1023, + .min_rate_inc_period = 1, + .max_rate_inc_period = 1023, + .min_alpha_dec_period = 1, + .max_alpha_dec_period = 1023, + .min_token_period = 4, + .max_token_period = 255, + .min_flow_min_rate = 1, + .max_flow_min_rate = 4095, + .min_rate_dec_period = 1, + .max_rate_dec_period = 255, + .min_min_cnp_period = 1, + .max_min_cnp_period = 255, + .min_factor_gita = 1, + .max_factor_gita = 15, + .min_rate_first_set = 128, + .max_rate_first_set = 50000, + .default_ldcp_wnd_min = 5, + .default_ldcp_wnd_default = 1024, + .default_ldcp_alpha = 2, + .default_ldcp_beta = 9, + .default_ldcp_gamma = 6, + .default_ldcp_eta = 6, + .min_ldcp_wnd_min = 1, + .max_ldcp_wnd_min = 0x8, + .min_ldcp_wnd_default = 1024, + .max_ldcp_wnd_default = 32768, + .min_ldcp_alpha = 0, + .max_ldcp_alpha = 0xf, + .min_ldcp_beta = 0, + .max_ldcp_beta = 0xf, + .min_ldcp_gamma = 0, + .max_ldcp_gamma = 0xf, + .min_ldcp_eta = 0, + .max_ldcp_eta = 0xf }, // v2 +}; + +static int roce5_update_ver_check(struct roce5_device *rdev, u32 cc_algo) +{ + u32 algo_bitmap = 0; + /* driver intercept not support algorithm. 1823V200 only supports LDCP algorithm, 1825V100 current only supports DCQCN, IPQCN, user custom A & B algorithm */ + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + algo_bitmap = BIT(ROCE_CC_DISABLE); + if (ROCE5_SUPPORT_CC_GET_DCQCN_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_DCQCN_ALGO); + } + if (ROCE5_SUPPORT_CC_GET_LDCP_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_LDCP_ALGO); + } + if (ROCE5_SUPPORT_CC_GET_IPQCN_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_IPQCN_ALGO); + } + if (ROCE5_SUPPORT_CC_GET_USER_ALGO_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_USER_A_ALGO); + algo_bitmap |= BIT(ROCE_CC_USER_B_ALGO); + } + } + + if (ROCE_CC_ALGO_TEST_BIT(algo_bitmap, cc_algo) == 0) { + ROCE_DEV_ERR(rdev, "func_id(0x%x), cc algo(0x%x) not support!", + rdev->glb_func_id, cc_algo); + return -EINVAL; + } + return 0; +} + +static int roce5_update_cfg_cc_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_cc_param_s cc_param; + int ret; + + ret = roce5_update_ver_check(rdev, ecn_ctx->cc_algo); + if (ret != 0) { + return ret; + } + + (void)memset_s(&cc_param, sizeof(cc_param), 0, sizeof(cc_param)); + + cc_param.dw0.bs.bypass = ecn_ctx->rp_ctx.bypass; + cc_param.dw0.bs.rtt_rsp_prio = ecn_ctx->np_ctx.rtt_rsp_prio; + cc_param.dw0.bs.rtt_rsp_prio_enable = + ecn_ctx->np_ctx.rtt_rsp_prio_enable; + cc_param.dw0.bs.rtt_version = ecn_ctx->np_ctx.rtt_version; + cc_param.dw0.bs.ecn_cnp_en = ecn_ctx->np_ctx.ecn_cnp_en; + cc_param.dw0.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + cc_param.dw0.bs.cnp_prio_enable = ecn_ctx->np_ctx.cnp_prio_enable; + cc_param.dw0.bs.slow_path_psn_threshold = + ecn_ctx->multipath_ctx.slow_path_psn_threshold; + cc_param.dw0.bs.rtt_req_event_mode = + ecn_ctx->multipath_ctx.rtt_req_event_mode; + cc_param.dw1.bs.cnp_prio = ecn_ctx->np_ctx.cnp_prio; + cc_param.dw1.bs.cnp_cos = + roce5_get_db_cos_from_vlan_pri(rdev, cc_param.dw1.bs.cnp_prio); + cc_param.dw1.bs.cc_appid = ecn_ctx->cc_algo; + + if (cc_param.dw1.bs.cc_appid == ROCE_CC_LDCP_ALGO) { + cc_param.dw0.bs.algo_mode = CCP_ALGO_MODE_CWND; + } else if (cc_param.dw1.bs.cc_appid == ROCE_CC_DCQCN_ALGO || + cc_param.dw1.bs.cc_appid == ROCE_CC_IPQCN_ALGO) { + cc_param.dw0.bs.algo_mode = CCP_ALGO_MODE_RATE; + } else { + cc_param.dw0.bs.algo_mode = ecn_ctx->ucc_ctx.algo_mode; + } + + if (cc_param.dw0.bs.algo_mode == CCP_ALGO_MODE_CWND) { + cc_param.dw0.bs.flow_ctrl_unit = FLOW_CTRL_UNIT_QP; + } else { + cc_param.dw0.bs.flow_ctrl_unit = ecn_ctx->np_ctx.flow_ctrl_unit; + } + + ROCE_DEV_INFO(rdev, "set cc_appid(%u), func_id(%u).", + cc_param.dw1.bs.cc_appid, rdev->glb_func_id); + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)&cc_param, + sizeof(roce_cc_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_CC_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set cc param, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce5_update_dcqcn_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + ccp_dcqcn_para_s dcqcn_param; + int ret; + + if (ecn_ctx->cc_algo != ROCE_CC_DCQCN_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not dcqcn, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + + (void)memset_s(&dcqcn_param, sizeof(dcqcn_param), 0, + sizeof(dcqcn_param)); + + dcqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + dcqcn_param.dw0.bs.min_rate = ecn_ctx->rp_ctx.min_rate; + dcqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + dcqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + dcqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + dcqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + dcqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + dcqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + dcqcn_param.dw3.bs.gita_shift = ecn_ctx->rp_ctx.factor_gita; + dcqcn_param.dw3.bs.rate_target_clamp = + ecn_ctx->rp_ctx.rate_target_clamp; + dcqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + // adjust_en + dcqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&dcqcn_param, + sizeof(ccp_dcqcn_para_s), + (u16)ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set dcqcn param, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce5_update_ldcp_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_ldcp_param_s ldcp_param; + int ret; + + if (ecn_ctx->cc_algo != ROCE_CC_LDCP_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not ldcp, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + + ROCE_DEV_INFO( + rdev, + "update ldcp param, func_id(%d), alpha(%u), beta(%u), gamma(%u), eta(%u) wnd_min(%u) wnd_default(%u)", + rdev->glb_func_id, ecn_ctx->ldcp_ctx.alpha, + ecn_ctx->ldcp_ctx.beta, ecn_ctx->ldcp_ctx.gamma, + ecn_ctx->ldcp_ctx.eta, ecn_ctx->ldcp_ctx.wnd_min, + ecn_ctx->ldcp_ctx.wnd_default); + + (void)memset_s(&ldcp_param, sizeof(ldcp_param), 0, sizeof(ldcp_param)); + + ldcp_param.dw0.bs.alpha = ecn_ctx->ldcp_ctx.alpha; + ldcp_param.dw0.bs.beta = ecn_ctx->ldcp_ctx.beta; + ldcp_param.dw0.bs.gamma = ecn_ctx->ldcp_ctx.gamma; + ldcp_param.dw0.bs.eta = ecn_ctx->ldcp_ctx.eta; + ldcp_param.dw0.bs.wnd_min = ecn_ctx->ldcp_ctx.wnd_min; + ldcp_param.dw1.bs.wnd_default = ecn_ctx->ldcp_ctx.wnd_default; + ldcp_param.dw0.bs.set_flag = 1; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ldcp_param, + sizeof(roce_ldcp_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_LDCP_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ldcp param, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce5_update_ipqcn_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_ipqcn_param_s ipqcn_param; + int ret; + + if (ecn_ctx->cc_algo != ROCE_CC_IPQCN_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not ipqcn, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + + (void)memset_s(&ipqcn_param, sizeof(ipqcn_param), 0, + sizeof(ipqcn_param)); + + ipqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + ipqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate; + ipqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + ipqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + ipqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + ipqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + ipqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + ipqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + ipqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita; + ipqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp; + ipqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + ipqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ipqcn_param, + sizeof(roce_ipqcn_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ipqcn param, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +typedef int (*roce5_update_param_t)(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx); + +static roce5_update_param_t roce5_update_param_funcs[] = { + roce5_update_cfg_cc_param, roce5_update_dcqcn_param, + roce5_update_ipqcn_param, roce5_update_ldcp_param, + roce5_update_ucc_param, +}; + +int roce5_update_ecn_param(const struct roce5_ecn_ctx *ecn_ctx) +{ + unsigned int i; + int ret = 0; + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + /* current only supports LDCP algorithm, its remaining algorithm can only modify its algorithm parameters, cannot switch */ + for (i = 0; i < (sizeof(roce5_update_param_funcs) / + sizeof(roce5_update_param_t)); + i++) { + ret = roce5_update_param_funcs[i](rdev, ecn_ctx); + if (ret != 0) { + ret = -EINVAL; + break; + } + } + + return ret; +} + +static ssize_t roce5_show_cc_algo(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + switch ((int)ecn_ctx->cc_algo) { + case ROCE_CC_DISABLE: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "disable"); + case ROCE_CC_DCQCN_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "dcqcn"); + case ROCE_CC_LDCP_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "ldcp"); + case ROCE_CC_IPQCN_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "hc3_1/ipqcn"); + case ROCE_CC_USER_A_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "user_cc_6"); + case ROCE_CC_USER_B_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "user_cc_7"); + default: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "error_type"); + } +} + +static ssize_t roce5_store_cc_algo(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + u32 old_cc_algo = 0; + int cc_algo = 0; + const char *ptr = NULL; + int ret; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + /* dcqcn -> 0x0, ldcp -> 0x2, hc3_1 -> 0x80 */ + if (strncmp(buf, "disable", strlen("disable")) == 0) { + cc_algo = ROCE_CC_DISABLE; + } else if (strncmp(buf, "dcqcn", strlen("dcqcn")) == 0) { + cc_algo = ROCE_CC_DCQCN_ALGO; + } else if ((strncmp(buf, "ipqcn", strlen("ipqcn")) == 0) || + (strncmp(buf, "hc3_1", strlen("hc3_1")) == 0)) { + cc_algo = ROCE_CC_IPQCN_ALGO; + } else if (strncmp(buf, "ldcp", strlen("ldcp")) == 0) { + cc_algo = ROCE_CC_LDCP_ALGO; + } else if (strncmp(buf, "user_cc_", strlen("user_cc_")) == 0) { + if (strlen(buf) > ROCE_USER_CC_ID_MAX_LEN) { + ROCE_ERR( + "Invalid buffer length, expected <= %d, got %zu, Buffer: '%s'", + ROCE_USER_CC_ID_MAX_LEN, strlen(buf), buf); + return -EINVAL; + } + ptr = buf + strlen("user_cc_"); + cc_algo = (unsigned char)(*ptr) - CHAR_0; + if (cc_algo != ROCE_CC_USER_A_ALGO && + cc_algo != ROCE_CC_USER_B_ALGO) { + ROCE_ERR("Invalid cc_algo(%d),buf(%s)", cc_algo, buf); + return -EINVAL; + } + } else { + ROCE_ERR("Invalid cc_algo(%d),buf(%s)", cc_algo, buf); + return -EIO; + } + + if (ecn_ctx->cc_algo != (u32)cc_algo) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cc_algo = ecn_ctx->cc_algo; + ecn_ctx->cc_algo = (u32)cc_algo; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Failed to update cc_algo(%d), ret(%d).", + cc_algo, ret); + ecn_ctx->cc_algo = old_cc_algo; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce5_show_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce5_prio_enable_ctx, enable); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)prio_enable_ctx->prio_en); +} + +static ssize_t roce5_store_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce5_ecn_rp_ctx *rp_ctx = NULL; + struct roce5_ecn_np_ctx *np_ctx = NULL; + struct roce5_ecn_ctx *ecn_ctx = NULL; + struct roce5_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce5_prio_enable_ctx, enable); + struct roce5_ecn_enable_ctx *ecn_enable_ctx = + (struct roce5_ecn_enable_ctx *)prio_enable_ctx->ecn_enable_ctx; + + if (ecn_enable_ctx->np_rp == ROCE_DCQCN_NP) { + np_ctx = container_of(ecn_enable_ctx, struct roce5_ecn_np_ctx, + enable_ctx); + ecn_ctx = container_of(np_ctx, struct roce5_ecn_ctx, np_ctx); + } else { + rp_ctx = container_of(ecn_enable_ctx, struct roce5_ecn_rp_ctx, + enable_ctx); + ecn_ctx = container_of(rp_ctx, struct roce5_ecn_ctx, rp_ctx); + } + + ret = sscanf_s(buf, "%d", &prio_en); + if (ret != 1) { + return -EIO; + } + + if (((u32)prio_en & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = prio_enable_ctx->prio_en; + prio_enable_ctx->prio_en = (u32)prio_en; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update prio_en(%d), ret(%d).", + prio_en, ret); + prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce5_show_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce5_ip_prio_enable_ctx, ip_enable); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ip_prio_enable_ctx->prio_en); +} + +static ssize_t roce5_store_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce5_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce5_ip_prio_enable_ctx, ip_enable); + struct roce5_ecn_ip_enable_ctx *ip_enable_ctx = + (struct roce5_ecn_ip_enable_ctx *) + ip_prio_enable_ctx->ecn_ip_enable_ctx; + struct roce5_ecn_ctx *ecn_ctx = container_of( + ip_enable_ctx, struct roce5_ecn_ctx, ip_enable_ctx); + + ret = sscanf_s(buf, "%d", &prio_en); + if (ret != 1) { + return -EIO; + } + + if (((u32)prio_en & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ip_prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = ip_prio_enable_ctx->prio_en; + ip_prio_enable_ctx->prio_en = (u32)prio_en; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update prio_en(%d), ret(%d).", + prio_en, ret); + ip_prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* rp_ctx parameters use macro generate show/store function */ +ROCE5_PARAM_SHOW_STORE_RP(bypass, bypass, min_bypass, max_bypass) +ROCE5_PARAM_SHOW_STORE_RP(rate_inc_hai, rate_inc_hai, min_rate_inc_hai, + max_rate_inc_hai) +ROCE5_PARAM_SHOW_STORE_RP(rate_inc_ai, rate_inc_ai, min_rate_inc_ai, + max_rate_inc_ai) +ROCE5_PARAM_SHOW_STORE_RP(initial_alpha, initial_alpha, min_initial_alpha, + max_initial_alpha) +ROCE5_PARAM_SHOW_STORE_RP(factor_gita, factor_gita, min_factor_gita, + max_factor_gita) +ROCE5_PARAM_SHOW_STORE_RP(rate_inc_period, rate_inc_period, min_rate_inc_period, + max_rate_inc_period) +ROCE5_PARAM_SHOW_STORE_RP(rate_dec_period, rate_dec_period, min_rate_dec_period, + max_rate_dec_period) +ROCE5_PARAM_SHOW_STORE_RP(token_period, token_period, min_token_period, + max_token_period) +ROCE5_PARAM_SHOW_STORE_RP(min_rate, min_rate, min_flow_min_rate, + max_flow_min_rate) +ROCE5_PARAM_SHOW_STORE_RP(alpha_dec_period, alpha_dec_period, + min_alpha_dec_period, max_alpha_dec_period) +ROCE5_PARAM_SHOW_STORE_RP(rate_first_set, rate_first_set, min_rate_first_set, + max_rate_first_set) + +/* rate_target_clamp special handle: use ROCE_ENABLE_CHECK_MASK but non- range check */ +static ssize_t roce5_show_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_rp_ctx *ecn_rp_ctx = to_roce5_ecn_rp_ctx(kobj); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_rp_ctx->rate_target_clamp); +} + +static ssize_t roce5_store_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int rate_target_clamp = 0; + u32 old_rate_target_clamp = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_rp_ctx *ecn_rp_ctx = to_roce5_ecn_rp_ctx(kobj); + + ret = sscanf_s(buf, "%d", &rate_target_clamp); + if (ret != 1) { + return -EIO; + } + + if (((u32)rate_target_clamp & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_rp_ctx->rate_target_clamp != (u32)rate_target_clamp) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_target_clamp = ecn_rp_ctx->rate_target_clamp; + ecn_rp_ctx->rate_target_clamp = (u32)rate_target_clamp; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update rate_target_clamp(%d), ret(%d).", + rate_target_clamp, ret); + ecn_rp_ctx->rate_target_clamp = old_rate_target_clamp; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* np_ctx parameters use macro generate show/store function */ +ROCE5_PARAM_SHOW_STORE_NP(min_cnp_period, min_cnp_period, min_min_cnp_period, + max_min_cnp_period) + +/* flow_ctrl_unit special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_flow_ctrl_unit(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->flow_ctrl_unit); +} +static ssize_t roce5_store_flow_ctrl_unit(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret, _val; + u32 _old; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + char remaining; + + ret = sscanf_s(buf, "%d %c", &_val, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, flow_ctrl_unit should be unsigned int, ret(%d), flow_ctrl_unit(%d)", + ret, _val); + return -EIO; + } + + if (((u32)_val & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->flow_ctrl_unit != (u32)_val) { + mutex_lock(&ecn_ctx->ecn_mutex); + _old = ecn_np_ctx->flow_ctrl_unit; + ecn_np_ctx->flow_ctrl_unit = (u32)_val; + ret = roce5_update_ecn_param(ecn_ctx); + + if (ret != 0) { + ecn_np_ctx->flow_ctrl_unit = _old; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + return (ssize_t)count; +} + +/* rtt_version special handle: only check maximum value */ +static ssize_t roce5_show_rtt_version(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->rtt_version); +} +static ssize_t roce5_store_rtt_version(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 _val, _old; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + char remaining; + + ret = sscanf_s(buf, "%u %c", &_val, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_version should be unsigned int, ret(%d), rtt_version(%u)", + ret, _val); + return -EIO; + } + + if (_val > + g_roce_cc_ecn_param[rdev->cc_param_version].max_rtt_version) { + ROCE_ERR("Invalid input, rtt_version(%u) over max rtt version", + _val); + return -EIO; + } + + if (ecn_np_ctx->rtt_version != _val) { + mutex_lock(&ecn_ctx->ecn_mutex); + _old = ecn_np_ctx->rtt_version; + ecn_np_ctx->rtt_version = _val; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->rtt_version = _old; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + return (ssize_t)count; +} + +/* rtt_rsp_prio special handle: use PRI_ARRAY_LEN - 1 check */ +static ssize_t roce5_show_rtt_rsp_prio(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + ecn_np_ctx->rtt_rsp_prio); +} + +static ssize_t roce5_store_rtt_rsp_prio(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + u32 rtt_rsp_prio = 0; + u32 old_rtt_rsp_prio = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + char remaining; + ret = sscanf_s(buf, "%u %c", &rtt_rsp_prio, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_rsp_prio should be unsigned int, ret(%d), rtt_rsp_prio(%u)\n", + ret, rtt_rsp_prio); + return -EIO; + } + + if (rtt_rsp_prio > (PRI_ARRAY_LEN - 1)) { + return -EIO; + } + + if (ecn_np_ctx->rtt_rsp_prio != (u32)rtt_rsp_prio) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rtt_rsp_prio = ecn_np_ctx->rtt_rsp_prio; + ecn_np_ctx->rtt_rsp_prio = (u32)rtt_rsp_prio; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update rtt_rsp_prio(%d), ret(%d).", + rtt_rsp_prio, ret); + ecn_np_ctx->rtt_rsp_prio = old_rtt_rsp_prio; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* rtt_rsp_prio_enable special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_rtt_rsp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + ecn_np_ctx->rtt_rsp_prio_enable); +} + +static ssize_t roce5_store_rtt_rsp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + u32 rtt_rsp_prio_enable = 0; + u32 old_rtt_rsp_prio_enable = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + char remaining; + ret = sscanf_s(buf, "%u %c", &rtt_rsp_prio_enable, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_rsp_prio_enable should be unsigned int, ret(%d), rtt_rsp_prio_enable(%u)", + ret, rtt_rsp_prio_enable); + return -EIO; + } + + if ((rtt_rsp_prio_enable & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->rtt_rsp_prio_enable != (u32)rtt_rsp_prio_enable) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rtt_rsp_prio_enable = ecn_np_ctx->rtt_rsp_prio_enable; + ecn_np_ctx->rtt_rsp_prio_enable = (u32)rtt_rsp_prio_enable; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update rtt_rsp_prio_enable(%d), ret(%d).", + rtt_rsp_prio_enable, ret); + ecn_np_ctx->rtt_rsp_prio_enable = + old_rtt_rsp_prio_enable; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* ecn_cnp_en special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_ecn_cnp_en(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->ecn_cnp_en); +} + +static ssize_t roce5_store_ecn_cnp_en(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + u32 ecn_cnp_en = 0; + u32 old_ecn_cnp_en = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + char remaining; + ret = sscanf_s(buf, "%u %c", &ecn_cnp_en, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, ecn_cnp_en should be unsigned int, ret(%d), ecn_cnp_en(%u)", + ret, ecn_cnp_en); + return -EIO; + } + + if ((ecn_cnp_en & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->ecn_cnp_en != (u32)ecn_cnp_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ecn_cnp_en = ecn_np_ctx->ecn_cnp_en; + ecn_np_ctx->ecn_cnp_en = (u32)ecn_cnp_en; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update ecn_cnp_en(%d), ret(%d).", + ecn_cnp_en, ret); + ecn_np_ctx->ecn_cnp_en = old_ecn_cnp_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* cnp_prio_enable special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->cnp_prio_enable); +} + +static ssize_t roce5_store_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int cnp_prio_enable = 0; + u32 old_cnp_prio_enable = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + ret = sscanf_s(buf, "%d", &cnp_prio_enable); + if (ret != 1) { + return -EIO; + } + + if (((u32)cnp_prio_enable & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->cnp_prio_enable != (u32)cnp_prio_enable) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio_enable = ecn_np_ctx->cnp_prio_enable; + ecn_np_ctx->cnp_prio_enable = (u32)cnp_prio_enable; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update cnp_prio_enable(%d), ret(%d).", + cnp_prio_enable, ret); + ecn_np_ctx->cnp_prio_enable = old_cnp_prio_enable; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* cnp_prio special handle: use PRI_ARRAY_LEN - 1 check */ +static ssize_t roce5_show_cnp_prio(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->cnp_prio); +} + +static ssize_t roce5_store_cnp_prio(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int cnp_prio = 0; + u32 old_cnp_prio = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + ret = sscanf_s(buf, "%d", &cnp_prio); + if (ret != 1) { + return -EIO; + } + + if (((u32)cnp_prio) > (PRI_ARRAY_LEN - 1)) { + return -EIO; + } + + if (ecn_np_ctx->cnp_prio != (u32)cnp_prio) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio = ecn_np_ctx->cnp_prio; + ecn_np_ctx->cnp_prio = (u32)cnp_prio; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update cnp_prio(%d), ret(%d).", + cnp_prio, ret); + ecn_np_ctx->cnp_prio = old_cnp_prio; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static int +roce5_init_ecn_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_enable_ctx *ecn_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = { "0", "1", "2", "3", "4", "5", "6", "7" }; + struct roce5_prio_enable_ctx *prio_enable = NULL; + + ecn_enable_ctx->enable_root = kobject_create_and_add("enable", kobj); + if (ecn_enable_ctx->enable_root == NULL) { + ROCE_ERR("Failed to create kobject for ecn enable.(ret:%d)", + ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + prio_enable = &ecn_enable_ctx->prio_enable[i]; + prio_enable->ecn_enable_ctx = (void *)ecn_enable_ctx; + prio_enable->prio = (u32)i; + prio_enable->prio_en = 1; + sysfs_attr_init(&prio_enable->enable.attr); + prio_enable->enable.attr.name = prio_arr[i]; + prio_enable->enable.attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + prio_enable->enable.show = roce5_show_ecn_prio_enable; + prio_enable->enable.store = roce5_store_ecn_prio_enable; + ret = sysfs_create_file(ecn_enable_ctx->enable_root, + &prio_enable->enable.attr); + if (ret != 0) { + ROCE_ERR( + "Failed to create file for ecn enable.(ret:%d)", + ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); + + return ret; +} + +static void +roce5_remove_ecn_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_enable_ctx *ecn_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); +} + +static int +roce5_init_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = { "0", "1", "2", "3", "4", "5", "6", "7" }; + struct roce5_ip_prio_enable_ctx *ip_prio_enable = NULL; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + ip_enable_ctx->ip_enable_root = + kobject_create_and_add("ip_enable", kobj); + if (ip_enable_ctx->ip_enable_root == NULL) { + ROCE_ERR("Failed to create kobject for ecn ip enable.(ret:%d)", + ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + ip_prio_enable = &ip_enable_ctx->ip_prio_enable[i]; + ip_prio_enable->ecn_ip_enable_ctx = (void *)ip_enable_ctx; + ip_prio_enable->prio = (u32)i; + ip_prio_enable->prio_en = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_ip_enable_en; + sysfs_attr_init(&ip_prio_enable->ip_enable.attr); + ip_prio_enable->ip_enable.attr.name = prio_arr[i]; + ip_prio_enable->ip_enable.attr.mode = + (S_IWUSR | S_IRUSR | S_IRGRP); + ip_prio_enable->ip_enable.show = roce5_show_ecn_ip_prio_enable; + ip_prio_enable->ip_enable.store = + roce5_store_ecn_ip_prio_enable; + ret = sysfs_create_file(ip_enable_ctx->ip_enable_root, + &ip_prio_enable->ip_enable.attr); + if (ret != 0) { + ROCE_ERR( + "Failed to create file for ecn ip enable.(ret:%d)", + ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file( + ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); + + return ret; +} + +static void +roce5_remove_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file( + ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); +} + +ROCE_ATTR_RW(bypass, roce5_show_bypass, roce5_store_bypass); +ROCE_ATTR_RW(alpha_dec_period, roce5_show_alpha_dec_period, + roce5_store_alpha_dec_period); +ROCE_ATTR_RW(rate_dec_period, roce5_show_rate_dec_period, + roce5_store_rate_dec_period); +ROCE_ATTR_RW(rate_inc_period, roce5_show_rate_inc_period, + roce5_store_rate_inc_period); +ROCE_ATTR_RW(factor_gita, roce5_show_factor_gita, roce5_store_factor_gita); +ROCE_ATTR_RW(initial_alpha, roce5_show_initial_alpha, + roce5_store_initial_alpha); +ROCE_ATTR_RW(rate_inc_ai, roce5_show_rate_inc_ai, roce5_store_rate_inc_ai); +ROCE_ATTR_RW(rate_inc_hai, roce5_show_rate_inc_hai, roce5_store_rate_inc_hai); +ROCE_ATTR_RW(rate_first_set, roce5_show_rate_first_set, + roce5_store_rate_first_set); +ROCE_ATTR_RW(rate_target_clamp, roce5_show_rate_target_clamp, + roce5_store_rate_target_clamp); +ROCE_ATTR_RW(token_period, roce5_show_token_period, roce5_store_token_period); +ROCE_ATTR_RW(min_rate, roce5_show_min_rate, roce5_store_min_rate); + +static struct attribute *ecn_rp_ctx_attrs[] = { + ROCE_ATTR_PTR(bypass), + ROCE_ATTR_PTR(alpha_dec_period), + ROCE_ATTR_PTR(rate_dec_period), + ROCE_ATTR_PTR(rate_inc_period), + ROCE_ATTR_PTR(factor_gita), + ROCE_ATTR_PTR(initial_alpha), + ROCE_ATTR_PTR(rate_inc_ai), + ROCE_ATTR_PTR(rate_inc_hai), + ROCE_ATTR_PTR(rate_first_set), + ROCE_ATTR_PTR(rate_target_clamp), + ROCE_ATTR_PTR(token_period), + ROCE_ATTR_PTR(min_rate), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_rp_ctx_attr_grp = { + .attrs = ecn_rp_ctx_attrs, +}; + +static const struct attribute_group *ecn_rp_ctx_attr_grps[] = { + &ecn_rp_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_rp_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_rp_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_rp_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_rp_ctx_attr_grps, +#else + .default_attrs = ecn_rp_ctx_attrs, +#endif +}; + +static int roce5_init_ecn_rp_sysfs(struct kobject *kobj, + struct roce5_ecn_rp_ctx *ecn_rp_ctx) +{ + int ret = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + ecn_rp_ctx->bypass = + g_roce_cc_ecn_param[rdev->cc_param_version].default_bypass; + ecn_rp_ctx->alpha_dec_period = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_alpha_dec_period; + ecn_rp_ctx->rate_dec_period = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_dec_period; + ecn_rp_ctx->rate_inc_period = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_inc_period; + ecn_rp_ctx->factor_gita = + g_roce_cc_ecn_param[rdev->cc_param_version].default_factor_gita; + ecn_rp_ctx->initial_alpha = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_initial_alpha; + ecn_rp_ctx->rate_inc_ai = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rate_inc_ai; + ecn_rp_ctx->rate_inc_hai = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rate_inc_hai; + ecn_rp_ctx->rate_first_set = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_first_set; + ecn_rp_ctx->rate_target_clamp = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_target_clamp; + ecn_rp_ctx->token_period = + g_roce_cc_ecn_param[rdev->cc_param_version].default_token_period; + ecn_rp_ctx->min_rate = + g_roce_cc_ecn_param[rdev->cc_param_version].default_min_rate; + ret = kobject_init_and_add(&ecn_rp_ctx->ecn_rp_root, &roce_ecn_rp_ktype, + kobj, "roce5_rp"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_rp.(ret:%d)", ret); + kobject_put(&ecn_rp_ctx->ecn_rp_root); + return ret; + } + + ecn_rp_ctx->enable_ctx.np_rp = ROCE_DCQCN_RP; + ret = roce5_init_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, + &ecn_rp_ctx->enable_ctx); + if (ret != 0) { + goto err_init_prio_enable_sysfs; + } + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_rp_ctx->ecn_rp_root); + + return ret; +} + +static void roce5_remove_ecn_rp_sysfs(struct kobject *kobj, + struct roce5_ecn_rp_ctx *ecn_rp_ctx) +{ + roce5_remove_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, + &ecn_rp_ctx->enable_ctx); + + kobject_put(&ecn_rp_ctx->ecn_rp_root); +} + +ROCE_ATTR_RW(rtt_rsp_prio, roce5_show_rtt_rsp_prio, roce5_store_rtt_rsp_prio); +ROCE_ATTR_RW(rtt_rsp_prio_enable, roce5_show_rtt_rsp_prio_enable, + roce5_store_rtt_rsp_prio_enable); +ROCE_ATTR_RW(rtt_version, roce5_show_rtt_version, roce5_store_rtt_version); +ROCE_ATTR_RW(ecn_cnp_en, roce5_show_ecn_cnp_en, roce5_store_ecn_cnp_en); +ROCE_ATTR_RW(flow_ctrl_unit, roce5_show_flow_ctrl_unit, + roce5_store_flow_ctrl_unit); +ROCE_ATTR_RW(min_cnp_period, roce5_show_min_cnp_period, + roce5_store_min_cnp_period); +ROCE_ATTR_RW(cnp_prio_enable, roce5_show_cnp_prio_enable, + roce5_store_cnp_prio_enable); +ROCE_ATTR_RW(cnp_prio, roce5_show_cnp_prio, roce5_store_cnp_prio); + +static struct attribute *ecn_np_ctx_attrs[] = { + ROCE_ATTR_PTR(min_cnp_period), + ROCE_ATTR_PTR(cnp_prio_enable), + ROCE_ATTR_PTR(cnp_prio), + ROCE_ATTR_PTR(rtt_rsp_prio), + ROCE_ATTR_PTR(rtt_rsp_prio_enable), + ROCE_ATTR_PTR(rtt_version), + ROCE_ATTR_PTR(ecn_cnp_en), + ROCE_ATTR_PTR(flow_ctrl_unit), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_np_ctx_attr_grp = { + .attrs = ecn_np_ctx_attrs, +}; + +static const struct attribute_group *ecn_np_ctx_attr_grps[] = { + &ecn_np_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_np_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_np_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_np_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_np_ctx_attr_grps, +#else + .default_attrs = ecn_np_ctx_attrs, +#endif +}; + +static int roce5_init_ecn_np_sysfs(struct kobject *kobj, + struct roce5_ecn_np_ctx *ecn_np_ctx) +{ + int ret = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + ecn_np_ctx->min_cnp_period = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_min_cnp_period; + ecn_np_ctx->cnp_prio_enable = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_cnp_prio_enable; + ecn_np_ctx->cnp_prio = + g_roce_cc_ecn_param[rdev->cc_param_version].default_cnp_prio; + ecn_np_ctx->ecn_cnp_en = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_ecn_cnp_enable; + ecn_np_ctx->flow_ctrl_unit = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_flow_ctrl_unit; + ecn_np_ctx->rtt_rsp_prio = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rtt_rsp_prio; + ecn_np_ctx->rtt_rsp_prio_enable = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rtt_rsp_prio_enable; + ecn_np_ctx->rtt_version = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rtt_version; + ret = kobject_init_and_add(&ecn_np_ctx->ecn_np_root, &roce_ecn_np_ktype, + kobj, "roce5_np"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_np.(ret:%d)", ret); + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; + } + + ecn_np_ctx->enable_ctx.np_rp = ROCE_DCQCN_NP; + ret = roce5_init_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, + &ecn_np_ctx->enable_ctx); + if (ret != 0) { + goto err_init_prio_enable_sysfs; + } + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; +} + +/* ldcp_ctx parameters use macro generate show/store function */ +ROCE5_PARAM_SHOW_STORE_LDCP(wnd_min, wnd_min, min_ldcp_wnd_min, + max_ldcp_wnd_min); +ROCE5_PARAM_SHOW_STORE_LDCP(wnd_default, wnd_default, min_ldcp_wnd_default, + max_ldcp_wnd_default); +ROCE5_PARAM_SHOW_STORE_LDCP(alpha, alpha, min_ldcp_alpha, max_ldcp_alpha); +ROCE5_PARAM_SHOW_STORE_LDCP(beta, beta, min_ldcp_beta, max_ldcp_beta); +ROCE5_PARAM_SHOW_STORE_LDCP(gamma, gamma, min_ldcp_gamma, max_ldcp_gamma); +ROCE5_PARAM_SHOW_STORE_LDCP(eta, eta, min_ldcp_eta, max_ldcp_eta); + +ROCE_ATTR_RW(wnd_min, roce5_show_wnd_min, roce5_store_wnd_min); +ROCE_ATTR_RW(wnd_default, roce5_show_wnd_default, roce5_store_wnd_default); +ROCE_ATTR_RW(alpha, roce5_show_alpha, roce5_store_alpha); +ROCE_ATTR_RW(beta, roce5_show_beta, roce5_store_beta); +ROCE_ATTR_RW(gamma, roce5_show_gamma, roce5_store_gamma); +ROCE_ATTR_RW(eta, roce5_show_eta, roce5_store_eta); + +static struct attribute *ecn_ldcp_ctx_attrs[] = { + ROCE_ATTR_PTR(wnd_min), + ROCE_ATTR_PTR(wnd_default), + ROCE_ATTR_PTR(alpha), + ROCE_ATTR_PTR(beta), + ROCE_ATTR_PTR(gamma), + ROCE_ATTR_PTR(eta), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_ldcp_ctx_attr_grp = { + .attrs = ecn_ldcp_ctx_attrs, +}; + +static const struct attribute_group *ecn_ldcp_ctx_attr_grps[] = { + &ecn_ldcp_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_ldcp_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_ldcp_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_ldcp_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_ldcp_ctx_attr_grps, +#else + .default_attrs = ecn_ldcp_ctx_attrs, +#endif +}; + +static int roce5_init_ecn_ldcp_sysfs(struct kobject *kobj, + roce5_ecn_ldcp_ctx_t *ecn_ldcp_ctx) +{ + int ret; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + ecn_ldcp_ctx->wnd_min = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_wnd_min; + ecn_ldcp_ctx->wnd_default = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_wnd_default; + ecn_ldcp_ctx->alpha = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_alpha; + ecn_ldcp_ctx->beta = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_beta; + ecn_ldcp_ctx->gamma = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_gamma; + ecn_ldcp_ctx->eta = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_eta; + ret = kobject_init_and_add(&ecn_ldcp_ctx->ecn_ldcp_root, + &roce_ecn_ldcp_ktype, kobj, "roce5_ldcp"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_ldcp.(ret:%d)", ret); + kobject_put(&ecn_ldcp_ctx->ecn_ldcp_root); + return ret; + } + + return 0; +} + +static void roce5_remove_ecn_ldcp_sysfs(roce5_ecn_ldcp_ctx_t *ecn_ldcp_ctx) +{ + kobject_put(&ecn_ldcp_ctx->ecn_ldcp_root); +} + +static void roce5_remove_ecn_np_sysfs(struct kobject *kobj, + struct roce5_ecn_np_ctx *ecn_np_ctx) +{ + roce5_remove_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, + &ecn_np_ctx->enable_ctx); + + kobject_put(&ecn_np_ctx->ecn_np_root); +} + +static ssize_t roce5_show_ecn_ver(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_ctx->ecn_ver); +} + +static ssize_t roce5_store_ecn_ver(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret = 0; + int ecn_ver = 0; + u32 old_ecn_ver = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + + ret = sscanf_s(buf, "%d", &ecn_ver); + if (ret != 1) { + return -EIO; + } + + if (ROCE5_PARAM_CHECK_BOUND(ecn_ver, ECN_VER_DCQCN, ECN_VER_PATHQCN)) { + return -EIO; + } + + if (ecn_ctx->ecn_ver != (u32)ecn_ver) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ecn_ver = ecn_ctx->ecn_ver; + ecn_ctx->ecn_ver = (u32)ecn_ver; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update ecn_ver(%d), ret(%d).", + ecn_ver, ret); + ecn_ctx->ecn_ver = old_ecn_ver; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_ATTR_RW(ecn_ver, roce5_show_ecn_ver, roce5_store_ecn_ver); +ROCE_ATTR_RW(cc_algo, roce5_show_cc_algo, roce5_store_cc_algo); + +static struct attribute *ecn_ctx_attrs[] = { + ROCE_ATTR_PTR(ecn_ver), + ROCE_ATTR_PTR(cc_algo), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_ctx_attr_grp = { + .attrs = ecn_ctx_attrs, +}; + +static const struct attribute_group *ecn_ctx_attr_grps[] = { + &ecn_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_ctx_attr_grps, +#else + .default_attrs = ecn_ctx_attrs, +#endif +}; + +static void roce5_init_cc_algo(struct roce5_device *rdev, + struct roce5_ecn_ctx *ecn_ctx) +{ + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ecn_ctx->cc_algo = ROCE_CC_DCQCN_ALGO; + } else { + ecn_ctx->cc_algo = ROCE_CC_DISABLE; + } +} + +static void roce5_init_cc_param_version(struct roce5_device *rdev) +{ + rdev->cc_param_version = + ROCE5_SUPPORT_CC_PARAM_V1(rdev) != 0 ? ROCE_CC_PARAM_V1 : + ROCE5_SUPPORT_CC_PARAM_V2(rdev) != 0 ? ROCE_CC_PARAM_V2 : + ROCE_CC_PARAM_INVALID; +} + +static int roce5_init_ecn_sysfs(struct roce5_device *rdev, + struct roce5_ecn_ctx *ecn_ctx) +{ + int ret = 0; + struct net_device *ndev = rdev->ndev; + + ret = memset_s(ecn_ctx, sizeof(*ecn_ctx), 0, sizeof(*ecn_ctx)); + if (ret != 0) { + ROCE_ERR("Failed to memset ecn_ctx.(ret:%d)", ret); + return -ENOMEM; + } + + mutex_init(&ecn_ctx->ecn_mutex); + + ecn_ctx->ecn_ver = ECN_VER_DCQCN; + roce5_init_cc_algo(rdev, ecn_ctx); + roce5_init_cc_param_version(rdev); + + ret = kobject_init_and_add(&ecn_ctx->ecn_root, &roce_ecn_ktype, + &ndev->dev.kobj, "ecn"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject ecn.(ret:%d)", ret); + kobject_put(&ecn_ctx->ecn_root); + return ret; + } + + ret = roce5_init_ecn_ldcp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ldcp_ctx); + if (ret != 0) { + goto err_init_ldcp_sysfs; + } + + ret = roce5_init_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + if (ret != 0) { + goto err_init_np_sysfs; + } + + ret = roce5_init_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + if (ret != 0) { + goto err_init_rp_sysfs; + } + + ret = roce5_init_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, + &ecn_ctx->ip_enable_ctx); + if (ret != 0) { + goto err_init_ip_enable_sysfs; + } + + ret = roce5_init_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx); + if (ret != 0) { + goto err_init_ecn_ucc_sysfs; + } + + ret = roce5_init_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root, + &ecn_ctx->multipath_ctx); + if (ret != 0) { + goto err_init_multipath_sysfs; + } + + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + goto err_update; + } + + return 0; + +err_update: + roce5_remove_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root, + &ecn_ctx->multipath_ctx); +err_init_multipath_sysfs: + roce5_remove_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx); +err_init_ecn_ucc_sysfs: + roce5_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, + &ecn_ctx->ip_enable_ctx); +err_init_ip_enable_sysfs: + roce5_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); +err_init_rp_sysfs: + roce5_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); +err_init_np_sysfs: + roce5_remove_ecn_ldcp_sysfs(&ecn_ctx->ldcp_ctx); +err_init_ldcp_sysfs: + kobject_put(&ecn_ctx->ecn_root); + return ret; +} + +static void roce5_prase_qpc(roce_verbs_qp_attr_s *qp_attr) +{ + qp_attr->com_info.rsvd = be32_to_cpu(qp_attr->com_info.rsvd); + qp_attr->ext_seg.dfx_info.dw0.value = + be32_to_cpu(qp_attr->ext_seg.dfx_info.dw0.value); + qp_attr->ext_seg.dfx_info.dw2.value = + be32_to_cpu(qp_attr->ext_seg.dfx_info.dw2.value); + qp_attr->ext_seg.dfx_info.dw4.value = + be32_to_cpu(qp_attr->ext_seg.dfx_info.dw4.value); +} + +static ssize_t roce5_store_dfx_qpc(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret; + u32 qpn; + struct roce5_dfx_qpc_ctx *qpc_ctx = NULL; + struct roce5_dfx_ctx *dfx_ctx = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_qp_attr_s qp_attr; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + ROCE_ERR("Invalid input para."); + return -EINVAL; + } + + qpc_ctx = container_of(attr, struct roce5_dfx_qpc_ctx, kattr); + dfx_ctx = container_of(qpc_ctx, struct roce5_dfx_ctx, qpc_ctx); + rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx); + + (void)memset_s(&qp_attr, sizeof(roce_verbs_qp_attr_s), 0, + sizeof(roce_verbs_qp_attr_s)); + + ret = (int)sscanf_s(buf, "%u", &qpn); + if (ret != 1) { + ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if ((qpn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_qps) || + (qpn < ROCE_MIN_QPN)) { + ROCE_DEV_ERR(rdev, "Invalid qpn(%u), max_qps(%u)", qpn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_qps); + return -EIO; + } + + ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)&qp_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query qp_attr, ret(%d)", ret); + return (ssize_t)ret; + } + + roce5_prase_qpc(&qp_attr); + + roce5_dfx_print("qpn(0x%x): sqc pi(0x%x), sqc ci(0x%x).", qpn, + qp_attr.ext_seg.dfx_info.dw0.bs.sq_pi, + qp_attr.ext_seg.dfx_info.dw0.bs.sq_ci); + roce5_dfx_print( + "sqac ci(0x%x), sqac wqe_prefetch_ci(0x%x), rqc pi(0x%x), rqc ci(0x%x).", + qp_attr.ext_seg.dfx_info.dw2.bs.sqa_ci, + qp_attr.ext_seg.dfx_info.dw2.bs.sqa_wqe_prefetch_ci, + qp_attr.ext_seg.dfx_info.dw4.bs.rq_pi, + qp_attr.ext_seg.dfx_info.dw4.bs.rq_ci); + + return (ssize_t)count; +} + +static ssize_t roce5_store_dfx_cqc(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret; + u32 cqn; + struct roce5_dfx_cqc_ctx *cqc_ctx = NULL; + struct roce5_dfx_ctx *dfx_ctx = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_cq_attr_s cq_attr; + roce5_cq_query_outbuf_s cq_query; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + ROCE_ERR("Invalid input para."); + return -EINVAL; + } + + cqc_ctx = container_of(attr, struct roce5_dfx_cqc_ctx, kattr); + dfx_ctx = container_of(cqc_ctx, struct roce5_dfx_ctx, cqc_ctx); + rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx); + + (void)memset_s(&cq_query, sizeof(roce5_cq_query_outbuf_s), 0, + sizeof(roce5_cq_query_outbuf_s)); + + ret = (int)sscanf_s(buf, "%u", &cqn); + if (ret != 1) { + ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (cqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs) { + ROCE_DEV_ERR(rdev, "Invalid cqn(%u), max_cqs(%u)", cqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs); + return -EIO; + } + + ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)&cq_query, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query cqc, ret(%d)", ret); + return (ssize_t)ret; + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + (void)memcpy_s(&cq_attr, sizeof(roce_verbs_cq_attr_s), + &cq_query, sizeof(roce_verbs_cq_attr_s)); + cq_attr.dfx_info.dw0.value = + be32_to_cpu(cq_attr.dfx_info.dw0.value); + cq_attr.dfx_info.dw1.value = + be32_to_cpu(cq_attr.dfx_info.dw1.value); + roce5_dfx_print("cqn(0x%x): pi(0x%x), ci(0x%x).", cqn, + cq_attr.dfx_info.dw1.bs.pi, + cq_attr.dfx_info.dw0.bs.ci); + } else { + ROCE_DEV_ERR(rdev, + "Unsupport rdev device_type, device_type(%d)", + rdev->device_type); + } + + return (ssize_t)count; +} + +static ssize_t roce5_store_dfx_srqc(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 srqn; + struct roce5_dfx_srqc_ctx *srqc_ctx = NULL; + struct roce5_dfx_ctx *dfx_ctx = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_srq_attr_s srq_attr; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + ROCE_ERR("Invalid input para."); + return -EINVAL; + } + + srqc_ctx = container_of(attr, struct roce5_dfx_srqc_ctx, kattr); + dfx_ctx = container_of(srqc_ctx, struct roce5_dfx_ctx, srqc_ctx); + rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx); + + (void)memset_s(&srq_attr, sizeof(roce_verbs_srq_attr_s), 0, + sizeof(roce_verbs_srq_attr_s)); + + ret = (int)sscanf_s(buf, "%u", &srqn); + if (ret != 1) { + ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (srqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs) { + ROCE_DEV_ERR(rdev, "Invalid srqn(%u), max_srqs(%u)", srqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs); + return -EIO; + } + + ret = roce5_dfx_cmd_query_srq(rdev, srqn, &srq_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query srq_attr, ret(%d)", ret); + return (ssize_t)ret; + } + + srq_attr.dfx_info.dw0.value = be32_to_cpu(srq_attr.dfx_info.dw0.value); + + roce5_dfx_print("srqn(0x%x): pcnt(0x%x), ccnt(0x%x).", srqn, + srq_attr.dfx_info.dw0.bs.pcnt, + srq_attr.dfx_info.dw0.bs.ccnt); + + return (ssize_t)count; +} + +static int roce5_init_dfx_sub_ctx_sysfs(struct kobject *kobj, + struct roce5_dfx_ctx *dfx_ctx) +{ + int ret; + struct kobj_attribute *qpc_kattr = NULL; + struct kobj_attribute *cqc_kattr = NULL; + struct kobj_attribute *srqc_kattr = NULL; + + if (kobj == NULL || dfx_ctx == NULL) { + ROCE_ERR("Invalid kobj or dfx_ctx."); + return -EINVAL; + } + + qpc_kattr = &dfx_ctx->qpc_ctx.kattr; + cqc_kattr = &dfx_ctx->cqc_ctx.kattr; + srqc_kattr = &dfx_ctx->srqc_ctx.kattr; + + sysfs_attr_init(&qpc_kattr->attr); + qpc_kattr->attr.name = "qpc"; + qpc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + qpc_kattr->store = roce5_store_dfx_qpc; + ret = sysfs_create_file(kobj, &qpc_kattr->attr); + if (ret != 0) { + ROCE_ERR("QPC failed to do sysfs_create_file, ret(%d)", ret); + return ret; + } + + sysfs_attr_init(&cqc_kattr->attr); + cqc_kattr->attr.name = "cqc"; + cqc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + cqc_kattr->store = roce5_store_dfx_cqc; + ret = sysfs_create_file(kobj, &cqc_kattr->attr); + if (ret != 0) { + ROCE_ERR("CQC failed to do sysfs_create_file, ret(%d)", ret); + goto err_create_cqc_sysfs; + } + + sysfs_attr_init(&srqc_kattr->attr); + srqc_kattr->attr.name = "srqc"; + srqc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + srqc_kattr->store = roce5_store_dfx_srqc; + ret = sysfs_create_file(kobj, &srqc_kattr->attr); + if (ret != 0) { + ROCE_ERR("SRQC failed to do sysfs_create_file, ret(%d)", ret); + goto err_create_srqc_sysfs; + } + + return 0; + +err_create_srqc_sysfs: + sysfs_remove_file(kobj, &cqc_kattr->attr); + +err_create_cqc_sysfs: + sysfs_remove_file(kobj, &qpc_kattr->attr); + + return ret; +} + +static int roce5_init_dfx_sysfs(struct net_device *ndev, + struct roce5_dfx_ctx *dfx_ctx) +{ + int ret; + + (void)memset_s(dfx_ctx, sizeof(*dfx_ctx), 0, sizeof(*dfx_ctx)); + + dfx_ctx->dfx_root = + kobject_create_and_add("roce5_dfx", &ndev->dev.kobj); + if (dfx_ctx->dfx_root == NULL) { + ROCE_ERR("Failed to do kobject_create_and_add."); + return -ENOMEM; + } + + ret = roce5_init_dfx_sub_ctx_sysfs(dfx_ctx->dfx_root, dfx_ctx); + if (ret != 0) { + ROCE_ERR("Failed to init dfx sub ctx sysfs, ret(%d)", ret); + kobject_put(dfx_ctx->dfx_root); + return ret; + } + + return 0; +} + +static void roce5_remove_ecn_sysfs(struct roce5_device *rdev) +{ + struct roce5_ecn_ctx *ecn_ctx = &rdev->ecn_ctx; + roce5_remove_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root, + &ecn_ctx->multipath_ctx); + + roce5_remove_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx); + + roce5_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, + &ecn_ctx->ip_enable_ctx); + + roce5_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + + roce5_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + + roce5_remove_ecn_ldcp_sysfs(&ecn_ctx->ldcp_ctx); + + kobject_put(&ecn_ctx->ecn_root); +} + +int roce5_init_sysfs(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_init_ecn_sysfs(rdev, &rdev->ecn_ctx); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init ecn sysfs, ret(%d)", ret); + return ret; + } + + ret = roce5_init_dfx_sysfs(rdev->ndev, &rdev->dfx_ctx); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init dfx sysfs, ret(%d)", ret); + return ret; + } + + return 0; +} + +static void roce5_remove_dfx_sub_sysfs(struct roce5_dfx_ctx *dfx_ctx) +{ + if (dfx_ctx->dfx_root != NULL) { + sysfs_remove_file(dfx_ctx->dfx_root, + &dfx_ctx->srqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, + &dfx_ctx->cqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, + &dfx_ctx->qpc_ctx.kattr.attr); + + kobject_put(dfx_ctx->dfx_root); + + dfx_ctx->dfx_root = NULL; + } +} + +void roce5_remove_sysfs(struct roce5_device *rdev) +{ + roce5_remove_dfx_sub_sysfs(&rdev->dfx_ctx); + roce5_remove_ecn_sysfs(rdev); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h new file mode 100644 index 000000000..814f29ddc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h @@ -0,0 +1,241 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SYSFS_H +#define ROCE_SYSFS_H + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/kobject.h> +#include "roce_ucc_sysfs.h" +#include "roce_multipath_sysfs.h" + +#define ROCE_MIN_QPN 2 +#define BYTE_TO_BIT 8 + +#define PRI_ARRAY_LEN 8 + +#define ROCE_MAX_PORT_CNT 8 +#define ROCE_MAX_PF_NUM 32 + +#define ROCE_ENABLE_CHECK_MASK 0xfffffffe + +/* ECN VER */ +#define ECN_VER_DCQCN 0 +#define ECN_VER_PATHQCN 1 + +#define FLOW_CTRL_UNIT_QP 0 +#define FLOW_CTRL_UNIT_IP 1 + +enum { ROCE_DCQCN_NP = 0, ROCE_DCQCN_RP = 1 }; + +enum { + ROCE_CC_PARAM_INVALID = 0, + ROCE_CC_PARAM_V1 = 1, + ROCE_CC_PARAM_V2 = 2, +}; + +#define HRN0_K_MAGIC_NUM_3_SYSFS 3 +#define HRN0_K_MAGIC_NUM_7_SYSFS 7 +#define HRN0_K_MAGIC_NUM_8_SYSFS 8 +#define HRN0_K_MAGIC_NUM_M_SYSFS 1000000 + +#define to_roce5_ecn_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_ctx, ecn_root) +#define to_roce5_ecn_np_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_np_ctx, ecn_np_root) +#define to_roce5_ecn_rp_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_rp_ctx, ecn_rp_root) +#define to_roce5_ecn_ldcp_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_ldcp_ctx, ecn_ldcp_root) + +#define ROCE5_PARAM_CHECK_BOUND(val, min, max) \ + ((val) < (min)) || ((val) > (max)) + +#define INIT_ROCE_KOBJ_ATTR(_name, _mode, _show, _store) \ + { \ + .attr = { .name = __stringify(_name), .mode = _mode }, \ + .show = (_show), .store = (_store), \ + } + +#define ROCE_ATTR_RW(_name, _show, _store) \ + static struct kobj_attribute roce_attr_##_name = INIT_ROCE_KOBJ_ATTR( \ + _name, (S_IWUSR | S_IRUSR | S_IRGRP), (_show), (_store)) + +#define ROCE_ATTR_PTR(_name) (&roce_attr_##_name.attr) + +#define ROCE_CC_ALGO_TEST_BIT(bitmap, cc_algo) (!!(bitmap & (1U << cc_algo))) + +struct roce5_prio_enable_ctx { + struct kobj_attribute enable; + void *ecn_enable_ctx; + u32 prio_en; + u32 prio; +}; + +struct roce5_ip_prio_enable_ctx { + struct kobj_attribute ip_enable; + void *ecn_ip_enable_ctx; + u32 prio_en; + u32 prio; +}; + +struct roce5_ecn_ip_enable_ctx { + struct kobject *ip_enable_root; + struct roce5_ip_prio_enable_ctx ip_prio_enable[PRI_ARRAY_LEN]; +}; + +struct roce5_ecn_enable_ctx { + struct kobject *enable_root; + struct roce5_prio_enable_ctx prio_enable[PRI_ARRAY_LEN]; + u32 np_rp; +}; + +struct roce5_dfx_qpc_ctx { + struct kobj_attribute kattr; +}; + +struct roce5_dfx_cqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce5_dfx_srqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce5_dfx_ctx { + struct kobject *dfx_root; + struct roce5_dfx_qpc_ctx qpc_ctx; + struct roce5_dfx_cqc_ctx cqc_ctx; + struct roce5_dfx_srqc_ctx srqc_ctx; +}; + +struct roce5_ecn_rp_ctx { + struct kobject ecn_rp_root; + u32 alpha_dec_period; + u32 rate_dec_period; + u32 rate_inc_period; + u32 factor_gita; + u32 initial_alpha; + u32 rate_inc_ai; + u32 rate_inc_hai; + u32 rate_first_set; + u32 rate_target_clamp; + u32 token_period; + u32 min_rate; + u32 bypass; + struct roce5_ecn_enable_ctx enable_ctx; +}; + +struct roce5_ecn_np_ctx { + struct kobject ecn_np_root; + u32 min_cnp_period; + u32 cnp_prio_enable; + u32 cnp_prio; + u32 rtt_rsp_prio; + u32 rtt_rsp_prio_enable; + u32 rtt_version; + u32 ecn_cnp_en; + u32 flow_ctrl_unit; + struct roce5_ecn_enable_ctx enable_ctx; +}; + +typedef struct roce5_ecn_ldcp_ctx { + struct kobject ecn_ldcp_root; + u32 wnd_min; + u32 wnd_default; /* initial default window size, unit: Byte */ + u32 alpha; + u32 beta; + u32 gamma; + u32 eta; +} roce5_ecn_ldcp_ctx_t; + +struct roce5_ecn_ctx { + struct kobject ecn_root; + struct mutex ecn_mutex; + u32 ecn_ver; + u32 cc_algo; + roce5_ecn_ldcp_ctx_t ldcp_ctx; + struct roce5_ecn_np_ctx np_ctx; + struct roce5_ecn_rp_ctx rp_ctx; + struct roce5_ecn_ip_enable_ctx ip_enable_ctx; + struct roce5_ecn_ucc_ctx ucc_ctx; + struct roce5_ecn_multipath_ctx multipath_ctx; +}; + +struct roce_cc_ecn_param { + u32 default_ip_enable_en; + u32 default_bypass; + u32 default_min_cnp_period; + u32 default_alpha_dec_period; + u32 default_rate_dec_period; + u32 default_rate_inc_period; + u32 default_factor_gita; + u32 default_initial_alpha; + u32 default_rate_inc_ai; + u32 default_rate_inc_hai; + u32 default_rate_first_set; + u32 default_rate_target_clamp; + u32 default_cnp_prio_enable; + u32 default_cnp_prio; + u32 default_token_period; + u32 default_min_rate; + u32 default_ecn_cnp_enable; + u32 default_flow_ctrl_unit; + u32 default_rtt_rsp_prio; + u32 default_rtt_rsp_prio_enable; + u32 default_rtt_version; + + u32 max_rtt_version; + u32 min_bypass; + u32 max_bypass; + u32 min_rate_inc_hai; + u32 max_rate_inc_hai; + u32 min_rate_inc_ai; + u32 max_rate_inc_ai; + u32 min_initial_alpha; + u32 max_initial_alpha; + u32 min_rate_inc_period; + u32 max_rate_inc_period; + u32 min_alpha_dec_period; + u32 max_alpha_dec_period; + u32 min_token_period; + u32 max_token_period; + u32 min_flow_min_rate; + u32 max_flow_min_rate; + u32 min_rate_dec_period; + u32 max_rate_dec_period; + u32 min_min_cnp_period; + u32 max_min_cnp_period; + u32 min_factor_gita; + u32 max_factor_gita; + u32 min_rate_first_set; + u32 max_rate_first_set; + + u32 default_ldcp_wnd_min; + u32 default_ldcp_wnd_default; + u32 default_ldcp_alpha; + u32 default_ldcp_beta; + u32 default_ldcp_gamma; + u32 default_ldcp_eta; + + u32 min_ldcp_wnd_min; + u32 max_ldcp_wnd_min; + u32 min_ldcp_wnd_default; + u32 max_ldcp_wnd_default; + u32 min_ldcp_alpha; + u32 max_ldcp_alpha; + u32 min_ldcp_beta; + u32 max_ldcp_beta; + u32 min_ldcp_gamma; + u32 max_ldcp_gamma; + u32 min_ldcp_eta; + u32 max_ldcp_eta; +}; + +int roce5_update_ecn_param(const struct roce5_ecn_ctx *ecn_ctx); + +#endif //ROCE_SYSFS_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c new file mode 100644 index 000000000..e163f895b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "roce_sysfs.h" +#include "roce_cc_format.h" +#include "roce_mpu_cmd.h" +#include "roce_cmd.h" +#include "ccp_algo_format.h" +#include "roce_kernel_compat_gen.h" + +#define MAX_STRING_FORMAT_LEN 64 +#define to_roce5_ecn_ucc_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_ucc_ctx, ecn_ucc_root) + +static char g_param_buffer[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN]; +static char g_param_show_tmp[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN]; +static char g_param_show_format[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN]; + +static int roce5_ucc_store_input_check(struct kobject *kobj, const char *buf, + u8 param_index) +{ + if (kobj == NULL || buf == NULL) { + ROCE_ERR("kobj or buf is null\n"); + return -EINVAL; + } + + if (param_index >= ROCE_UCC_MAX_PARAM_NUM) { + ROCE_ERR("invalid index:%hhu.\n", param_index); + return -EINVAL; + } + + if (strlen(buf) > (ROCE_UCC_PARAM_BUFF_LEN - 1)) { + ROCE_ERR("input buffer is beyond 1022.\n"); + return -EIO; + } + + return 0; +} + +static int roce5_init_ucc_global_buffer(u8 param_index, const char *buf) +{ + int ret; + ret = memset_s(g_param_show_tmp[param_index], + sizeof(g_param_show_tmp[param_index]), 0, + sizeof(g_param_show_tmp[param_index])); + if (ret != 0) { + ROCE_ERR("Failed to memset buffer.(ret:%d)\n", ret); + return -EIO; + } + + ret = strncpy_s(g_param_buffer[param_index], + sizeof(g_param_buffer[param_index]), buf, + sizeof(g_param_buffer[param_index]) - 1); + if (ret != 0) { + ROCE_ERR("Failed to strncpy buffer.(ret:%d)\n", ret); + return -EIO; + } + + g_param_buffer[param_index][ROCE_UCC_PARAM_BUFF_LEN - 1] = '\0'; + + return 0; +} + +static int check_key_unique(char (*pkey)[ROCE_UCC_MAX_KEY_SIZE], u8 key_num) +{ + u8 key_index = 0; + + if (key_num >= ROCE_UCC_MAX_KEY_SIZE) { + ROCE_ERR("invalid key value:%hhu.\n", key_num); + return -EINVAL; + } + + for (key_index = 0; key_index < key_num; key_index++) { + if (strcmp(pkey[key_index], pkey[key_num]) == 0) { + ROCE_ERR("key:%s is not unique.\n", pkey[key_num]); + return -EINVAL; + } + } + return 0; +} + +static int parse_each_line(char *param_line, + char (*pkey)[ROCE_UCC_MAX_KEY_SIZE], u8 key_num, + u32 *param_value, u8 *param_bits, u16 *sum_bits) +{ + u64 temp_value = 0; + u64 temp_bits = 0; + u64 temp_bit_sum = *sum_bits; + char extra = 0; + char value_str[ROCE_UCC_PARAM_VALUE_LEN] = { 0 }; + char bits_str[ROCE_UCC_PARAM_BITS_LEN] = { 0 }; + + if (key_num >= ROCE_UCC_MAX_KEY_SIZE) { + ROCE_ERR("invalid key value:%hhu.\n", key_num); + return -EINVAL; + } + + /* input of each line per per key:value:length perform parse, obtain value and length size, and check key whether repeat */ + if (sscanf_s(param_line, " %31[^: ] : %10[0-9] : %2[0-9] %c", + pkey[key_num], (unsigned)sizeof(pkey[key_num]), value_str, + (unsigned)sizeof(value_str), bits_str, + (unsigned)sizeof(bits_str), &extra, + 1) == ROCE_UCC_LINE_PARAM_TH) { + if (kstrtoull(value_str, NUM_10, &temp_value) < 0) { + ROCE_ERR("invalid input value:%s.\n", value_str); + return -EINVAL; + } + + if (kstrtoull(bits_str, NUM_10, &temp_bits) < 0) { + ROCE_ERR("invalid input bits:%s.\n", bits_str); + return -EINVAL; + } + + if ((temp_bits == 0) || (temp_bits > TYPE_BITS(u32)) || + (temp_value > BITS_VALUE(temp_bits))) { + ROCE_ERR("bits:%llu or value:%llu is invalid.\n", + temp_bits, temp_value); + return -EINVAL; + } + + temp_bit_sum += temp_bits; + if (temp_bit_sum > TYPE_BITS(u32)) { + ROCE_ERR("the length of bits:%llu beyond 32.\n", + temp_bit_sum); + return -EINVAL; + } + + if (check_key_unique(pkey, key_num) != 0) { + return -EINVAL; + } + + *param_value = (u32)temp_value; + *param_bits = (u8)temp_bits; + *sum_bits = (u16)temp_bit_sum; + return 0; + } + + ROCE_ERR("param format is wrong, please enter key:value:length param, " + "value less than 4294967296, length less than 33.\n"); + return -EINVAL; +} + +static int roce5_parse_ucc_param(u32 *ucc_param, char *param_buffer, + char *show_tmpkey) +{ + u8 key_num = 0; + u32 temp_param = 0; + /* param_value\param_bits used to store each line parse of values size and so occupy bit width result */ + u32 param_value = 0; + u8 param_bits = 0; + u16 sum_bits = 0; + /* param_line\ppbuffer used for input parameters partinterval parse */ + char *param_line = NULL; + char **ppbuffer = ¶m_buffer; + + char(*pkey)[ROCE_UCC_MAX_KEY_SIZE] = NULL; + char arr_tempkey[ROCE_UCC_MAX_KEY_NUM][ROCE_UCC_MAX_KEY_SIZE] = { 0 }; + char append_buff[ROCE_UCC_LINE_MAX_LEN] = { 0 }; + pkey = arr_tempkey; + + while ((param_line = strsep(ppbuffer, "\n")) != NULL) { + if (*param_line == '\0') { + break; + } + /* per per key:value:length parse input of each one line */ + if (parse_each_line(param_line, pkey, key_num, ¶m_value, + ¶m_bits, &sum_bits) != 0) { + return -EIO; + } + + if (snprintf_s(append_buff, ROCE_UCC_LINE_MAX_LEN, + ROCE_UCC_LINE_MAX_LEN - 1, "%s:%u:%hhu\n", + pkey[key_num], param_value, param_bits) < 0) { + ROCE_ERR("snprintf_s err!\n"); + return -EIO; + } + + if (strncat_s(show_tmpkey, ROCE_UCC_PARAM_BUFF_LEN, append_buff, + ROCE_UCC_PARAM_BUFF_LEN - strlen(show_tmpkey) - + 1) != 0) { + ROCE_ERR("strncat_s err!\n"); + return -EIO; + } + key_num++; + temp_param = key_num > 1 ? ((temp_param << param_bits) + + param_value) : + param_value; + } + + if (sum_bits < TYPE_BITS(u32)) { + temp_param = temp_param << (TYPE_BITS(u32) - sum_bits); + } + + if (ucc_param != NULL) { + *ucc_param = temp_param; + } + return 0; +} + +static int roce5_store_ucc_param_inner(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, u8 param_index) +{ + int ret = 0; + u32 ucc_param = 0; + u32 old_ucc_param = 0; + struct roce5_ecn_ctx *ecn_ctx = NULL; + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = NULL; + + if (roce5_ucc_store_input_check(kobj, buf, param_index) != 0) { + return -EINVAL; + } + + ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj); + + if (roce5_init_ucc_global_buffer(param_index, buf) != 0) { + return -EIO; + } + + if (roce5_parse_ucc_param(&ucc_param, g_param_buffer[param_index], + g_param_show_tmp[param_index]) != 0) { + return -EIO; + } + + mutex_lock(&ecn_ctx->ecn_mutex); + if (strncpy_s(g_param_show_format[param_index], + sizeof(g_param_show_format[param_index]), + g_param_show_tmp[param_index], + sizeof(g_param_show_format[param_index]) - 1) != 0) { + ROCE_ERR("Failed to strncpy buffer.\n"); + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + + mutex_unlock(&ecn_ctx->ecn_mutex); + + if ((ecn_ucc_ctx != NULL) && + (ecn_ucc_ctx->param[param_index] != ucc_param)) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ucc_param = ecn_ucc_ctx->param[param_index]; + ecn_ucc_ctx->param[param_index] = ucc_param; + ret = (ssize_t)roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_ucc_ctx->param[param_index] = old_ucc_param; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return 0; +} + +ssize_t roce5_show_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ucc_param_ctx *ucc_param = + (struct roce5_ucc_param_ctx *)to_roce5_ucc_param_ctx(attr); + return sprintf_s(buf, sizeof(g_param_show_format[ucc_param->param_id]), + "%s", g_param_show_format[ucc_param->param_id]); +} + +ssize_t roce5_store_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct roce5_ucc_param_ctx *ucc_param = + (struct roce5_ucc_param_ctx *)to_roce5_ucc_param_ctx(attr); + return (roce5_store_ucc_param_inner(kobj, attr, buf, + ucc_param->param_id) == 0) ? + (ssize_t)count : + -EIO; +} + +static ssize_t roce5_show_algo_mode(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + switch ((int)ecn_ucc_ctx->algo_mode) { + case CCP_ALGO_MODE_CWND: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + (u32)CCP_ALGO_MODE_CWND); + case CCP_ALGO_MODE_RATE: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + (u32)CCP_ALGO_MODE_RATE); + default: + break; + } + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "error_type"); +} + +static ssize_t roce5_store_algo_mode(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj); + u32 old_algo_mode = 0; + int algo_mode = 0; + int ret; + char remaining; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + ret = sscanf_s(buf, "%d %c", &algo_mode, &remaining, 1); + if (ret != 1) { + ROCE_ERR("Invalid input, algo_mode should be unsigned int\n"); + return -EIO; + } + + if (algo_mode != CCP_ALGO_MODE_CWND && + algo_mode != CCP_ALGO_MODE_RATE) { + ROCE_ERR("Invalid algo_mode(%d),buf(%s)\n", algo_mode, buf); + return -EIO; + } + + if (ecn_ucc_ctx->algo_mode != (u32)algo_mode) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_algo_mode = ecn_ucc_ctx->algo_mode; + ecn_ucc_ctx->algo_mode = (u32)algo_mode; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Failed to update algo_mode(%d), ret(%d).", + algo_mode, ret); + ecn_ucc_ctx->algo_mode = old_algo_mode; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_UCC_PARAM_RW(param0, 0); +ROCE_UCC_PARAM_RW(param1, 1); +ROCE_UCC_PARAM_RW(param2, 2); +ROCE_UCC_PARAM_RW(param3, 3); +ROCE_UCC_PARAM_RW(param4, 4); +ROCE_UCC_PARAM_RW(param5, 5); +ROCE_UCC_PARAM_RW(param6, 6); +ROCE_UCC_PARAM_RW(param7, 7); +ROCE_UCC_PARAM_RW(param8, 8); +ROCE_UCC_PARAM_RW(param9, 9); +ROCE_UCC_PARAM_RW(param10, 10); +ROCE_UCC_PARAM_RW(param11, 11); +ROCE_UCC_PARAM_RW(param12, 12); +ROCE_UCC_PARAM_RW(param13, 13); +ROCE_UCC_PARAM_RW(param14, 14); +ROCE_UCC_PARAM_RW(param15, 15); +ROCE_ATTR_RW(algo_mode, roce5_show_algo_mode, roce5_store_algo_mode); + +static struct attribute *ecn_ucc_ctx_attrs[] = { + ROCE_UCC_PARAM_ATTR_PTR(param0), ROCE_UCC_PARAM_ATTR_PTR(param1), + ROCE_UCC_PARAM_ATTR_PTR(param2), ROCE_UCC_PARAM_ATTR_PTR(param3), + ROCE_UCC_PARAM_ATTR_PTR(param4), ROCE_UCC_PARAM_ATTR_PTR(param5), + ROCE_UCC_PARAM_ATTR_PTR(param6), ROCE_UCC_PARAM_ATTR_PTR(param7), + ROCE_UCC_PARAM_ATTR_PTR(param8), ROCE_UCC_PARAM_ATTR_PTR(param9), + ROCE_UCC_PARAM_ATTR_PTR(param10), ROCE_UCC_PARAM_ATTR_PTR(param11), + ROCE_UCC_PARAM_ATTR_PTR(param12), ROCE_UCC_PARAM_ATTR_PTR(param13), + ROCE_UCC_PARAM_ATTR_PTR(param14), ROCE_UCC_PARAM_ATTR_PTR(param15), + ROCE_ATTR_PTR(algo_mode), NULL, +}; + +static void roce_ecn_ucc_sysfs_release(struct kobject *kobj) +{ +} + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_ucc_ctx_attr_grp = { + .attrs = ecn_ucc_ctx_attrs, +}; + +static const struct attribute_group *ecn_ucc_ctx_groups[] = { + &ecn_ucc_ctx_attr_grp, + NULL, +}; +#endif + +static struct kobj_type roce_ecn_ucc_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_ucc_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_ucc_ctx_groups, +#else + .default_attrs = ecn_ucc_ctx_attrs, +#endif +}; + +int roce5_update_ucc_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_ucc_param_s ucc_param; + int ret; + int param_idx = 0; + + if (ecn_ctx->cc_algo != ROCE_CC_USER_A_ALGO && + ecn_ctx->cc_algo != ROCE_CC_USER_B_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not user algo, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + (void)memset_s(&ucc_param, sizeof(ucc_param), 0, sizeof(ucc_param)); + + for (; param_idx < ROCE_UCC_MAX_PARAM_NUM; param_idx++) { + ucc_param.param[param_idx] = ecn_ctx->ucc_ctx.param[param_idx]; + } + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ucc_param, + sizeof(roce_ucc_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_UCC_PARAM); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set user algo param, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +int roce5_init_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx) +{ + int ret = 0; + int param_idx = 0; + + if (ecn_ucc_ctx == NULL) { + pr_err_ratelimited("[ROCE, ERR] %s: ecn_ucc_ctx is null\n", + __func__); + return -EINVAL; + } + + ecn_ucc_ctx->algo_mode = CCP_ALGO_MODE_CWND; + for (; param_idx < ROCE_UCC_MAX_PARAM_NUM; param_idx++) { + ecn_ucc_ctx->param[param_idx] = ROCE_DEFAULT_UCC_PARAM; + } + + ret = kobject_init_and_add(&ecn_ucc_ctx->ecn_ucc_root, + &roce_ecn_ucc_ktype, kobj, "roce5_ucc"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_ucc.(ret:%d)\n", ret); + kobject_put(&ecn_ucc_ctx->ecn_ucc_root); + return ret; + } + + return 0; +} + +void roce5_remove_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx) +{ + if (ecn_ucc_ctx == NULL) { + ROCE_ERR("ecn_ucc_ctx is null\n"); + return; + } + + kobject_put(&ecn_ucc_ctx->ecn_ucc_root); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h new file mode 100644 index 000000000..bbbe92b30 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h @@ -0,0 +1,73 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef ROCE_UCC_SYSFS_H +#define ROCE_UCC_SYSFS_H + +#include <linux/types.h> +#include <linux/stat.h> +#include <linux/kobject.h> + +#define ROCE_UCC_MAX_PARAM_NUM 16 +#define ROCE_UCC_PARAM_BUFF_LEN 1024 +#define ROCE_DEFAULT_UCC_PARAM 0 +#define ROCE_UCC_MAX_KEY_SIZE 32 +#define ROCE_UCC_MAX_KEY_NUM 32 +#define ROCE_UCC_LINE_MAX_LEN 64 +#define ROCE_UCC_PARAM_VALUE_LEN 11 +#define ROCE_UCC_PARAM_BITS_LEN 3 +#define ROCE_UCC_LINE_PARAM_TH 3 + +#define CHAR_0 '0' +#define CHAR_9 '9' +#define NUM_10 10 +#define ROCE_USER_CC_ID_LABLE_LEN 8 +#define ROCE_USER_CC_ID_MAX_LEN (ROCE_USER_CC_ID_LABLE_LEN + 2) + +#define TYPE_BITS(type) (sizeof(type) * 8) +#define BITS_VALUE(bits) (((u64)1 << (bits)) - 1) + +/* forward-declaration */ +struct roce5_device; +struct roce5_ecn_ctx; + +typedef struct roce_ucc_param { + u32 param[ROCE_UCC_MAX_PARAM_NUM]; +} roce_ucc_param_s; + +struct roce5_ecn_ucc_ctx { + struct kobject ecn_ucc_root; + u32 algo_mode; + u32 param[ROCE_UCC_MAX_PARAM_NUM]; +}; + +struct roce5_ucc_param_ctx { + struct kobj_attribute param; + u32 param_id; +}; + +ssize_t roce5_show_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + char *buf); +ssize_t roce5_store_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count); +#define ROCE_UCC_PARAM_RW(_name, _idx) \ + static struct roce5_ucc_param_ctx roce_ucc_attr_##_name = { \ + .param = INIT_ROCE_KOBJ_ATTR( \ + _name, (S_IWUSR | S_IRUSR | S_IRGRP), \ + roce5_show_ucc_param, roce5_store_ucc_param), \ + .param_id = (_idx), \ + } + +#define ROCE_UCC_PARAM_ATTR_PTR(_name) (&roce_ucc_attr_##_name.param.attr) +#define to_roce5_ucc_param_ctx(_kattr) \ + container_of(_kattr, struct roce5_ucc_param_ctx, param) + +int roce5_update_ucc_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx); +int roce5_init_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx); +void roce5_remove_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx); + +#endif /* ROCE_UCC_SYSFS_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c new file mode 100644 index 000000000..145904486 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c @@ -0,0 +1,13 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + + +#include "roce_main_extension.h" +#include "roce_mr_chip_ext.h" + +void roce5_init_dev_ext_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.check_mr_status = roce5_check_mr_status; + rdev->ib_dev.reg_user_fast_mr = roce5_reg_user_fast_mr; + rdev->ib_dev.map_frmr_sg = roce5_map_user_frmr_sg; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h new file mode 100644 index 000000000..a5f0847f8 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_MAIN_CHIP_EXT_C +#define ROCE_MAIN_CHIP_EXT_C + +#define ROCE_UVERBS_CMD_EXT_MASK \ + ((1ULL << IB_USER_VERBS_CMD_CREATE_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_ARM_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_MR) | \ + (1ULL << IB_USER_VERBS_CMD_REG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_MAP_FRMR_SG)) + +#endif /* ROCE_MAIN_CHIP_EXT_C */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c new file mode 100644 index 000000000..c460ca8a5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c @@ -0,0 +1,348 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_mr.h" +#include "roce_infra.h" +#include "roce_mr_extension.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "hmm_common.h" +#include "roce_user.h" + +int roce5_check_alloc_mr_type(enum ib_mr_type mr_type) +{ + int ret = 0; + + if (mr_type != IB_MR_TYPE_MEM_REG && mr_type != IB_MR_INDIRECT_REG && + mr_type != IB_MR_TYPE_FAST_MR) { + ret = -EINVAL; + ROCE_ERR("mr_type is invalid. mr_type:%u", mr_type); + } + + return ret; +} + +enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type) +{ + switch (ib_mr_type) { + case IB_MR_TYPE_MEM_REG: + return RDMA_DMA_MR; + + case IB_MR_INDIRECT_REG: + return RDMA_INDIRECT_MR; + + case IB_MR_TYPE_FAST_MR: + return RDMA_FRMR; + + default: + return RDMA_DMA_MR; + } +} + +static int roce5_alloc_usr_frmr(struct ib_pd *ibpd, struct ib_udata *udata, + struct roce5_reg_frmr_cmd *ucmd, + struct roce5_device **rdev, + struct roce5_mr **rmr) +{ + struct ib_umem **fumem = NULL; + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return -EINVAL; + } + + if (udata == NULL) { + ROCE_ERR("Udata is null"); + return -EINVAL; + } + + *rdev = to_roce5_dev(ibpd->device); + + if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd))) { + ROCE_DEV_ERR((*rdev), + "Failed to copy_from_udata while creating frmr"); + return -EFAULT; + } + + *rmr = (struct roce5_mr *)kzalloc(sizeof(struct roce5_mr), GFP_KERNEL); + if (*rmr == NULL) { + ROCE_DEV_ERR((*rdev), "Failed to alloc memory for roce mr"); + return -ENOMEM; + } + + fumem = (struct ib_umem **)kzalloc( + sizeof(struct ib_umem *) * ROCE5_FRMR_UMEM_NUM, GFP_KERNEL); + if (fumem == NULL) { + ROCE_DEV_ERR((*rdev), "Failed to alloc memory for frmr umem"); + kfree(*rmr); + return -ENOMEM; + } + (*rmr)->fumem = fumem; + return 0; +} + +static int roce5_get_frmr_umem(struct roce5_mr *rmr, struct ib_pd *ibpd, + struct roce5_reg_frmr_cmd *ucmd, + struct roce5_device *rdev) +{ + rmr->umem = ib_umem_get(ibpd->uobject->context, ucmd->pbl_vaddr, + PAGE_SIZE_4K, 0, 0); + if (rmr->umem == NULL) { + ROCE_DEV_ERR(rdev, "Rmr->umem is null"); + return -ENOMEM; + } + + if (IS_ERR(rmr->umem)) { + ROCE_DEV_ERR(rdev, "Failed to get ib umem, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + return 0; +} + +static int roce5_init_frmr_attr(u64 *pbl_paddr, struct roce5_mr *rmr, + struct roce5_device *rdev, struct ib_pd *ibpd) +{ + int ret; + u32 mpt_xid_mode; + struct roce5_pd *rpd = NULL; + + *pbl_paddr = sg_dma_address(rmr->umem->sg_head.sgl); + if (*pbl_paddr == 0ULL) { + ROCE_DEV_ERR(rdev, "Invalid pbl_paddr"); + return -EINVAL; + } + + rmr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + ret = roce5_alloc_tpt(rdev, &rmr->rdmamr, ROCE_FRMR_MAX_PAGES, + PAGE_SHIFT, mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, ret(%d)", ret); + return ret; + } + + rpd = to_roce5_pd(ibpd); + roce5_set_rdma_mr(&rmr->rdmamr, RDMA_FRMR, rpd->pdn, 0ULL, 0ULL, + ROCE5_MR_DEFAULT_ACCESS); + + return 0; +} + +struct ib_mr *roce5_reg_user_fast_mr(struct ib_pd *ibpd, struct ib_udata *udata) +{ + int ret; + struct roce5_device *rdev = NULL; + struct roce5_mr *rmr = NULL; + struct roce5_reg_frmr_cmd ucmd; + u64 pbl_paddr; + + ret = roce5_alloc_usr_frmr(ibpd, udata, &ucmd, &rdev, &rmr); + if (ret != 0) { + ROCE_ERR("Failed to pre check, ret(%d)", ret); + goto err_out; + } + + ret = roce5_get_frmr_umem(rmr, ibpd, &ucmd, rdev); + if (ret != 0) { + goto err_ib_umem_get; + } + + /* Use valloc to allocate a page of memory, just need to get the first physical address */ + ret = roce5_init_frmr_attr(&pbl_paddr, rmr, rdev, ibpd); + if (ret != 0) { + goto err_pbl_addr; + } + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(rmr->rdmamr), + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to enable mpt of frmr, ret(%d)", + ret); + goto err_enable_mpt; + } + + /* User mode also needs to send out pbl_paddr */ + if (ib_copy_to_udata(udata, &pbl_paddr, sizeof(u64)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + goto err_enable_mpt; + } + + rmr->ibmr.lkey = rmr->rdmamr.key; + rmr->ibmr.rkey = rmr->rdmamr.key; + + return &rmr->ibmr; + +err_enable_mpt: + roce5_free_tpt(rdev, &rmr->rdmamr); + +err_pbl_addr: + ib_umem_release(rmr->umem); + +err_ib_umem_get: + kfree(rmr->fumem, sizeof(struct ib_umem *) * ROCE5_FRMR_UMEM_NUM); + kfree(rmr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +int roce5_frmr_check_user_param(struct ib_mr *ibmr, unsigned int page_num, + struct ib_udata *udata) +{ + if (ibmr == NULL) { + ROCE_ERR("Ibmr is null"); + return -EINVAL; + } + + if (page_num == 0 || page_num > ROCE5_FRMR_UMEM_NUM) { + ROCE_ERR("page_num(%d) is invalid", page_num); + return -EINVAL; + } + + if (udata == NULL) { + ROCE_ERR("udata is NULL"); + return -EINVAL; + } + return 0; +} + +void roce5_frmr_free_addr_list(unsigned long long *va_list, + unsigned long long *pa_list) +{ + vfree(va_list); + vfree(pa_list); +} + +int roce5_frmr_alloc_addr_list(struct roce5_device *rdev, + unsigned long long **va_list, + unsigned long long **pa_list, u32 len) +{ + int ret; + unsigned long long *val = NULL; + unsigned long long *pal = NULL; + + val = (unsigned long long *)vmalloc(len); + if (val == NULL) { + ROCE_DEV_ERR(rdev, "Failed to vmalloc va_list"); + return -EAGAIN; + } + pal = (unsigned long long *)vmalloc(len); + if (pal == NULL) { + ROCE_DEV_ERR(rdev, "Failed to vmalloc pa_list"); + vfree(val); + return -EAGAIN; + } + ret = memset_s(val, len, 0, len); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to memset va_list.(ret:%d)", ret); + ret = -ENOMEM; + goto err_free; + } + ret = memset_s(pal, len, 0, len); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to memset pa_list.(ret:%d)", ret); + ret = -ENOMEM; + goto err_free; + } + + *va_list = val; + *pa_list = pal; + return 0; + +err_free: + roce5_frmr_free_addr_list(val, pal); + return ret; +} + +static int process_va_list_and_allocate_umem( + struct roce5_mr *rmr, struct roce5_device *rdev, struct ib_mr *ibmr, + unsigned long long *va_list, unsigned long long *pa_list, u32 page_num) +{ + u32 i; + int ret; + for (i = 0; i < page_num; i++) { + ROCE_DEV_INFO(rdev, "i:%d, va(%llu)", i, va_list[i]); + rmr->fumem[i] = ib_umem_get(ibmr->pd->uobject->context, + va_list[i], FRMR_PAGE_SIZE, 0, 0); + if (IS_ERR(rmr->fumem[i])) { + ret = (int)PTR_ERR(rmr->fumem[i]); + roce5_free_frmr_umem(rmr->fumem, i - 1); + ROCE_DEV_ERR(rdev, + "Alloc umem failed, func_id(%d) err:%d", + rdev->glb_func_id, ret); + return ret; + } + + /* use valloc allocate a page of memory, only needs need get the a physical address */ + pa_list[i] = sg_dma_address(rmr->fumem[i]->sg_head.sgl); + ROCE_DEV_INFO(rdev, "PA_LIST:%llu", pa_list[i]); + if (pa_list[i] == 0ULL) { + ret = -EINVAL; + roce5_free_frmr_umem(rmr->fumem, i); + ROCE_DEV_ERR(rdev, "Invalid physicaL addr index(%d)", + i); + return ret; + } + } + return 0; +} + +int roce5_map_user_frmr_sg(struct ib_mr *ibmr, unsigned int page_num, + struct ib_udata *udata) +{ + struct roce5_mr *rmr = NULL; + struct roce5_device *rdev = NULL; + int ret; + unsigned long long *va_list = NULL; + unsigned long long *pa_list = NULL; + u32 len; + + ret = roce5_frmr_check_user_param(ibmr, page_num, udata); + if (ret) { + return ret; + } + + len = page_num * sizeof(unsigned long long); + rmr = to_roce5_mr(ibmr); + rdev = to_roce5_dev(ibmr->device); + ret = roce5_frmr_alloc_addr_list(rdev, &va_list, &pa_list, len); + if (ret) { + return ret; + } + + ROCE_DEV_INFO(rdev, "func_id(%d), page_num(%u)", rdev->glb_func_id, + page_num); + + ret = ib_copy_from_udata((void *)va_list, udata, len); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to copy udata, ret(%d)", ret); + goto err_free; + } + + ret = process_va_list_and_allocate_umem(rmr, rdev, ibmr, va_list, + pa_list, page_num); + if (ret != 0) { + goto err_free; + } + + /* 讲 pa information pass to user mode */ + if (ib_copy_to_udata(udata, pa_list, len)) { + ret = -EFAULT; + ROCE_DEV_ERR(rdev, "Failed to copy udata, func_id(%d)", + rdev->glb_func_id); + goto err_free; + } + + rmr->fumem_nr = page_num; + roce5_frmr_free_addr_list(va_list, pa_list); + return 0; + +err_free: + rmr->fumem_nr = 0; + roce5_frmr_free_addr_list(va_list, pa_list); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h new file mode 100644 index 000000000..c206ca7f6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MR_CHIP_EXT_H +#define ROCE_MR_CHIP_EXT_H + +#include <linux/types.h> +#include <linux/io-mapping.h> +#include <linux/list.h> +#include <linux/cdev.h> + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> + +int roce5_check_mr_status(struct ib_mr *ibmr, u32 check_mask, + struct ib_mr_status *mr_status); +struct ib_mr *roce5_reg_user_fast_mr(struct ib_pd *ibpd, + struct ib_udata *udata); +int roce5_map_user_frmr_sg(struct ib_mr *ibmr, unsigned int page_num, + struct ib_udata *udata); + +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c new file mode 100644 index 000000000..903da9898 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c @@ -0,0 +1,44 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#include "roce_qp_extension.h" + +void roce5_set_qp_dif_attr(struct roce5_qp *rqp, + const struct ib_qp_init_attr *init_attr, + const struct roce5_device *rdev) +{ + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EN) != 0) { + rqp->signature_en = true; + ROCE_DEV_INFO(rdev, + "func(%d) qp(%u) roce5_create_qp signature_en.", + rdev->glb_func_id, rqp->qpn); + } + + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_DOUBLE_SGL) != 0) { + rqp->double_sgl_mode = 1; + ROCE_DEV_INFO( + rdev, + "func(%d) qp(%u) roce5_create_qp double_sgl_mode.", + rdev->glb_func_id, rqp->qpn); + } + + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EXT_MTU) != 0) { + rqp->ext_mtu = 1; + ROCE_DEV_INFO(rdev, "func(%d) qp(%u) roce5_create_qp ext_mtu.", + rdev->glb_func_id, rqp->qpn); + } + + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EXT_MTU_MODE) != 0) { + rqp->ext_mtu_mode = 1; + ROCE_DEV_INFO(rdev, + "func(%d) qp(%u) roce5_create_qp ext_mtu_mode.", + rdev->glb_func_id, rqp->qpn); + } + + return; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c new file mode 100644 index 000000000..a6323afd0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c @@ -0,0 +1,26 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + + +#include "roce_srq_extension.h" + +void roce5_srq_container_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, struct roce5_device *rdev) +{ + u8 srq_cont_en = 0; + srq_cont_en = init_attr->container_flag; + + rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC); + rsrq->container_flag = srq_cont_en ? init_attr->container_flag : + rdev->cfg_info.srq_container_en; + rsrq->container_mode = + srq_cont_en ? + init_attr->container_mode : + (rsrq->xrc_en ? rdev->cfg_info.xrc_srq_container_mode : + rdev->cfg_info.srq_container_mode); + rsrq->container_warn_th = init_attr->attr.srq_limit; + rsrq->rqe_cnt_th = srq_cont_en ? init_attr->container_warn_th : + rdev->cfg_info.warn_th; + + rsrq->container_size = roce5_get_container_sz(rsrq->container_mode); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h new file mode 100644 index 000000000..b956ce82c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h @@ -0,0 +1,13 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_COMMON +#define HYPER_ROCE_COMMON + +enum roce5_hyper_mode { + ROCE_HYPER_MODE_DISABLE, + ROCE_HYPER_MODE_UDP, + ROCE_HYPER_MODE_AR, +}; + +#endif /* HYPER_ROCE_COMMON */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c new file mode 100644 index 000000000..ca38186cc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c @@ -0,0 +1,511 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/types.h> +#include "hinic5_rdma.h" +#include "rdma_comp.h" +#include "roce_hyper_npu_cmd.h" +#include "roce_npu_cmd.h" +#include "roce_hyper_npu_cmd.h" +#include "comm_defs.h" + +#include "roce.h" +#include "comm_defs.h" +#include "roce_qp.h" +#include "roce_qp_ext.h" +#include "roce_netlink.h" +#include "roce_cqm_cmd.h" +#include "hyper_roce_common.h" +#include "hyper_roce_qp.h" +#include "hyper_roce_netlink.h" + +#define HYPER_ROCE_RDMARC_ENTRY_SIZE_BYTES 32 + +static int roce5_send_cmd_set_qp_hyper_mode(struct roce5_device *rdev, + struct roce5_hyper_set_in *msg_in) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_enable_hyper_roce_s *input = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_enable_hyper_roce_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_enable_hyper_roce_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(msg_in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_ENABLE_HYPER_ROCE_QP); + + input->cmd_info.dw0.bs.hyper_mode = msg_in->mode; + input->cmd_info.dw0.bs.srp_en = msg_in->sack_en; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int +roce5_send_cmd_modify_qp_extend(struct roce5_device *rdev, + struct roce5_hyper_extend_config_in *msg_in) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_enable_hyper_roce_s *input = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_enable_hyper_roce_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_enable_hyper_roce_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(msg_in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_ENABLE_HYPER_ROCE_QP); + + switch (msg_in->attr_mask) { + case IBV_QP_ATTR_EXTEND_UDP_SRC_PORT: + input->cmd_info.dw0.bs.base_src_port = msg_in->udp_src_port; + break; + case IBV_QP_ATTR_EXTEND_HYROCE_FEATURE: + input->cmd_info.dw0.bs.hyper_mode = msg_in->feature.type; + input->cmd_info.dw0.bs.srp_en = msg_in->feature.sack_enable; + break; + case IBV_QP_ATTR_EXTEND_LB_MODE: + input->cmd_info.dw0.bs.lb_mode = msg_in->lb_mode; + break; + case IBV_QP_ATTR_EXTEND_MP_CONFIG: + input->cmd_info.dw1.bs.sport_repeat_num = + msg_in->mp.flowlet_pkg_num; + input->cmd_info.dw1.bs.path_num = msg_in->mp.path_num; + input->cmd_info.dw1.bs.sport_interval = msg_in->mp.interval; + input->cmd_info.dw1.bs.path_rr_enable = + msg_in->mp.path_rr_enable; + input->cmd_info.dw1.bs.cc_mode = msg_in->mp.cc_mode; + break; + case IBV_QP_ATTR_EXTEND_AR_CONFIG: + input->cmd_info.dw1.bs.port_rr_enable = + msg_in->ar.port_rr_enable; + break; + case IBV_QP_ATTR_EXTEND_SACK_CONFIG: + input->cmd_info.dw2.bs.oor_range_m = msg_in->sack.oor_range; + input->cmd_info.dw2.bs.srp_range_n = msg_in->sack.srp_range; + break; + default: + ROCE_DEV_ERR(rdev, "Unsupport attr mask!\n"); + break; + } + + input->cmd_info.dw0.bs.attr_ext_mask = msg_in->attr_mask; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + input->cmd_info.dw1.value = cpu_to_be32(input->cmd_info.dw1.value); + input->cmd_info.dw2.value = cpu_to_be32(input->cmd_info.dw2.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int +roce5_send_cmd_set_qp_msnc(struct roce5_device *rdev, u32 qpn, + struct roce5_qp_hyper_extend *rqp_hyper_ext) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_init_msnc_s *input = NULL; + u64 bitmap_addr = 0; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_init_msnc_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_init_msnc_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_INIT_MSNC_QP); + + input->cmd_info.dw0.bs.sack_bitmap_size = + HYPER_ROCE_TX_SACK_BITMAP_SIZE; + input->cmd_info.dw0.bs.txmsn_entry_size = HYPER_ROCE_MSN_ENTRY_SIZE; + input->cmd_info.dw0.bs.txmsn_table_size = HYPER_ROCE_MSN_TABLE_SIZE; + input->cmd_info.dw0.bs.txmsn_table_gpa_h = + (rqp_hyper_ext->tx_msnc.dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & + 0xffffff; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + input->cmd_info.txmsn_table_gpa_l = + cpu_to_be32((u32)(rqp_hyper_ext->tx_msnc.dma_addr >> + ROCE5_RDMARC_GPA_L_SHIFT)); + + input->cmd_info.dw2.bs.rxmsn_entry_size0 = HYPER_ROCE_MSN_ENTRY_SIZE; + input->cmd_info.dw2.bs.rxmsn_table_size0 = HYPER_ROCE_MSN_TABLE_SIZE; + input->cmd_info.dw2.bs.rxmsn_table_gpa0_h = + (rqp_hyper_ext->rx_msnc.dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & + 0xffffff; + input->cmd_info.dw2.value = cpu_to_be32(input->cmd_info.dw2.value); + + input->cmd_info.rxmsn_table_gpa0_l = + cpu_to_be32((u32)(rqp_hyper_ext->rx_msnc.dma_addr >> + ROCE5_RDMARC_GPA_L_SHIFT)); + + bitmap_addr = (u64)rqp_hyper_ext->rx_msnc.dma_addr + + HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE * + HYPER_ROCE_RDMARC_ENTRY_SIZE_BYTES; + + input->cmd_info.dw4.bs.sack_bitmap_size = + HYPER_ROCE_RX_SACK_BITMAP_SIZE; + input->cmd_info.dw4.bs.rxmsn_entry_size1 = HYPER_ROCE_MSN_ENTRY_SIZE; + input->cmd_info.dw4.bs.rxmsn_table_size1 = HYPER_ROCE_MSN_TABLE_SIZE; + input->cmd_info.dw4.bs.rxmsn_table_gpa1_h = + (bitmap_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & 0xffffff; + input->cmd_info.dw4.value = cpu_to_be32(input->cmd_info.dw4.value); + + input->cmd_info.rxmsn_table_gpa1_l = + cpu_to_be32((u32)(bitmap_addr >> ROCE5_RDMARC_GPA_L_SHIFT)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_INIT_MSNC_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int roce5_alloc_msnc(struct roce5_device *rdev, + struct roce5_qp_hyper_extend *rqp_hyper_ext) +{ + int ret; + ret = roce5_rdma_rdmarc_alloc(rdev->hwdev, + HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE, + &rqp_hyper_ext->tx_msnc); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdmarc, func_id(%u).", + rdev->glb_func_id); + return ret; + } + + ret = roce5_rdma_rdmarc_alloc( + rdev->hwdev, + HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE + + HYPER_ROCE_RXMSN_BITMAP_RDMARC_ALLOC_SIZE, + &rqp_hyper_ext->rx_msnc); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdmarc, func_id(%u).", + rdev->glb_func_id); + roce5_rdma_rdmarc_free(rdev->hwdev, &rqp_hyper_ext->tx_msnc); + return ret; + } + + rqp_hyper_ext->msnc_en = true; + + return 0; +} + +static int32_t roce5_genl_set_qp_hyper_mode(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_hyper_set_in *msg_in = NULL; + struct roce5_qp *rqp = NULL; + struct roce5_qp_hyper_extend *rqp_hyper_ext = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_hyper_set_in) || out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_hyper_set_in *)in; + + if (msg_in->qpn == ROCE_QP_SMI_QP_NUM || + msg_in->qpn == ROCE_QP_GSI_QP_NUM) { + ROCE_DEV_ERR(rdev, "Invalid qpn(%u).", msg_in->qpn); + return -EINVAL; + } + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp."); + return -EINVAL; + } + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state >= IB_QPS_RTR) { + ROCE_DEV_ERR(rdev, "Invalid qp state, current state(%u).", + rqp->qp_state); + ret = -EINVAL; + goto out; + } + + ret = roce5_send_cmd_set_qp_hyper_mode(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + goto out; + } + + rqp_hyper_ext = (struct roce5_qp_hyper_extend *)roce5_qp_get_ext_buf( + rqp, ROCE_EXT_HYPER_ROCE); + if (rqp_hyper_ext == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to the buf of qp hyper extend, qpn(%u).", + rqp->qpn); + ret = -EINVAL; + goto out; + } + + rqp_hyper_ext->mode = msg_in->mode; + rqp_hyper_ext->sack_en = msg_in->sack_en; + + /* ROCE_HYPER_MODE_DISABLE indicates not enable out-of-order receive, not need allocate rdmarc resource; enable sack_en selective retransmission, must enable out-of-order receive */ + if (rqp_hyper_ext->mode != ROCE_HYPER_MODE_DISABLE) { + ret = roce5_alloc_msnc(rdev, rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc msnc, ret(%d).", + ret); + goto out; + } + + ret = roce5_send_cmd_set_qp_msnc(rdev, msg_in->qpn, + rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + goto out; + } + } +out: + mutex_unlock(&rqp->mutex); + roce5_rqp_put(rqp); + return ret; +} + +static int32_t roce5_genl_set_default_multipath_config( + struct roce5_device *rdev, struct roce5_hyper_extend_config_in *msg_in) +{ + int ret = 0; + + msg_in->lb_mode = HYPER_ROCE_DEFAULT_LB_MODE; + msg_in->mp.flowlet_pkg_num = HYPER_ROCE_DEFAULT_MULTI_PKG_NUM; + msg_in->mp.path_num = HYPER_ROCE_DEFAULT_MULTI_PATH_NUM; + msg_in->mp.interval = HYPER_ROCE_DEFAULT_MULTI_INTERVAL; + msg_in->mp.path_rr_enable = HYPER_ROCE_DEFAULT_MULTI_PATHRR_ENABLE; + msg_in->mp.cc_mode = HYPER_ROCE_DEFAULT_MULTI_CC_MODE; + + msg_in->attr_mask = IBV_QP_ATTR_EXTEND_LB_MODE; + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + + msg_in->attr_mask = IBV_QP_ATTR_EXTEND_MP_CONFIG; + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + + return ret; +} + +static int32_t roce5_modify_hyper_config( + struct roce5_device *rdev, struct roce5_hyper_extend_config_in *msg_in, + struct roce5_qp_hyper_extend *rqp_hyper_ext, struct roce5_qp *rqp) +{ + int ret = 0; + + if (msg_in->attr_mask == IBV_QP_ATTR_EXTEND_HYROCE_FEATURE) { + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + + rqp_hyper_ext->mode = (u8)msg_in->feature.type; + rqp_hyper_ext->version = (u8)msg_in->feature.version; + rqp_hyper_ext->sack_en = (u8)msg_in->feature.sack_enable; + + /* ROCE_HYPER_MODE_DISABLE indicates not enable out-of-order receive, not need allocate rdmarc resource; enable sack_en selective retransmission, must enable out-of-order receive */ + if (rqp_hyper_ext->mode != ROCE_HYPER_MODE_DISABLE) { + ret = roce5_alloc_msnc(rdev, rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc msnc, ret(%d).", + ret); + return ret; + } + + ret = roce5_send_cmd_set_qp_msnc(rdev, msg_in->qpn, + rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to send CMDQ, ret(%d)", + ret); + return ret; + } + } + + ret = roce5_genl_set_default_multipath_config(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set default mp config, ret(%d).", + ret); + return ret; + } + } else { + if (rqp_hyper_ext->mode == ROCE_HYPER_MODE_DISABLE) { + ROCE_DEV_ERR(rdev, + "Hyper_roce is disable, current mode(%u).", + rqp_hyper_ext->mode); + ret = -EINVAL; + return ret; + } + + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + } + + return ret; +} + +static int32_t roce5_genl_modify_qp_extend(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_hyper_extend_config_in *msg_in = NULL; + struct roce5_qp *rqp = NULL; + struct roce5_qp_hyper_extend *rqp_hyper_ext = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_hyper_extend_config_in) || + out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_hyper_extend_config_in *)in; + + if (msg_in->qpn == ROCE_QP_SMI_QP_NUM || + msg_in->qpn == ROCE_QP_GSI_QP_NUM) { + ROCE_DEV_ERR(rdev, "Invalid qpn(%u).", msg_in->qpn); + return -EINVAL; + } + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp."); + return -EINVAL; + } + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state >= IB_QPS_RTR) { + ROCE_DEV_ERR(rdev, "Invalid qp state, current state(%u).", + rqp->qp_state); + ret = -EINVAL; + goto out; + } + + rqp_hyper_ext = (struct roce5_qp_hyper_extend *)roce5_qp_get_ext_buf( + rqp, ROCE_EXT_HYPER_ROCE); + if (rqp_hyper_ext == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to the buf of qp hyper extend, qpn(%u).", + rqp->qpn); + ret = -EINVAL; + goto out; + } + + ret = roce5_modify_hyper_config(rdev, msg_in, rqp_hyper_ext, rqp); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to modify hyper roce config."); + } +out: + mutex_unlock(&rqp->mutex); + roce5_rqp_put(rqp); + return ret; +} + +static roce5_netlink_cmd_process g_cmd_process[] = { + { USRNL_ROCE_CMD_HYPER_QP_SET, roce5_genl_set_qp_hyper_mode }, + { USRNL_ROCE_CMD_MODIFY_QP_EXTEND, roce5_genl_modify_qp_extend }, +}; + +int roce5_hyper_netlink_init(void) +{ + int ret; + + ret = roce5_netlink_register_cbs( + g_cmd_process, + sizeof(g_cmd_process) / sizeof(roce5_netlink_cmd_process)); + if (ret != 0) { + ROCE_ERR( + "Failed to register hyper roce netlink callbacks, ret(%d).", + ret); + return ret; + } + + return 0; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h new file mode 100644 index 000000000..27de1b070 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_NETLINK_H +#define HYPER_ROCE_NETLINK_H + +#include <linux/types.h> + +#define HYPER_ROCE_DEFAULT_LB_MODE 1 +#define HYPER_ROCE_DEFAULT_MULTI_PKG_NUM 4 +#define HYPER_ROCE_DEFAULT_MULTI_PATH_NUM 4 +#define HYPER_ROCE_DEFAULT_MULTI_INTERVAL 1 +#define HYPER_ROCE_DEFAULT_MULTI_PATHRR_ENABLE 0 +#define HYPER_ROCE_DEFAULT_MULTI_CC_MODE 2 + +struct roce5_hyper_set_in { + uint32_t qpn; + uint8_t mode; /* @see enum roce_hyper_mode */ + uint8_t sack_en; + uint8_t rsvd[2]; +}; + +int roce5_hyper_netlink_init(void); + +enum ibv_qp_attr_extend_mask { + IBV_QP_ATTR_EXTEND_UDP_SRC_PORT = 1 << 0, /* source UDP port number */ + IBV_QP_ATTR_EXTEND_HYROCE_FEATURE = 1 << 1, /* high-order RoCE of feature */ + IBV_QP_ATTR_EXTEND_LB_MODE = 1 << 2, /* load balance mode */ + IBV_QP_ATTR_EXTEND_MP_CONFIG = 1 << 3, /* Multi-Path multipath mode configuration */ + IBV_QP_ATTR_EXTEND_AR_CONFIG = + 1 << 4, /* Adaptive-Routing multipath mode configuration */ + IBV_QP_ATTR_EXTEND_SACK_CONFIG = + 1 << 5, /* Selective Ack selective retransmission parameters configuration */ +}; + +/* high-order RoCE feature configuration structure */ +struct ibv_hyroce_feature { + uint32_t type; /* high-order RoCE type, value range: enum ibv_hyroce_feature_type */ + uint32_t version; /* high-order RoCE version, value range: enum ibv_hyroce_feature_version */ + uint32_t sack_enable; /* selective retransmission switch, 0- disable, 1- enable */ + + uint32_t resv[32]; /* reserved field, used for not to extension feature */ +}; + +/* multipath (Multi-Path) configuration parameters structure */ +struct ibv_mp_config { + uint32_t flowlet_pkg_num; /* each sub-flow (flowlet) of packet count, used for flow splitting */ + uint32_t path_num; /* multipath of path count */ + uint32_t interval; /* UDP port number increment interval, used for distinguish different path */ + uint32_t path_rr_enable; /* path whether supports poll (RR) select network port, 0- not support, 1- supports */ + uint32_t cc_mode; + + uint32_t resv[32]; /* reserved field, used for not to extension */ +}; + +/* adaptive path by (Adaptive Routing) configuration structure */ +struct ibv_ar_config { + uint32_t port_rr_enable; /* whether enable network port side port poll (RR) per-packet function, 0- disable, 1- enable */ + + uint32_t resv[32]; /* reserved field, used for not to extension */ +}; + +/* selective retransmission (SACK - Selective Ack) configuration structure */ +struct ibv_sack_config { + uint32_t srp_range; /* TX(send) direction maximum repass packet count */ + uint32_t oor_range; /* RX(receive) direction out-of-order repass window of packet count */ + + uint32_t resv[32]; /* reserved field, used for not to extension */ +}; + +struct roce5_hyper_extend_config_in { + uint32_t qpn; + uint32_t udp_src_port; /* source UDP port number */ + struct ibv_hyroce_feature feature; /* high-order RoCE feature */ + uint32_t lb_mode; /* load balance mode, value range: enum ibv_lb_mode type */ + struct ibv_mp_config mp; /* multipath (Multi-Path) configuration */ + struct ibv_ar_config ar; /* adaptive path by (Adaptive Routing) configuration */ + struct ibv_sack_config sack; /* selective retransmission (SACK) configuration */ + enum ibv_qp_attr_extend_mask attr_mask; +}; + +#endif /* HYPER_ROCE_NETLINK_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c new file mode 100644 index 000000000..1cc831f16 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "roce_qp.h" +#include "roce_restore.h" +#include "hyper_roce_common.h" +#include "hyper_roce_qp.h" + +int roce5_hyper_qp_init(void *parent, void *ext_data) +{ + struct roce5_qp_hyper_extend *hyper_qp_ext = + (struct roce5_qp_hyper_extend *)ext_data; + hyper_qp_ext->mode = ROCE_HYPER_MODE_DISABLE; + hyper_qp_ext->sack_en = false; + return 0; +} + +void roce5_hyper_qp_cleanup(void *parent, void *ext_data) +{ + struct roce5_qp *rqp = (struct roce5_qp *)parent; + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + struct roce5_qp_hyper_extend *hyper_qp_ext = + (struct roce5_qp_hyper_extend *)ext_data; + + if (hyper_qp_ext->msnc_en) { + roce5_rdma_rdmarc_free(rdev->hwdev, &(hyper_qp_ext->tx_msnc)); + roce5_rdma_rdmarc_free(rdev->hwdev, &(hyper_qp_ext->rx_msnc)); + } +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h new file mode 100644 index 000000000..52c38f0a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h @@ -0,0 +1,35 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_QP_H +#define HYPER_ROCE_QP_H + +#include <linux/types.h> +#include "base_type.h" +#include "hinic5_rdma.h" + +/* Entry_size for TXMSN/RXMSN Table 00: 32B 01: 64B Others: reserved. */ +#define HYPER_ROCE_MSN_ENTRY_SIZE 0u +/* Entry number for TXMSN/RXMSN Table 00: 256 01: 512 10: 1024 11: 2048 Others: reserved. */ +#define HYPER_ROCE_MSN_TABLE_SIZE 0u +/* 000: 128bit 001: 256bit 010: 512bit 011: 1024bit 100: 2048bit Others: reserved. */ +#define HYPER_ROCE_TX_SACK_BITMAP_SIZE 0u +/* 011: 1024bit 100: 2048bit Others: reserved. */ +#define HYPER_ROCE_RX_SACK_BITMAP_SIZE 3u +#define HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE \ + (256u << (HYPER_ROCE_MSN_ENTRY_SIZE & 0x3)) +#define HYPER_ROCE_RXMSN_BITMAP_RDMARC_ALLOC_SIZE (4) + +struct roce5_qp_hyper_extend { + u8 mode; /* @see enum roce_hyper_mode */ + u8 version; + u8 sack_en; + struct rdma_rdmarc tx_msnc; + struct rdma_rdmarc rx_msnc; + bool msnc_en; +}; + +int roce5_hyper_qp_init(void *parent, void *ext_data); +void roce5_hyper_qp_cleanup(void *parent, void *ext_data); + +#endif /* HYPER_ROCE_QP_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c new file mode 100644 index 000000000..fe4e5cf64 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c @@ -0,0 +1,60 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "roce.h" +#include "roce_dfx.h" +#include "hyper_roce_tool_pktdrop.h" +#include "hyper_roce_tool.h" + +enum roce5_hyper_cmds { + ROCE_HYPER_CMD_START = 1, + ROCE_HYPER_CMD_START_DROP_QP, + ROCE_HYPER_CMD_STOP_DROP_QP, + ROCE_HYPER_CMD_MAX +}; + +const roce5_adm_func_t hyper_roce5_adm_func[] = { + [ROCE_HYPER_CMD_START_DROP_QP] = roce5_hyper_tool_start_drop_qp, + [ROCE_HYPER_CMD_STOP_DROP_QP] = roce5_hyper_tool_stop_drop_qp, +}; + +int roce5_hyper_tool_dispatch(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct roce5_hyper_cmd_head *head = NULL; + const void *data_ptr = NULL; + u32 data_len; + int ret; + + if (in_size < ROCE_HYPER_CMD_HEAD_SIZE) { + ROCE_DEV_ERR(rdev, "Invalid command length(%d).", in_size); + return -EINVAL; + } + + data_ptr = (void *)((char *)buf_in + ROCE_HYPER_CMD_HEAD_SIZE); + data_len = in_size - ROCE_HYPER_CMD_HEAD_SIZE; + head = data_ptr; + + if (head->sub_cmd < ROCE_HYPER_CMD_START || + head->sub_cmd >= ROCE_HYPER_CMD_MAX) { + ROCE_DEV_ERR(rdev, "Unknown hyper roce command(%d).", + head->sub_cmd); + return -EINVAL; + } + + if (hyper_roce5_adm_func[head->sub_cmd] == NULL) { + ROCE_DEV_ERR(rdev, "The callback of cmd(%d) is NULL.", + head->sub_cmd); + return -EINVAL; + } + + ret = hyper_roce5_adm_func[head->sub_cmd](rdev, buf_in, in_size, + buf_out, out_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to execute the callback of cmd(%d)", + head->sub_cmd); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h new file mode 100644 index 000000000..2438bbb78 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h @@ -0,0 +1,21 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_TOOL_H +#define HYPER_ROCE_TOOL_H + +#include "base_type.h" +#include "roce.h" + +#define ROCE_CMD_HYPER 0x220 + +struct roce5_hyper_cmd_head { + u32 sub_cmd; +}; + +#define ROCE_HYPER_CMD_HEAD_SIZE sizeof(struct roce5_hyper_cmd_head) + +int roce5_hyper_tool_dispatch(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* HYPER_ROCE_TOOL_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c new file mode 100644 index 000000000..7d305e069 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c @@ -0,0 +1,142 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/types.h> +#include "hinic5_rdma.h" +#include "rdma_comp.h" +#include "roce_npu_cmd.h" +#include "roce_hyper_npu_cmd.h" +#include "comm_defs.h" + +#include "roce.h" +#include "roce_qp.h" +#include "roce_qp_ext.h" +#include "roce_netlink.h" +#include "roce_cqm_cmd.h" +#include "hyper_roce_tool_pktdrop.h" + +int roce5_hyper_tool_start_drop_qp(struct roce5_device *rdev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret = 0; + u32 drop_pkt_ratio = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_modify_qp_drop_percent_s *input = NULL; + struct pktdrop_cmd_inbuf *in = NULL; + + if (in_size < sizeof(struct pktdrop_cmd_inbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct pktdrop_cmd_inbuf)); + return -EINVAL; + } + + if (*out_size < sizeof(union pktdrop_outbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, (int)sizeof(union pktdrop_outbuf)); + return -ENOMEM; + } + + in = (struct pktdrop_cmd_inbuf *)buf_in; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_qp_drop_percent_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + drop_pkt_ratio = in->data.attr.dw15.bs.drop_pkt_ratio; + input = (roce_uni_cmd_modify_qp_drop_percent_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_MODIFY_QP_DROP_PERCENT); + + input->cmd_info.dw0.bs.drop_precent = drop_pkt_ratio; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int roce5_hyper_tool_stop_drop_qp(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret = 0; + u32 drop_pkt_ratio = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_modify_qp_drop_percent_s *input = NULL; + struct pktdrop_cmd_inbuf *in = NULL; + + if (in_size < sizeof(struct pktdrop_cmd_inbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct pktdrop_cmd_inbuf)); + return -EINVAL; + } + + if (*out_size < sizeof(union pktdrop_outbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, (int)sizeof(union pktdrop_outbuf)); + return -ENOMEM; + } + + in = (struct pktdrop_cmd_inbuf *)buf_in; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_qp_drop_percent_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_modify_qp_drop_percent_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_MODIFY_QP_DROP_PERCENT); + + input->cmd_info.dw0.bs.drop_precent = drop_pkt_ratio; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h new file mode 100644 index 000000000..4d3dbd1d5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_TOOL_PKTDROP_H +#define HYPER_ROCE_TOOL_PKTDROP_H +#include "hyper_roce_pub_cmd.h" +#include "roce.h" + +struct pktdrop_cmd_inbuf { + u32 cmd_type; + u32 hyper_roce_sub_cmd; + u32 mode; + u32 qpn; + u32 drop_level; + struct pktdrop_l2d_tbl_dfx data; +}; + +int roce5_hyper_tool_start_drop_qp(struct roce5_device *rdev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce5_hyper_tool_stop_drop_qp(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce.h new file mode 100644 index 000000000..3fbc3343b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce.h @@ -0,0 +1,665 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_H +#define ROCE_H + +#include <linux/types.h> +#include <linux/io-mapping.h> +#include <linux/list.h> + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> +#include "roce_verbs_compat.h" + +#include "hinic5_hw.h" +#include "hinic5_crm.h" +#include "hinic5_hw_cfg.h" +#include "hinic5_lld.h" +#include "hinic5_cqm.h" +#include "hinic5_rdma.h" +#include "nic_cfg_comm.h" +#include "hmm_common.h" + +#include "roce_infra.h" +#include "roce_db.h" +#include "roce_sysfs.h" + +#include "roce_uld_kernel_api.h" +#include "roce_npu_cmd_defs.h" +#include "roce_npu_cmd_ext_defs.h" + +#define HIROCE5_DRV_NAME "roce5_drv" +#define HIROCE5_DRV_AUTHOR "Huawei Technologies CO., Ltd" +#define HIROCE5_DRV_DESC \ + "Huawei(R) Intelligent Network Interface Card, RoCE Driver" +#define HIROCE5_DRV_VERSION GLOBAL_VERSION_STR + +#define RDMA_DRIVER_HRN 8395 +#define ROCE_IB_UVERBS_ABI_VERSION 1 +#define ROCE_ULD_DEV_NAME_LEN 16 + +#define MAX_CEQ_NEED 256 + +#define DEV_ADDR_FIRST_BYTE_VAL_MASK 2 + +#define ROCE_NODE_DESC_LEN 5 + +#define ROCE_SQ_WQEBB_SIZE 64 + +#define ROCE_GID_LEN 16 + +#define ROCE_PCI_CFG_REGS_BAR0 0 +#define ROCE_PCI_CFG_REGS_BAR3 3 + +#define DEFAULT_ROCE_DEV_NODE_PRI 0666 + +#define ROCE_MAX_PORT_NUM 8 + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#define MAX_ROCE_DEV (28 * 4) + +#define CHECK_1LAYER_IS_NULL(ptr) ((ptr) == NULL) +#define CHECK_2LAYER_IS_NULL(ptr, l2_member) \ + (CHECK_1LAYER_IS_NULL(ptr) || (((ptr)->l2_member) == NULL)) + +static inline bool roce5_func_is_vf(void *hwdev) +{ + return hinic5_func_type(hwdev) == TYPE_VF; +} + +static inline bool roce5_func_is_pf(void *hwdev) +{ + return hinic5_func_type(hwdev) == TYPE_PF; +} + +enum { + ROCE5_1_PORT_NUM = 1, //EVB single die 形mode + ROCE5_2_PORT_NUM = 2, + ROCE5_4_PORT_NUM = 4 +}; + +enum { + ROCE5_25G_PORT_SPEED = 25, + ROCE5_100G_PORT_SPEED = 100, + ROCE5_200G_PORT_SPEED = 200, + ROCE5_400G_PORT_SPEED = + 201, //because MPU bit width not 够衍generate of custom enum, refer to port_speed_mode_e + ROCE5_800G_PORT_SPEED = 202 +}; + +enum { + ROCE5_INVALID_HCA = -1, + ROCE5_2_100G_HCA = 0, + ROCE5_4_25G_HCA = 1, + ROCE5_2_25G_HCA = 2, + ROCE5_2_200G_HCA = 3, + ROCE5_1_100G_HCA = 4, + ROCE5_2_400G_HCA = 5, + ROCE5_4_200G_HCA = 6 +}; + +enum ROCE5_100G_BW_PARAM_E { + ROCE5_100G_CIR = 46500000, + ROCE5_100G_PIR = 52500000, + ROCE5_100G_CNP = 100 +}; + +enum ROCE5_25G_BW_PARAM_E { + ROCE5_25G_CIR = 23200000, + ROCE5_25G_PIR = 25500000, + ROCE5_25G_CNP = 3 +}; + +enum roce_bitshift_e { + BYTES_TO_2B_SHIFT = 1, + BYTES_TO_4B_SHIFT, + BYTES_TO_8B_SHIFT, + BYTES_TO_16B_SHIFT, + BYTES_TO_32B_SHIFT +}; + +/* BIG/LITTLE ENGIAN switch */ +#ifdef HW_CONVERT_ENDIAN +#define roce5_convert_be32(val) (val) +#define roce5_convert_cpu32(val) (val) +#else +#define roce5_convert_be32(val) cpu_to_be32(val) +#define roce5_convert_cpu32(val) be32_to_cpu(val) +#endif + +enum roce5_aeq_type { + /* ofed err */ + OFED_ET_PATH_MIG = 0, + OFED_ET_COMM_EST, + OFED_ET_SQ_DRAINED, + OFED_ET_SRQ_QP_LAST_WQE, + OFED_ET_WQ_CATAS_ERR, + + OFED_ET_PATH_MIG_FAILED = 5, + OFED_ET_WQ_INVAL_REQ_ERR, + OFED_ET_WQ_ACCESS_ERR, + OFED_ET_CQ_ERR, + OFED_ET_SRQ_LIMIT, + OFED_ET_SRQ_CATAS_ERR, + + /* non ofed err */ + NON_OFED_ET_QPC_LOOKUP_ERR = 11, + NON_OFED_ET_OTHER_TYPE_ERR, + + /* non ofed extend err */ + NON_OFED_ET_DEBUG_ERR = 32, + + /* NOF AA err */ + OFED_NOF_AA_QP_DISCONNECT = 64, + OFED_NOF_AA_MASTER_CHANGE, + + INVAL_ET_ERR +}; + +enum { + ROCE_CMD_TIME_CLASS_A = 30000, //30s, TODO: use ASIC card after need modify is 3s + ROCE_CMD_TIME_CLASS_B = 40000, //40s, TODO: use ASIC card after need modify is 4s + ROCE_CMD_TIME_CLASS_C = 50000 //50s, TODO: use ASIC card after need modify is 5s +}; + +enum roce5_ctrl_status { ROCE5_PORT_EVENT = BIT(0) }; + +#define ROCE_DEFAULT_PORT_NUM 1 + +#define ROCE_UVERBS_CMD_BASE_MASK \ + ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \ + (1ULL << IB_USER_VERBS_CMD_REG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_CLOSE_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_QP)) + +#define ROCE_UVERBS_EXT_CMD_MASK \ + ((1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ) | \ + (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_QP)) + +enum roce5_load_balance_mode_e { + ROCE_LB_MODE_0 = 0, + ROCE_LB_MODE_1, + ROCE_LB_MODE_2, + ROCE_LB_MODE_N, +}; + +enum push_ofed_device_status { + ROCE_DEV_STATUS_NORMAL = 0, + ROCE_DEV_STATUS_CMDQ_TIMEOUT +}; + +enum roce5_func_state { ROCE_FUNC_DISABLE = 0, ROCE_FUNC_ENABLE }; + +enum { + ROCE_Cl_TYPE_QPC = 0x0, /* cl_start: 0x0, cl_end: 0x7f, cl_size: 0 */ + ROCE_CL_TYPE_MPT = 0x1, /* cl_start: 0x150, cl_end: 0x15f, cl_size: 0 */ + ROCE_CL_TYPE_SQ_WQE = + 0x2, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_RQ_WQE = + 0x3, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_CQC_SRQC = + 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_RDMARC = + 0x5, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_CMTT = + 0x6, /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ + ROCE_CL_TYPE_DMTT = 0x7 /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ +}; + +#define XRC_CONTAINER_FLAG ((u32)(1L << 20)) + +struct roce5_buf_list { + void *buf; + dma_addr_t map; +}; + +struct roce5_buf { + struct roce5_buf_list direct; + struct roce5_buf_list *page_list; + int nbufs; + + int npages; + int page_shift; +}; + +struct shadow_hw_db { + u32 page_idx; + u32 refcnt; +}; + +struct roce5_ucontext { + struct ib_ucontext ibucontext; + void __iomem *db_map; + void __iomem *dwqe_map; + u64 db_dma_addr; + u64 dwqe_dma_addr; + struct list_head db_page_list; + struct mutex db_page_mutex; + + struct shadow_hw_db shadow_db_ctx; +}; + +struct roce5_qp_cnt { + struct mutex cur_qps_mutex; + u32 alloc_qp_cnt; + u32 del_qp_cnt; +}; + +struct roce5_dev_hw_info { + int config_num_ports; /* Number of ports from configuration file */ + int hca_type; /* HCA version: 4x25G or 2x100G */ + u8 phy_port; + u8 ep_id; /* EP ID */ + u8 cpu_endian; + u8 rsvd; + + bool is_vf; +}; + +struct roce5_dev_cfg_info { + u8 scence_id; /* load scenes ID as aa_en */ + u8 lb_en; /* load balance enable */ + u8 lb_mode; /* load balance mode */ + struct { + u8 pcie_atomic_en : 1; + u8 rsvd : 7; + }; + + u8 srq_container_en; + u8 srq_container_mode; + u8 xrc_srq_container_mode; + u8 warn_th; + + u8 fake_en; + u8 page_bit; + u8 pf_start_bit; + u8 pf_end_bit; + + u8 port_num; /* cfg data port num */ + u8 host_num; /* cfg data host num */ + u8 rsvd1[2]; + roce_get_chip_ability_s feature_cap; +}; +#define ROCE_LB_QPN_MAX \ + 1 //less qp, same a smf on of qp will distribution to same a icq queue, because this decrease low lb of qp count +struct roce5_qp_lb_scqn_ctrl { + struct mutex qp_lb_scqn_ctrl_mutex; + //record device in lb of qpn + u32 qp[ROCE_LB_QPN_MAX]; +}; + +struct roce5_cq_ctrl { + struct mutex cq_ctrl_mutex; + u8 alloc_cq_cnt; + u8 high_perf_cq_cnt; + u8 rsvd_cq_step; + unsigned long cq_bitmap; +}; + +enum roce5_device_type { ROCE_DEV_TYPE_UNKNOWN, ROCE_DEV_TYPE_HI1825_V100 }; + +enum roce5_perf_cq_step { ROCE_1825V100_STEP = 8 }; + +#define ROCE_IS_DEV_1825V100(rdev) \ + (((rdev)->device_type) == ROCE_DEV_TYPE_HI1825_V100) + +#ifdef __ROCE_DFX__ +struct roce5_list { + struct list_head node; + struct mutex node_lock; +}; +struct roce5_device_res { + struct roce5_list mr_list; +}; +#endif //__ROCE_DFX__ + +//uld service can extension structure space +#define ROCE_DEVICE_ULD_DEF_LEN 64 +#define ROCE_DEVICE_MAGIC 0xaabacada +struct roce5_device_uld_def { + u8 self_def[ROCE_DEVICE_ULD_DEF_LEN]; +}; + +struct roce5_device { + struct ib_device ib_dev; + struct hinic5_lld_dev *lld_dev; + struct net_device *ndev; + struct device *dev; + void *hwdev; + void *hwdev_hdl; + //struct dev_version_info dev_ver; /*version info */ + struct hinic5_board_info board_info; + + void __iomem *kernel_db_map; + void __iomem *kernel_dwqe_map; + + spinlock_t node_desc_lock; + struct mutex cap_mask_mutex; + struct rdma_service_cap rdma_cap; + + u8 mac[6]; /* Mac addr. */ + u16 glb_func_id; + unsigned long status; + int ceq_num; + int ceqn[MAX_CEQ_NEED]; + int try_times; + int destory_flow_timeout; + bool ib_active; + enum roce5_device_type device_type; + + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; + u8 mpt_xid_mode; + struct roce5_qp_lb_scqn_ctrl qp_lb_scqn_ctrl; + struct roce5_dev_hw_info hw_info; /* Hw info read from nic/pcie */ + struct roce5_dev_cfg_info cfg_info; /* Cfg data info read from MPU */ + +#ifdef ROCE_BONDING_EN + enum ib_port_state port_state; + struct roce5_bond_device *bond_dev; + atomic_t next_port; +#endif + + struct mutex mac_vlan_mutex; + struct list_head mac_vlan_list_head; + struct net_device **gid_dev; + + struct roce5_ecn_ctx ecn_ctx; + struct roce5_dfx_ctx dfx_ctx; + + struct roce5_qp_cnt qp_cnt; + struct roce5_cq_ctrl cqn_ctrl; + + struct srcu_struct mr_srcu; + atomic_t num_prefetch; + struct completion comp_prefetch; + enum push_ofed_device_status dev_status_to_ofed; + + spinlock_t reset_flow_resource_lock; + struct list_head qp_list; + + void *fake_data_buf; + dma_addr_t fake_data_page_addr; + void *pri_roce_dev + [ROCE_MAX_PORT_NUM]; //mailbox, ppf store all pf's roce device, to improve the effiency. + int hca_vf_mode; //HCA uses mode to init MAC/IP vf table + bool local_en; //STUB local + bool full_qp_spec; + bool huge_page_en; + bool is_vroce; + u8 cc_param_version; + u8 shadow; + u8 rsvd[2]; + u32 shadow_vfid; + u16 shadow_vf_num; + + rwlock_t rwlock_func_tbl_bond; + u32 func_tbl_bond; + +#ifdef __ROCE_DFX__ + struct roce5_device_res *device_res; + struct roce5_ucontext *ucontext; +#endif //__ROCE_DFX__ + + struct roce5_device_uld_def uld_def; + u32 rdev_magic; /* used for uld check, preventstop踩踏 */ +}; + +int roce5_db_map_user(struct roce5_ucontext *context, unsigned long virt, + struct roce5_db *db, u8 shadow_db); +void roce5_db_unmap_user(struct roce5_ucontext *context, struct roce5_db *db, + u8 shadow_db); + +int roce5_buf_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct tag_cqm_buf *buf); +int roce5_umem_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct ib_umem *umem, bool use_huge); + +int roce5_query_device(struct ib_device *ibdev, struct ib_device_attr *props, + struct ib_udata *uhw); + +int roce5_query_port_compat(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props); + +int roce5_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props); + +enum rdma_link_layer roce5_port_link_layer_compat(struct ib_device *ibdev, + u8 port_num); + +enum rdma_link_layer roce5_port_link_layer(struct ib_device *ibdev, + u32 port_num); + +int roce5_query_gid_compat(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid); + +int roce5_query_gid(struct ib_device *ibdev, u32 port, int index, + union ib_gid *gid); + +int roce5_query_pkey_compat(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey); + +int roce5_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); + +int roce5_modify_device(struct ib_device *ibdev, int mask, + struct ib_device_modify *props); + +int roce5_modify_port_compat(struct ib_device *ibdev, u8 port, int mask, + struct ib_port_modify *props); + +int roce5_modify_port(struct ib_device *ibdev, u32 port, int mask, + struct ib_port_modify *props); + +int roce5_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma); + +int roce5_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); + +int roce5_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); + +int roce5_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); + +int roce5_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int roce5_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +void roce5_dealloc_ucontext(struct ib_ucontext *ibcontext); + +int roce5_alloc_ucontext(struct ib_ucontext *ibucontext, + struct ib_udata *udata); + +int roce5_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); + +int roce5_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); +int roce5_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); + +int roce5_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); + +int roce5_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int roce5_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period); + +int roce5_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); + +int roce5_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); + +int roce5_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); + +struct ib_qp *roce5_create_qp_compat(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata); + +int roce5_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); + +int roce5_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); + +int roce5_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_destroy_ah(struct ib_ah *ibah, u32 flags); + +int roce5_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + +int roce5_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + +int roce5_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); + +int roce5_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); + +struct ib_mr *roce5_get_dma_mr(struct ib_pd *ibpd, int access); +struct ib_mr *roce5_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, + u32 max_num_sg); +struct ib_mr *roce5_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, + u64 virt_addr, int access, + struct ib_udata *udata); + +int roce5_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); + +int roce5_dealloc_mw(struct ib_mw *ibmw); + +int roce5_port_immutable_compat(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); +int roce5_port_immutable(struct ib_device *ibdev, u32 port_num, + struct ib_port_immutable *immutable); +int roce5_query_device_status(struct ib_device *ibdev, int *dev_status); + +int roce5_init_sysfs(struct roce5_device *rdev); +void roce5_remove_sysfs(struct roce5_device *rdev); + +struct net_device *roce5_ib_get_netdev_compat(struct ib_device *ibdev, + u8 port_num); +struct net_device *roce5_ib_get_netdev(struct ib_device *ibdev, u32 port_num); + +void roce5_async_event(void *svc_hd, u8 event_type, u8 *val); +u8 roce5_async_event_level(void *svc_hd, u8 event_type, u8 *val); +void roce5_ifconfig_up_down_event_report(struct roce5_device *rdev, + u8 net_event); +int roce5_register_netdev_notifier(void); +void roce5_unregister_netdev_notifier(void); +void roce5_clean_vlan_device_mac(struct roce5_device *rdev); +void roce5_clean_real_device_mac(struct roce5_device *rdev); + +int roce5_ib_add_gid(const struct ib_gid_attr *attr, + __always_unused void **context); + +int roce5_ib_del_gid(const struct ib_gid_attr *attr, + __always_unused void **context); + +/* OFA - OpenFabrics Alliance */ +struct ib_pd *roce5_alloc_pd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata); + +int roce5_dealloc_pd_ofa(struct ib_pd *ibpd); + +struct ib_srq *roce5_create_srq_ofa(struct ib_pd *ibpd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); + +struct ib_ucontext *roce5_alloc_ucontext_ofa(struct ib_device *ibdev, + struct ib_udata *udata); + +int roce5_dealloc_ucontext_ofa(struct ib_ucontext *ibcontext); + +struct ib_xrcd *roce5_alloc_xrcd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata); + +int roce5_dealloc_xrcd_ofa(struct ib_xrcd *ibxrcd); + +int roce5_destroy_srq_ofa(struct ib_srq *ibsrq); + +int roce5_post_srq_recv_ofa(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +int roce5_post_recv_ofa(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +struct ib_cq *roce5_create_cq_ofa(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, + struct ib_udata *udata); + +int roce5_destroy_cq_ofa(struct ib_cq *ibcq); + +struct ib_ah *roce5_create_ah_ofa(struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata); + +int roce5_destroy_ah_ofa(struct ib_ah *ibah); + +struct ib_mw *roce5_alloc_mw_ofa(struct ib_pd *ibpd, enum ib_mw_type type, + struct ib_udata *udata); + +int roce5_dealloc_mw_ofa(struct ib_mw *ibmw); + +int roce5_dereg_mr_ofa(struct ib_mr *ibmr); + +int roce5_destroy_qp_ofa(struct ib_qp *ibqp); + +int roce5_ib_add_gid_ofa(const union ib_gid *gid, + const struct ib_gid_attr *attr, + __always_unused void **context); + +int roce5_g_rdev_add(struct roce5_device *rdev); + +void roce5_g_rdev_remove(struct roce5_device *rdev); + +struct roce5_device *roce5_get_rdev_by_card_func_id(uint64_t card_id, + uint16_t func_id); + +struct roce5_device *roce5_get_rdev_by_name(const char *dev_name); + +#endif //ROCE_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h new file mode 100644 index 000000000..da2b4f65a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h @@ -0,0 +1,87 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_COMPAT_H +#define ROCE_COMPAT_H + +#include <linux/delay.h> +#include <linux/types.h> + +#define ROCE_LIKELY(x) likely(x) + +#define ROCE_UNLIKELY(x) unlikely(x) + +static inline u32 roce_ilog2(u32 n) +{ + return ilog2(n); +} +#define ROCE_ILOG2(n) roce_ilog2(n) + +static inline u32 roce_roundup_pow_of_two(u32 n) +{ + return roundup_pow_of_two(n); +} +#define ROCE_ROUNDUP_POW_OF_TWO(n) roce_roundup_pow_of_two(n) + +#ifndef _lint +#define ROCE_MDELAY(n) mdelay(n) +#define ROCE_UDELAY(n) udelay(n) +#else +#define ROCE_MDELAY(n) ((void)n) +#define ROCE_UDELAY(n) ((void)n) +#endif + +#define US_PERF_DELAY 100 +#define ROCE_DELAY() ROCE_UDELAY(US_PERF_DELAY) + +#define ROCE_ALIGN(a, b) ALIGN(a, b) +#define ROCE_MAX(a, b) max(a, b) +#define ROCE_MIN(a, b) min(a, b) +#define ROCE_DIV_ROUND_UP(a, b) DIV_ROUND_UP(a, b) +#define ROCE_FLS(n) fls(n) + +#define ROCE_MEMCMP(a, b, count) memcmp(a, b, count) +#define ROCE_IO_MAPPING_MAP_WC(mapping, offset) \ + io_mapping_map_wc(mapping, offset) +#define ROCE_IOREMAP(phys_addr, size) ioremap(phys_addr, size) +#define ROCE_IOUNMAP(addr) iounmap(addr) +#define ROCE_IO_MAPPING_UNMAP(vaddr) io_mapping_unmap(vaddr) + +#ifndef wc_wmb + +#if defined(__i386__) +#define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") +#elif defined(__x86_64__) +#define wc_wmb() asm volatile("sfence" ::: "memory") +#elif defined(__ia64__) +#define wc_wmb() asm volatile("fwb" ::: "memory") +#else +#define wc_wmb() wmb() +#endif + +#endif + +typedef struct tag_roce_verbs_cmd_com { + union { + u32 value; + + struct { + u32 version : 8; + u32 rsvd : 8; + u32 cmd_bitmask : 16; //CMD_TYPE_BITMASK_E + } bs; + } dw0; + + u32 index; //qpn/cqn/srqn/mpt_index/gid idx +} roce_verbs_cmd_com_s; + +typedef struct tag_roce_xq_mtt_info { + u32 mtt_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + u32 mtt_num; /* Number of cmtt, which needs to be assigned by the driver when the is kicked out by queue. */ + u32 mtt_cache_line_start; /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_end; /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_size; /* 0:256B,1:512B */ +} roce_xq_mtt_info_s; + +#endif //ROCE_COMPAT_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h new file mode 100644 index 000000000..649a4275b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#ifndef ROCE_EVENT_EXTENSION_H +#define ROCE_EVENT_EXTENSION_H + +#include "roce.h" +#include "roce_event.h" + +int to_unkown_event_str_index(uint8_t event_type, const uint8_t *val); +void roce5_event_report_extend(const struct roce5_device *rdev, u8 *val, + int event_str_index); +int roce5_async_event_handle_extend(u8 event_type, u8 *val, + struct roce5_device *rdev); + +#endif /* ROCE_EVENT_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h new file mode 100644 index 000000000..54cfc7bf6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#ifndef ROCE_GID_H +#define ROCE_GID_H + +#include <linux/types.h> + +struct rdma_gid_entry { + union { + u8 raw[16]; + struct { + __be64 subnet_prefix; + __be64 interface_id; + } global; + }; + union { + struct { + u32 rsvd : 8; + u32 cvlan : 12; /* in layer vlan customer vlan */ + u32 svlan : 12; /* outside layer vlan */ + } bs; + u32 value; + } dw4; + + union { + u32 hdr_len_value; + }; + + union { + struct { + u16 tag : 2; /* 0: no has vlan; 1: one layer vlan; 2: 2 layer vlan; 3: stag */ + u16 tunnel : 1; // rsvd for ppe, don't use. 'tunnel' + u16 gid_type : 2; + u16 ppe_rsvd1 : 1; + u16 outer_tag : 2; // rsvd for ppe, don't use. 'outer_tag' + u16 ppe_rsvd3 : 1; // rsvd for ppe, don't use. 'stag' + u16 gid_update : 1; + u16 rsvd : 6; + } bs; + u16 value; + } dw6_h; + + u8 smac[6]; +}; + +#endif // ROCE_GID_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h new file mode 100644 index 000000000..b055d5c92 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h @@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + +#ifndef ROCE_INFRA_H +#define ROCE_INFRA_H + +#include <linux/kern_levels.h> +#include <linux/sched.h> + +#define ROCE_INFO(format, ...) \ + printk(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_WARN(format, ...) \ + printk(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_ERR(format, ...) \ + printk(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_INFO_LIMIT(format, ...) \ + printk_ratelimited(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format \ + "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_WARN_LIMIT(format, ...) \ + printk_ratelimited(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format \ + "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_ERR_LIMIT(format, ...) \ + printk_ratelimited(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_INFO_ONCE(format, ...) \ + printk_once(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_WARN_ONCE(format, ...) \ + printk_once(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_ERR_ONCE(format, ...) \ + printk_once(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +/* Importent, ensure dev is not null pointer when using ROCE_DEV_XXX() !!! */ +#define ROCE_DEV_INFO(dev, format, ...) \ + dev_info((dev)->hwdev_hdl, "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_WARN(dev, format, ...) \ + dev_warn((dev)->hwdev_hdl, "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_ERR(dev, format, ...) \ + dev_err((dev)->hwdev_hdl, "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_DEV_INFO_LIMIT(dev, format, ...) \ + dev_info_ratelimited((dev)->hwdev_hdl, \ + "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_WARN_LIMIT(dev, format, ...) \ + dev_warn_ratelimited((dev)->hwdev_hdl, \ + "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_ERR_LIMIT(dev, format, ...) \ + dev_err_ratelimited((dev)->hwdev_hdl, \ + "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_PRINT_GID(dev, raw, format, ...) \ + ROCE_DEV_WARN( \ + (dev), \ + "gid:%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x==>" format, \ + *((u16 *)(raw) + IP_INDEX_0), *((u16 *)(raw) + IP_INDEX_1), \ + *((u16 *)(raw) + IP_INDEX_2), *((u16 *)(raw) + IP_INDEX_3), \ + *((u16 *)(raw) + IP_INDEX_4), *((u16 *)(raw) + IP_INDEX_5), \ + *((u16 *)(raw) + IP_INDEX_6), *((u16 *)(raw) + IP_INDEX_7), \ + ##__VA_ARGS__) + +#endif //ROCE_INFRA_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h new file mode 100644 index 000000000..b1bca23b0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h @@ -0,0 +1,6 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_KERNEL_COMPAT_GEN_H +#define ROCE_KERNEL_COMPAT_GEN_H +#endif /* ROCE_KERNEL_COMPAT_GEN_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h new file mode 100644 index 000000000..2244bb236 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h @@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MAIN_EXTENSION_H +#define ROCE_MAIN_EXTENSION_H + +#include <linux/pci.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "hinic5_crm.h" +#include "hinic5_lld.h" +#include "hinic5_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_dfx.h" +#include "roce_cmd.h" +#include "roce_user.h" +#include "rdma_comp.h" +#ifdef ROCE_CHIP_TEST +#include "roce_main_chip_ext.h" +#endif + +#define ROCE_MAX_WQES_COMPUTE (4 * K_UNIT - 1) +#define RDMA_MAX_SQ_DESC_SZ_COMPUTE 512 +#define ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE 448 + +#define ROCE5_ULP_CMD(rdev, bitmask) \ + (((rdev)->is_vroce) ? (VERBS_CMD_TYPE_VROCE_BITMASK) : (bitmask)) + +static inline u64 roce5_get_uverbs_cmd_mask(void) +{ + u64 mask = 0; + mask |= ROCE_UVERBS_CMD_BASE_MASK; +#ifdef ROCE_CHIP_TEST + mask |= ROCE_UVERBS_CMD_EXT_MASK; +#endif + return mask; +} + +struct hinic5_uld_info *roce5_info_get(void); + +void roce5_service_init_pre(void); + +void roce5_lock_rdev(void); + +void roce5_unlock_rdev(void); + +int roce5_get_rdev_by_uld(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, + struct hinic5_event_info *event); + +void roce5_init_dev_ext_handlers(struct roce5_device *rdev); + +int roce5_board_cfg_check(struct roce5_device *rdev); + +void roce5_remove_clean_res_ext(struct roce5_device *rdev); + +int roce5_set_comm_event(const struct roce5_device *rdev, + const struct hinic5_event_info *event); + +void *roce5_rdev_alloc_ext(void); + +void roce5_fill_gid_entry_ext(const struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry); + +void roce5_cfg_info_update_ext(struct roce5_device *rdev, + roce_get_cfg_info_resp_s *resp); + +int roce5_add_check_ext(const struct hinic5_lld_dev *lld_dev); + +#endif /* ROCE_MAIN_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h new file mode 100644 index 000000000..ff0464a31 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MR_EXTEND_H +#define ROCE_MR_EXTEND_H + +#include <rdma/ib_verbs.h> +#include "roce.h" + +int roce5_check_alloc_mr_type(enum ib_mr_type mr_type); +enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type); + +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h new file mode 100644 index 000000000..6b8a3113a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h @@ -0,0 +1,237 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_OS_COMPAT_H +#define ROCE_OS_COMPAT_H +#include <rdma/ib_verbs.h> +#include "roce.h" +#include "roce_restore.h" +#include "roce_verbs_compat.h" +#include "roce_kernel_compat_gen.h" +#if defined(UVERBS_IOCTL_HAVE_RDMA_UDATA_TO_DRV_CONTEXT) +#include <rdma/uverbs_ioctl.h> +#endif + +#define PAGE_SIZE_4KB (1ULL << PAGE_SHIFT_4KB) +#define PAGE_SIZE_64KB (1ULL << PAGE_SHIFT_64KB) +#define PAGE_SIZE_2MB (1ULL << PAGE_SHIFT_2MB) + +#define BIT_NUM_PER_BYTE (8) +#define PAGE_SHIFT_4KB (12) +#define PAGE_SHIFT_64KB (16) +#define PAGE_SHIFT_2MB (21) + +#if !defined(HAVE_IB_UMEM_NUM_DMA_BLOCKS) +static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem, + unsigned long pgsz) +{ + return (size_t)((ALIGN(umem->address + umem->length, pgsz) - + ALIGN_DOWN(umem->address, pgsz))) / + pgsz; +} +#endif + +#if !defined(HAVE_RDMA_UMEM_FOR_EACH_DMA_BLOCK) +#define rdma_umem_for_each_dma_block(umem, biter, pgsz) \ + for (__rdma_umem_block_iter_start(biter, umem, pgsz); \ + __rdma_block_iter_next(biter);) + +static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter, + struct ib_umem *umem, + unsigned long pgsz) +{ + __rdma_block_iter_start(biter, umem->sg_head.sgl, umem->nmap, pgsz); +} +static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter) +{ + return __rdma_block_iter_next(biter) && biter->__sg_numblocks--; +} +#endif + +#if defined(IB_UMEM_NEED_IB_UMEM_FIND_BEST_PGSZ) +static inline unsigned int rdma_find_pg_bit(unsigned long addr, + unsigned long pgsz_bitmap) +{ + unsigned long align; + unsigned long pgsz; + + align = addr & -addr; + + pgsz = pgsz_bitmap & ~(-align << 1); + if (!pgsz) + return __ffs(pgsz_bitmap); + + return __fls(pgsz); +} + +static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, + unsigned long pgsz_bitmap, + unsigned long virt) +{ + struct scatterlist *sg; + unsigned int best_pg_bit; + unsigned long va, pgoff; + dma_addr_t mask; + int i; + + if (WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0)))) + return 0; + + va = virt; + mask = roundup_pow_of_two(umem->length); + pgoff = umem->address & ~PAGE_MASK; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + mask |= (sg_dma_address(sg) + pgoff) ^ va; + if (i && i != (umem->nmap - 1)) + mask |= sg_dma_len(sg); + va += sg_dma_len(sg) - pgoff; + pgoff = 0; + } + best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap); + + return BIT_ULL(best_pg_bit); +} +#endif + +static inline u32 roce5_umem_get_page_count(struct ib_umem *umem) +{ +#if defined(HAVE_IB_UMEM_PAGE_COUNT) + return (u32)ib_umem_page_count(umem); +#elif defined(HAVE_IB_UMEM_NUM_PAGES) + return (u32)ib_umem_num_pages(umem); +#endif +} + +static inline u32 roce5_umem_get_page_shift(struct ib_umem *umem) +{ +#if defined(UMEM_HAVE_PAGE_SHIFT) + return umem->page_shift; +#else + return PAGE_SHIFT; +#endif +} + +static inline u32 roce5_umem_get_page_size(struct ib_umem *umem) +{ +#if defined(UMEM_HAVE_PAGE_SHIFT) + return BIT((unsigned int)umem->page_shift); +#else + return PAGE_SIZE; +#endif +} + +static inline u32 roce5_get_page_size(struct roce5_device *rdev, bool use_huge) +{ + if (use_huge && rdev->huge_page_en) { + return PAGE_SIZE_2MB; + } + return PAGE_SIZE; +} + +static inline size_t roce5_get_page_size_cap(size_t page_size_cap, + size_t length) +{ + size_t max_shift; + + if (length == 0) { + return 0; + } + + max_shift = fls_long(length) - 1; + if (max_shift == sizeof(size_t) * BIT_NUM_PER_BYTE - 1) + return page_size_cap; + else + return page_size_cap & (((size_t)1 << (max_shift + 1)) - 1); +} + +static inline u32 roce5_umem_get_buf_page_shift(struct roce5_device *rdev, + struct ib_umem *umem, + u64 buf_addr) +{ + u32 real_page_size_cap, page_shift; + + real_page_size_cap = (u32)roce5_get_page_size_cap( + (size_t)rdev->rdma_cap.page_size_cap, umem->length); + + if (rdev->huge_page_en) { + page_shift = (u32)order_base_2(ib_umem_find_best_pgsz( + umem, real_page_size_cap, buf_addr)); + } else { + page_shift = roce5_umem_get_page_shift(umem); + } + + return page_shift; +} + +static inline u32 roce5_umem_get_mtt_npages(struct roce5_device *rdev, + struct ib_umem *umem, + u32 page_shift) +{ + u32 npages; + + if (rdev->huge_page_en) { + npages = ib_umem_num_dma_blocks(umem, BIT(page_shift)); + } else { + npages = roce5_umem_get_page_count(umem); + } + + return npages; +} + +/* These APIs can be reused by different OSs */ +static inline struct ib_umem *roce5_umem_get(struct ib_device *ibdev, + struct ib_ucontext *context, + unsigned long addr, size_t size, + int access, int dmasync) +{ +#if (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(MMU_INTERVAL_NOTIFIER)) || \ + (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(IB_UMEM_GET_HAVE_IB_DEVICE)) + struct uverbs_attr_bundle *attrs; +#endif + +#if (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(MMU_INTERVAL_NOTIFIER)) || \ + (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(IB_UMEM_GET_HAVE_IB_DEVICE)) + attrs = container_of(&context, struct uverbs_attr_bundle, context); +#endif + +#if defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) + return ib_umem_get(context, addr, size, access, dmasync); +#elif defined(IB_UMEM_GET_HAVE_IB_DEVICE) && defined(MMU_INTERVAL_NOTIFIER) + return ib_umem_get(ibdev, addr, size, access); +#elif defined(IB_UMEM_GET_HAVE_DMASYNC) + return ib_umem_get(&attrs->driver_udata, addr, size, access, dmasync); +#else + return ib_umem_get(&attrs->driver_udata, addr, size, access); +#endif +} + +#if defined(IB_UMEM_GET_PEER) +static inline struct ib_umem * +roce5_mr_umem_get_peer(struct roce5_device *rdev, size_t length, int access, + u64 addr, struct ib_udata *udata, bool use_huge) +{ + u64 page_size = PAGE_SIZE; + size_t buf_size = length; + + page_size = roce5_get_page_size(rdev, use_huge); + buf_size = (size_t)ROCE_ALIGN(buf_size, page_size); + +#if defined(IB_UMEM_GET_PEER_HAVE_IBDEVICE) && defined(MMU_INTERVAL_NOTIFIER) + return ib_umem_get_peer(&rdev->ib_dev, addr, buf_size, access, + (IB_PEER_MEM_ALLOW | IB_PEER_MEM_INVAL_SUPP)); +#else + return ib_umem_get_peer(udata, addr, buf_size, access, + (IB_PEER_MEM_ALLOW | IB_PEER_MEM_INVAL_SUPP)); +#endif +} +#endif + +struct roce5_ucontext * +roce5_queue_to_ucontext(void *ibq, struct ib_udata *udata, int q_type); + +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h new file mode 100644 index 000000000..2cbcd5d71 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h @@ -0,0 +1,61 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_EXTENSION_H +#define ROCE_QP_EXTENSION_H + +#include <rdma/ib_verbs.h> + +#include "roce_qp.h" +#include "roce_qp_exp.h" + +int to_roce5_qp_type(enum ib_qp_type qp_type); + +bool roce5_check_qp_modify_ok(enum ib_qp_state cur_state, + enum ib_qp_state next_state, enum ib_qp_type type, + enum ib_qp_attr_mask mask, + enum rdma_link_layer ll); + +int roce5_is_qp_normal(struct roce5_qp *rqp, struct ib_qp_init_attr *init_attr); + +void roce5_set_qp_dif_attr(struct roce5_qp *rqp, + const struct ib_qp_init_attr *init_attr, + const struct roce5_device *rdev); + +#ifdef ROCE_EXTEND + +#define ROCE_QP_VBS_FLAG (1U << 22) +#define ROCE_QP_OSD_FLAG (1U << 23) +#define QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC 204800 //200K + +struct roce5_set_qp_ext_attr_cmd { + u32 qpn; + u32 attr_mask; +}; + +struct roce5_modify_qp_vbs_cmd { + u32 qpn; + u64 ci_record_addr; +}; + +struct roce5_vbs_qp { + cqm_qpc_mpt_s *vbs_sqpc_info; /* vbs private data */ + struct roce5_db db; /* to record ci_record_pa */ +}; + +long roce5_vbs_create_sqpc(struct roce5_device *rdev, void *buf); +#endif + +void roce5_destroy_pcqc_ext(struct roce5_qp *rqp); +int roce5_modify_qp_pre_ext(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata); +void roce5_set_attr_ext(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, int attr_mask); +void roce5_transmit_ci_record_addr_ext(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr); +void roce5_cos_update_ext(struct roce5_device *rdev, u8 *cos, + enum ib_qp_type qp_type); + +#endif /* ROCE_QP_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h new file mode 100644 index 000000000..c772472a6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h @@ -0,0 +1,88 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_RESTORE_H +#define ROCE_RESTORE_H +#include <rdma/ib_verbs.h> + +#include "roce.h" +#include "roce_qp.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_mr.h" +#include "roce_mix.h" +#include "roce_pd.h" +#include "roce_xrc.h" +#include "hinic5_cqm.h" + +static inline struct roce5_device *to_roce5_dev(const struct ib_device *ibdev) +{ + return container_of(ibdev, struct roce5_device, ib_dev); +} + +static inline struct roce5_ucontext * +to_roce5_ucontext(const struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct roce5_ucontext, ibucontext); +} + +static inline struct roce5_qp *to_roce5_qp(const struct ib_qp *ibqp) +{ + return container_of(ibqp, struct roce5_qp, ibqp); +} + +static inline struct roce5_qp *cqmobj_to_roce_qp(const cqm_object_s *object) +{ + cqm_qpc_mpt_s *qpc_info = NULL; + qpc_info = container_of(object, cqm_qpc_mpt_s, object); + return (struct roce5_qp *)qpc_info->priv; +} + +static inline struct roce5_srq *to_roce5_srq(const struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct roce5_srq, ibsrq); +} + +static inline struct roce5_srq *cqmobj_to_roce5_srq(const cqm_object_s *object) +{ + cqm_queue_s *cqm_srq = NULL; + cqm_srq = container_of(object, cqm_queue_s, object); + return (struct roce5_srq *)cqm_srq->priv; +} + +static inline struct roce5_cq *to_roce5_cq(const struct ib_cq *ibcq) +{ + return container_of(ibcq, struct roce5_cq, ibcq); +} + +static inline struct roce5_mr *to_roce5_mr(const struct ib_mr *ibmr) +{ + return container_of(ibmr, struct roce5_mr, ibmr); +} + +static inline struct roce5_mw *to_roce5_mw(const struct ib_mw *ibmw) +{ + return container_of(ibmw, struct roce5_mw, ibmw); +} + +static inline struct roce5_mr * +rdma_mr_to_roce5_mr(const struct hmm_rdma *rdmamr) +{ + return container_of(rdmamr, struct roce5_mr, rdmamr); +} + +static inline struct roce5_ah *to_roce5_ah(const struct ib_ah *ibah) +{ + return container_of(ibah, struct roce5_ah, ibah); +} + +static inline struct roce5_pd *to_roce5_pd(const struct ib_pd *ibpd) +{ + return container_of(ibpd, struct roce5_pd, ibpd); +} + +static inline struct roce5_xrcd *to_roce5_xrcd(const struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct roce5_xrcd, ibxrcd); +} +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h new file mode 100644 index 000000000..b931f9426 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h @@ -0,0 +1,38 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SRQ_EXTENSION_H +#define ROCE_SRQ_EXTENSION_H + +#include <linux/types.h> +#include "roce_srq.h" + +/** + * container_mode: + * mode: 0 -> container_size: 16 + * mode: 1 -> container_size: 8 + * mode: 2 -> container_size: 4 + * mode: 3 -> container_size: 2 + */ +enum roce5_srq_mode { + ROCE_SRQ_MODE_0 = 0, + ROCE_SRQ_MODE_1, + ROCE_SRQ_MODE_2, + ROCE_SRQ_MODE_3 +}; + +enum roce5_srq_cont_num_mode { + ROCE_SRQ_CONT_NUM_MODE3 = 2, + ROCE_SRQ_CONT_NUM_MODE2 = 4, + ROCE_SRQ_CONT_NUM_MODE1 = 8, + ROCE_SRQ_CONT_NUM_MODE0 = 16 +}; + +u8 roce5_get_container_sz(u32 container_mode); +u8 roce5_calculate_cont_th(u32 srq_limit); +void roce5_srq_container_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, + struct roce5_device *rdev); + +#endif /* ROCE_SRQ_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h new file mode 100644 index 000000000..81effef16 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h @@ -0,0 +1,100 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_USER_H +#define ROCE_USER_H + +#include <linux/types.h> + +struct roce5_alloc_ucontext_resp { + uint32_t num_qps; /* the maximum of qps */ + uint32_t num_xsrqs; /* the maximum of xsrqs */ + uint32_t cqe_size; /* the cqe entry size that device support, the valid value is 32B or 64B; */ + uint32_t wqebb_size; /* the wqebb entry size that device support, the valid value is 64B or 128B */ + uint32_t dwqe_size; /* the direct maximum wqe size that device support, 256B */ + uint32_t max_msg_size; /* the maximum message size that device support, (2G -1)B */ + uint32_t max_comp_vector; /* the maximum completion vector */ + uint32_t max_inline_size; /* the maximum of qp inline size that device support */ + uint32_t dev_caps; + uint8_t phy_port; + uint8_t storage_aa_en; + uint16_t func_id; + uint64_t card_id; + uint16_t db_offset; + uint16_t dwqe_offset; + struct { + uint16_t pcie_atomic_en : 1; + uint16_t rsvd1 : 15; + }; + uint8_t srq_container_en; + uint8_t srq_container_mode; + uint8_t xrc_srq_container_mode; + uint8_t warn_th; + uint8_t huge_page_en; + uint8_t rsvd; + uint8_t max_send_sge; /* the maximum of sq sge */ + uint8_t max_recv_sge; /* the maximum of rq sge */ + uint32_t max_sq_wqe_len; /* the maximum sq wqe length that device support */ + uint64_t roce_user_ext_ability; + uint64_t roce_uld_ext_ability; +}; + +struct roce5_create_cq_cmd { + u64 buf_addr; + u64 db_addr; +}; + +struct roce5_resize_cq_cmd { + u64 buf_addr; /* resize cq's 'buf va */ + u64 stage; /* describe the resize stage,0 or 1; */ +}; + +struct roce5_create_srq_cmd { + u64 buf_addr; + u64 db_addr; + u32 rsvd; +}; + +struct create_qp_cmd { + u64 buf_addr; /* describe the qp buf address that used to store cqe; */ + u64 db_addr; /* describe the qp sw db address */ + u32 comp_mask; /* kernel can judge whether handle receive inline through comp_mask */ + u8 log_sq_bb_count; /* wqebb number = 2 << log_sq_bb_count */ + u8 log_sq_stride; /* wqebb size = 2 << log_sq_stride */ + u16 reserved; /* reserved is used to aligned cmd */ +}; + +struct roce5_modify_srq_cmd { + u8 container_flag; + u8 container_warn_th; + u16 rsvd; +}; + +struct roce5_reg_frmr_cmd { + u64 pbl_vaddr; +}; + +struct roce5_modify_qp_ex_resp_data { + u8 is_cos_valid; + u8 cos; + u8 is_ssn_valid; + u8 rsvd0; + u32 ssn; + u32 rsvd[6]; +#ifdef ROCE_LATENCY_PROF_EN + uint8_t latency_prof_en; + uint8_t rsvd1[3]; + uint32_t frequency_reduction_ratio; + uint64_t non_ptp_time_diff; +#endif +}; + +struct roce5_create_ah_resp { + u8 dmac[ETH_ALEN]; + u16 vlan_id; + u8 cos; + u8 rsvd[3]; +}; + +#endif //ROCE_USER_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h new file mode 100644 index 000000000..cd87856ea --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_VERBS_COMPAT_H +#define ROCE_VERBS_COMPAT_H + +#include <rdma/ib_verbs.h> +#include "roce_kernel_compat_gen.h" + +enum { ROCE_Q_TYPE_QP, ROCE_Q_TYPE_SRQ, ROCE_Q_TYPE_CQ }; + +#if !defined(HAVE_RDMA_AH_INIT_ATTR) +/* kernel 4.19 not defined but kernel 5.10 defined, reused for ah creating */ +struct rdma_ah_init_attr { + struct rdma_ah_attr *ah_attr; +}; + +enum ib_wr_opcode_compat { + IB_WR_REG_MR_INTEGRITY = IB_WR_REG_SIG_MR, +}; + +#endif + +#endif /* ROCE_VERBS_COMPAT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h new file mode 100644 index 000000000..099b76a1a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h @@ -0,0 +1,79 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_BOND_H +#define ROCE_BOND_H + +#include <net/bonding.h> +#include <linux/netdevice.h> + +#include "hinic5_lld.h" +#include "drv_bond_api.h" + +#include "roce.h" +#include "roce_qp.h" +#include "roce_cmd.h" + +#define ROCE_BOND_MIN_SLAVE 1 +#define ROCE_BOND_MAX_SLAVE BOND_PORT_MAX_NUM + +#define ROCE_BOND_RSVD_VLAN_ID 4095 + +#define ROCE_BOND_FWD_ID_TBL_ALL_BITS 32U +#define ROCE_BOND_FWD_ID_TBL_PER_BITS 3U +#define ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK 0x7U + +struct roce5_bond_slave { + struct net_device *netdev; + struct hinic5_lld_dev *lld_dev; + struct netdev_lag_lower_state_info netdev_state; + u32 update_cnt; + u16 func_id; + u8 er_id; +}; + +struct roce5_bond_transition_res { + struct hinic5_lld_dev *ppf_lld; +}; + +struct roce5_bond_device { + char name[IFNAMSIZ]; /* the name of nic bond device */ + struct list_head entry; + struct hinic5_lld_dev *bond_lld_dev; + struct roce5_bond_slave slaves[ROCE_BOND_MAX_SLAVE]; + struct mutex slave_lock; /* lock for slaves */ + u32 slave_cnt; + atomic_t next_port; + struct work_struct detach_work; + struct roce5_device *attached_rdev; + struct bond_attr attr; + struct roce5_bond_transition_res transition_res; + u8 need_clean_on_remove; +}; + +bool roce5_bond_is_active(struct roce5_device *rdev); +void roce5_bond_detach(struct roce5_device *rdev); +struct net_device *roce5_bond_get_netdev(struct roce5_device *rdev); + +int roce5_add_bond_real_slave_mac(struct roce5_device *rdev, u8 *mac); +int roce5_add_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id); +void roce5_del_bond_real_slave_mac(struct roce5_device *rdev); +void roce5_del_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id); + +void roce5_bond_rr_set_flow(struct roce5_device *rdev, struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr); + +int roce5_bond_event_cfg_rdev(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, bool hold_rdev); +void roce5_handle_bonded_port_state_event(struct roce5_device *rdev); + +bool roce5_get_bond_ipsurx_en(void); +void roce5_set_bond_ipsurx_en(bool ipsurx_en); + +int roce5_bond_attach(struct roce5_device *rdev); +int roce5_bond_init(void); +void roce5_bond_exit(void); +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c new file mode 100644 index 000000000..f295f241d --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "bond_common_defs.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +static int roce5_bond_check_slave_port(struct hinic5_lld_dev *lld_dev, + u8 slave_ports[], u8 slave_ports_num, + bool *result) +{ + int ret, i; + roce_bond_check_slave_port_req_s req = { 0 }; + roce_bond_check_slave_port_rsp_s rsp = { 0 }; + u16 out_size = (u16)sizeof(rsp); + + for (i = 0; i < slave_ports_num; ++i) { + req.slave_ports[i] = slave_ports[i]; + } + req.slave_ports_num = slave_ports_num; + + ret = roce5_msg_to_mgmt_sync(lld_dev->hwdev, + ROCE_MPU_CMD_BOND_CHECK_SLAVE_PORT, &req, + sizeof(req), &rsp, &out_size); + if ((ret != 0) || (out_size == 0) || (rsp.head.status != 0)) { + ROCE_ERR( + "Failed to check slave port, ret(%d), out size(%u), status(%u)", + ret, out_size, rsp.head.status); + return -EINVAL; + } + + *result = false; + for (i = 0; i < slave_ports_num; ++i) { + *result |= (rsp.slave_ports_result[i] != false) ? true : false; + } + + return 0; +} + +/* + * roce5_can_do_bond - Check if RoCE can attach to bond device + * @bond: bond device to check + * Return: true if can attach, false otherwise + */ +bool roce5_can_do_bond(struct bonding *bond) +{ + bool ret = false; + int slave_cnt = 0; + int err; + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + char chip_name[IFNAMSIZ] = { 0 }; + u8 slave_ports[ROCE_BOND_PORT_MAX_NUM]; + bool result; + + if (bond == NULL) { + ROCE_ERR("Invalid Parameter, ptr is NULL."); + return false; + } + + if (roce5_bond_mode_is_supported(bond->params.mode) == false) { + ROCE_ERR("Invalid bond mode(%u).", bond->params.mode); + return false; + } + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + lld_dev = hinic5_get_lld_dev_by_netdev(slave->dev); + if (lld_dev == NULL) { + goto out; + } + + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + ROCE_WARN("VF device %s skipped", slave->dev->name); + continue; + } + + err = roce5_bond_slave_check(lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond slave check err %d.", err); + goto out; + } + + if (slave_cnt < ROCE_BOND_PORT_MAX_NUM) { + slave_ports[slave_cnt] = + hinic5_physical_port_id(lld_dev->hwdev); + } + + slave_cnt++; + ROCE_INFO("can do bond? slave_cnt(%d), slave_name(%s).", + slave_cnt, slave->dev->name); + } + + if (((slave_cnt >= ROCE_BOND_MIN_SLAVE) && + (slave_cnt <= ROCE_BOND_MAX_SLAVE)) == false) { + goto out; + } + + if (lld_dev == NULL) { + ROCE_INFO("lld dev is NULL."); + goto out; + } + + err = roce5_bond_check_slave_port(lld_dev, slave_ports, slave_cnt, + &result); + if (err != 0) { + ROCE_ERR("Failed to check slave ports, err(%d).", err); + goto out; + } + + if (result == false) { + ROCE_INFO("Ports don't meet requirements."); + goto out; + } + + ret = true; + +out: + rcu_read_unlock(); + return ret; +} + +static int +roce5_bond_create_transition_store(struct roce5_bond_device *bond_dev, + const struct bond_tracker *tracker) +{ + int ret; + struct hinic5_lld_dev *ppf_lld = NULL; + + ppf_lld = roce5_bond_get_ppf_lld(tracker); + if (ppf_lld == NULL) { + ROCE_ERR("Failed to get lld of the PPF."); + return -EINVAL; + } + + ret = roce5_bond_transition_store(bond_dev, ppf_lld); + if (ret != 0) { + ROCE_ERR("Failed to store resources of transition, ret(%d).", + ret); + hinic5_lld_dev_put(ppf_lld); + return ret; + } + + return 0; +} + +static void +roce5_bond_create_transition_free(struct roce5_bond_device *bond_dev) +{ + roce5_bond_transition_free(bond_dev); +} + +/* + * roce5_before_bond_active - Pre-active callback for bond creation + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_before_bond_active(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + u16 bond_id; + int i, slave_cnt; + int ret; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR( + "Invalid parameter, roce bond_name(%d) or attr(%d) is NULL.", + !!bond_name, !!attr); + return; + } + + bond_id = attr->bond_id; + ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to get bond tracker, bond name(%s), ret(%d).", + bond_name, ret); + goto err; + } + + if (roce5_bond_before_active_check(&tracker, attr) == false) { + ROCE_ERR("Failed to check before active bond."); + goto err; + } + + bond_dev = roce5_bond_create_bond_device(bond_name); + if (IS_ERR(bond_dev)) { + ROCE_ERR("Failed to create roce bond device."); + goto err; + } + + /* Store transition resources during the bond creating period to prevent abnormal resources using. */ + ret = roce5_bond_create_transition_store(bond_dev, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to store resources of transition."); + goto err; + } + + /* Remove the functions of slaves. */ + for (i = 0, slave_cnt = 0; + i < BOND_PORT_MAX_NUM && slave_cnt < tracker.cnt; ++i) { + if (tracker.ndev[i] == NULL) { + continue; + } + + ret = roce5_bond_add_slave(bond_dev, tracker.ndev[i], + &tracker.netdev_state[i], false); + if (ret != 0) { + ROCE_ERR( + "Failed to add slave, name(%s), slave name(%s).", + bond_dev->name, tracker.ndev[i]->name); + goto err_with_transition; + } + + ++slave_cnt; + } + + /* Free transition resources during the bond creating period. */ + roce5_bond_create_transition_free(bond_dev); + bond_dev->attr = *attr; + + return; + +err_with_transition: + roce5_bond_create_transition_free(bond_dev); +err: + roce5_bond_destroy(bond_name, bond_id); +} + +/* + * roce5_after_bond_active - Post-active callback for bond creation + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_after_bond_active(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR( + "Invalid parameter, bond name(%d) or attr(%d) is NULL.", + !!bond_name, !!attr); + return; + } + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("Can't find roce bond device by name(%s).", bond_name); + return; + } + + roce5_update_rdev_func_val(bond_dev); +} + +/* + * roce5_before_bond_modify - Pre-modify callback for bond modification + * @bond_name: name of bond device + * @attr: new bond attributes + * @err: error code from previous operation + */ +void roce5_before_bond_modify(const char *bond_name, struct bond_attr *attr, + int err) +{ + int ret; + struct roce5_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR("roce bond_name or attr is NULL."); + return; + } + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("Can't find roce bond device by name(%s).", bond_name); + return; + } + + ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to get bond tracker, bond name(%s), ret(%d).", + bond_name, ret); + return; + } + + ret = roce5_handle_diff_slave(bond_dev, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to handle different slave."); + return; + } + + return; +} + +/* + * roce5_after_bond_modify - Post-modify callback for bond modification + * @bond_name: name of bond device + * @attr: new bond attributes + * @err: error code from previous operation + */ +void roce5_after_bond_modify(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR("roce bond_name or attr is NULL."); + return; + } + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("can't find bond device by name(%s).", bond_name); + return; + } + + bond_dev->attr = *attr; + + roce5_update_rdev_func_val(bond_dev); +} + +/* + * roce5_before_bond_deactive - Pre-deactive callback for bond deletion + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_before_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("can't find bond device by name(%s).", bond_name); + return; + } + + roce5_bond_destroy(bond_dev->name, attr->bond_id); +} + +/* + * roce5_after_bond_deactive - Post-deactive callback for bond deletion + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_after_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err) +{ + /* destroy switch thread handle, bond device may already be delete, current thread not disable care handle result */ +} + +/* + * roce5_bond_mode_is_supported - Check if bond mode is supported + * @bond_mode: bond mode to check + * Return: true if supported, false otherwise + */ +bool roce5_bond_mode_is_supported(u16 bond_mode) +{ + if (bond_mode != BOND_MODE_8023AD && bond_mode != BOND_MODE_XOR && + bond_mode != BOND_MODE_ACTIVEBACKUP) { + ROCE_ERR( + "Bond mode only support 802.3ad/xor/active-backup, current bond mode(%u).", + bond_mode); + return false; + } + return true; +} + +/* + * roce5_bond_slave_check - Check if slave device is valid for RoCE bond + * @lld_dev: LLD device to check + * @chip_name: buffer to store chip name + * @chip_name_size: size of chip_name buffer + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_slave_check(struct hinic5_lld_dev *lld_dev, char *chip_name, + size_t chip_name_size) +{ + int err; + char tmp_name[IFNAMSIZ] = { 0 }; + + if (lld_dev == NULL) { + ROCE_ERR("get lld dev err."); + return -ENODEV; + } + + if (roce5_func_is_vf(lld_dev->hwdev)) { + ROCE_ERR("Bond Slave device is VF."); + return -EINVAL; + } + + err = hinic5_get_chip_name(lld_dev, tmp_name, sizeof(tmp_name)); + if (err != 0) { + ROCE_ERR("Bond Slave get chip name err %d.", err); + return err; + } + + if (strlen(chip_name) == 0) { + err = memcpy_s(chip_name, chip_name_size, tmp_name, + sizeof(tmp_name)); + if (err != 0) { + ROCE_ERR("Failed to memcpy chip_name.(ret:%d)", err); + return -ENOMEM; + } + } else if (strcmp(tmp_name, chip_name) != 0) { + ROCE_ERR( + "Creating bond cross cards is not supported, bond dev chip_name %s, slave chip name %s.", + chip_name, tmp_name); + return -EINVAL; + } + + return 0; +} + +/* + * roce5_bond_before_active_check - Pre-active check for bond creation + * @tracker: bond tracker from CFM framework + * @attr: bond attributes + * Return: true if check passes, false otherwise + */ +bool roce5_bond_before_active_check(struct bond_tracker *tracker, + struct bond_attr *attr) +{ + int i, slave_cnt; + int err; + struct hinic5_lld_dev *lld_dev = NULL; + char chip_name[IFNAMSIZ] = { 0 }; + + if (roce5_bond_mode_is_supported(attr->bond_mode) == false) { + ROCE_ERR("Invalid bond mode(%u).", attr->bond_mode); + return false; + } + + if (tracker->cnt < ROCE_BOND_MIN_SLAVE || + tracker->cnt > ROCE_BOND_MAX_SLAVE) { + ROCE_ERR("Not support the slave number(%u) of the bond.", + tracker->cnt); + return false; + } + + for (i = 0, slave_cnt = 0; + i < BOND_PORT_MAX_NUM && slave_cnt < tracker->cnt; ++i) { + if (tracker->ndev[i] == NULL) { + continue; + } + + lld_dev = hinic5_get_lld_dev_by_netdev(tracker->ndev[i]); + if (lld_dev == NULL) { + ROCE_ERR("get lld dev err."); + return false; + } + + err = roce5_bond_slave_check(lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond slave check err %d.", err); + return false; + } + + ++slave_cnt; + } + return true; +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c new file mode 100644 index 000000000..cb3612778 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "bond_common_defs.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* Global variables: g_roce5_bond_mutex protects g_roce5_bond_list */ +LIST_HEAD(g_roce5_bond_list); +DEFINE_MUTEX(g_roce5_bond_mutex); + +static bool g_roce5_bond_ipsurx_en = true; + +int roce5_bond_transition_store(struct roce5_bond_device *bond_dev, + struct hinic5_lld_dev *ppf_lld) +{ + int ret = 0; + + ret = hinic5_stateful_init(ppf_lld->hwdev); + if (ret != 0) { + ROCE_ERR( + "Failed to init stateful resource of the PPF, ret(%d).", + ret); + goto out; + } + + bond_dev->transition_res.ppf_lld = ppf_lld; +out: + return ret; +} + +void roce5_bond_transition_free(struct roce5_bond_device *bond_dev) +{ + if (bond_dev->transition_res.ppf_lld == NULL) { + return; + } + + hinic5_stateful_deinit(bond_dev->transition_res.ppf_lld->hwdev); + hinic5_lld_dev_put(bond_dev->transition_res.ppf_lld); + bond_dev->transition_res.ppf_lld = NULL; +} + +/* + * Caller has to call hinic5_lld_dev_put() to free resource when finish using ppf lld. + * There's no need to call hinic5_lld_dev_put() to free resource when this function return NULL. + */ +struct hinic5_lld_dev * +roce5_bond_get_ppf_lld(const struct bond_tracker *tracker) +{ + int i; + struct net_device *first_slave = NULL; + struct hinic5_lld_dev *first_slave_lld = NULL, *ppf_lld = NULL; + + for (i = 0; i < BOND_PORT_MAX_NUM; ++i) { + if (tracker->ndev[i] != NULL) { + first_slave = tracker->ndev[i]; + break; + } + } + + if (first_slave == NULL) { + ROCE_ERR("Failed to get net device of the first slave."); + return NULL; + } + + first_slave_lld = hinic5_get_lld_dev_by_netdev(first_slave); + if (first_slave_lld == NULL) { + ROCE_ERR("Failed to get lld of the first slave."); + return NULL; + } + + ppf_lld = hinic5_get_ppf_lld_dev(first_slave_lld); + if (ppf_lld == NULL) { + ROCE_ERR("Failed to get lld of the PPF."); + return NULL; + } + + return ppf_lld; +} + +/* + * roce5_get_bond_dev - Get bond device by name (internal) + * @bond_name: name of bond device + * Return: pointer to roce5_bond_device if found, NULL otherwise + * Lock: caller must hold g_roce5_bond_mutex + */ +struct roce5_bond_device *roce5_get_bond_dev(const char *bond_name) +{ + struct roce5_bond_device *bdev = NULL; + + list_for_each_entry(bdev, &g_roce5_bond_list, entry) { + if (strcmp(bdev->name, bond_name) == 0) { + return bdev; + } + } + + return NULL; +} + +/* + * roce5_get_bond_dev_by_name - Get bond device by name (public wrapper) + * @bond_name: name of bond device + * Return: pointer to roce5_bond_device if found, NULL otherwise + */ +struct roce5_bond_device *roce5_get_bond_dev_by_name(const char *bond_name) +{ + struct roce5_bond_device *bdev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + bdev = roce5_get_bond_dev(bond_name); + mutex_unlock(&g_roce5_bond_mutex); + return bdev; +} + +struct roce5_bond_device *roce5_bond_create_bond_device(const char *bond_name) +{ + struct roce5_bond_device *rbdev = NULL; + errno_t ret; + + if (bond_name == NULL) { + ROCE_ERR("Invalid parameter, ptr is NULL."); + return ERR_PTR(-EINVAL); + } + + rbdev = roce5_get_bond_dev_by_name(bond_name); + if (rbdev != NULL) { + ROCE_ERR("Bond device(%s) already exists.", bond_name); + return ERR_PTR(-EINVAL); + } + + rbdev = kzalloc(sizeof *rbdev, GFP_KERNEL); + if (rbdev == NULL) { + ROCE_ERR("Failed to alloc memory."); + return ERR_PTR(-ENOMEM); + } + + ret = strncpy_s(rbdev->name, IFNAMSIZ, bond_name, strlen(bond_name)); + if (ret != EOK) { + ROCE_ERR("Failed to copy string, ret(%d).", ret); + kfree(rbdev); + return ERR_PTR(-EFAULT); + } + + rbdev->need_clean_on_remove = true; + mutex_init(&rbdev->slave_lock); + + mutex_lock(&g_roce5_bond_mutex); + list_add_tail(&rbdev->entry, &g_roce5_bond_list); + mutex_unlock(&g_roce5_bond_mutex); + + return rbdev; +} + +static int +roce5_bond_destroy_transition_store(struct roce5_bond_device *bond_dev) +{ + int ret; + struct hinic5_lld_dev *ppf_lld = NULL; + + if (bond_dev->slave_cnt == 0) { + ROCE_WARN("There's no slave in roce bond."); + return 0; + } + + ppf_lld = hinic5_get_ppf_lld_dev(bond_dev->slaves[0].lld_dev); + if (ppf_lld == NULL) { + ROCE_ERR("Failed to get lld of the PPF."); + return -EINVAL; + } + + ret = roce5_bond_transition_store(bond_dev, ppf_lld); + if (ret != 0) { + ROCE_ERR("Failed to store resources of transition, ret(%d).", + ret); + hinic5_lld_dev_put(ppf_lld); + return ret; + } + + return 0; +} + +static void +roce5_bond_destroy_transition_free(struct roce5_bond_device *bond_dev) +{ + roce5_bond_transition_free(bond_dev); +} +/* + * roce5_bond_destroy - Destroy bond device + * @bond_name: name of bond device + * @bond_id: bond ID + */ +void roce5_bond_destroy(const char *bond_name, u16 bond_id) +{ + int ret; + u32 i; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + bond_dev = roce5_get_bond_dev(bond_name); + if (bond_dev == NULL) { + /* destroy switch thread handle, bond device may already be delete, current thread not disable care handle result */ + ROCE_INFO("bond name(%s) has been detached", bond_name); + mutex_unlock(&g_roce5_bond_mutex); + return; + } + + list_del(&bond_dev->entry); + mutex_unlock(&g_roce5_bond_mutex); + + if (bond_dev->bond_lld_dev != NULL) { + ret = roce5_bond_destroy_transition_store(bond_dev); + if (ret != 0) { + ROCE_ERR( + "Failed to store transition resources on destroy."); + } + + bond_dev->need_clean_on_remove = false; + hinic5_detach_service(bond_dev->bond_lld_dev, SERVICE_T_ROCE); + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + ret = hinic5_attach_service(bond_dev->slaves[i].lld_dev, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR( + "Failed to attach roce device, ret(%d).", + ret); + } + } + bond_dev->slave_cnt = 0; + roce5_bond_destroy_transition_free(bond_dev); + mutex_unlock(&bond_dev->slave_lock); + } + + kfree(bond_dev); + return; +} + +/* + * roce5_bond_attach - Attach bond device to RoCE + * @rdev: RoCE device to attach + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_attach(struct roce5_device *rdev) +{ + u32 i; + int ret = 0; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (rdev->ndev != bond_dev->slaves[i].netdev) { + continue; + } + + if (bond_dev->attached_rdev == NULL) { + bond_dev->attached_rdev = rdev; + rdev->bond_dev = bond_dev; + } else { + ret = -EEXIST; + } + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + mutex_unlock(&bond_dev->slave_lock); + } +out: + mutex_unlock(&g_roce5_bond_mutex); + return ret; +} + +/* + * roce5_bond_is_active - Check if bond is active for RoCE device + * @rdev: RoCE device to check + * Return: true if bond is active, false otherwise + */ +bool roce5_bond_is_active(struct roce5_device *rdev) +{ + return (rdev->bond_dev != NULL); +} + +void roce5_bond_detach(struct roce5_device *rdev) +{ + int ret; + u32 i; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + mutex_unlock(&g_roce5_bond_mutex); + ROCE_INFO("bond_dev is NULL."); + return; + } + + if (bond_dev->need_clean_on_remove == false) { + mutex_unlock(&g_roce5_bond_mutex); + ROCE_INFO("bond_dev no need to detached on roce remove."); + return; + } + + if (roce5_get_bond_dev(bond_dev->name) == bond_dev) { + list_del(&bond_dev->entry); + } + + mutex_unlock(&g_roce5_bond_mutex); + ret = roce5_bond_destroy_transition_store(bond_dev); + if (ret != 0) { + ROCE_ERR("Failed to store transition resources on destroy."); + } + hinic5_bond_detach(bond_dev->attr.bond_id, HINIC5_BOND_USER_ROCE); + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->bond_lld_dev == bond_dev->slaves[i].lld_dev) { + continue; + } + ret = hinic5_attach_service(bond_dev->slaves[i].lld_dev, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to attach roce device, ret(%d).", ret); + } + } + bond_dev->slave_cnt = 0; + mutex_unlock(&bond_dev->slave_lock); + roce5_bond_destroy_transition_free(bond_dev); + mutex_lock(&g_roce5_bond_mutex); + kfree(bond_dev); + rdev->bond_dev = NULL; + mutex_unlock(&g_roce5_bond_mutex); +} +/* + * roce5_bond_event_cfg_rdev - Configure RoCE device for bond events + * @lld_dev: LLD device + * @uld_dev: ULD device + * @rdev: pointer to RoCE device pointer (output) + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_event_cfg_rdev(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, bool hold_rdev) +{ + u32 i = 0; + struct roce5_bond_device *bond_dev = NULL; + + if (lld_dev == NULL) { + ROCE_ERR("lld_dev is null."); + return -EINVAL; + } + + if (uld_dev != NULL) { + *rdev = (struct roce5_device *)uld_dev; + return 0; + } + + mutex_lock(&g_roce5_bond_mutex); + list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) { + if (bond_dev == NULL) { + ROCE_ERR("bond_dev is null"); + continue; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->slaves[i].lld_dev == lld_dev) { + *rdev = bond_dev->attached_rdev; + if (hold_rdev == true && *rdev != NULL) { + uld5_dev_hold((*rdev)->lld_dev, + SERVICE_T_ROCE); + } + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + } + mutex_unlock(&bond_dev->slave_lock); + } + +out: + mutex_unlock(&g_roce5_bond_mutex); + return *rdev ? 0 : -EINVAL; +} + +/* + * roce5_update_rdev_func_val - Update function table values for RoCE device + * @bond_dev: bond device + */ +void roce5_update_rdev_func_val(struct roce5_bond_device *bond_dev) +{ + struct roce5_device *rdev = NULL; + u32 func_tbl_bond; + int ret; + + if (bond_dev->bond_lld_dev == NULL) { + return; + } + + rdev = hinic5_get_uld_dev(bond_dev->bond_lld_dev, SERVICE_T_ROCE); + if (rdev == NULL) { + ROCE_ERR("Failed to find roce device."); + return; + } + + ret = roce5_get_func_table(rdev->hwdev, rdev->glb_func_id, + &func_tbl_bond); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to get func table value, func_id(%d).", + rdev->glb_func_id); + uld5_dev_put(bond_dev->bond_lld_dev, SERVICE_T_ROCE); + return; + } + + write_lock(&rdev->rwlock_func_tbl_bond); + rdev->func_tbl_bond = func_tbl_bond; + write_unlock(&rdev->rwlock_func_tbl_bond); + + uld5_dev_put(bond_dev->bond_lld_dev, SERVICE_T_ROCE); +} + +bool roce5_get_bond_ipsurx_en(void) +{ + return g_roce5_bond_ipsurx_en; +} + +void roce5_set_bond_ipsurx_en(bool ipsurx_en) +{ + g_roce5_bond_ipsurx_en = ipsurx_en; +} + +/* + * roce5_bond_init - Initialize RoCE bond module + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_init(void) +{ + int ret; + struct bond_srv_func roce_func = { + .before_active = roce5_before_bond_active, + .after_active = roce5_after_bond_active, + .before_modify = roce5_before_bond_modify, + .after_modify = roce5_after_bond_modify, + .before_deactive = roce5_before_bond_deactive, + .after_deactive = roce5_after_bond_deactive, + .can_attach = roce5_can_do_bond, + }; + + ret = hinic5_bond_register(HINIC5_BOND_USER_ROCE, &roce_func); + if (ret != 0) { + ROCE_ERR("Failed to register bond, ret(%d).", ret); + return ret; + } + + return 0; +} + +void roce5_bond_exit(void) +{ + int ret; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) { + bond_dev->need_clean_on_remove = false; + } + mutex_unlock(&g_roce5_bond_mutex); + + ret = hinic5_bond_unregister(HINIC5_BOND_USER_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to unregister service func, ret(%d).", ret); + } +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h new file mode 100644 index 000000000..fd9853215 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h @@ -0,0 +1,97 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifndef ROCE_BOND_INTERNAL_H +#define ROCE_BOND_INTERNAL_H + +#ifdef ROCE_BONDING_EN + +#include <net/bonding.h> +#include <linux/types.h> +#include <linux/netdevice.h> +#include <linux/mutex.h> +#include <linux/workqueue.h> + +#include "hinic5_lld.h" +#include "drv_bond_api.h" +#include "bond_common_defs.h" +#include "roce_bond.h" + +/* Slave synchronization structure for bond modify */ +struct roce5_bond_slave_sync { + bool exist_arr + [ROCE_BOND_MAX_SLAVE]; /* Indicate which slave need to be deleted */ + struct net_device * + new_ndev[ROCE_BOND_MAX_SLAVE]; /* New slaves need to be added */ + struct netdev_lower_state_info new_ndev_state[BOND_PORT_MAX_NUM]; + u32 new_ndev_num; +}; + +/* Global variables declaration */ +extern struct list_head g_roce5_bond_list; //TODO: later via interface access +extern struct mutex g_roce5_bond_mutex; //TODO: later via interface access +extern struct workqueue_struct *g_bond_wq; //TODO: later via interface access + +/* CFM Interface Layer - Internal Functions */ +bool roce5_bond_mode_is_supported(u16 bond_mode); +int roce5_bond_slave_check(struct hinic5_lld_dev *lld_dev, char *chip_name, + size_t chip_name_size); +bool roce5_bond_before_active_check(struct bond_tracker *tracker, + struct bond_attr *attr); +bool roce5_can_do_bond(struct bonding *bond); + +/* CFM Callback Functions (called by CFM framework) */ +void roce5_before_bond_active(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_after_bond_active(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_before_bond_modify(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_after_bond_modify(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_before_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_after_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err); + +/* Device Management Layer - Internal Functions */ +struct roce5_bond_device *roce5_get_bond_dev(const char *bond_name); +struct roce5_bond_device *roce5_get_bond_dev_by_name(const char *bond_name); +void roce5_bond_destroy(const char *bond_name, u16 bond_id); +void roce5_update_rdev_func_val(struct roce5_bond_device *bond_dev); +struct hinic5_lld_dev * +roce5_bond_get_ppf_lld(const struct bond_tracker *tracker); +struct roce5_bond_device *roce5_bond_create_bond_device(const char *bond_name); + +/* Slave Management Layer - Internal Functions */ +struct roce5_bond_slave * +roce5_find_next_slave_slot(struct roce5_bond_device *rbdev); +int roce5_get_bond_slave_index_by_ndev(struct roce5_bond_device *rbdev, + struct net_device *ndev); +int roce5_bond_add_slave(struct roce5_bond_device *rbdev, + struct net_device *ndev, + struct netdev_lower_state_info *ndev_info, + bool is_update_table); +int roce5_bond_del_slave(struct roce5_bond_device *rbdev, u32 slave_index); +int roce5_find_same_slave_and_update(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker, + struct roce5_bond_slave_sync *slave_sync); +int roce5_handle_diff_slave(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker); + +/* MAC Table Operation Layer - Internal Functions */ +int roce5_bond_add_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id); +int roce5_bond_del_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id); + +/* TX Port Selection Layer - Internal Functions (declared in roce_bond.h as public) */ + +int roce5_bond_transition_store(struct roce5_bond_device *bond_dev, + struct hinic5_lld_dev *ppf_lld); +void roce5_bond_transition_free(struct roce5_bond_device *bond_dev); + +#endif /* ROCE_BONDING_EN */ + +#endif /* ROCE_BOND_INTERNAL_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c new file mode 100644 index 000000000..75264917f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c @@ -0,0 +1,260 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* + * roce5_add_bond_real_slave_mac - Add real slave MAC to table + * @rdev: RoCE device + * @mac: MAC address to add + * Return: 0 on success, negative error code on failure + */ +int roce5_add_bond_real_slave_mac(struct roce5_device *rdev, u8 *mac) +{ + struct roce5_bond_device *bond_dev = rdev->bond_dev; + struct roce5_bond_slave *slave = NULL; + int ret; + u32 i; + + if (bond_dev == NULL) { + ROCE_DEV_ERR(rdev, "Failed to find bond_dev, func_id(%u).", + rdev->glb_func_id); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + + ret = roce5_add_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, + ROCE_BOND_RSVD_VLAN_ID, + slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + ROCE_DEV_ERR(rdev, + "Failed to add mac_vlan entry, ret(%d).", + ret); + return ret; + } + + if (slave->func_id != rdev->glb_func_id) { + /* The IPSU MAC table is used for fast forwarding. Even if the addition fails, the forwarding information + can still be obtained by checking the MAC table later, without judging the execution result. */ + roce5_add_ipsu_tbl_mac_entry(rdev, mac, 0, + rdev->glb_func_id, + slave->er_id, + slave->func_id); + } + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +/* + * roce5_add_bond_vlan_slave_mac - Add VLAN slave Mac to table + * @rdev: RoCE device + * @mac: MAC address to add + * @vlan_id: VLAN ID + * Return: 0 on success, negative error code on failure + */ +int roce5_add_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id) +{ + struct roce5_bond_device *bond_dev = rdev->bond_dev; + struct roce5_bond_slave *slave = NULL; + int ret; + u32 i; + + if (bond_dev == NULL) { + ROCE_DEV_ERR(rdev, "Failed to find bond_dev"); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) { + continue; + } + + ret = roce5_add_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id, + slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + ROCE_DEV_ERR(rdev, + "Failed to add mac_vlan entry, ret(%d).", + ret); + return ret; + } + + /* The IPSU MAC table is used for fast forwarding. Even if the addition fails, the forwarding information + can still be obtained by checking the MAC table later, without judging the execution result. */ + roce5_add_ipsu_tbl_mac_entry(rdev, mac, vlan_id, + rdev->glb_func_id, slave->er_id, + slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +/* + * roce5_del_bond_real_slave_mac - Delete real slave Mac from table + * @rdev: RoCE device + */ +void roce5_del_bond_real_slave_mac(struct roce5_device *rdev) +{ + u32 i; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_device *bond_dev = rdev->bond_dev; + + if (bond_dev == NULL) { + return; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id != rdev->glb_func_id) { + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, + 0, slave->er_id, + slave->func_id); + } + + (void)roce5_del_mac_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, + ROCE_BOND_RSVD_VLAN_ID, + slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); +} + +/* + * roce5_del_bond_vlan_slave_mac - Delete VLAN slave Mac from table + * @rdev: RoCE device + * @mac: MAC address to delete + * @vlan_id: VLAN ID + */ +void roce5_del_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id) +{ + u32 i; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_device *bond_dev = rdev->bond_dev; + + if (bond_dev == NULL) { + return; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) { + continue; + } + + roce5_del_ipsu_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id, + slave->er_id, slave->func_id); + + (void)roce5_del_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id, + slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); +} + +/* + * roce5_bond_add_slave_table - Add slave table entry + * @rbdev: bond device + * @slave_er_id: slave ER ID + * @slave_func_id: slave function ID + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_add_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id) +{ + struct roce5_device *rdev = rbdev->attached_rdev; + struct roce5_vlan_dev_list *vlan_dev_list = NULL; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + ROCE_ERR("Missing attached roce device."); + return -EINVAL; + } + + /* Add entry of non-vlan device */ + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)rdev->mac, 0, + rdev->glb_func_id, slave_er_id, + slave_func_id); + /* Add entries of vlan device */ + mutex_lock(&rdev->mac_vlan_mutex); + if (rdev->mac_vlan_list_head.next == NULL) { + ROCE_ERR("mac vlan list not init."); + mutex_unlock(&rdev->mac_vlan_mutex); + return -EINVAL; + } + list_for_each_entry(vlan_dev_list, &rdev->mac_vlan_list_head, list) { + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)rdev->mac, + vlan_dev_list->vlan_id, + rdev->glb_func_id, slave_er_id, + slave_func_id); + } + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +/* + * roce5_bond_del_slave_table - Delete slave table entry + * @rbdev: bond device + * @slave_er_id: slave ER ID + * @slave_func_id: slave function ID + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_del_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id) +{ + struct roce5_device *rdev = rbdev->attached_rdev; + struct roce5_vlan_dev_list *vlan_dev_list = NULL; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + ROCE_ERR("Missing attached roce device."); + return -EINVAL; + } + + /* Delete entry of non-vlan device */ + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, slave_er_id, + slave_func_id); + /* Delete entries of vlan device */ + mutex_lock(&rdev->mac_vlan_mutex); + if (rdev->mac_vlan_list_head.next == NULL) { + ROCE_ERR("mac vlan list not init."); + mutex_unlock(&rdev->mac_vlan_mutex); + return -EINVAL; + } + list_for_each_entry(vlan_dev_list, &rdev->mac_vlan_list_head, list) { + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, + vlan_dev_list->vlan_id, + slave_er_id, slave_func_id); + } + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c new file mode 100644 index 000000000..a0fee96a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c @@ -0,0 +1,459 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "bond_common_defs.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* + * roce5_find_next_slave_slot - Find next available slave slot + * @rbdev: bond device + * Return: pointer to available slave slot, NULL if no slot available + * Lock: caller must hold rbdev->slave_lock + */ +struct roce5_bond_slave * +roce5_find_next_slave_slot(struct roce5_bond_device *rbdev) +{ + if (rbdev->slave_cnt >= ROCE_BOND_MAX_SLAVE) { + ROCE_ERR( + "RoCE bond device slave slots are full, max number of slaves is %d.", + ROCE_BOND_MAX_SLAVE); + return NULL; + } + + if (rbdev->slaves[rbdev->slave_cnt].netdev != NULL) { + ROCE_ERR("Error occur! Slave slot is broken, slot index(%d).", + rbdev->slave_cnt); + return NULL; + } + + return &rbdev->slaves[rbdev->slave_cnt]; +} + +/* + * roce5_get_bond_slave_index_by_ndev - Get slave index by netdev + * @rbdev: bond device + * @ndev: netdev to search for + * Return: slave index if found, negative error code otherwise + * Lock: caller must hold rbdev->slave_lock + */ +int roce5_get_bond_slave_index_by_ndev(struct roce5_bond_device *rbdev, + struct net_device *ndev) +{ + u32 index; + + for (index = 0; index < rbdev->slave_cnt; ++index) { + if (rbdev->slaves[index].netdev == ndev) { + return (int)index; + } + } + + return -ENODEV; +} + +static void roce5_bond_slave_init(struct roce5_bond_slave *slave, + struct net_device *ndev, + struct hinic5_lld_dev *ndev_lld, + struct netdev_lower_state_info *ndev_info, + bool is_update_table, + struct roce5_bond_device *rbdev) +{ + int ret; + + slave->netdev = ndev; + slave->netdev_state.link_up = ndev_info->link_up; + slave->netdev_state.tx_enabled = ndev_info->tx_enabled; + + slave->lld_dev = ndev_lld; + slave->func_id = hinic5_global_func_id(slave->lld_dev->hwdev); + slave->er_id = hinic5_er_id(slave->lld_dev->hwdev); + + hinic5_detach_service(slave->lld_dev, SERVICE_T_ROCE); + + if (is_update_table) { + ret = roce5_bond_add_slave_table(rbdev, slave->er_id, + slave->func_id); + if (ret != 0) { + ROCE_WARN( + "Failed to add table entries of slave(%u), ret(%d).", + slave->func_id, ret); + } + } +} + +static void roce5_bond_slave_deinit(struct roce5_bond_slave *slave, + bool is_update_table, + struct roce5_bond_device *rbdev) +{ + int ret; + + if (is_update_table) { + ret = roce5_bond_del_slave_table(rbdev, slave->er_id, + slave->func_id); + if (ret != 0) { + ROCE_ERR( + "Failed to delete slave table entries, ret(%d).", + ret); + } + } + + ret = hinic5_attach_service(slave->lld_dev, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR( + "Failed to attach roce function, func_id(%u), ret(%d).", + slave->func_id, ret); + } +} + +/* + * roce5_bond_add_slave - Add slave to bond device + * @rbdev: bond device + * @ndev: slave netdev + * @ndev_info: slave state info + * @is_update_table: whether to update MAC table + * Return: 0 on success, negative error code on failure + * Lock: caller must hold rbdev->slave_lock + */ +int roce5_bond_add_slave(struct roce5_bond_device *rbdev, + struct net_device *ndev, + struct netdev_lower_state_info *ndev_info, + bool is_update_table) +{ + int ret, err_ret; + struct hinic5_lld_dev *ndev_lld = NULL; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_slave slave_temp; + + ndev_lld = hinic5_get_lld_dev_by_netdev(ndev); + if (ndev_lld == NULL) { + ROCE_ERR("Failed to get lld dev by netdev %s.", ndev->name); + return -ENODEV; + } + + if (hinic5_support_roce(ndev_lld->hwdev, NULL) == false) { + return 0; + } + + roce5_bond_slave_init(&slave_temp, ndev, ndev_lld, ndev_info, + is_update_table, rbdev); + + /* Add slave to roce bond device. */ + mutex_lock(&rbdev->slave_lock); + + slave = roce5_find_next_slave_slot(rbdev); + if (slave == NULL) { + ROCE_ERR("Failed to find next slave slot."); + ret = -ENXIO; + mutex_unlock(&rbdev->slave_lock); + goto err_without_slot; + } + + *slave = slave_temp; + ++rbdev->slave_cnt; + + mutex_unlock(&rbdev->slave_lock); + + /* Try to attach roce device to roce bond device. */ + if (rbdev->attached_rdev == NULL) { + if (rbdev->bond_lld_dev != NULL) { + ROCE_ERR("Impossible situation! bond lld isn't NULL."); + ret = -EINVAL; + goto err_check; + } + + rbdev->bond_lld_dev = ndev_lld; + + ret = hinic5_attach_service(rbdev->bond_lld_dev, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR( + "Failed to attach roce service, ret(%d), bond name(%s).", + ret, rbdev->name); + ret = -EINVAL; + goto err_attach; + } + + if (rbdev->attached_rdev == NULL) { + ROCE_ERR( + "Failed to attach roce bond device with roce device, bond name(%s).", + rbdev->name); + ret = -EINVAL; + goto err_attach_bond; + } + } + + return 0; + +err_attach_bond: + hinic5_detach_service(rbdev->bond_lld_dev, SERVICE_T_ROCE); +err_attach: + rbdev->bond_lld_dev = NULL; +err_check: + mutex_lock(&rbdev->slave_lock); + err_ret = memset_s(slave, sizeof(struct roce5_bond_slave), 0, + sizeof(struct roce5_bond_slave)); + if (err_ret != 0) { + ROCE_ERR("Failed to clean memory of slave, ret(%d).", err_ret); + } + --rbdev->slave_cnt; + mutex_unlock(&rbdev->slave_lock); +err_without_slot: + roce5_bond_slave_deinit(&slave_temp, is_update_table, rbdev); + return ret; +} + +/* + * roce5_bond_del_slave - Delete slave from bond device + * @rbdev: bond device + * @slave_index: index of slave to delete + * Return: 0 on success, negative error code on failure + * Lock: caller must hold rbdev->slave_lock + */ +int roce5_bond_del_slave(struct roce5_bond_device *rbdev, u32 slave_index) +{ + int ret; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_slave slave_temp; + size_t mem_copy_len; + + mutex_lock(&rbdev->slave_lock); + + if (slave_index >= rbdev->slave_cnt) { + ROCE_ERR("Invalid parameter, slave index(%u), slave count(%u).", + slave_index, rbdev->slave_cnt); + ret = -EINVAL; + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + + slave = &rbdev->slaves[slave_index]; + if (rbdev->bond_lld_dev != NULL && + slave->lld_dev == rbdev->bond_lld_dev) { + ROCE_WARN( + "Slave(%s) lld dev is the same as the roce bond device(%s), skip removing.", + slave->netdev->name, + (rbdev->attached_rdev != NULL) ? + rbdev->attached_rdev->ib_dev.name : + "NULL"); + ret = 0; + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + + slave_temp = *slave; + + ret = memset_s(slave, sizeof(struct roce5_bond_slave), 0, + sizeof(struct roce5_bond_slave)); + if (ret != 0) { + ROCE_WARN("Failed to clear slave information."); + } + + if (slave_index != ROCE_BOND_MAX_SLAVE - 1) { + mem_copy_len = sizeof(struct roce5_bond_slave) * + (ROCE_BOND_MAX_SLAVE - slave_index - 1); + ret = memmove_s((void *)&rbdev->slaves[slave_index], + mem_copy_len, + (void *)&rbdev->slaves[slave_index + 1], + mem_copy_len); + if (ret != EOK) { + ROCE_ERR("Failed to execute memmove, ret(%d).", ret); + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + } + + ret = memset_s((void *)&rbdev->slaves[ROCE_BOND_MAX_SLAVE - 1], + sizeof(struct roce5_bond_slave), 0, + sizeof(struct roce5_bond_slave)); + if (ret != 0) { + ROCE_ERR("Failed to execute memmove, ret(%d).", ret); + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + + --rbdev->slave_cnt; + + mutex_unlock(&rbdev->slave_lock); + + if (rbdev->slave_cnt == 0 && rbdev->attached_rdev != NULL) { + if (rbdev->bond_lld_dev == NULL) { + ROCE_ERR("Impossible situation! bond lld is NULL."); + ret = -EINVAL; + goto err_out; + } + + hinic5_detach_service(rbdev->bond_lld_dev, SERVICE_T_ROCE); + if (rbdev->attached_rdev != NULL) { + ROCE_ERR( + "Failed to detach roce service, bond name(%s).", + rbdev->name); + ret = -EINVAL; + goto err_out; + } + + rbdev->bond_lld_dev = NULL; + } + + ret = roce5_bond_del_slave_table(rbdev, slave_temp.er_id, + slave_temp.func_id); + if (ret != 0) { + ROCE_WARN( + "Failed to delete table entries of slave(%u), ret(%d).", + slave_temp.func_id, ret); + ret = 0; + } + + ret = hinic5_attach_service(slave_temp.lld_dev, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to attach roce function, func_id(%u).", + slave_temp.func_id); + goto err_out; + } + + return 0; + +err_out: + return ret; +} + +/* + * roce5_find_same_slave_and_update - Find and update existing slave + * @rbdev: bond device + * @tracker: bond tracker from CFM framework + * @slave_sync: slave synchronization info + * Return: 0 on success, negative error code on failure + */ +int roce5_find_same_slave_and_update(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker, + struct roce5_bond_slave_sync *slave_sync) +{ + int ret = 0; + int i; + int roce_slave_index; + int tracker_slave_cnt = 0; + struct hinic5_lld_dev *lld_dev = NULL; + char chip_name[IFNAMSIZ] = { 0 }; + int err; + + ret = memset_s(slave_sync, sizeof(struct roce5_bond_slave_sync), 0, + sizeof(struct roce5_bond_slave_sync)); + if (ret != 0) { + ROCE_ERR("Failed to init slave sync info, ret(%d).", ret); + return ret; + } + + mutex_lock(&rbdev->slave_lock); + + if (rbdev->bond_lld_dev != NULL) { + err = hinic5_get_chip_name(rbdev->bond_lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond dev get chip name err %d.", err); + mutex_unlock(&rbdev->slave_lock); + return err; + } + } + + for (i = 0; i < BOND_PORT_MAX_NUM && tracker_slave_cnt < tracker->cnt; + ++i) { + if (tracker->ndev[i] == NULL) { + continue; + } + + roce_slave_index = roce5_get_bond_slave_index_by_ndev( + rbdev, tracker->ndev[i]); + if (roce_slave_index == -ENODEV) { + lld_dev = + hinic5_get_lld_dev_by_netdev(tracker->ndev[i]); + err = roce5_bond_slave_check(lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond slave check err %d.", err); + mutex_unlock(&rbdev->slave_lock); + return err; + } + + slave_sync->new_ndev[slave_sync->new_ndev_num] = + tracker->ndev[i]; + slave_sync->new_ndev_state[slave_sync->new_ndev_num] = + tracker->netdev_state[i]; + ++slave_sync->new_ndev_num; + } else { + slave_sync->exist_arr[roce_slave_index] = true; + + /* update state of the slave */ + rbdev->slaves[roce_slave_index].netdev_state.link_up = + tracker->netdev_state[i].link_up; + rbdev->slaves[roce_slave_index].netdev_state.tx_enabled = + tracker->netdev_state[i].tx_enabled; + } + + ++tracker_slave_cnt; + } + + mutex_unlock(&rbdev->slave_lock); + + return ret; +} + +/* + * roce5_handle_diff_slave - Handle different slaves during bond modify + * @rbdev: bond device + * @tracker: bond tracker from CFM framework + * Return: 0 on success, negative error code on failure + */ +int roce5_handle_diff_slave(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker) +{ + u32 i; + int ret = 0; + struct roce5_bond_slave_sync slave_sync; + + ret = roce5_find_same_slave_and_update(rbdev, tracker, &slave_sync); + if (ret != 0) { + ROCE_ERR("Failed to find same netdev."); + return ret; + } + + /* delete slaves */ + for (i = rbdev->slave_cnt; i > 0; --i) { + if (slave_sync.exist_arr[i - 1] == false) { + ret = roce5_bond_del_slave(rbdev, i - 1); + if (ret != 0) { + ROCE_ERR("Failed to delete slave, ret(%d).", + ret); + return ret; + } + } + } + + /* add new slaves */ + for (i = 0; i < slave_sync.new_ndev_num; ++i) { + ret = roce5_bond_add_slave(rbdev, slave_sync.new_ndev[i], + &slave_sync.new_ndev_state[i], true); + if (ret != 0) { + ROCE_ERR("Failed to add new slave, ret(%d).", ret); + return ret; + } + } + + return ret; +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c new file mode 100644 index 000000000..dd34d319e --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c @@ -0,0 +1,94 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> +#include <linux/atomic.h> + +#include "securec.h" + +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* + * roce5_bond_get_netdev - Get bond netdev for TX + * @rdev: RoCE device + * Return: netdev with increased reference count, NULL if no bond + * Refcount: caller must call dev_put() when done + */ +struct net_device *roce5_bond_get_netdev(struct roce5_device *rdev) +{ + u32 i; + struct roce5_bond_device *bond_dev = rdev->bond_dev; + struct net_device *ret_dev = NULL; + struct slave *slave = NULL; + + mutex_lock(&g_roce5_bond_mutex); + if (bond_dev == NULL) { + mutex_unlock(&g_roce5_bond_mutex); + return ret_dev; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + rcu_read_lock(); + slave = bond_slave_get_rcu(bond_dev->slaves[i].netdev); + rcu_read_unlock(); + if (slave == NULL) { + continue; + } + if (bond_is_active_slave(slave)) { + if (netif_running(bond_dev->slaves[i].netdev) && + netif_carrier_ok(bond_dev->slaves[i].netdev)) { + ret_dev = bond_dev->slaves[i].netdev; + } else if (netif_running( + bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt] + .netdev) && + netif_carrier_ok( + bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt] + .netdev)) { + ret_dev = bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt] + .netdev; + } else { + ret_dev = bond_dev->slaves[i].netdev; + } + dev_hold(ret_dev); + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce5_bond_mutex); + return ret_dev; + } + } + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce5_bond_mutex); + return ret_dev; +} + +/* + * roce5_bond_rr_set_flow - Set flow hash for round-robin TX selection + * @rdev: RoCE device + * @rqp: RoCE QP + * @qp_attr: QP attributes + */ +void roce5_bond_rr_set_flow(struct roce5_device *rdev, struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr) +{ + rqp->tx_hash_value = (u32)atomic_add_return(1, &rdev->next_port); + qp_attr->path_info.dw0.bs.bond_tx_hash_value = + (rqp->tx_hash_value & 0xf); +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h b/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h new file mode 100644 index 000000000..6b919d753 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_DEBUG_H +#define ROCE_DEBUG_H + +/* + * DEBUG_ONLY: statement is only valid in debug mode. + * + * RELEASE_ONLY: statement is only valid in release mode. + * + * DEBUG_REPLACE: used in functions of release to indicate the function may be replaced by another function with the + * same name defined in the debug mode. + */ +#ifdef ENABLE_DEBUG +#define DEBUG_ONLY(statement) statement +#define RELEASE_ONLY(statement) +#define DEBUG_REPLACE __attribute__((weak)) +#else +#define DEBUG_ONLY(statement) +#define RELEASE_ONLY(statement) statement +#define DEBUG_REPLACE +#endif + +#endif /* ROCE_DEBUG_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c new file mode 100644 index 000000000..787a86791 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c @@ -0,0 +1,375 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. + + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> + +#include "roce.h" +#include "hinic5_mt.h" +#include "roce_cc_format.h" +#include "roce_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_mpu_cmd_defs.h" +#include "securec.h" +#include "roce_dfx.h" + +int roce5_get_drv_version(void *buf_out, u32 *out_size) +{ + struct drv_version_info *ver_info = buf_out; + int rc; + + if (!buf_out || !out_size) { + ROCE_ERR("Input params is NULL."); + return -EINVAL; + } + + if (*out_size != sizeof(*ver_info)) { + ROCE_ERR("Unexpect out buf size from user :%u, expect: %lu", + *out_size, sizeof(*ver_info)); + return -EINVAL; + } + + rc = snprintf_s(ver_info->ver, sizeof(ver_info->ver), + sizeof(ver_info->ver) - 1, "%s %s", + HIROCE5_DRV_VERSION, __TIME_STR__); + if (rc == -1) { + ROCE_ERR("Snprintf roce version err"); + return -EFAULT; + } + + return 0; +} + +void * +roce5_adm_generic_find_cmd_func(const roce5_adm_generic_cmd_proc_t *cmd_array, + int elem_count, u32 target_cmd) +{ + int i; + + if (cmd_array == NULL || elem_count <= 0) { + return NULL; + } + + for (i = 0; i < elem_count; i++) { + if (cmd_array[i].cmd_type == target_cmd && + cmd_array[i].proc_func != NULL) { + return cmd_array[i].proc_func; + } + } + return NULL; +} + +static int roce5_dfx_enable_bw_ctrl(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + if (rdev->hw_info.hca_type == ROCE5_2_100G_HCA) { + inbuf->ctrl_param.cir = ROCE5_100G_CIR; + inbuf->ctrl_param.pir = ROCE5_100G_PIR; + inbuf->ctrl_param.cnp = ROCE5_100G_CNP; + } else { + inbuf->ctrl_param.cir = ROCE5_25G_CIR; + inbuf->ctrl_param.pir = ROCE5_25G_PIR; + inbuf->ctrl_param.cnp = ROCE5_25G_CNP; + } + return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_EN, inbuf); +} + +static int roce5_dfx_disable_bw_ctrl(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + inbuf->ctrl_param.cir = 0; + inbuf->ctrl_param.pir = 0; + inbuf->ctrl_param.cnp = 0; + return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_DIS, inbuf); +} + +static int roce5_dfx_change_bw_ctrl_param(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_RESET, inbuf); +} + +static int roce5_dfx_query_bw_ctrl_param(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + return roce5_query_bw_ctrl_state(rdev, &outbuf->bw_ctrl_param); +} + +static const roce5_adm_dfx_bw_ctrl_cmd_proc_t + g_roce5_adm_dfx_bw_ctrl_cmd_proc[] = { + { ROCE_CMD_ENABLE_BW_CTRL, roce5_dfx_enable_bw_ctrl }, + { ROCE_CMD_DISABLE_BW_CTRL, roce5_dfx_disable_bw_ctrl }, + { ROCE_CMD_CHANGE_BW_CTRL_PARAM, + roce5_dfx_change_bw_ctrl_param }, + { ROCE_CMD_QUERY_BW_CTRL_PARAM, roce5_dfx_query_bw_ctrl_param }, + }; + +int roce5_adm_dfx_bw_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + struct roce5_bw_ctrl_inbuf *inbuf = + (struct roce5_bw_ctrl_inbuf *)buf_in; + struct roce5_bw_ctrl_outbuf *outbuf = + (struct roce5_bw_ctrl_outbuf *)buf_out; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *) + g_roce5_adm_dfx_bw_ctrl_cmd_proc; + int elem_count = sizeof(g_roce5_adm_dfx_bw_ctrl_cmd_proc) / + sizeof(g_roce5_adm_dfx_bw_ctrl_cmd_proc[0]); + roce5_adm_dfx_bw_ctrl_func_t roce5_adm_dfx_bw_ctrl_func = NULL; + + if (in_size < sizeof(struct roce5_bw_ctrl_inbuf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct roce5_bw_ctrl_inbuf)); + return -ENOMEM; + } + if (*out_size < sizeof(struct roce5_bw_ctrl_outbuf)) { + ROCE_ERR( + "Input parameter(out_zise: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, + (int)sizeof(struct roce5_bw_ctrl_outbuf)); + return -ENOMEM; + } + ret = memset_s(buf_out, sizeof(struct roce5_bw_ctrl_outbuf), 0, + sizeof(struct roce5_bw_ctrl_outbuf)); + if (ret != 0) { + ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret); + return -ENOMEM; + } + *out_size = sizeof(struct roce5_bw_ctrl_outbuf); + + if (inbuf->cmd_type >= VM_COMPAT_TEST) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + roce5_adm_dfx_bw_ctrl_func = + (roce5_adm_dfx_bw_ctrl_func_t)roce5_adm_generic_find_cmd_func( + generic_array, elem_count, inbuf->cmd_type); + if (roce5_adm_dfx_bw_ctrl_func == NULL) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + return roce5_adm_dfx_bw_ctrl_func(rdev, inbuf, outbuf); +} + +int roce5_adm_dfx_set_ldcp_param(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + roce_ldcp_param_s ldcp_param; + const struct roce5_ldcp_param *inbuf = + (struct roce5_ldcp_param *)buf_in; + + if (in_size < sizeof(struct roce5_ldcp_param)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct roce5_ldcp_param)); + return -ENOMEM; + } + if (inbuf->cmd_type != ROCE_CMD_SET_LDCP_PARAM) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + ldcp_param.dw0.bs.alpha = inbuf->ldcp_param.alpha & 0xf; + ldcp_param.dw0.bs.beta = inbuf->ldcp_param.beta & 0xf; + ldcp_param.dw0.bs.gamma = inbuf->ldcp_param.gamma & 0xf; + ldcp_param.dw0.bs.eta = inbuf->ldcp_param.eta & 0xf; + ldcp_param.dw0.bs.set_flag = inbuf->ldcp_param.set_flag & 0x1; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ldcp_param, + sizeof(roce_ldcp_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_LDCP_PARAM); + if (ret != 0) { + ROCE_ERR("Failed to set ldcp param.(ret:%d)", ret); + } else { + ROCE_WARN("Set ldcp param 0x%x", ldcp_param.dw0.value); + } + return ret; +} + +int roce5_adm_dfx_port_traffic(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + roce_mpu_port_statics_cmd_inbuf_t inbuf; + roce_port_sta_outbuf_t *outbuf = (roce_port_sta_outbuf_t *)buf_out; + int ret; + + memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf)); + inbuf.cmd = *(roce_port_sta_inbuf_t *)buf_in; + outbuf->head.status = 0xFF; + + if (in_size < sizeof(roce_port_sta_inbuf_t)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(roce_port_sta_inbuf_t)); + return -ENOMEM; + } + if (inbuf.cmd.cmd_type != ROCE_CMD_PORT_TRAFFIC) { + ROCE_DEV_ERR(rdev, "Not support this type(%d).", + inbuf.cmd.cmd_type); + return -EINVAL; + } + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DFX_PORT_TRAFFIC, + (void *)&inbuf, (u16)sizeof(inbuf), + buf_out, (u16 *)(void *)out_size); + if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) { + ROCE_ERR( + "Failed to query port traffic, err(%d), out size(0x%x) status(0x%x)", + ret, *out_size, outbuf->head.status); + return -EINVAL; + } + + return 0; +} + +#ifdef ROCE_CHIP_TEST +static const roce5_adm_dfx_vnic_waterline_cmd_proc_t + g_roce5_adm_dfx_vnic_wl_cmd_proc[] = { + { ROCE_CMD_SET_VNIC_WATERLINE, roce5_set_vnic_waterline_state }, + { ROCE_CMD_GET_VNIC_WATERLINE, roce5_get_vnic_waterline_state }, + }; + +int roce5_adm_dfx_vnic_wl_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + struct roce5_vnic_waterline_ctrl_buf *inbuf = + (struct roce5_vnic_waterline_ctrl_buf *)buf_in; + struct roce5_vnic_waterline_ctrl_buf *outbuf = + (struct roce5_vnic_waterline_ctrl_buf *)buf_out; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *) + g_roce5_adm_dfx_vnic_wl_cmd_proc; + int elem_count = sizeof(g_roce5_adm_dfx_vnic_wl_cmd_proc) / + sizeof(g_roce5_adm_dfx_vnic_wl_cmd_proc[0]); + roce5_adm_dfx_vnic_waterline_ctrl_func_t roce5_adm_dfx_vnic_wl_ctrl_func; + + if (in_size < sizeof(struct roce5_vnic_waterline_ctrl_buf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, + (int)sizeof(struct roce5_vnic_waterline_ctrl_buf)); + return -EINVAL; + } + + if (inbuf->cmd_type >= VM_COMPAT_TEST) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + ret = memset_s(buf_out, sizeof(struct roce5_vnic_waterline_ctrl_buf), 0, + sizeof(struct roce5_vnic_waterline_ctrl_buf)); + if (ret != 0) { + ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret); + return -ENOMEM; + } + + roce5_adm_dfx_vnic_wl_ctrl_func = + (roce5_adm_dfx_vnic_waterline_ctrl_func_t) + roce5_adm_generic_find_cmd_func( + generic_array, elem_count, inbuf->cmd_type); + if (roce5_adm_dfx_vnic_wl_ctrl_func == NULL) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + return roce5_adm_dfx_vnic_wl_ctrl_func(rdev, inbuf, outbuf); +} + +#endif /* ROCE_CHIP_TEST */ + +int roce5_adm_dfx_roce_set(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) + +{ + roce_dfx_set_cmd_s inbuf; + roce_dfx_set_outbuf_s *outbuf = (roce_dfx_set_outbuf_s *)buf_out; + int ret; + + if (in_size < sizeof(roce_dfx_set_inbuf_s)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(roce_dfx_set_inbuf_s)); + return -ENOMEM; + } + + memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf)); + inbuf.cmd = *(roce_dfx_set_inbuf_s *)buf_in; + inbuf.cmd.func_id = rdev->glb_func_id; + + if (inbuf.cmd.cmd_type != ROCE_CMD_ROCE_SET) { + ROCE_DEV_ERR(rdev, "Not support this type(%d).", + inbuf.cmd.cmd_type); + return -EINVAL; + } + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DFX_ROCE_SET, + (void *)&inbuf, (u16)sizeof(inbuf), outbuf, + (u16 *)(void *)out_size); + if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) { + ROCE_ERR( + "Failed to set roce configs, err(%d), out size(0x%x) status(0x%x)", + ret, *out_size, outbuf->head.status); + return -EINVAL; + } + return 0; +} + +int roce5_adm_dfx_roce_set_debug(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) + +{ + roce_dfx_set_debug_cmd_s inbuf; + roce_dfx_set_debug_outbuf_s *outbuf = + (roce_dfx_set_debug_outbuf_s *)buf_out; + int ret; + + if (in_size < sizeof(roce_dfx_set_debug_inbuf_s)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(roce_dfx_set_debug_inbuf_s)); + return -ENOMEM; + } + + memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf)); + inbuf.cmd = *(roce_dfx_set_debug_inbuf_s *)buf_in; + inbuf.cmd.func_id = rdev->glb_func_id; + + if (inbuf.cmd.cmd_type != ROCE_CMD_ROCE_SET_DEBUG) { + ROCE_DEV_ERR(rdev, "Not support this type(%d).", + inbuf.cmd.cmd_type); + return -EINVAL; + } + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, + ROCE_MPU_CMD_DFX_ROCE_SET_DEBUG, + (void *)&inbuf, (u16)sizeof(inbuf), outbuf, + (u16 *)(void *)out_size); + if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) { + ROCE_ERR( + "Failed to set roce configs, err(%d), out size(0x%x) status(0x%x)", + ret, *out_size, outbuf->head.status); + return -EINVAL; + } + return 0; +} +#endif /* __ROCE_DFX__ */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h new file mode 100644 index 000000000..139867581 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. + + +#ifndef ROCE_DFX_H +#define ROCE_DFX_H + +#include <linux/types.h> + +#include "hinic5_rdma.h" + +#include "roce_sysfs.h" +#include "roce.h" +#include "roce_gid.h" +#include "roce_compat.h" +#include "roce_pub_cmd.h" +#include "roce_npu_cmd_dfx_defs.h" +#include "roce_debug.h" + +#include "securec.h" + +#define MR_KEY_2_INDEX_SHIFT 8 + +#define ROCE_IO_DFX_CFG_VADDR_ID 0 +#define ROCE_IO_DFX_CFG_PADDR_ID 1 +#define ROCE_IO_DFX_CFG_ADDR_NUM 2 + +#define roce5_dfx_print(fmt, args...) \ + do { \ + (void)printk("[roce5_dfx] : " fmt "\n", ##args); \ + } while (0) + +typedef struct tag_roce_cmd_mpt_query { + roce_verbs_cmd_com_s com; +} roce_cmd_mpt_query_s; + +typedef struct roce5_mpt_query_outbuf { + u32 value[16]; +} roce5_mpt_query_outbuf_s; + +typedef struct tag_roce_cmd_roce_cq_query { + roce_verbs_cmd_com_s com; +} roce_cmd_cq_query_s; + +typedef struct roce5_cq_query_outbuf { + u32 value[32]; +} roce5_cq_query_outbuf_s; + +typedef struct tag_roce_cmd_qp_query { + roce_verbs_cmd_com_s com; +} roce_cmd_qp_query_s; + +typedef struct roce5_qp_query_outbuf { + u32 context[128]; +} roce5_qp_query_outbuf_s; + +typedef struct roce5_srq_query_outbuf { + u32 value[24]; +} roce5_srq_query_outbuf_s; + +struct roce5_dfx_query_inbuf { + u32 cmd_type; + union { + u32 qpn; + u32 cqn; + u32 srqn; + u32 mpt_key; + u32 gid_index; + struct { + u32 qpn; + u32 cqn; + } query_pi_ci; + }; +}; + +#define ROCE_DEV_NAME_LEN_MAX 64 + +struct roce5_dfx_dbg_inbuf { + u32 cmd_type; + union { + u32 qpn; + u32 srqn; + }; + char dev_name[ROCE_DEV_NAME_LEN_MAX]; +}; + +#define ROCE_CTX_MAX_SIZE 512 // adaptation lock store referencein of structure + +union roce5_dfx_dbg_outbuf { + roce_verbs_qp_attr_s qp_attr; + roce_verbs_srq_attr_dfx_s srq_attr; + u32 buf[ROCE_CTX_MAX_SIZE]; + u32 algo_type; + u32 dev_type; +}; + +struct roce5_dfx_common_inbuf { + u32 cmd_type; + char dev_name[ROCE_DEV_NAME_LEN_MAX]; +}; + +union roce5_dfx_common_outbuf { + u32 dev_type; +}; + +struct roce5_dfx_pi_ci { + u32 qpc_sq_pi_on_chip; + u32 qpc_sq_pi; + u32 qpc_sq_load_pi; + u32 qpc_rq_pi_on_chip; + u32 qpc_rq_load_pi; + u32 qpc_rq_pi; + u32 qpc_rc_pi; + u32 qpc_sq_ci; + u32 qpc_sq_wqe_prefetch_ci; + u32 qpc_sq_mtt_prefetch_wqe_ci; + u32 qpc_sqa_ci; + u32 qpc_sqa_wqe_prefetch_ci; + u32 qpc_rq_ci; + u32 qpc_rq_wqe_prefetch_ci; + u32 qpc_rq_mtt_prefetch_wqe_ci; + u32 qpc_rq_base_ci; + u32 qpc_rc_ci; + u32 qpc_rc_prefetch_ci; + u32 cq_ci_on_chip; + u32 cq_ci; + u32 cq_load_ci; + u64 cq_ci_record_gpa_at_hop_num; + u32 cq_last_solicited_pi; + u32 cq_pi; + u32 cq_last_notified_pi; +}; + +struct roce5_dfx_qp_count { + u32 qp_alloced; + u32 qp_deleted; + u32 qp_alive; +}; + +#define ROCE_GID_ENTRY_NUM 17 // 1 host fe + 16 shadow fe +union roce5_dfx_query_outbuf { + /* query cache context */ + roce_verbs_qp_attr_s qp_attr; + roce_verbs_cq_attr_s cq_attr; + roce_verbs_srq_attr_s srq_attr; + roce_verbs_mr_attr_s mr_attr; + /* query host context */ + roce5_qp_query_outbuf_s qp_query; + roce5_cq_query_outbuf_s cq_query; + roce5_srq_query_outbuf_s srq_query; + struct roce_mpt_context mpt; + struct rdma_gid_entry gid_entry[ROCE_GID_ENTRY_NUM]; + struct roce5_dfx_pi_ci pi_ci; + struct roce5_dfx_qp_count qp_count; + u32 algo_type; + struct roce_uni_cmd_latch_data latch_data_outbuf; +}; + +enum roce5_bw_ctrl_cmd_e { + ROCE_BW_CTRL_DIS, + ROCE_BW_CTRL_EN, + ROCE_BW_CTRL_RESET +}; + +enum roce5_qos_wl_ctrl_cmd_e { + ROCE_QOS_WL_CTRL_CMD_SET, + ROCE_QOS_WL_CTRL_CMD_GET +}; + +struct rdma_gid_query_outbuf { + struct rdma_gid_entry gid_entry[ROCE_GID_ENTRY_NUM]; +}; + +struct roce5_bw_ctrl_inbuf { + u32 cmd_type; + struct { + u32 cir; + u32 pir; + u32 cnp; + } ctrl_param; +}; + +struct roce5_bw_set_inbuf { + u32 cmd_type; + struct { + u32 mod; + u32 cir; + u32 cbs; + u32 pir; + u32 pbs; + } set_param; +}; + +struct roce5_bw_ctrl_param { + u8 color_type; + u16 ptype; + u8 hw_wred_mode; + + u32 cir; + u32 pir; + u32 cbs; + u32 xbs; + u32 cnp; + u32 enable; +}; + +struct roce5_bw_set_param { + u16 ptype; + u8 color_type; + u8 hw_wred_mode; + + u32 cir; + u32 cbs; + u32 pir; + u32 pbs; + u32 enable; +}; + +struct roce5_ldcp_param { + u32 cmd_type; + struct { + u8 alpha; + u8 beta; + u8 gamma; + u8 eta; + u8 set_flag; + } ldcp_param; +}; + +struct roce5_bw_ctrl_outbuf { + struct roce5_bw_ctrl_param bw_ctrl_param; +}; + +struct roce5_bw_set_outbuf { + struct roce5_bw_set_param bw_set_param; +}; + +enum roce5_dfx_io_cmd_type { ROCE_IO_CTRL_DIS, ROCE_IO_CTRL_EN }; + +struct roce5_dfx_io_alarm { + enum roce5_dfx_io_cmd_type en_flag; + u16 pf_id; + u16 rsvd; + u16 io_latency_thd; + u16 exec_time; + u32 exp_qpn; + void *rcd_uaddr; + struct timespec64 start; + struct mutex io_alarm_mutex; +}; + +struct roce5_dfx_io_inbuf { + u32 cmd_type; + struct roce5_dfx_io_alarm io_alarm; +}; + +struct roce5_dfx_io_outbuf { + struct roce5_dfx_io_alarm io_alarm; +}; + +typedef struct roce5_adm_generic_cmd_proc { + u32 cmd_type; + void *proc_func; +} roce5_adm_generic_cmd_proc_t; + +typedef int (*roce5_adm_func_t)(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +typedef struct roce5_adm_cmd_proc { + u32 cmd_type; + roce5_adm_func_t proc_func; +} roce5_adm_cmd_proc_t; + +typedef int (*roce5_adm_dfx_dbg_t)(struct roce5_device *rdev, + const struct roce5_dfx_dbg_inbuf *inbuf, + union roce5_dfx_dbg_outbuf *outbuf); + +typedef struct roce5_adm_dfx_dbg_cmd_proc { + u32 cmd_type; + roce5_adm_dfx_dbg_t proc_func; +} roce5_adm_dfx_dbg_cmd_proc_t; + +typedef int (*roce5_adm_dfx_query_t)(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf); + +typedef struct roce5_adm_dfx_query_cmd_proc { + u32 cmd_type; + roce5_adm_dfx_query_t proc_func; +} roce5_adm_dfx_query_cmd_proc_t; + +typedef int (*roce5_adm_dfx_bw_ctrl_func_t)(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf); + +typedef struct roce5_adm_dfx_bw_ctrl_cmd_proc { + u32 cmd_type; + roce5_adm_dfx_bw_ctrl_func_t proc_func; +} roce5_adm_dfx_bw_ctrl_cmd_proc_t; + +#ifdef ROCE_CHIP_TEST +typedef struct roce5_vnic_waterline_ctrl_param { + u16 red_waterline; + u16 yellow_waterline; + u32 ip_val; +} roce5_vnic_waterline_ctrl_param_s; + +struct roce5_vnic_waterline_ctrl_buf { + u32 cmd_type; + roce5_vnic_waterline_ctrl_param_s waterline_param; +}; + +typedef int (*roce5_adm_dfx_vnic_waterline_ctrl_func_t)( + struct roce5_device *rdev, struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf); + +typedef struct roce5_adm_dfx_vnic_waterline_cmd_proc { + u32 cmd; + roce5_adm_dfx_vnic_waterline_ctrl_func_t proc_func; +} roce5_adm_dfx_vnic_waterline_cmd_proc_t; + +int roce5_adm_dfx_vnic_wl_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +#endif /* ROCE_CHIP_TEST */ + +/* and above next is V100 compatible data structure */ +typedef struct tag_roce_qurey_gid { + roce_verbs_cmd_com_s com; + __be32 port; +} roce_cmd_query_gid_s; + +typedef int (*roce_adm_dfx_attack_func_t)(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf); +struct roce_dfx_attack_cmd_func { + u32 cmd_id; + roce_adm_dfx_attack_func_t cmd_func; +}; + +DEBUG_REPLACE int roce5_get_drv_version(void *buf_out, u32 *out_size); +void * +roce5_adm_generic_find_cmd_func(const roce5_adm_generic_cmd_proc_t *cmd_array, + int elem_count, u32 target_cmd); +int roce5_adm_dfx_query(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_bw_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_set_ldcp_param(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_port_traffic(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_dbg(struct roce5_device *rdev, const void *buf_in, + void *buf_out, u32 *out_size); +int roce5_adm_dfx_roce_set(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_roce_set_debug(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +int roce5_dfx_attack_func(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +void roce5_dfx_clean_up(struct roce5_device *rdev); + +void *global_roce5_io_alarm_va_get(void); + +void global_roce5_io_alarm_va_set(u64 va); + +void global_roce5_io_alarm_pa_set(dma_addr_t pa); + +dma_addr_t global_roce5_io_alarm_pa_get(void); + +int roce5_dfx_query_cq(struct roce5_device *rdev, u32 cqn, u32 *context, + int cqc_size); + +int roce5_dfx_cmd_query_srq(struct roce5_device *rdev, u32 srqn, + roce_verbs_srq_attr_s *srq_attr); + +#endif // __ROCE_DFX_H__ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c new file mode 100644 index 000000000..84cc1455b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c @@ -0,0 +1,828 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/semaphore.h> + +#include "hinic5_hw.h" +#include "hinic5_hwdev.h" +#include "hisdk5_hwif.h" +#include "hinic5_mt.h" +#include "rdma_comp.h" +#include "roce.h" +#include "roce_qp.h" +#include "roce_infra.h" +#include "roce_mr.h" +#include "roce_dfx.h" +#include "roce_kernel_compat_gen.h" + +#define DB_BUFF_SIZE 2048 +#define DWQE_SIZE 256 +#define DWQE_TYPE_NUM 4 +#define ALIGN_UP_8B(length) (((length) + 7) >> 3) +#define ALIGN_DOWN_8B(offset) ((offset)&0xfffffff8) +#define IS_OFFSET_8B_ALIGN(offset) (((offset)&7) != 0) +#define UPPER_HALF_BIT_OFFSET 32 +#define U32S_PER_U64 2 + +static void roce5_copy_mask_attack_value_to_addr(u32 *addr, u32 attack_value, + u32 mask) +{ + u32 val; + + val = *addr; + val &= ~mask; + val |= (attack_value & mask); + *addr = val; +} + +static void roce5_copy_mask_attack_value_to_u64_addr(u64 *addr, + u32 attack_value, u32 mask, + bool is_upper_half) +{ + u64 val; + + val = *addr; + + if (is_upper_half) { + val &= ~((u64)mask << UPPER_HALF_BIT_OFFSET); + val |= ((u64)(attack_value & mask) << UPPER_HALF_BIT_OFFSET); + } else { + val &= ~mask; + val |= (attack_value & mask); + } + + *addr = val; +} + +static int roce5_copy_data_to_page(struct roce5_device *rdev, const u32 *val, + struct ib_umem *umem, u32 copy_len, + u32 offset, u32 attack_len, u32 mask) +{ + struct scatterlist *sg = NULL; + u32 *inbuf_tmp = NULL; + u32 sg_len; + u32 page_copy_len = 0; + u32 copied_len = 0; + u32 sgl_offset = 0; + u64 buf_offset = 0; + void *va = NULL; + u32 i = 0; + int ret; + + inbuf_tmp = (u32 *)kzalloc(PAGE_SIZE, GFP_KERNEL); + if (inbuf_tmp == NULL) { + ROCE_ERR("Failed to alloc inbuf temp."); + return -ENOMEM; + } + + if (mask == 0) { + for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) { + inbuf_tmp[i] = val[i % attack_len]; + } + } + + ROCE_INFO( + "copy_data_to_page va:%lx, length:%lu, offset:%u, copy_len:%u", + umem->address, umem->length, offset, copy_len); + +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + for_each_sg(umem->sgt_append.sgt.sgl, sg, umem->sgt_append.sgt.nents, i) +#else + //scatterlist of traverse use nents, orig_nents >= nents >= nmap + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, i) +#endif + { + sg_len = sg->length; + if (buf_offset + sg_len <= + offset) { //not found offset of scatterlist, accumulate sg_len, continue 往 next find + buf_offset += sg_len; + } else { //found offset of scatterlist + sgl_offset = + offset - + (u32)buf_offset; //get based on current scatterlist of offset + buf_offset = + offset; //found current page after, later of page of offset all max at offset + + va = sg_virt(sg); + if (va == NULL) { + ROCE_ERR("Failed to get va"); + kfree(inbuf_tmp); + return -ENOMEM; + } + + va += sgl_offset; + page_copy_len = ((copy_len - copied_len) < + (sg_len - sgl_offset)) ? + (copy_len - copied_len) : + (sg_len - sgl_offset); + + if (mask != 0) { + roce5_copy_mask_attack_value_to_addr( + (u32 *)va, val[0], mask); + ROCE_INFO( + "copy mask data to buf, buf_offset:%llu, sgl_offset:%u, mask_val:%u\n", + buf_offset, sgl_offset, + (val[0] & mask)); + break; + } + + ret = memcpy_s(va, page_copy_len, inbuf_tmp, + page_copy_len); + if (ret != 0) { + ROCE_ERR( + "Failed to memcpy attack value, ret(%d)", + ret); + kfree(inbuf_tmp); + return -EFAULT; + } + + ROCE_INFO( + "copy_data_from_page buf_offset:%llu, sgl_offset:%u, copied_len:%u, page_copy_len:%u, i:%u\n", + buf_offset, sgl_offset, copied_len, + page_copy_len, i); + + copied_len += page_copy_len; + if (copied_len >= copy_len) { + break; + } + } + } + + kfree(inbuf_tmp); + return 0; +} + +static void roce5_copy_attack_value_to_addr(u64 *addr, u64 attack_value, + u32 length) +{ + int i; + int count = ALIGN_UP_8B(length); + + for (i = 0; i < count; i++) { + wmb(); + *((u64 *)addr + i) = attack_value; + wmb(); + } +} + +static int roce5_dfx_attack_sqdb_hard(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_ucontext *ucontext = NULL; + u64 db_phy_start; + void __iomem *db_virt_start = NULL; + void __iomem *db_virt_offset = NULL; + void *hwdev = NULL; + struct roce5_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + union roce_sq_db sq_db; + u32 db_offset; + u32 db_8B_offset; + bool is_upper_half; + u32 length; + + hwdev = rdev->hwdev; + if (hwdev == NULL) { + ROCE_ERR("Roce hwdev is NULL"); + return -ENOMEM; + } + + ucontext = rdev->ucontext; + if (ucontext == NULL) { + ROCE_ERR("Roce device context is null"); + return -ENOMEM; + } + + db_offset = inbuf->offset; + if (db_offset >= DB_BUFF_SIZE) { + ROCE_ERR( + "roce attack sqdb_hard offset is too large, should be less than %u", + DB_BUFF_SIZE); + return -EINVAL; + } + + db_8B_offset = ALIGN_DOWN_8B(db_offset); + db_phy_start = ucontext->db_dma_addr; + db_virt_start = (db_phy_start - + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base_phy) + + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base; + db_virt_offset = db_virt_start + db_8B_offset; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + if (inbuf->mask != 0) { + is_upper_half = IS_OFFSET_8B_ALIGN(db_offset); + roce5_copy_mask_attack_value_to_u64_addr((u64 *)db_virt_offset, + inbuf->attack_value[0], + inbuf->mask, + is_upper_half); + ROCE_INFO("roce attack sqdb, mask_val:0x%x, offset:%u", + (inbuf->attack_value[0] & inbuf->mask), db_offset); + } else { + sq_db.sq_db_val = 0; + sq_db.sq_db_seg.dw0.value = + roce5_convert_be32(inbuf->attack_value[0]); + sq_db.sq_db_seg.dw1.value = + roce5_convert_be32(inbuf->attack_value[1]); + + length = (db_offset + inbuf->length > DB_BUFF_SIZE) ? + (DB_BUFF_SIZE - db_offset) : + inbuf->length; + roce5_copy_attack_value_to_addr((u64 *)db_virt_offset, + sq_db.sq_db_val, length); + + ROCE_INFO("roce attack sqdb length:%u, offset:%u", length, + db_offset); + ROCE_INFO("roce attack sqdb type:%u, cos:%u, qpn:%u\n", + sq_db.sq_db_seg.dw0.bs.type, + sq_db.sq_db_seg.dw0.bs.cos, + sq_db.sq_db_seg.dw0.bs.qpn); + ROCE_INFO("roce attack sqdb mtu_shift:%u, pi:%u\n", + sq_db.sq_db_seg.dw1.bs.mtu_shift, + sq_db.sq_db_seg.dw1.bs.pi); + } + + roce5_rqp_put(rqp); + return 0; +} + +static int roce5_copy_attack_value_to_dwqe_addr(u64 *addr, const u32 *val, + u32 length, u32 attack_len) +{ + int i; + int count = ALIGN_UP_8B(length); + u64 *inbuf_tmp = NULL; + u32 step = 2; + + inbuf_tmp = (u64 *)kzalloc(DWQE_SIZE, GFP_KERNEL); + if (inbuf_tmp == NULL) { + ROCE_ERR("Failed to alloc inbuf temp."); + return -ENOMEM; + } + + for (i = 0; i < count * U32S_PER_U64; i += step) { + inbuf_tmp[i / step] = ((u64)val[(i + 1) % attack_len] + << UPPER_HALF_BIT_OFFSET) | + val[i % attack_len]; + } + + for (i = 0; i < count; i++) { + wmb(); + *((u64 *)addr + i) = inbuf_tmp[i]; + wmb(); + } + + kfree(inbuf_tmp); + return 0; +} + +static int roce5_dfx_attack_dwqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_ucontext *ucontext = NULL; + u64 dwqe_phy_start; + void __iomem *dwqe_virt_start = NULL; + void __iomem *dwqe_virt_offset = NULL; + void *hwdev = NULL; + struct roce5_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + u32 dwqe_offset; + u32 length; + u32 dwqe_8B_offset; + bool is_upper_half; + int i; + int ret; + + hwdev = rdev->hwdev; + if (hwdev == NULL) { + ROCE_ERR("Roce hwdev is NULL"); + return -ENOMEM; + } + + ucontext = rdev->ucontext; + if (ucontext == NULL) { + ROCE_ERR("Roce device context is null"); + return -ENOMEM; + } + + dwqe_offset = inbuf->offset; + if (dwqe_offset >= DWQE_SIZE) { + ROCE_ERR( + "roce attack dwqe offset is too large, should be less than %u", + DWQE_SIZE); + return -EINVAL; + } + + dwqe_phy_start = ucontext->dwqe_dma_addr; + //dwqe physical base address and virtual base address all is based on db perform offset of offset, so corresponding when at db of va basebase on to before offset offset + dwqe_virt_start = + (dwqe_phy_start - + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base_phy) + + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + length = (dwqe_offset + inbuf->length > DWQE_SIZE) ? + (DWQE_SIZE - dwqe_offset) : + (inbuf->length); + dwqe_8B_offset = ALIGN_DOWN_8B(dwqe_offset); + for (i = 0; i < DWQE_TYPE_NUM; i++) { + dwqe_virt_offset = + dwqe_virt_start + i * DWQE_SIZE + dwqe_8B_offset; + if (inbuf->mask != 0) { + is_upper_half = IS_OFFSET_8B_ALIGN(dwqe_offset); + roce5_copy_mask_attack_value_to_u64_addr( + (u64 *)dwqe_virt_offset, inbuf->attack_value[0], + inbuf->mask, is_upper_half); + } else { + ret = roce5_copy_attack_value_to_dwqe_addr( + (u64 *)dwqe_virt_offset, inbuf->attack_value, + length, inbuf->attack_len); + if (ret != 0) { + ROCE_ERR( + "Failed to copy attack value to dwqe, ret(%d)", + ret); + return -EFAULT; + } + } + } + + if (inbuf->mask != 0) { + ROCE_INFO("roce attack dwqe, mask_val:0x%x, offset:%u", + (inbuf->attack_value[0] & inbuf->mask), dwqe_offset); + } else { + ROCE_INFO("roce attack dwqe, length:%u, offset:%u", length, + dwqe_offset); + } + + roce5_rqp_put(rqp); + return 0; +} + +static int roce5_dfx_attack_sqwqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + u32 sq_buf_sz; + u32 length; + u32 sq_wqe_offset; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + sq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->sq.wqebb_cnt + << (unsigned)rqp->sq.wqe_shift, + PAGE_SIZE); + sq_wqe_offset = inbuf->offset; + if (sq_wqe_offset >= sq_buf_sz) { + ROCE_ERR( + "roce attack sq wqe offset is too large, should be less than %u", + sq_buf_sz); + ret = -EINVAL; + goto sq_wqe_err; + } + + length = (sq_wqe_offset + inbuf->length > sq_buf_sz) ? + (sq_buf_sz - sq_wqe_offset) : + inbuf->length; + ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, rqp->umem, + length, sq_wqe_offset, inbuf->attack_len, + inbuf->mask); + if (ret != 0) { + ROCE_ERR("Failed to copy to sq page, qpn:0x%x", qpn); + } + +sq_wqe_err: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_attack_rqwqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + u32 rq_buf_sz; + u32 length; + u32 rq_attack_offset; + u32 rq_wqe_offset; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + rq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift, + PAGE_SIZE); + rq_attack_offset = inbuf->offset; + if (rq_attack_offset >= rq_buf_sz) { + ROCE_ERR( + "roce attack rq wqe offset is too large, should be less than %u", + rq_buf_sz); + ret = -EINVAL; + goto rq_wqe_err; + } + + length = (rq_attack_offset + inbuf->length > rq_buf_sz) ? + (rq_buf_sz - rq_attack_offset) : + inbuf->length; + rq_wqe_offset = rqp->rq.offset + rq_attack_offset; + ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, rqp->umem, + length, rq_wqe_offset, inbuf->attack_len, + inbuf->mask); + if (ret != 0) { + ROCE_ERR("Failed to copy to rq page, qpn:0x%x", qpn); + } + +rq_wqe_err: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_attack_mtt(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct mtt *roce_mtt) +{ + struct hmm_comp_priv *comp_priv = NULL; + u64 *mtts = NULL; + u64 *mtts_offset = NULL; + u64 page_size; + u32 mtt_offset; + u32 length; + u64 val = ((u64)inbuf->attack_value[1] << 32) | + (u64)inbuf->attack_value[0]; + u32 mtt_8B_offset; + bool is_upper_half; + + if (roce_mtt == NULL) { + ROCE_ERR("roce qp mtt is null"); + return -ENOMEM; + } + + if (roce_mtt->mtt_vaddr == NULL) { + ROCE_ERR("roce mtt vaddr is null, layers:%u", + roce_mtt->mtt_layers); + return -ENOMEM; + } + + comp_priv = get_hmm_comp_priv(rdev->hwdev, SERVICE_T_ROCE); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -ENOMEM; + } + + mtts = (u64 *)hmm_em_table_find( + &comp_priv->mtt_em_table, + roce_mtt->mtt_seg[roce_mtt->mtt_layers - 1]->offset, NULL); + if (mtts == NULL) { + ROCE_ERR("Can't find va and pa of mtt entry"); + return -ENOMEM; + } + + page_size = 1L << roce_mtt->mtt_page_shift; + mtt_offset = inbuf->offset; + if (mtt_offset >= page_size) { + ROCE_ERR( + "roce attack mtt offset is too large, should be less than %llu", + page_size); + return -EINVAL; + } + + mtt_8B_offset = ALIGN_DOWN_8B(mtt_offset); + mtts_offset = mtts + mtt_8B_offset / sizeof(u64); + if (inbuf->mask != 0) { + is_upper_half = IS_OFFSET_8B_ALIGN(mtt_offset); + roce5_copy_mask_attack_value_to_u64_addr(mtts_offset, + inbuf->attack_value[0], + inbuf->mask, + is_upper_half); + ROCE_INFO("roce attack mtt, mask_val:0x%x, offset:%u\n", + (inbuf->attack_value[0] & inbuf->mask), mtt_offset); + } else { + length = (inbuf->length + mtt_offset > page_size) ? + (page_size - mtt_offset) : + inbuf->length; + roce5_copy_attack_value_to_addr(mtts_offset, val, length); + ROCE_INFO("roce attack mtt, attack_len:%u, offset:%u", length, + mtt_offset); + } + + return 0; +} + +static int roce5_dfx_attack_cmtt(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + ret = roce5_dfx_attack_mtt(rdev, inbuf, &(rqp->mtt)); + if (ret != 0) { + ROCE_ERR("Failed to attack cmtt"); + } + + roce5_rqp_put(rqp); + return ret; +} + +struct roce5_mr *roce5_mr_find(struct roce5_device *rdev, u32 key_id) +{ + struct roce5_mr *mr = NULL; + + mutex_lock(&rdev->device_res->mr_list.node_lock); + list_for_each_entry(mr, &rdev->device_res->mr_list.node, node) { + if (mr->rdmamr.mpt.mpt_index == key_id) { + mutex_unlock(&rdev->device_res->mr_list.node_lock); + ROCE_INFO("Find mr, it's mpt index is %u", + mr->rdmamr.mpt.mpt_index); + return mr; + } + } + + mutex_unlock(&rdev->device_res->mr_list.node_lock); + ROCE_ERR("Don't find correct mr, key_id:%u", key_id); + + return NULL; +} + +static int roce5_dfx_attack_dmtt(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_mr *mr = NULL; + int ret = 0; + + mr = roce5_mr_find(rdev, inbuf->mpt_index); + if (mr == NULL) { + ROCE_ERR( + "Failed to attack dmtt, don't find valid mr, mpt_index:%u", + inbuf->mpt_index); + return -EINVAL; + } + + ret = roce5_dfx_attack_mtt(rdev, inbuf, &(mr->rdmamr.mtt)); + if (ret != 0) { + ROCE_ERR("Failed to attack dmtt"); + } + + return ret; +} + +static void roce5_attack_value_to_rdmarc(u32 *vaddr, u32 length, const u32 *val, + u32 attack_len) +{ + int cnt = (length + 3) >> 2; + int i; + + for (i = 0; i < cnt; i++) { + vaddr[i] = val[i % attack_len]; + } +} + +static int roce5_dfx_attack_rdmarc(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + struct rdma_rdmarc *rdmarc = NULL; + void *rdmarc_vaddr = NULL; + u32 *rdmarc_offset_vaddr = NULL; + u32 rdmarc_entry_num; + u32 rdmarc_offset; + u32 rdmarc_size; + u32 length; + int ret = 0; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + rdmarc = &(rqp->rdmarc); + rdmarc_vaddr = rdmarc->vaddr; + if (rdmarc_vaddr == NULL) { + ROCE_ERR("Rdmarc vaddr is null"); + ret = -ENOMEM; + goto rdmarc_err; + } + + rdmarc_entry_num = (1 << rdmarc->order); + rdmarc_size = rdmarc_entry_num * ROCE_RDMARC_ENTRY_SZ; + rdmarc_offset = inbuf->offset; + if (rdmarc_offset >= rdmarc_size) { + ROCE_ERR( + "roce attack rdmarc offset is too large, should be less than %u", + rdmarc_size); + ret = -EINVAL; + goto rdmarc_err; + } + + rdmarc_offset_vaddr = (u32 *)((u8 *)rdmarc_vaddr + rdmarc_offset); + + if (inbuf->mask != 0) { + roce5_copy_mask_attack_value_to_addr(rdmarc_offset_vaddr, + inbuf->attack_value[0], + inbuf->mask); + ROCE_INFO("roce attack rdmarc, mask_val:0x%x, offset:%u", + (inbuf->attack_value[0] & inbuf->mask), + rdmarc_offset); + } else { + length = (rdmarc_offset + inbuf->length > rdmarc_size) ? + (rdmarc_size - rdmarc_offset) : + inbuf->length; + roce5_attack_value_to_rdmarc(rdmarc_offset_vaddr, length, + inbuf->attack_value, + inbuf->attack_len); + ROCE_INFO("roce attack rdmarc, attack_length:%u, offset:%u", + length, rdmarc_offset); + } + +rdmarc_err: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_attack_cq(struct roce5_cq *cq, struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf) +{ + u32 ret; + u32 cq_attack_offset; + u32 cq_buf_sz; + u32 length; + u32 cq_depth = cq->cqe_mask + 1; + u32 cqe_size = rdev->rdma_cap.cqe_size; + + cq_buf_sz = cqe_size * cq_depth; + if (inbuf->offset >= cq_buf_sz) { + ROCE_ERR( + "Roce attack cq cqe offset is too large, should be less than %u", + cq_buf_sz); + return -EINVAL; + } + + cq_attack_offset = inbuf->offset; + length = ((cq_attack_offset + inbuf->length) > cq_buf_sz) ? + (cq_buf_sz - cq_attack_offset) : + inbuf->length; + ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, cq->umem, + length, cq_attack_offset, + inbuf->attack_len, inbuf->mask); + if (ret != 0) { + ROCE_ERR("Failed to copy to cq page"); + } + + return ret; +} + +static int roce5_dfx_attack_cqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + roce5_get_cqs(rqp, &send_cq, &recv_cq); + if (send_cq != NULL) { + ret = roce5_dfx_attack_cq(send_cq, rdev, inbuf); + if (ret != 0) { + ROCE_ERR("Failed to attack send_cq, qpn:0x%x", qpn); + goto cq_wqe_err; + } + } + + if (recv_cq != NULL) { + ret = roce5_dfx_attack_cq(recv_cq, rdev, inbuf); + if (ret != 0) { + ROCE_ERR("Failed to attack recv_cq, qpn:0x%x", qpn); + goto cq_wqe_err; + } + } + +cq_wqe_err: + roce5_rqp_put(rqp); + return ret; +} + +static struct roce_dfx_attack_cmd_func g_roce_attack_cmd_funcs[] = { + { ROCE_ATTACK_TYPE_SQDB_HARD, roce5_dfx_attack_sqdb_hard }, + { ROCE_ATTACK_TYPE_DWQE, roce5_dfx_attack_dwqe }, + { ROCE_ATTACK_TYPE_SQWQE, roce5_dfx_attack_sqwqe }, + { ROCE_ATTACK_TYPE_RQWQE, roce5_dfx_attack_rqwqe }, + { ROCE_ATTACK_TYPE_CMTT, roce5_dfx_attack_cmtt }, + { ROCE_ATTACK_TYPE_DMTT, roce5_dfx_attack_dmtt }, + { ROCE_ATTACK_TYPE_RDMARC, roce5_dfx_attack_rdmarc }, + { ROCE_ATTACK_TYPE_CQE, roce5_dfx_attack_cqe }, +}; + +void roce5_cpu_to_be32(void *data, u32 len) +{ + u32 chunk_sz = sizeof(u32); + u32 *mem = (u32 *)data; + u32 data_len; + u32 i; + + if (data == NULL) + return; + + data_len = len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = cpu_to_be32(*mem); + mem++; + } +} + +int roce5_dfx_attack_func(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct roce_attack_inbuf *inbuf = (struct roce_attack_inbuf *)buf_in; + struct roce_attack_outbuf *outbuf = + (struct roce_attack_outbuf *)buf_out; + roce_adm_dfx_attack_func_t func = NULL; + u32 cmd_type; + u32 i; + + if ((inbuf == NULL) || (outbuf == NULL)) { + ROCE_ERR("Roce dfx attack buf in or buf out is null."); + return -EINVAL; + } + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct roce_attack_inbuf)) { + ROCE_ERR("roce dfx attack buffer in size too small"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(struct roce_attack_outbuf)) { + ROCE_ERR("roce dfx attack buffer out size too small"); + return -EINVAL; + } + + *out_size = 0; + + if (inbuf->convert_endian != 0) + roce5_cpu_to_be32(&(inbuf->attack_value), + sizeof(inbuf->attack_value)); + + for (i = 0; i < sizeof(g_roce_attack_cmd_funcs) / + sizeof(g_roce_attack_cmd_funcs[0]); + i++) { + cmd_type = g_roce_attack_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->attack_type) { + func = g_roce_attack_cmd_funcs[i].cmd_func; + if (func == NULL) { + ROCE_ERR( + "function of attack cmd(0x%x) is not supported.", + cmd_type); + return -EINVAL; + } + + *out_size = sizeof(struct roce_attack_outbuf); + + return func(rdev, inbuf, outbuf); + } + } + + ROCE_ERR("attack cmd(0x%x) does't exist.", cmd_type); + + return -EINVAL; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c new file mode 100644 index 000000000..371fa3211 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c @@ -0,0 +1,35 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/semaphore.h> +#include "hinic5_mt.h" +#include "securec.h" +#include "roce_infra.h" +#include "roce.h" + +int roce5_get_drv_version(void *buf_out, u32 *out_size) +{ + struct drv_version_info *ver_info = buf_out; + int rc; + + if (!buf_out) { + ROCE_ERR("Buf_out is NULL."); + return -EINVAL; + } + + if (*out_size != sizeof(*ver_info)) { + ROCE_ERR("Unexpect out buf size from user :%u, expect: %lu", + *out_size, sizeof(*ver_info)); + return -EINVAL; + } + + rc = snprintf_s(ver_info->ver, sizeof(ver_info->ver), + sizeof(ver_info->ver) - 1, "%s %s(debug)", + HIROCE5_DRV_VERSION, __TIME_STR__); + if (rc == -1) { + ROCE_ERR("Snprintf roce version err"); + return -EFAULT; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c new file mode 100644 index 000000000..ef4b2da57 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c @@ -0,0 +1,851 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. + + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/kthread.h> + +#include <rdma/ib_verbs.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "hinic5_mt.h" + +#include "roce_compat.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_cq.h" +#include "roce_cmd.h" +#include "roce_mix.h" +#include "roce_infra.h" +#include "roce_npu_cmd.h" +#include "roce_cqm_cmd.h" +#include "roce_restore.h" +#include "roce_main_extension.h" +#include "roce_com.h" +#include "securec.h" +#include "hmm_common.h" +#include "roce_npu_cmd_gid_defs.h" + +static int roce5_dfx_get_dev_algo(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + u32 cc_algo; + struct roce5_ecn_ctx ecn_ctx; + u32 *algo_type = &outbuf->algo_type; + + if (rdev == NULL) { + ROCE_ERR("Failed to get roce device."); + return -EINVAL; + } + ecn_ctx = rdev->ecn_ctx; + cc_algo = ecn_ctx.cc_algo; + *algo_type = cc_algo; + + return 0; +} + +static int roce5_dfx_get_qp_count(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_dfx_qp_count *qp_count = &outbuf->qp_count; + + qp_count->qp_alloced = rdev->qp_cnt.alloc_qp_cnt; + qp_count->qp_deleted = rdev->qp_cnt.del_qp_cnt; + qp_count->qp_alive = + rdev->qp_cnt.alloc_qp_cnt - rdev->qp_cnt.del_qp_cnt; + + return 0; +} + +/* QPC is statically allocated, so it is not necessary to judge whether it exists when querying the cache content */ +static int roce5_dfx_query_cache_qpc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + u32 qpn = inbuf->qpn; + roce_verbs_qp_attr_s *qp_attr = &outbuf->qp_attr; + + ret = memset_s(qp_attr, sizeof(roce_verbs_qp_attr_s), 0, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)qp_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + roce5_dfx_print("Failed to query QPC from cache!"); + roce5_dfx_print( + "******************From Cache: qpn(%#x) ********************", + qpn); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY QPC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: qpn(%#x) ********************", + qpn); + + return ret; + } + + return 0; +} + +static int roce5_dfx_query_host_qpc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + roce5_qp_query_outbuf_s *qp_query = &outbuf->qp_query; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + roce5_dfx_print("Failed to get roce qp."); + roce5_dfx_print( + "******************From Host: qpn(%#x) ********************", + qpn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY QPC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: qpn(%#x) ********************", + qpn); + return -EINVAL; + } + + ret = memcpy_s(qp_query, sizeof(roce5_qp_query_outbuf_s), + rqp->qpc_info->vaddr, sizeof(roce5_qp_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy qp_ctx, ret(%d).", ret); + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_uni_cmd_query_cq(struct roce5_device *rdev, u32 cqn, + u32 *verbs_cq_attr, int cq_attr_size) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_cq_query_s *cq_query_inbuf = NULL; + roce_uni_cmd_cq_query_outbuf_s *cq_query_outbuf = NULL; + struct roce5_cq *rcq = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_cq_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_cq_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + rcq = roce5_get_rcq_by_cqn(rdev, cqn); + if (rcq == NULL) { + roce5_dfx_print("Failed to get roce cq."); + goto err_rcq; + } + cq_query_inbuf = (roce_uni_cmd_cq_query_s *)cqm_cmd_inbuf->buf; + cq_query_inbuf->com.index = cpu_to_be32(cqn); + cq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (rcq->shadow != 0) { + cq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_query_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rcq->shadow_vfid); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_QUERY_CQ, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + ROCE_CMD_TIME_CLASS_B, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + roce5_dfx_print("Failed to send cmd QUERY_CQ, ret=%d", ret); + ret = -1; + goto out; + } + + cq_query_outbuf = (roce_uni_cmd_cq_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)verbs_cq_attr, (size_t)cq_attr_size, + (void *)&cq_query_outbuf->cq_info.context[0], + (size_t)cq_attr_size); + if (ret != 0) { + ROCE_ERR("Failed to memcpy verbs_cq_attr.(ret:%d)", ret); + ret = -EINVAL; + goto out; + } +out: + roce5_rcq_put(rcq); +err_rcq: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int roce5_dfx_query_cq(struct roce5_device *rdev, u32 cqn, u32 *context, + int cqc_size) +{ + if (ROCE5_SUPPORT_DFX_CMD_QUERY_CQ_MSG_V2(rdev) != 0) { + return roce5_dfx_uni_cmd_query_cq(rdev, cqn, context, cqc_size); + } + return -EINVAL; +} + +/* CQC is dynamic allocation. When querying cache content, you need to determine whether it exists or not. */ +static int roce5_dfx_query_cache_cqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + cqm_object_s *cqm_obj_cq = NULL; + u32 cqn = inbuf->cqn; + roce5_cq_query_outbuf_s *cq_query = &outbuf->cq_query; + + cqm_obj_cq = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_cq == NULL) { + roce5_dfx_print("Failed to get cqm_obj_cq."); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + + return -EINVAL; + } + + cqm5_object_put(cqm_obj_cq); + + ret = memset_s(cq_query, sizeof(roce5_cq_query_outbuf_s), 0, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset cq_context.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)cq_query, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to query cq from cache!"); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY CQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + + return -EINVAL; + } + + return 0; +} + +static int roce5_dfx_query_host_cqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_cq *rcq = NULL; + int ret = 0; + u32 cqn = inbuf->cqn; + roce5_cq_query_outbuf_s *cq_query = &outbuf->cq_query; + + rcq = roce5_get_rcq_by_cqn(rdev, cqn); + if (rcq == NULL) { + roce5_dfx_print("Failed to get roce cq."); + roce5_dfx_print( + "******************From Host: cqn(%#x) ********************", + cqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY CQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: cqn(%#x) ********************", + cqn); + + return -EINVAL; + } + + ret = memcpy_s(cq_query, sizeof(roce5_cq_query_outbuf_s), + rcq->cqm_cq->q_ctx_vaddr, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy cq_ctx.(ret:%d)", ret); + goto err_out; + } + +err_out: + roce5_rcq_put(rcq); + return ret; +} + +int roce5_dfx_cmd_query_srq(struct roce5_device *rdev, u32 srqn, + roce_verbs_srq_attr_s *srq_attr) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_srq_query_s *srq_query_inbuf = NULL; + roce_uni_cmd_srq_query_outbuf_s *srq_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_srq_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_srq_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + srq_query_inbuf = (roce_uni_cmd_srq_query_s *)cqm_cmd_inbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32(srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_SRQ, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + roce5_dfx_print("Failed to send cmd QUERY_SRQ, ret=%d", ret); + ret = -1; + goto out; + } + + srq_query_outbuf = + (roce_uni_cmd_srq_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s(srq_attr, sizeof(roce_verbs_srq_attr_s), + &srq_query_outbuf->com.srq_attr, + sizeof(roce_verbs_srq_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy srq_attr.(ret:%d)", ret); + ret = -EINVAL; + goto out; + } +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +/* SRQC is dynamic allocation. When querying the cache content, you need to determine whether it exists or not. */ +static int roce5_dfx_query_cache_srqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + cqm_object_s *cqm_obj_srq = NULL; + u32 srqn = inbuf->srqn; + roce_verbs_srq_attr_s *srq_attr = &outbuf->srq_attr; + + cqm_obj_srq = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + roce5_dfx_print("Failed to get cqm_obj_srq"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY SRQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + cqm5_object_put(cqm_obj_srq); + + ret = memset_s(srq_attr, sizeof(roce_verbs_srq_attr_s), 0, + sizeof(roce_verbs_srq_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset srq_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_cmd_query_srq(rdev, srqn, srq_attr); + if (ret != 0) { + roce5_dfx_print("Failed to query srq from cache"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY SRQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + return 0; +} + +static int roce5_dfx_query_host_srqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_srq *rsrq = NULL; + int ret = 0; + u32 srqn = inbuf->srqn; + roce5_srq_query_outbuf_s *srq_query = &outbuf->srq_query; + + rsrq = roce5_get_rsrq_by_srqn(rdev, srqn); + if (rsrq == NULL) { + roce5_dfx_print("Failed to get roce srq."); + roce5_dfx_print( + "******************From Host: srqn(%#x) ********************", + srqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY SRQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + ret = memcpy_s(srq_query, sizeof(roce5_srq_query_outbuf_s), + rsrq->cqm_srq->q_ctx_vaddr, + sizeof(roce5_srq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy srq_ctx.(ret:%d)", ret); + goto err_out; + } + +err_out: + roce5_rsrq_put(rsrq); + return ret; +} + +static int roce5_dfx_uni_cmd_query_mpt(struct roce5_device *rdev, u32 key, + struct roce_mpt_context *mpt_entry) +{ + int ret; + u32 mpt_index; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_mpt_query_s *mpt_query_inbuf = NULL; + roce_uni_cmd_mpt_query_outbuf_s *mpt_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_mpt_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_mpt_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + mpt_query_inbuf = (roce_uni_cmd_mpt_query_s *)cqm_cmd_inbuf->buf; + mpt_index = (key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + mpt_query_inbuf->com.index = cpu_to_be32(mpt_index); + mpt_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_MR_BITMASK)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_MPT, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + roce5_dfx_print("Failed to send cmd QUERY_MPT"); + ret = -1; + goto out; + } + + mpt_query_outbuf = + (roce_uni_cmd_mpt_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s(mpt_entry, sizeof(struct roce_mpt_context), + &mpt_query_outbuf->mpt_info.context[0], + sizeof(struct roce_mpt_context)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mpt_entry.(ret:%d)", ret); + ret = -EINVAL; + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static int roce5_dfx_query_mpt(struct roce5_device *rdev, u32 key, + struct roce_mpt_context *mpt_entry) +{ + if (ROCE5_SUPPORT_DFX_CMD_QUERY_MPT_MSG_V2(rdev) != 0) { + return roce5_dfx_uni_cmd_query_mpt(rdev, key, mpt_entry); + } + return -EINVAL; +} + +static int roce5_dfx_uni_cmd_query_gid(struct roce5_device *rdev, u32 port, + u32 gid_index, + struct rdma_gid_entry *gid_entry) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_query_gid_s *gid_query_inbuf = NULL; + struct rdma_gid_query_outbuf *gid_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_query_gid_s), &cqm_cmd_outbuf, + (u16)sizeof(struct rdma_gid_query_outbuf)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + gid_query_inbuf = (roce_uni_cmd_query_gid_s *)cqm_cmd_inbuf->buf; + gid_query_inbuf->com.index = cpu_to_be32(gid_index); + gid_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_GID_BITMASK)); + gid_query_inbuf->com.opt = cpu_to_be32(port); + + if (rdev->shadow != 0) { + gid_query_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rdev->shadow_vfid); + gid_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + gid_query_inbuf->gid_query.dw0.shadow.shadow_vf_num = + rdev->shadow_vf_num; + gid_query_inbuf->gid_query.dw0.value = + cpu_to_be32(gid_query_inbuf->gid_query.dw0.value); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_GID, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send cmd QUERY_GID, ret(%d)", + ret); + ret = -1; + goto out; + } + + gid_query_outbuf = (struct rdma_gid_query_outbuf *)cqm_cmd_outbuf->buf; + ret = memcpy_s(gid_entry, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM, + gid_query_outbuf->gid_entry, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy gid_entry.(ret:%d)", ret); + } +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int roce5_dfx_query_gid(struct roce5_device *rdev, u32 port, + u32 gid_index, struct rdma_gid_entry *gid_entry) +{ + if (ROCE5_SUPPORT_DFX_CMD_QUERY_GID_MSG_V2(rdev) != 0) { + return roce5_dfx_uni_cmd_query_gid(rdev, port, gid_index, + gid_entry); + } + return -EINVAL; +} + +/* MPT is statically allocated, so it is not necessary to judge whether it exists when querying the cache content */ +static int roce5_dfx_query_cache_mpt(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + ret = memset_s(mpt_entry, sizeof(struct roce_mpt_context), 0, + sizeof(struct roce_mpt_context)); + if (ret != 0) { + ROCE_ERR("Failed to memset mpt_entry.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_mpt(rdev, mpt_key, mpt_entry); + if (ret != 0) { + roce5_dfx_print("Failed to query mpt from cache!"); + roce5_dfx_print( + "******************From Cache: mpt_key(%#x) ********************", + mpt_key); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY MPT FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: mpt_key(%#x) ********************", + mpt_key); + + return ret; + } + + return 0; +} + +static int roce5_dfx_query_host_mpt(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + u32 mpt_index = 0; + struct mpt *rmpt = NULL; + int ret = 0; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + mpt_index = (mpt_key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + rmpt = roce5_get_mpt_by_index(rdev, mpt_index); + if (rmpt == NULL) { + roce5_dfx_print("Failed to get roce mpt."); + roce5_dfx_print( + "******************From Host: mpt_key(%#x) ********************", + mpt_key); + roce5_dfx_print( + ">>>>>>>>>>> QUERY MPT FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: mpt_key(%#x) ********************", + mpt_key); + + return -EINVAL; + } + + ret = memcpy_s(mpt_entry, sizeof(struct roce_mpt_context), rmpt->vaddr, + sizeof(struct roce_mpt_context)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mpt_entry.(ret:%d)", ret); + goto err_out; + } + +err_out: + roce5_mpt_put(rmpt); + return ret; +} + +static int roce5_dfx_query_cache_gid(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + u32 gid_index = inbuf->gid_index; + struct rdma_gid_entry *gid_entry = outbuf->gid_entry; + + ret = memset_s(gid_entry, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM, 0, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_gid(rdev, 0, gid_index, gid_entry); + if (ret != 0) { + roce5_dfx_print( + "***************port(1), gid_index(%d) *****************", + gid_index); + roce5_dfx_print("Failed to query gid from cache!"); + roce5_dfx_print( + "***************port(1), gid_index(%d) *****************", + gid_index); + + return -EINVAL; + } + + return 0; +} + +static void roce5_dfx_query_pi_ci_set(const roce_verbs_qp_attr_s *qp_attr, + struct roce5_dfx_pi_ci *pi_ci) +{ + pi_ci->qpc_sq_pi_on_chip = qp_attr->chip_seg.dw5.bs.sq_pi_on_chip; + pi_ci->qpc_sq_pi = qp_attr->ext_seg.dfx_info.dw0.bs.sq_pi; + pi_ci->qpc_sq_load_pi = qp_attr->ext_seg.dfx_info.dw1.bs.sq_load_pi; + pi_ci->qpc_rq_pi_on_chip = qp_attr->chip_seg.dw8.bs.rq_pi_on_chip; + pi_ci->qpc_rq_load_pi = qp_attr->ext_seg.dfx_info.dw1.bs.rq_load_pi; + pi_ci->qpc_rq_pi = qp_attr->ext_seg.dfx_info.dw4.bs.rq_pi; + pi_ci->qpc_rc_pi = qp_attr->ext_seg.dfx_info.dw5.bs.rc_pi; + pi_ci->qpc_sq_ci = qp_attr->ext_seg.dfx_info.dw0.bs.sq_ci; + pi_ci->qpc_sq_wqe_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw1.bs.sq_wqe_prefetch_ci; + pi_ci->qpc_sq_mtt_prefetch_wqe_ci = + qp_attr->ext_seg.dfx_info.dw3.bs.sq_mtt_prefetch_wqe_ci; + pi_ci->qpc_sqa_ci = qp_attr->ext_seg.dfx_info.dw2.bs.sqa_ci; + pi_ci->qpc_sqa_wqe_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw2.bs.sqa_wqe_prefetch_ci; + pi_ci->qpc_rq_ci = qp_attr->ext_seg.dfx_info.dw4.bs.rq_ci; + pi_ci->qpc_rq_wqe_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw5.bs.rq_wqe_prefetch_ci; + pi_ci->qpc_rq_mtt_prefetch_wqe_ci = + qp_attr->ext_seg.dfx_info.dw3.bs.rq_mtt_prefetch_wqe_ci; + pi_ci->qpc_rq_base_ci = qp_attr->chip_seg.dw4.bs.rq_base_ci; + pi_ci->qpc_rc_ci = qp_attr->ext_seg.dfx_info.dw5.bs.rc_ci; + pi_ci->qpc_rc_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw1.bs.rc_prefetch_ci; +} + +static int roce5_dfx_query_pi_ci(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + roce_verbs_qp_attr_s be_attr; /* BE context structure */ + roce_verbs_qp_attr_s qp_attr; /* LE context structure */ + roce_verbs_cq_attr_s cq_attr; + roce5_cq_query_outbuf_s cq_query; + u32 qpn = inbuf->query_pi_ci.qpn; + u32 cqn = inbuf->query_pi_ci.cqn; + struct roce5_dfx_pi_ci *pi_ci = &outbuf->pi_ci; + + ret = memset_s(pi_ci, sizeof(struct roce5_dfx_pi_ci), 0, + sizeof(struct roce5_dfx_pi_ci)); + if (ret != 0) { + ROCE_ERR("Failed to memset pi_ci.(ret:%d)", ret); + return -ENOMEM; + } + + ret = memset_s(&qp_attr, sizeof(qp_attr), 0, sizeof(qp_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)&be_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + return ret; + } + + roce5_be32_2_le32(&be_attr, (u32 *)(void *)&qp_attr, + (sizeof(roce_verbs_qp_attr_s) / sizeof(u32))); + + roce5_dfx_query_pi_ci_set(&qp_attr, pi_ci); + + ret = memset_s(&cq_query, sizeof(roce5_cq_query_outbuf_s), 0, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset cq_query.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)&cq_query, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + return ret; + } + + if (ROCE5_SUPPORT_DFX_CMD_QUERY_PI_CI_MSG_V2(rdev) != 0) { + (void)memcpy_s(&cq_attr, sizeof(roce_verbs_cq_attr_s), + &cq_query, sizeof(roce_verbs_cq_attr_s)); + cq_attr.dw0.value = be32_to_cpu(cq_attr.dw0.value); + cq_attr.dfx_info.dw0.value = + be32_to_cpu(cq_attr.dfx_info.dw0.value); + cq_attr.dfx_info.dw1.value = + be32_to_cpu(cq_attr.dfx_info.dw1.value); + cq_attr.dfx_info.dw2.value = + be32_to_cpu(cq_attr.dfx_info.dw2.value); + cq_attr.dfx_info.dw3.value = + be32_to_cpu(cq_attr.dfx_info.dw3.value); + cq_attr.ci_record_gpa_at_hop_num = + be64_to_cpu(cq_attr.ci_record_gpa_at_hop_num); + + pi_ci->cq_ci_on_chip = cq_attr.dw0.bs.ci_on_chip; + pi_ci->cq_ci = cq_attr.dfx_info.dw0.bs.ci; + pi_ci->cq_load_ci = cq_attr.dfx_info.dw2.bs.load_ci; + pi_ci->cq_ci_record_gpa_at_hop_num = + cq_attr.ci_record_gpa_at_hop_num; + pi_ci->cq_last_solicited_pi = + cq_attr.dfx_info.dw2.bs.last_solicited_pi; + pi_ci->cq_pi = cq_attr.dfx_info.dw1.bs.pi; + pi_ci->cq_last_notified_pi = + cq_attr.dfx_info.dw3.bs.last_notified_pi; + } + return 0; +} + +static roce5_adm_dfx_query_cmd_proc_t g_roce5_adm_dfx_query_cmd_proc[] = { + { ROCE_CMD_GET_QPC_FROM_CACHE, roce5_dfx_query_cache_qpc }, + { ROCE_CMD_GET_QPC_FROM_HOST, roce5_dfx_query_host_qpc }, + { ROCE_CMD_GET_CQC_FROM_CACHE, roce5_dfx_query_cache_cqc }, + { ROCE_CMD_GET_CQC_FROM_HOST, roce5_dfx_query_host_cqc }, + { ROCE_CMD_GET_SRQC_FROM_CACHE, roce5_dfx_query_cache_srqc }, + { ROCE_CMD_GET_SRQC_FROM_HOST, roce5_dfx_query_host_srqc }, + { ROCE_CMD_GET_MPT_FROM_CACHE, roce5_dfx_query_cache_mpt }, + { ROCE_CMD_GET_MPT_FROM_HOST, roce5_dfx_query_host_mpt }, + { ROCE_CMD_GET_GID_FROM_CACHE, roce5_dfx_query_cache_gid }, + { ROCE_CMD_GET_QPC_CQC_PI_CI, roce5_dfx_query_pi_ci }, + { ROCE_CMD_GET_QP_COUNT, roce5_dfx_get_qp_count }, + { ROCE_CMD_GET_DEV_ALGO, roce5_dfx_get_dev_algo }, +}; + +int roce5_adm_dfx_query(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + const struct roce5_dfx_query_inbuf *inbuf = + (struct roce5_dfx_query_inbuf *)buf_in; + union roce5_dfx_query_outbuf *outbuf = + (union roce5_dfx_query_outbuf *)buf_out; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *) + g_roce5_adm_dfx_query_cmd_proc; + int elem_count = sizeof(g_roce5_adm_dfx_query_cmd_proc) / + sizeof(g_roce5_adm_dfx_query_cmd_proc[0]); + roce5_adm_dfx_query_t roce5_adm_dfx_query_func = NULL; + + if (in_size < sizeof(struct roce5_dfx_query_inbuf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, + (int)sizeof(struct roce5_dfx_query_inbuf)); + return -ENOMEM; + } + if (*out_size < sizeof(union roce5_dfx_query_outbuf)) { + ROCE_ERR( + "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, + (int)sizeof(union roce5_dfx_query_outbuf)); + return -ENOMEM; + } + ret = memset_s(buf_out, sizeof(union roce5_dfx_query_outbuf), 0, + sizeof(union roce5_dfx_query_outbuf)); + if (ret != 0) { + ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret); + return -ENOMEM; + } + *out_size = (u32)sizeof(union roce5_dfx_query_outbuf); + + if (inbuf->cmd_type >= VM_COMPAT_TEST) { + roce5_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + roce5_adm_dfx_query_func = + (roce5_adm_dfx_query_t)roce5_adm_generic_find_cmd_func( + generic_array, elem_count, inbuf->cmd_type); + if (roce5_adm_dfx_query_func == NULL) { + roce5_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + return roce5_adm_dfx_query_func(inbuf, rdev, outbuf); +} + +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c new file mode 100644 index 000000000..7c641f422 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c @@ -0,0 +1,83 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/slab.h> +#include "securec.h" +#include "roce_infra.h" +#include "roce_obj_ext.h" + +int roce5_obj_ext_init(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext) +{ + int i, ret; + + ret = memset_s(obj_ext->bufs, sizeof(void *) * ROCE_EXT_FEATURE_MAX, 0, + sizeof(void *) * ROCE_EXT_FEATURE_MAX); + if (ret != 0) { + ROCE_ERR( + "Failed to execute memset to object extension buffer, ret(%d).", + ret); + return ret; + } + + for (i = 0; i < item_num; ++i) { + if (reg_items[i].ext_id >= ROCE_EXT_FEATURE_MAX) { + ROCE_ERR("Invalid extension id(%u).", + reg_items[i].ext_id); + ret = -EINVAL; + goto err_out; + } + + if (obj_ext->bufs[reg_items[i].ext_id] != NULL) { + ROCE_ERR("Repeated register, extension id(%u).", + reg_items[i].ext_id); + ret = -EINVAL; + goto err_out; + } + + obj_ext->bufs[reg_items[i].ext_id] = + kzalloc(reg_items[i].data_size, GFP_KERNEL); + if (obj_ext->bufs[reg_items[i].ext_id] == NULL) { + ROCE_ERR("Failed alloc memory for extension."); + ret = -ENOMEM; + goto err_out; + } + + ret = reg_items[i].init_func( + parent, obj_ext->bufs[reg_items[i].ext_id]); + if (ret != 0) { + ROCE_ERR("Failed to init extension, ret(%d).", ret); + goto err_out; + } + } + + return 0; + +err_out: + if (obj_ext->bufs[reg_items[i].ext_id] != NULL) { + kfree(obj_ext->bufs[reg_items[i].ext_id]); + } + + for (--i; i >= 0; --i) { + reg_items[i].cleanup_func(parent, + obj_ext->bufs[reg_items[i].ext_id]); + kfree(obj_ext->bufs[reg_items[i].ext_id]); + } + return ret; +} + +void roce5_obj_ext_deinit(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext) +{ + int i; + for (i = 0; i < item_num; ++i) { + if (reg_items[i].ext_id >= ROCE_EXT_FEATURE_MAX || + obj_ext->bufs[reg_items[i].ext_id] == NULL) { + continue; + } + + reg_items[i].cleanup_func(parent, + obj_ext->bufs[reg_items[i].ext_id]); + kfree(obj_ext->bufs[reg_items[i].ext_id]); + } +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h new file mode 100644 index 000000000..3e5f7196c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_OBJ_EXT_H +#define ROCE_OBJ_EXT_H + +#include <linux/types.h> +#include <linux/stddef.h> + +typedef int (*ext_init_func)(void *parent, void *ext_data); +typedef void (*ext_cleanup_func)(void *parent, void *ext_data); + +enum ROCE_EXT_ID { ROCE_EXT_HYPER_ROCE, ROCE_EXT_FEATURE_MAX }; + +struct roce5_ext_reg { + u8 ext_id; + size_t data_size; + ext_init_func init_func; + ext_cleanup_func cleanup_func; +}; + +struct roce5_obj_ext { + void *bufs[ROCE_EXT_FEATURE_MAX]; +}; + +static inline void *roce5_obj_ext_get_buf(struct roce5_obj_ext *obj_ext, + enum ROCE_EXT_ID ext_id) +{ + if (ext_id >= ROCE_EXT_FEATURE_MAX) { + return NULL; + } + return obj_ext->bufs[ext_id]; +} + +int roce5_obj_ext_init(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext); +void roce5_obj_ext_deinit(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext); + +#endif /* ROCE_OBJ_EXT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c new file mode 100644 index 000000000..6cfed12be --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c @@ -0,0 +1,125 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/jiffies.h> + +#include "roce_infra.h" +#include "securec.h" +#include "roce_option.h" + +#define ROCE_WAIT_TIME_SEC 60 + +struct roce5_option *roce5_some(void *value) +{ + int ret; + + struct roce5_option *opt = + kmalloc(sizeof(struct roce5_option), GFP_KERNEL); + if (opt == NULL) { + ROCE_ERR("Failed to alloc memory."); + return NULL; + } + + ret = strcpy_s(opt->waiting_msg, ROCE_WAITING_MSG_MAX_SIZE, + "Waiting for object release..."); + if (ret != EOK) { + ROCE_ERR("Failed to init waiting message."); + kfree(opt); + return NULL; + } + + spin_lock_init(&opt->spin_lock); + kref_init(&opt->refcount); + opt->value = value; + + return opt; +} + +static void roce5_wait_release(struct kref *refcount) +{ + struct roce5_option *opt = + container_of(refcount, struct roce5_option, refcount); + + opt->value = NULL; + complete(&opt->release_comp); +} + +static void roce5_timer_print_wait_msg(struct timer_list *tl) +{ + struct roce5_option *opt = from_timer(opt, tl, msg_timer); + + ROCE_WARN("%s", opt->waiting_msg); + + mod_timer(&opt->msg_timer, jiffies + ROCE_WAIT_TIME_SEC * HZ); +} + +void roce5_option_unset(struct roce5_option *opt, char *waiting_msg, + size_t waiting_msg_size) +{ + int ret; + unsigned long flags; /* type required by kernel API */ + + if (opt == NULL) { + ROCE_ERR("Invalid parameter, opt is NULL."); + return; + } + + if (waiting_msg != NULL && waiting_msg_size > 0) { + ret = strcpy_s(opt->waiting_msg, ROCE_WAITING_MSG_MAX_SIZE, + waiting_msg); + if (ret != EOK) { + ROCE_WARN("Failed to set waiting message, ret(%d).", + ret); + } + } + + init_completion(&opt->release_comp); + + timer_setup(&opt->msg_timer, roce5_timer_print_wait_msg, 0); + mod_timer(&opt->msg_timer, jiffies + ROCE_WAIT_TIME_SEC * HZ); + + spin_lock_irqsave(&opt->spin_lock, flags); + kref_put(&opt->refcount, roce5_wait_release); + spin_unlock_irqrestore(&opt->spin_lock, flags); + + wait_for_completion(&opt->release_comp); + + del_timer_sync(&opt->msg_timer); +} + +void *roce5_option_get_value(struct roce5_option *opt) +{ + void *ans = NULL; + unsigned long flags; /* type required by kernel API */ + + if (opt == NULL) { + ROCE_ERR("Invalid parameter, opt is NULL."); + return NULL; + } + + spin_lock_irqsave(&opt->spin_lock, flags); + if (opt->value != NULL) { + kref_get(&opt->refcount); + ans = opt->value; + } + spin_unlock_irqrestore(&opt->spin_lock, flags); + + return ans; +} + +void roce5_option_put_value(struct roce5_option *opt) +{ + unsigned long flags; /* type required by kernel API */ + if (opt == NULL) { + ROCE_ERR("Invalid parameter, opt is NULL."); + return; + } + + spin_lock_irqsave(&opt->spin_lock, flags); + if (opt->value != NULL) { + kref_put(&opt->refcount, roce5_wait_release); + } + spin_unlock_irqrestore(&opt->spin_lock, flags); +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h new file mode 100644 index 000000000..dba51cd68 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_OPTION_H +#define ROCE_OPTION_H + +#include <linux/kref.h> +#include <linux/completion.h> +#include <linux/timer.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> + +#define ROCE_WAITING_MSG_MAX_SIZE 100 + +struct roce5_option { + spinlock_t spin_lock; + struct kref refcount; /* count for the value ptr. */ + struct completion release_comp; + struct timer_list msg_timer; + char waiting_msg[ROCE_WAITING_MSG_MAX_SIZE]; + void *value; +}; + +struct roce5_option *roce5_some(void *value); +void roce5_option_unset(struct roce5_option *opt, char *waiting_msg, + size_t waiting_msg_size); +void *roce5_option_get_value(struct roce5_option *opt); +void roce5_option_put_value(struct roce5_option *opt); + +#endif /* ROCE_OPTION_H */ -- 2.37.7